From 927184efee39dc45e730a60e1e92270b5e7a17d6 Mon Sep 17 00:00:00 2001 From: geffzhang Date: Thu, 1 May 2025 16:47:16 +0800 Subject: [PATCH 01/26] Refactor InvokeFunction code Refactor the InvokeFunction code by categorizing the dummy function, function callback, and MCP tool invocation logic into three distinct classes --- BotSharp.sln | 11 --- IFunctionExecutor.cs | 15 ++++ .../BotSharp.Core.MCP.csproj | 18 ---- .../BotSharpMCPExtensions.cs | 64 -------------- src/Infrastructure/BotSharp.Core.MCP/Using.cs | 18 ---- .../BotSharp.Core/BotSharp.Core.csproj | 1 + .../MCP/BotSharpMCPExtensions.cs | 35 ++++++++ .../MCP}/Helpers/AiFunctionHelper.cs | 0 .../MCP}/Hooks/MCPToolAgentHook.cs | 0 .../MCP}/Managers/McpClientManager.cs | 0 .../MCP}/Services/McpService.cs | 13 ++- .../MCP}/Settings/MCPSettings.cs | 0 .../Routing/Executor/DummyFunctionExecutor.cs | 37 ++++++++ .../Executor/FunctionCallbackExecutor.cs | 23 +++++ .../Executor/FunctionExecutorFactory.cs | 41 +++++++++ .../Routing/Executor/IFunctionExecutor.cs | 8 ++ .../Routing/Executor/MCPFunctionExecutor.cs} | 48 +++++----- .../Routing/RoutingService.InvokeFunction.cs | 88 ++++--------------- src/Infrastructure/BotSharp.Core/Using.cs | 77 ++++++++-------- src/WebStarter/WebStarter.csproj | 1 - 20 files changed, 245 insertions(+), 253 deletions(-) create mode 100644 IFunctionExecutor.cs delete mode 100644 src/Infrastructure/BotSharp.Core.MCP/BotSharp.Core.MCP.csproj delete mode 100644 src/Infrastructure/BotSharp.Core.MCP/BotSharpMCPExtensions.cs delete mode 100644 src/Infrastructure/BotSharp.Core.MCP/Using.cs create mode 100644 src/Infrastructure/BotSharp.Core/MCP/BotSharpMCPExtensions.cs rename src/Infrastructure/{BotSharp.Core.MCP => BotSharp.Core/MCP}/Helpers/AiFunctionHelper.cs (100%) rename src/Infrastructure/{BotSharp.Core.MCP => BotSharp.Core/MCP}/Hooks/MCPToolAgentHook.cs (100%) rename src/Infrastructure/{BotSharp.Core.MCP => BotSharp.Core/MCP}/Managers/McpClientManager.cs (100%) rename src/Infrastructure/{BotSharp.Core.MCP => BotSharp.Core/MCP}/Services/McpService.cs (69%) rename src/Infrastructure/{BotSharp.Core.MCP => BotSharp.Core/MCP}/Settings/MCPSettings.cs (100%) create mode 100644 src/Infrastructure/BotSharp.Core/Routing/Executor/DummyFunctionExecutor.cs create mode 100644 src/Infrastructure/BotSharp.Core/Routing/Executor/FunctionCallbackExecutor.cs create mode 100644 src/Infrastructure/BotSharp.Core/Routing/Executor/FunctionExecutorFactory.cs create mode 100644 src/Infrastructure/BotSharp.Core/Routing/Executor/IFunctionExecutor.cs rename src/Infrastructure/{BotSharp.Core.MCP/Functions/McpToolAdapter.cs => BotSharp.Core/Routing/Executor/MCPFunctionExecutor.cs} (72%) diff --git a/BotSharp.sln b/BotSharp.sln index a692f403..972dc35f 100644 --- a/BotSharp.sln +++ b/BotSharp.sln @@ -131,8 +131,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Core.Realtime", "s EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.PizzaBot.MCPServer", "tests\BotSharp.PizzaBot.MCPServer\BotSharp.PizzaBot.MCPServer.csproj", "{8D2AD45F-836A-516F-DE6A-71443CEBB18A}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Core.MCP", "src\Infrastructure\BotSharp.Core.MCP\BotSharp.Core.MCP.csproj", "{534D1DD5-9D62-115C-F230-47080D76CE52}" -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Test.ComputerUse", "tests\BotSharp.Test.ComputerUse\BotSharp.Test.ComputerUse.csproj", "{C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.OsDriver", "src\Plugins\BotSharp.Plugin.OsDriver\BotSharp.Plugin.OsDriver.csproj", "{B268E2F0-060F-8466-7D81-ABA4D735CA59}" @@ -553,14 +551,6 @@ Global {8D2AD45F-836A-516F-DE6A-71443CEBB18A}.Release|Any CPU.Build.0 = Release|Any CPU {8D2AD45F-836A-516F-DE6A-71443CEBB18A}.Release|x64.ActiveCfg = Release|Any CPU {8D2AD45F-836A-516F-DE6A-71443CEBB18A}.Release|x64.Build.0 = Release|Any CPU - {534D1DD5-9D62-115C-F230-47080D76CE52}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {534D1DD5-9D62-115C-F230-47080D76CE52}.Debug|Any CPU.Build.0 = Debug|Any CPU - {534D1DD5-9D62-115C-F230-47080D76CE52}.Debug|x64.ActiveCfg = Debug|Any CPU - {534D1DD5-9D62-115C-F230-47080D76CE52}.Debug|x64.Build.0 = Debug|Any CPU - {534D1DD5-9D62-115C-F230-47080D76CE52}.Release|Any CPU.ActiveCfg = Release|Any CPU - {534D1DD5-9D62-115C-F230-47080D76CE52}.Release|Any CPU.Build.0 = Release|Any CPU - {534D1DD5-9D62-115C-F230-47080D76CE52}.Release|x64.ActiveCfg = Release|Any CPU - {534D1DD5-9D62-115C-F230-47080D76CE52}.Release|x64.Build.0 = Release|Any CPU {C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Debug|Any CPU.Build.0 = Debug|Any CPU {C19D9AC1-97DD-8E65-E8DB-D295A095AA2D}.Debug|x64.ActiveCfg = Debug|Any CPU @@ -672,7 +662,6 @@ Global {AF329442-B48E-4B48-A18A-1C869D1BA6F5} = {D5293208-2BEF-42FC-A64C-5954F61720BA} {781F1465-365C-0F22-1775-25025DAFA4C7} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749} {8D2AD45F-836A-516F-DE6A-71443CEBB18A} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC} - {534D1DD5-9D62-115C-F230-47080D76CE52} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749} {C19D9AC1-97DD-8E65-E8DB-D295A095AA2D} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC} {B268E2F0-060F-8466-7D81-ABA4D735CA59} = {51AFE054-AE99-497D-A593-69BAEFB5106F} {970BE341-9AC8-99A5-6572-E703C1E02FCB} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749} diff --git a/IFunctionExecutor.cs b/IFunctionExecutor.cs new file mode 100644 index 00000000..a3380e70 --- /dev/null +++ b/IFunctionExecutor.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BotSharp.Core.Routing +{ + public interface IFunctionExecutor + { + public Task Execute(RoleDialogModel message); + + public Task GetIndication(RoleDialogModel message); + } +} diff --git a/src/Infrastructure/BotSharp.Core.MCP/BotSharp.Core.MCP.csproj b/src/Infrastructure/BotSharp.Core.MCP/BotSharp.Core.MCP.csproj deleted file mode 100644 index 5f513dd9..00000000 --- a/src/Infrastructure/BotSharp.Core.MCP/BotSharp.Core.MCP.csproj +++ /dev/null @@ -1,18 +0,0 @@ - - - - $(TargetFramework) - $(LangVersion) - $(BotSharpVersion) - $(GeneratePackageOnBuild) - $(SolutionDir)packages - - - - - - - - - - diff --git a/src/Infrastructure/BotSharp.Core.MCP/BotSharpMCPExtensions.cs b/src/Infrastructure/BotSharp.Core.MCP/BotSharpMCPExtensions.cs deleted file mode 100644 index 8760935d..00000000 --- a/src/Infrastructure/BotSharp.Core.MCP/BotSharpMCPExtensions.cs +++ /dev/null @@ -1,64 +0,0 @@ -using BotSharp.Core.MCP.Functions; -using BotSharp.Core.MCP.Hooks; -using BotSharp.Core.MCP.Managers; -using BotSharp.Core.MCP.Services; -using BotSharp.Core.MCP.Settings; -using Microsoft.Extensions.Configuration; -using ModelContextProtocol.Client; - -namespace BotSharp.Core.MCP; - -public static class BotSharpMcpExtensions -{ - /// - /// Add mcp - /// - /// - /// - /// - public static IServiceCollection AddBotSharpMCP(this IServiceCollection services, IConfiguration config) - { - services.AddScoped(); - var settings = config.GetSection("MCP").Get(); - services.AddScoped(provider => settings); - - if (settings != null && settings.Enabled && !settings.McpServerConfigs.IsNullOrEmpty()) - { - var clientManager = new McpClientManager(settings); - services.AddSingleton(clientManager); - - foreach (var server in settings.McpServerConfigs) - { - RegisterFunctionCall(services, server, clientManager) - .ConfigureAwait(false) - .GetAwaiter() - .GetResult(); - } - - // Register hooks - services.AddScoped(); - } - return services; - } - - private static async Task RegisterFunctionCall(IServiceCollection services, McpServerConfigModel server, McpClientManager clientManager) - { - try - { - var client = await clientManager.GetMcpClientAsync(server.Id); - var tools = await client.ListToolsAsync(); - - foreach (var tool in tools) - { - services.AddScoped(provider => tool); - - services.AddScoped(provider => - { - var funcTool = new McpToolAdapter(provider, server.Name, tool, clientManager); - return funcTool; - }); - } - } - catch { } - } -} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core.MCP/Using.cs b/src/Infrastructure/BotSharp.Core.MCP/Using.cs deleted file mode 100644 index 0c5dcf37..00000000 --- a/src/Infrastructure/BotSharp.Core.MCP/Using.cs +++ /dev/null @@ -1,18 +0,0 @@ -global using System; -global using System.Collections.Generic; -global using System.Linq; -global using System.Threading.Tasks; -global using Microsoft.Extensions.DependencyInjection; - -global using BotSharp.Abstraction.Agents; -global using BotSharp.Abstraction.Agents.Enums; -global using BotSharp.Abstraction.Agents.Models; -global using BotSharp.Abstraction.Agents.Settings; -global using BotSharp.Abstraction.Conversations; -global using BotSharp.Abstraction.Conversations.Models; -global using BotSharp.Abstraction.Functions; -global using BotSharp.Abstraction.Functions.Models; -global using BotSharp.Abstraction.Utilities; -global using BotSharp.Abstraction.MCP.Models; -global using BotSharp.Abstraction.MCP.Services; - diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 5c27be38..0deff57d 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -221,6 +221,7 @@ + diff --git a/src/Infrastructure/BotSharp.Core/MCP/BotSharpMCPExtensions.cs b/src/Infrastructure/BotSharp.Core/MCP/BotSharpMCPExtensions.cs new file mode 100644 index 00000000..b3c4e6de --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/MCP/BotSharpMCPExtensions.cs @@ -0,0 +1,35 @@ +using BotSharp.Core.MCP.Hooks; +using BotSharp.Core.MCP.Managers; +using BotSharp.Core.MCP.Services; +using BotSharp.Core.MCP.Settings; +using Microsoft.Extensions.Configuration; + +namespace BotSharp.Core.MCP; + +public static class BotSharpMcpExtensions +{ + /// + /// Add mcp + /// + /// + /// + /// + public static IServiceCollection AddBotSharpMCP(this IServiceCollection services, IConfiguration config) + { + var settings = config.GetSection("MCP").Get(); + services.AddScoped(provider => settings); + + if (settings != null && settings.Enabled && !settings.McpServerConfigs.IsNullOrEmpty()) + { + services.AddScoped(); + + var clientManager = new McpClientManager(settings); + services.AddScoped(provider => clientManager); + + // Register hooks + services.AddScoped(); + } + return services; + } + +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core.MCP/Helpers/AiFunctionHelper.cs b/src/Infrastructure/BotSharp.Core/MCP/Helpers/AiFunctionHelper.cs similarity index 100% rename from src/Infrastructure/BotSharp.Core.MCP/Helpers/AiFunctionHelper.cs rename to src/Infrastructure/BotSharp.Core/MCP/Helpers/AiFunctionHelper.cs diff --git a/src/Infrastructure/BotSharp.Core.MCP/Hooks/MCPToolAgentHook.cs b/src/Infrastructure/BotSharp.Core/MCP/Hooks/MCPToolAgentHook.cs similarity index 100% rename from src/Infrastructure/BotSharp.Core.MCP/Hooks/MCPToolAgentHook.cs rename to src/Infrastructure/BotSharp.Core/MCP/Hooks/MCPToolAgentHook.cs diff --git a/src/Infrastructure/BotSharp.Core.MCP/Managers/McpClientManager.cs b/src/Infrastructure/BotSharp.Core/MCP/Managers/McpClientManager.cs similarity index 100% rename from src/Infrastructure/BotSharp.Core.MCP/Managers/McpClientManager.cs rename to src/Infrastructure/BotSharp.Core/MCP/Managers/McpClientManager.cs diff --git a/src/Infrastructure/BotSharp.Core.MCP/Services/McpService.cs b/src/Infrastructure/BotSharp.Core/MCP/Services/McpService.cs similarity index 69% rename from src/Infrastructure/BotSharp.Core.MCP/Services/McpService.cs rename to src/Infrastructure/BotSharp.Core/MCP/Services/McpService.cs index bbc2c3f4..3bff4442 100644 --- a/src/Infrastructure/BotSharp.Core.MCP/Services/McpService.cs +++ b/src/Infrastructure/BotSharp.Core/MCP/Services/McpService.cs @@ -1,5 +1,7 @@ +using BotSharp.Core.MCP.Managers; using BotSharp.Core.MCP.Settings; using Microsoft.Extensions.Logging; +using ModelContextProtocol.Client; namespace BotSharp.Core.MCP.Services; @@ -7,13 +9,16 @@ public class McpService : IMcpService { private readonly IServiceProvider _services; private readonly ILogger _logger; + private readonly McpClientManager _mcpClientManager; public McpService( IServiceProvider services, - ILogger logger) + ILogger logger, + McpClientManager mcpClient) { _services = services; _logger = logger; + _mcpClientManager = mcpClient; } public IEnumerable GetServerConfigs() @@ -24,9 +29,9 @@ public class McpService : IMcpService foreach (var config in configs) { - var tools = _services.GetServices() - .Where(x => x.Provider == config.Name) - .Select(x => x.Name); + var tools = _mcpClientManager.GetMcpClientAsync(config.Id) + .Result.ListToolsAsync() + .Result.Select(x=> x.Name); options.Add(new McpServerOptionModel { diff --git a/src/Infrastructure/BotSharp.Core.MCP/Settings/MCPSettings.cs b/src/Infrastructure/BotSharp.Core/MCP/Settings/MCPSettings.cs similarity index 100% rename from src/Infrastructure/BotSharp.Core.MCP/Settings/MCPSettings.cs rename to src/Infrastructure/BotSharp.Core/MCP/Settings/MCPSettings.cs diff --git a/src/Infrastructure/BotSharp.Core/Routing/Executor/DummyFunctionExecutor.cs b/src/Infrastructure/BotSharp.Core/Routing/Executor/DummyFunctionExecutor.cs new file mode 100644 index 00000000..d075e185 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Routing/Executor/DummyFunctionExecutor.cs @@ -0,0 +1,37 @@ +using BotSharp.Abstraction.Templating; + +namespace BotSharp.Core.Routing.Executor; + +public class DummyFunctionExecutor: IFunctionExecutor +{ + private FunctionDef functionDef; + private readonly IServiceProvider _services; + + public DummyFunctionExecutor(FunctionDef function, IServiceProvider services) + { + functionDef = function; + _services = services; + } + + + public async Task ExecuteAsync(RoleDialogModel message) + { + var render = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + + var dict = new Dictionary(); + foreach (var item in state.GetStates()) + { + dict[item.Key] = item.Value; + } + + var text = render.Render(functionDef.Output, dict); + message.Content = text; + return true; + } + + public async Task GetIndicatorAsync(RoleDialogModel message) + { + return "Running"; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Routing/Executor/FunctionCallbackExecutor.cs b/src/Infrastructure/BotSharp.Core/Routing/Executor/FunctionCallbackExecutor.cs new file mode 100644 index 00000000..939f4fb3 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Routing/Executor/FunctionCallbackExecutor.cs @@ -0,0 +1,23 @@ +using BotSharp.Abstraction.Functions; + +namespace BotSharp.Core.Routing.Executor; + +public class FunctionCallbackExecutor : IFunctionExecutor +{ + IFunctionCallback functionCallback; + + public FunctionCallbackExecutor(IFunctionCallback functionCallback) + { + this.functionCallback = functionCallback; + } + + public async Task ExecuteAsync(RoleDialogModel message) + { + return await functionCallback.Execute(message); + } + + public async Task GetIndicatorAsync(RoleDialogModel message) + { + return await functionCallback.GetIndication(message); + } +} diff --git a/src/Infrastructure/BotSharp.Core/Routing/Executor/FunctionExecutorFactory.cs b/src/Infrastructure/BotSharp.Core/Routing/Executor/FunctionExecutorFactory.cs new file mode 100644 index 00000000..c0c8bec3 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Routing/Executor/FunctionExecutorFactory.cs @@ -0,0 +1,41 @@ +using BotSharp.Abstraction.Functions; + +namespace BotSharp.Core.Routing.Executor; + +internal class FunctionExecutorFactory +{ + public static IFunctionExecutor Create(string functionName, Agent agent, IFunctionCallback functioncall, IServiceProvider serviceProvider) + { + if(functioncall != null) + { + return new FunctionCallbackExecutor(functioncall); + } + + var funDef = agent?.Functions?.FirstOrDefault(x => x.Name == functionName); + if (funDef != null) + { + if (!string.IsNullOrWhiteSpace(funDef?.Output)) + { + return new DummyFunctionExecutor(funDef,serviceProvider); + } + } + else + { + funDef = agent?.SecondaryFunctions?.FirstOrDefault(x => x.Name == functionName); + if (funDef != null) + { + if (!string.IsNullOrWhiteSpace(funDef?.Output)) + { + return new DummyFunctionExecutor(funDef, serviceProvider); + } + else + { + var mcpServerId = agent?.McpTools?.Where(x => x.Functions.Any(y => y.Name == funDef.Name)) + .FirstOrDefault().ServerId; + return new MCPFunctionExecutor(mcpServerId, functionName, serviceProvider); + } + } + } + return null; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Routing/Executor/IFunctionExecutor.cs b/src/Infrastructure/BotSharp.Core/Routing/Executor/IFunctionExecutor.cs new file mode 100644 index 00000000..4ba2e69f --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Routing/Executor/IFunctionExecutor.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Core.Routing.Executor; + +public interface IFunctionExecutor +{ + public Task ExecuteAsync(RoleDialogModel message); + + public Task GetIndicatorAsync(RoleDialogModel message); +} diff --git a/src/Infrastructure/BotSharp.Core.MCP/Functions/McpToolAdapter.cs b/src/Infrastructure/BotSharp.Core/Routing/Executor/MCPFunctionExecutor.cs similarity index 72% rename from src/Infrastructure/BotSharp.Core.MCP/Functions/McpToolAdapter.cs rename to src/Infrastructure/BotSharp.Core/Routing/Executor/MCPFunctionExecutor.cs index 0e433597..0ed6f0ea 100644 --- a/src/Infrastructure/BotSharp.Core.MCP/Functions/McpToolAdapter.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Executor/MCPFunctionExecutor.cs @@ -1,46 +1,34 @@ -using System.Text.Json; using BotSharp.Core.MCP.Managers; using ModelContextProtocol.Client; -namespace BotSharp.Core.MCP.Functions; +namespace BotSharp.Core.Routing.Executor; -public class McpToolAdapter : IFunctionCallback +public class MCPFunctionExecutor: IFunctionExecutor { - private readonly string _provider; - private readonly McpClientTool _tool; private readonly McpClientManager _clientManager; + private string mcpServer; + private string funcName; private readonly IServiceProvider _services; - public McpToolAdapter( - IServiceProvider services, - string serverName, - McpClientTool tool, - McpClientManager client) - { - _services = services ?? throw new ArgumentNullException(nameof(services)); - _tool = tool ?? throw new ArgumentNullException(nameof(tool)); - _clientManager = client ?? throw new ArgumentNullException(nameof(client)); - _provider = serverName; + public MCPFunctionExecutor(string mcpserver, string functionName, IServiceProvider services) + { + _services = services; + this.mcpServer = mcpserver; + this.funcName = functionName; + _clientManager = services.GetRequiredService(); } - public string Provider => _provider; - public string Name => _tool.Name; - - public async Task Execute(RoleDialogModel message) + public async Task ExecuteAsync(RoleDialogModel message) { try { // Convert arguments to dictionary format expected by mcpdotnet - Dictionary argDict = JsonToDictionary(message.FunctionArgs); - var currentAgentId = message.CurrentAgentId; - var agentService = _services.GetRequiredService(); - var agent = await agentService.LoadAgent(currentAgentId); - var serverId = agent.McpTools.Where(t => t.Functions.Any(f => f.Name == Name)).FirstOrDefault().ServerId; + Dictionary argDict = JsonToDictionary(message.FunctionArgs); - var client = await _clientManager.GetMcpClientAsync(serverId); + var client = await _clientManager.GetMcpClientAsync(mcpServer); // Call the tool through mcpdotnet - var result = await client.CallToolAsync(_tool.Name, !argDict.IsNullOrEmpty() ? argDict : []); + var result = await client.CallToolAsync(funcName, !argDict.IsNullOrEmpty() ? argDict : []); // Extract the text content from the result var json = string.Join("\n", result.Content.Where(c => c.Type == "text").Select(c => c.Text)); @@ -51,11 +39,17 @@ public class McpToolAdapter : IFunctionCallback } catch (Exception ex) { - message.Content = $"Error when calling tool {Name} of MCP server {Provider}. {ex.Message}"; + message.Content = $"Error when calling tool {funcName} of MCP server {mcpServer}. {ex.Message}"; return false; } } + public async Task GetIndicatorAsync(RoleDialogModel message) + { + return message.Indication ?? string.Empty; + } + + private static Dictionary JsonToDictionary(string? json) { if (string.IsNullOrEmpty(json)) diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs index 8c37536c..126453e8 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Functions; using BotSharp.Abstraction.Templating; +using BotSharp.Core.Routing.Executor; namespace BotSharp.Core.Routing; @@ -8,22 +9,22 @@ public partial class RoutingService public async Task InvokeFunction(string name, RoleDialogModel message) { var function = _services.GetServices().FirstOrDefault(x => x.Name == name); + + var currentAgentId = message.CurrentAgentId; + var agentService = _services.GetRequiredService(); + var agent = await agentService.GetAgent(currentAgentId); - var isFillDummyContent = false; - var dummyFuncResponse = string.Empty; - if (function == null) + IFunctionExecutor funcExecutor = FunctionExecutorFactory.Create(name, agent, function, _services); + + if (funcExecutor == null) { - dummyFuncResponse = await GetDummyFunctionOutput(name, message); - isFillDummyContent = !string.IsNullOrEmpty(dummyFuncResponse); - if (!isFillDummyContent) - { - message.StopCompletion = true; - message.Content = $"Can't find function implementation of {name}."; - _logger.LogError(message.Content); - return false; - } + message.StopCompletion = true; + message.Content = $"Can't find function implementation of {name}."; + _logger.LogError(message.Content); + return false; } + // Clone message var clonedMessage = RoleDialogModel.From(message); clonedMessage.FunctionName = name; @@ -34,23 +35,13 @@ public partial class RoutingService var progressService = _services.GetService(); - // Before executing functions - if (!isFillDummyContent) - { - clonedMessage.Indication = await function.GetIndication(message); - } - else - { - clonedMessage.Indication = "Running"; - } + clonedMessage.Indication = await funcExecutor.GetIndicatorAsync(message); if (progressService?.OnFunctionExecuting != null) { await progressService.OnFunctionExecuting(clonedMessage); } - - var agentService = _services.GetRequiredService(); - var agent = await agentService.GetAgent(clonedMessage.CurrentAgentId); + foreach (var hook in hooks) { hook.SetAgent(agent); @@ -61,19 +52,7 @@ public partial class RoutingService try { - if (clonedMessage.Handled) - { - clonedMessage.Content = clonedMessage.Content; - } - else if (!isFillDummyContent) - { - result = await function.Execute(clonedMessage); - } - else - { - clonedMessage.Content = dummyFuncResponse; - result = true; - } + result = await funcExecutor.ExecuteAsync(clonedMessage); // After functions have been executed foreach (var hook in hooks) @@ -111,41 +90,6 @@ public partial class RoutingService message.Content = JsonSerializer.Serialize(message.Data); } - // Save to Storage as well - /*if (!message.StopCompletion && message.FunctionName != "route_to_agent") - { - var storage = _services.GetRequiredService(); - storage.Append(Context.ConversationId, message); - }*/ - return result; } - - private async Task GetDummyFunctionOutput(string functionName, RoleDialogModel message) - { - if (string.IsNullOrEmpty(message.CurrentAgentId)) - { - return null; - } - - var agentService = _services.GetRequiredService(); - var agent = await agentService.GetAgent(message.CurrentAgentId); - var found = agent?.Functions?.FirstOrDefault(x => x.Name == functionName); - if (string.IsNullOrWhiteSpace(found?.Output)) - { - return null; - } - - var render = _services.GetRequiredService(); - var state = _services.GetRequiredService(); - - var dict = new Dictionary(); - foreach (var item in state.GetStates()) - { - dict[item.Key] = item.Value; - } - - var text = render.Render(found.Output, dict); - return text; - } } diff --git a/src/Infrastructure/BotSharp.Core/Using.cs b/src/Infrastructure/BotSharp.Core/Using.cs index 0afac090..00d0a224 100644 --- a/src/Infrastructure/BotSharp.Core/Using.cs +++ b/src/Infrastructure/BotSharp.Core/Using.cs @@ -1,51 +1,52 @@ -global using System; -global using System.Collections.Generic; -global using System.Text; -global using System.Threading.Tasks; -global using System.Linq; -global using System.Text.Json; -global using System.Net.Mime; -global using System.Net.Http; -global using System.Threading; -global using Microsoft.Extensions.DependencyInjection; -global using Microsoft.Extensions.Logging; -global using EntityFrameworkCore.BootKit; -global using BotSharp.Abstraction.Routing; -global using BotSharp.Abstraction.Plugins; global using BotSharp.Abstraction.Agents; -global using BotSharp.Abstraction.Conversations; -global using BotSharp.Abstraction.Crontab.Models; -global using BotSharp.Abstraction.Users; -global using BotSharp.Abstraction.Roles; -global using BotSharp.Abstraction.Roles.Models; -global using BotSharp.Abstraction.Utilities; -global using BotSharp.Abstraction.Conversations.Models; -global using BotSharp.Abstraction.Agents.Settings; -global using BotSharp.Abstraction.Conversations.Settings; global using BotSharp.Abstraction.Agents.Enums; global using BotSharp.Abstraction.Agents.Models; +global using BotSharp.Abstraction.Agents.Settings; +global using BotSharp.Abstraction.Conversations; +global using BotSharp.Abstraction.Conversations.Models; +global using BotSharp.Abstraction.Conversations.Settings; +global using BotSharp.Abstraction.Crontab.Models; +global using BotSharp.Abstraction.Files; +global using BotSharp.Abstraction.Files.Enums; +global using BotSharp.Abstraction.Files.Models; +global using BotSharp.Abstraction.Files.Utilities; global using BotSharp.Abstraction.Functions.Models; +global using BotSharp.Abstraction.Infrastructures.Events; +global using BotSharp.Abstraction.Knowledges.Models; +global using BotSharp.Abstraction.Loggers.Services; +global using BotSharp.Abstraction.MCP.Models; +global using BotSharp.Abstraction.MCP.Services; +global using BotSharp.Abstraction.Messaging.Enums; +global using BotSharp.Abstraction.Plugins; global using BotSharp.Abstraction.Repositories; global using BotSharp.Abstraction.Repositories.Filters; -global using BotSharp.Abstraction.Translation; -global using BotSharp.Abstraction.Files; -global using BotSharp.Abstraction.Files.Models; -global using BotSharp.Abstraction.Files.Enums; -global using BotSharp.Abstraction.Files.Utilities; -global using BotSharp.Abstraction.Translation.Attributes; -global using BotSharp.Abstraction.Messaging.Enums; -global using BotSharp.Abstraction.Knowledges.Models; +global using BotSharp.Abstraction.Roles; +global using BotSharp.Abstraction.Roles.Models; +global using BotSharp.Abstraction.Routing; global using BotSharp.Abstraction.SideCar.Attributes; -global using BotSharp.Abstraction.Statistics.Models; global using BotSharp.Abstraction.Statistics.Enums; +global using BotSharp.Abstraction.Statistics.Models; global using BotSharp.Abstraction.Statistics.Services; -global using BotSharp.Abstraction.Loggers.Services; -global using BotSharp.Abstraction.Infrastructures.Events; -global using BotSharp.Core.Repository; -global using BotSharp.Core.Routing; +global using BotSharp.Abstraction.Translation; +global using BotSharp.Abstraction.Translation.Attributes; +global using BotSharp.Abstraction.Users; +global using BotSharp.Abstraction.Utilities; global using BotSharp.Core.Agents.Services; global using BotSharp.Core.Conversations.Services; global using BotSharp.Core.Infrastructures; -global using BotSharp.Core.Users.Services; +global using BotSharp.Core.Loggers.Services; +global using BotSharp.Core.Repository; global using BotSharp.Core.Statistics.Services; -global using BotSharp.Core.Loggers.Services; \ No newline at end of file +global using BotSharp.Core.Users.Services; +global using EntityFrameworkCore.BootKit; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Logging; +global using System; +global using System.Collections.Generic; +global using System.Linq; +global using System.Net.Http; +global using System.Net.Mime; +global using System.Text; +global using System.Text.Json; +global using System.Threading; +global using System.Threading.Tasks; diff --git a/src/WebStarter/WebStarter.csproj b/src/WebStarter/WebStarter.csproj index 6020f043..bc33290f 100644 --- a/src/WebStarter/WebStarter.csproj +++ b/src/WebStarter/WebStarter.csproj @@ -36,7 +36,6 @@ - From aac1ae1195e4f711e15712e7ab2bcf6046c88cd1 Mon Sep 17 00:00:00 2001 From: geffzhang Date: Thu, 1 May 2025 16:54:42 +0800 Subject: [PATCH 02/26] upgrade ModelContextProtocol 0.1.0-preview11 --- Directory.Packages.props | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 7cdfc1a6..bca2caa7 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -113,8 +113,8 @@ - - + + From 5087b46e663817a136ec20ff5c798b7048aace55 Mon Sep 17 00:00:00 2001 From: geffzhang Date: Thu, 1 May 2025 17:14:02 +0800 Subject: [PATCH 03/26] rename MCPFunctionExecutor --- .../BotSharp.Core/Routing/Executor/FunctionExecutorFactory.cs | 2 +- .../Executor/{MCPFunctionExecutor.cs => MCPToolExecutor.cs} | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename src/Infrastructure/BotSharp.Core/Routing/Executor/{MCPFunctionExecutor.cs => MCPToolExecutor.cs} (96%) diff --git a/src/Infrastructure/BotSharp.Core/Routing/Executor/FunctionExecutorFactory.cs b/src/Infrastructure/BotSharp.Core/Routing/Executor/FunctionExecutorFactory.cs index c0c8bec3..3fb89509 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Executor/FunctionExecutorFactory.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Executor/FunctionExecutorFactory.cs @@ -32,7 +32,7 @@ internal class FunctionExecutorFactory { var mcpServerId = agent?.McpTools?.Where(x => x.Functions.Any(y => y.Name == funDef.Name)) .FirstOrDefault().ServerId; - return new MCPFunctionExecutor(mcpServerId, functionName, serviceProvider); + return new MCPToolExecutor(mcpServerId, functionName, serviceProvider); } } } diff --git a/src/Infrastructure/BotSharp.Core/Routing/Executor/MCPFunctionExecutor.cs b/src/Infrastructure/BotSharp.Core/Routing/Executor/MCPToolExecutor.cs similarity index 96% rename from src/Infrastructure/BotSharp.Core/Routing/Executor/MCPFunctionExecutor.cs rename to src/Infrastructure/BotSharp.Core/Routing/Executor/MCPToolExecutor.cs index 0ed6f0ea..f7625b48 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Executor/MCPFunctionExecutor.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Executor/MCPToolExecutor.cs @@ -3,14 +3,14 @@ using ModelContextProtocol.Client; namespace BotSharp.Core.Routing.Executor; -public class MCPFunctionExecutor: IFunctionExecutor +public class MCPToolExecutor: IFunctionExecutor { private readonly McpClientManager _clientManager; private string mcpServer; private string funcName; private readonly IServiceProvider _services; - public MCPFunctionExecutor(string mcpserver, string functionName, IServiceProvider services) + public MCPToolExecutor(string mcpserver, string functionName, IServiceProvider services) { _services = services; this.mcpServer = mcpserver; From 3dda7afc7c7448c7bfb1551b79d242d044a88b3a Mon Sep 17 00:00:00 2001 From: "aden.chen" Date: Mon, 12 May 2025 10:37:25 +0800 Subject: [PATCH 04/26] Add null-check for `agent` in `StreamingLogHook` --- src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs index 720f3fd0..2bc9c9e6 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs @@ -293,7 +293,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR var routing = _services.GetRequiredService(); var agentId = routing.Context.GetCurrentAgentId(); var agent = await _agentService.GetAgent(agentId); - + if (agent == null) return; var input = new ContentLogInputModel() { Name = agent.Name, From 710a1e19c2292232e510c5ac3981bf17b72e3003 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 12 May 2025 15:26:05 -0500 Subject: [PATCH 05/26] integrate --- .../Controllers/ConversationController.cs | 59 +++++++------------ .../View/ConversationViewModel.cs | 4 ++ .../Models/Realtime/RealtimeSessionBody.cs | 2 +- 3 files changed, 26 insertions(+), 39 deletions(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 5e180f22..a5a96a12 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -143,55 +143,29 @@ public class ConversationController : ControllerBase { var service = _services.GetRequiredService(); var userService = _services.GetRequiredService(); + var settings = _services.GetRequiredService(); + var (isAdmin, user) = await userService.IsAdminUser(_user.Id); - if (user == null) - { - return null; - } var filter = new ConversationFilter { Id = conversationId, - UserId = !isAdmin ? user.Id : null, + UserId = !isAdmin ? user?.Id : null, IsLoadLatestStates = isLoadStates }; + var conversations = await service.GetConversations(filter); - if (conversations.Items.IsNullOrEmpty()) - { - return null; - } + var conv = !conversations.Items.IsNullOrEmpty() + ? ConversationViewModel.FromSession(conversations.Items.First()) + : new(); - var result = ConversationViewModel.FromSession(conversations.Items.First()); - var state = _services.GetRequiredService(); - user = await userService.GetUser(result.User.Id); - result.User = UserViewModel.FromUser(user); + user = !string.IsNullOrEmpty(conv?.User?.Id) + ? await userService.GetUser(conv.User.Id) + : null; - return result; - } - - [HttpPost("/conversation/summary")] - public async Task GetConversationSummary([FromBody] ConversationSummaryModel input) - { - var service = _services.GetRequiredService(); - return await service.GetConversationSummary(input.ConversationIds); - } - - [HttpGet("/conversation/{conversationId}/user")] - public async Task GetConversationUser([FromRoute] string conversationId) - { - var service = _services.GetRequiredService(); - var conversations = await service.GetConversations(new ConversationFilter - { - Id = conversationId - }); - - var userService = _services.GetRequiredService(); - var conversation = conversations?.Items?.FirstOrDefault(); - var userId = conversation == null ? _user.Id : conversation.UserId; - var user = await userService.GetUser(userId); if (user == null) { - return new UserViewModel + user = new User { Id = _user.Id, UserName = _user.UserName, @@ -202,7 +176,16 @@ public class ConversationController : ControllerBase }; } - return UserViewModel.FromUser(user); + conv.User = UserViewModel.FromUser(user); + conv.IsRealtimeEnabled = settings?.Assemblies?.Contains("BotSharp.Core.Realtime") ?? false; + return conv; + } + + [HttpPost("/conversation/summary")] + public async Task GetConversationSummary([FromBody] ConversationSummaryModel input) + { + var service = _services.GetRequiredService(); + return await service.GetConversationSummary(input.ConversationIds); } [HttpPut("/conversation/{conversationId}/update-title")] diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/View/ConversationViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/View/ConversationViewModel.cs index 6300f661..e7110f44 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/View/ConversationViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/View/ConversationViewModel.cs @@ -1,9 +1,13 @@ using BotSharp.Abstraction.Conversations.Dtos; +using System.Text.Json.Serialization; namespace BotSharp.OpenAPI.ViewModels.Conversations; public class ConversationViewModel : ConversationDto { + [JsonPropertyName("is_realtime_enabled")] + public bool IsRealtimeEnabled { get; set; } + public static ConversationViewModel FromSession(Conversation sess) { return new ConversationViewModel diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs index a5ed764e..f767a6f1 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs @@ -76,7 +76,7 @@ public class RealtimeSessionTurnDetection public string Type { get; set; } = "semantic_vad"; [JsonPropertyName("eagerness")] - public string eagerness { get;set; } = "auto"; + public string Eagerness { get;set; } = "auto"; } public class InputAudioTranscription From b2bd14856efb41822db1cb5facd11172cef77f8a Mon Sep 17 00:00:00 2001 From: Mackinnon Buck Date: Mon, 12 May 2025 14:33:58 -0700 Subject: [PATCH 06/26] Update MEAI version to 9.4.4-preview.1.25259.16 --- Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index d9fee6f8..a2b0a119 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -60,7 +60,7 @@ - + From c837dd6611025edfaf82c601b0076c371805e789 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 13 May 2025 11:45:19 -0500 Subject: [PATCH 07/26] temp save --- .../BotSharp.Core/BotSharp.Core.csproj | 10 + .../BotSharp.Core/Functions/GetLocationFn.cs | 25 +++ .../BotSharp.Core/Functions/GetWeatherFn.cs | 56 +++++ .../AsyncWebsocketDataResultEnumerator.cs | 1 + .../Session/LlmRealtimeSession.cs | 9 +- .../functions/get_location.json | 20 ++ .../functions/get_weather.json | 19 ++ .../Realtime/RealTimeCompletionProvider.cs | 208 +++++++++++++++--- .../Realtime/RealTimeCompletionProvider.cs | 2 +- 9 files changed, 316 insertions(+), 34 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Core/Functions/GetLocationFn.cs create mode 100644 src/Infrastructure/BotSharp.Core/Functions/GetWeatherFn.cs create mode 100644 src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_location.json create mode 100644 src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_weather.json diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 89619484..de2c8909 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -96,6 +96,8 @@ + + @@ -204,6 +206,14 @@ PreserveNewest + + + + PreserveNewest + + + PreserveNewest + diff --git a/src/Infrastructure/BotSharp.Core/Functions/GetLocationFn.cs b/src/Infrastructure/BotSharp.Core/Functions/GetLocationFn.cs new file mode 100644 index 00000000..cabfbdbb --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Functions/GetLocationFn.cs @@ -0,0 +1,25 @@ +using BotSharp.Abstraction.Functions; +using BotSharp.Abstraction.Options; + +namespace BotSharp.Core.Functions; + +public class GetLocationFn : IFunctionCallback +{ + private readonly IServiceProvider _services; + + public GetLocationFn(IServiceProvider services) + { + _services = services; + } + + public string Name => "get_location"; + public string Indication => "Finding location"; + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs, BotSharpOptions.defaultJsonOptions); + + message.Content = $"There are a lot of fun events here in {args.City}"; + return true; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Functions/GetWeatherFn.cs b/src/Infrastructure/BotSharp.Core/Functions/GetWeatherFn.cs new file mode 100644 index 00000000..759bc68c --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Functions/GetWeatherFn.cs @@ -0,0 +1,56 @@ +using BotSharp.Abstraction.Functions; +using BotSharp.Abstraction.Models; +using BotSharp.Abstraction.Options; +using BotSharp.Abstraction.SideCar; +using System.Text.Json.Serialization; + +namespace BotSharp.Core.Functions; + +public class GetWeatherFn : IFunctionCallback +{ + private readonly IServiceProvider _services; + + public GetWeatherFn(IServiceProvider services) + { + _services = services; + } + + public string Name => "get_weather"; + public string Indication => "Querying weather"; + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs, BotSharpOptions.defaultJsonOptions); + + var sidecar = _services.GetService(); + var states = GetSideCarStates(); + + var userMessage = $"Please find the information at location {args.City}, {args.State}"; + var response = await sidecar.SendMessage(BuiltInAgentId.Chatbot, userMessage, states: states); + message.Content = $"It is a sunny day {response.Content}."; + return true; + } + + private List GetSideCarStates() + { + var sideCarStates = new List() + { + new("channel", "email") + }; + return sideCarStates; + } +} + +class Location +{ + [JsonPropertyName("city")] + public string? City { get; set; } + + [JsonPropertyName("state")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? State { get; set; } + + [JsonPropertyName("county")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? County { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs index 692e7220..af2abf8f 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs @@ -44,6 +44,7 @@ internal class AsyncWebsocketDataResultEnumerator : IAsyncEnumerator headers, CancellationToken cancellationToken = default) + public async Task ConnectAsync(Uri uri, Dictionary? headers = null, CancellationToken cancellationToken = default) { _webSocket?.Dispose(); _webSocket = new ClientWebSocket(); - foreach (var header in headers) + if (!headers.IsNullOrEmpty()) { - _webSocket.Options.SetRequestHeader(header.Key, header.Value); + foreach (var header in headers) + { + _webSocket.Options.SetRequestHeader(header.Key, header.Value); + } } await _webSocket.ConnectAsync(uri, cancellationToken); diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_location.json b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_location.json new file mode 100644 index 00000000..b0716218 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_location.json @@ -0,0 +1,20 @@ +{ + "name": "get_location", + "description": "Get location information for user.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "visibility_expression": "{% if states.channel == 'email' %}visible{% endif %}", + "description": "The location city that user wants to know about." + }, + "county": { + "type": "string", + "visibility_expression": "{% if states.channel != 'email' %}visible{% endif %}", + "description": "The location county that user wants to know about." + } + }, + "required": [ "city", "county" ] + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_weather.json b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_weather.json new file mode 100644 index 00000000..bdb679a2 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_weather.json @@ -0,0 +1,19 @@ +{ + "name": "get_weather", + "description": "Get weather information for user.", + "visibility_expression": "{% if states.channel != 'email' %}visible{% endif %}", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "The city where the user wants to get weather information." + }, + "state": { + "type": "string", + "description": "The state where the user wants to get weather information." + } + }, + "required": [ "city", "state" ] + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs index 98896ac3..d901f981 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -1,8 +1,15 @@ +using BotSharp.Abstraction.Options; +using BotSharp.Abstraction.Realtime.Models.Session; +using BotSharp.Core.Session; using GenerativeAI; using GenerativeAI.Core; using GenerativeAI.Live; using GenerativeAI.Live.Extensions; using GenerativeAI.Types; +using GenerativeAI.Types.Converters; +using Google.Ai.Generativelanguage.V1Beta2; +using Google.Api; +using System.Threading; namespace BotSharp.Plugin.GoogleAi.Providers.Realtime; @@ -18,14 +25,18 @@ public class GoogleRealTimeProvider : IRealTimeCompletion private readonly ILogger _logger; private List renderedInstructions = []; + private LlmRealtimeSession _session; + private readonly BotSharpOptions _botsharpOptions; private readonly GoogleAiSettings _settings; public GoogleRealTimeProvider( IServiceProvider services, GoogleAiSettings settings, + BotSharpOptions botSharpOptions, ILogger logger) { _settings = settings; + _botsharpOptions = botSharpOptions; _services = services; _logger = logger; } @@ -66,8 +77,48 @@ public class GoogleRealTimeProvider : IRealTimeCompletion _onInputAudioTranscriptionCompleted = onInputAudioTranscriptionCompleted; _onUserInterrupted = onUserInterrupted; + var settingsService = _services.GetRequiredService(); var realtimeModelSettings = _services.GetRequiredService(); + _model = realtimeModelSettings.Model; + var modelSettings = settingsService.GetSetting(Provider, _model); + + //if (_session != null) + //{ + // _session.Dispose(); + //} + + //_session = new LlmRealtimeSession(_services, new ChatSessionOptions + //{ + // JsonOptions = new JsonSerializerOptions + // { + // PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + // PropertyNameCaseInsensitive = true, + // Converters = { new JsonStringEnumConverter(), new DateOnlyJsonConverter(), new TimeOnlyJsonConverter() }, + // DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + // TypeInfoResolver = TypesSerializerContext.Default, + // UnknownTypeHandling = JsonUnknownTypeHandling.JsonElement, + + // } + //}); + + //await _session.ConnectAsync( + // uri: new Uri($"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent?key={modelSettings.ApiKey}"), + // cancellationToken: CancellationToken.None); + + ////await UpdateSession(conn, true); + + //_ = ReceiveMessage( + // conn, + // onModelReady, + // onModelAudioDeltaReceived, + // onModelAudioResponseDone, + // onModelAudioTranscriptDone, + // onModelResponseDone, + // onConversationItemCreated, + // onInputAudioTranscriptionCompleted, + // onUserInterrupted); + var client = ProviderHelper.GetGeminiClient(Provider, _model, _services); _chatClient = client.CreateGenerativeModel(_model); @@ -75,7 +126,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion config: new GenerationConfig { ResponseModalities = [Modality.AUDIO], - }, + }, systemInstruction: "You are a helpful assistant.", logger: _logger); @@ -84,37 +135,107 @@ public class GoogleRealTimeProvider : IRealTimeCompletion await _client.ConnectAsync(false); } + + private async Task ReceiveMessage( + RealtimeHubConnection conn, + Action onModelReady, + Action onModelAudioDeltaReceived, + Action onModelAudioResponseDone, + Action onModelAudioTranscriptDone, + Action> onModelResponseDone, + Action onConversationItemCreated, + Action onUserAudioTranscriptionCompleted, + Action onInterruptionDetected) + { + await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None)) + { + var receivedText = update?.RawResponse; + Console.WriteLine($"Received text: {receivedText}"); + + if (string.IsNullOrEmpty(receivedText)) + { + continue; + } + + + } + + _session.Dispose(); + } + + public async Task Disconnect() { + //if (_session != null) + //{ + // await _session.Disconnect(); + //} + if (_client != null) + { await _client.DisconnectAsync(); + } } public async Task AppenAudioBuffer(string message) { await _client.SendAudioAsync(Convert.FromBase64String(message)); + + //await SendEventToModel(new BidiClientPayload + //{ + // RealtimeInput = new() + // { + // MediaChunks = [ new() { Data = message, MimeType = "audio/pcm; rate=16000" } ] + // } + //}); } public async Task AppenAudioBuffer(ArraySegment data, int length) { var buffer = data.AsSpan(0, length).ToArray(); - await _client.SendAudioAsync(buffer,"audio/pcm;rate=16000"); + await _client.SendAudioAsync(buffer, "audio/pcm; rate=16000"); + + //await SendEventToModel(new BidiClientPayload + //{ + // RealtimeInput = new() + // { + // MediaChunks = [new() { Data = Convert.ToBase64String(buffer), MimeType = "audio/pcm; rate=16000" }] + // } + //}); } public async Task TriggerModelInference(string? instructions = null) { + var content = !string.IsNullOrWhiteSpace(instructions) + ? new Content(instructions, AgentRole.User) + : null; + await _client.SendClientContentAsync(new BidiGenerateContentClientContent() { + Turns = content != null ? [content] : null, TurnComplete = true, }); + + + + //await SendEventToModel(new BidiClientPayload + //{ + // ClientContent = new() + // { + // Turns = content != null ? [content] : null, + // TurnComplete = true + // } + //}); } public async Task CancelModelResponse() { + } public async Task RemoveConversationItem(string itemId) { + } private Task AttachEvents(MultiModalLiveClient client) @@ -236,6 +357,10 @@ public class GoogleRealTimeProvider : IRealTimeCompletion public async Task SendEventToModel(object message) { //todo Send Audio Chunks to Model, Botsharp RealTime Implementation seems to be incomplete + + //if (_session == null) return; + + //await _session.SendEventToModel(message); } public async Task UpdateSession(RealtimeHubConnection conn, bool isInit = false) @@ -246,13 +371,13 @@ public class GoogleRealTimeProvider : IRealTimeCompletion var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(conn.CurrentAgentId); - var (prompt, request) = PrepareOptions(_chatClient, agent, new List()); + var (prompt, request) = PrepareOptions(agent, []); var config = request.GenerationConfig; //Output Modality can either be text or audio if (config != null) { - config.ResponseModalities = new List([Modality.AUDIO]); + config.ResponseModalities = [Modality.AUDIO]; var words = new List(); HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent))); @@ -270,10 +395,10 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { Name = x.Name ?? string.Empty, Description = x.Description ?? string.Empty, + Parameters = x.Parameters != null + ? JsonSerializer.Deserialize(JsonSerializer.Serialize(x.Parameters)) + : null }; - fn.Parameters = x.Parameters != null - ? JsonSerializer.Deserialize(JsonSerializer.Serialize(x.Parameters)) - : null; return fn; }).ToArray(); @@ -282,33 +407,39 @@ public class GoogleRealTimeProvider : IRealTimeCompletion if (_settings.Gemini.UseGoogleSearch) { - if (request.Tools == null) - request.Tools = new List(); + request.Tools ??= []; request.Tools.Add(new Tool() { GoogleSearch = new GoogleSearchTool() }); } - // if(request.Tools.Count == 0) - // request.Tools = null; - // config.MaxOutputTokens = null; - await _client.SendSetupAsync(new BidiGenerateContentSetup() { GenerationConfig = config, Model = Model.ToModelId(), SystemInstruction = request.SystemInstruction, - Tools = request.Tools?.ToArray(), + //Tools = request.Tools?.ToArray(), }); + //await SendEventToModel(new BidiClientPayload + //{ + // Setup = new BidiGenerateContentSetup() + // { + // GenerationConfig = config, + // Model = $"models/{_model}", + // SystemInstruction = new Content(agent.Instruction, AgentRole.System), + // //Tools = request.Tools?.ToArray(), + // } + //}); + return prompt; } public async Task InsertConversationItem(RoleDialogModel message) { - if (_client == null) - throw new Exception("Client is not initialized"); + //if (_client == null) + // throw new Exception("Client is not initialized"); if (message.Role == AgentRole.Function) { var function = new FunctionResponse() @@ -321,13 +452,38 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { FunctionResponses = [function] }); + + //await SendEventToModel(new BidiClientPayload + //{ + // ToolResponse = new() + // { + // FunctionResponses = [function] + // } + //}); } else if (message.Role == AgentRole.Assistant) { + //await SendEventToModel(new BidiClientPayload + //{ + // ClientContent = new() + // { + // Turns = [new Content(message.Content, AgentRole.Model)], + // TurnComplete = true + // } + //}); } else if (message.Role == AgentRole.User) { await _client.SentTextAsync(message.Content); + + //await SendEventToModel(new BidiClientPayload + //{ + // ClientContent = new() + // { + // Turns = [new Content(message.Content, AgentRole.User)], + // TurnComplete = true + // } + //}); } else { @@ -335,33 +491,24 @@ public class GoogleRealTimeProvider : IRealTimeCompletion } } - public Task> OnResponsedDone(RealtimeHubConnection conn, string response) + public async Task> OnResponsedDone(RealtimeHubConnection conn, string response) { - throw new NotImplementedException(""); + return []; } - public Task OnConversationItemCreated(RealtimeHubConnection conn, string response) + public async Task OnConversationItemCreated(RealtimeHubConnection conn, string response) { - return Task.FromResult(new RoleDialogModel(AgentRole.User, response)); + return await Task.FromResult(new RoleDialogModel(AgentRole.User, response)); } - private (string, GenerateContentRequest) PrepareOptions(GenerativeModel aiModel, Agent agent, + private (string, GenerateContentRequest) PrepareOptions(Agent agent, List conversations) { var agentService = _services.GetRequiredService(); var googleSettings = _settings; renderedInstructions = []; - // Add settings - aiModel.UseGoogleSearch = googleSettings.Gemini.UseGoogleSearch; - aiModel.UseGrounding = googleSettings.Gemini.UseGrounding; - - aiModel.FunctionCallingBehaviour = new FunctionCallingBehaviour() - { - AutoCallFunction = false - }; - // Assembly messages var contents = new List(); var tools = new List(); @@ -458,6 +605,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion var maxTokens = int.TryParse(state.GetState("max_tokens"), out var tokens) ? tokens : agent.LlmConfig?.MaxOutputTokens ?? LlmConstant.DEFAULT_MAX_OUTPUT_TOKEN; + var request = new GenerateContentRequest { SystemInstruction = !systemPrompts.IsNullOrEmpty() diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index b32c0e1c..3257e9bf 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -671,7 +671,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return outputs; } - public async Task OnUserAudioTranscriptionCompleted(RealtimeHubConnection conn, string response) + private async Task OnUserAudioTranscriptionCompleted(RealtimeHubConnection conn, string response) { var data = JsonSerializer.Deserialize(response); return new RoleDialogModel(AgentRole.User, data.Transcript) From da9c3c6ab0273a52564f2b85380b1455aa1c97a9 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 13 May 2025 15:43:08 -0500 Subject: [PATCH 08/26] temp save --- .../MLTasks/IRealTimeCompletion.cs | 18 +- .../Services/RealtimeHub.cs | 6 +- .../AsyncWebsocketDataResultEnumerator.cs | 4 +- .../BotSharp.Plugin.GoogleAI.csproj | 2 +- .../Models/Realtime/RealtimeServerResponse.cs | 61 +++ .../Realtime/RealTimeCompletionProvider.cs | 365 ++++++++++-------- .../Realtime/RealTimeCompletionProvider.cs | 52 +-- .../BotSharp.LLM.Tests/GoogleRealTimeTests.cs | 15 +- 8 files changed, 319 insertions(+), 204 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index 11174356..315f4f5a 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -1,4 +1,6 @@ using BotSharp.Abstraction.Realtime.Models; +using System; +using static System.Runtime.InteropServices.JavaScript.JSType; namespace BotSharp.Abstraction.MLTasks; @@ -10,14 +12,14 @@ public interface IRealTimeCompletion Task Connect( RealtimeHubConnection conn, - Action onModelReady, - Action onModelAudioDeltaReceived, - Action onModelAudioResponseDone, - Action onAudioTranscriptDone, - Action> onModelResponseDone, - Action onConversationItemCreated, - Action onInputAudioTranscriptionCompleted, - Action onInterruptionDetected); + Func onModelReady, + Func onModelAudioDeltaReceived, + Func onModelAudioResponseDone, + Func onModelAudioTranscriptDone, + Func, Task> onModelResponseDone, + Func onConversationItemCreated, + Func onInputAudioTranscriptionDone, + Func onInterruptionDetected); Task AppenAudioBuffer(string message); Task AppenAudioBuffer(ArraySegment data, int length); diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 60729758..93363878 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -76,8 +76,8 @@ public class RealtimeHub : IRealtimeHub { var data = _conn.OnModelAudioResponseDone(); await (responseToUser?.Invoke(data) ?? Task.CompletedTask); - }, - onAudioTranscriptDone: async transcript => + }, + onModelAudioTranscriptDone: async transcript => { }, @@ -117,7 +117,7 @@ public class RealtimeHub : IRealtimeHub { }, - onInputAudioTranscriptionCompleted: async message => + onInputAudioTranscriptionDone: async message => { // append input audio transcript to conversation dialogs.Add(message); diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs index af2abf8f..548850e9 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs @@ -44,7 +44,9 @@ internal class AsyncWebsocketDataResultEnumerator : IAsyncEnumerator + $(TargetFramework) diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs new file mode 100644 index 00000000..25cf59e4 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs @@ -0,0 +1,61 @@ +using GenerativeAI.Types; + +namespace BotSharp.Plugin.GoogleAI.Models.Realtime; + +internal class RealtimeServerResponse +{ + [JsonPropertyName("setupComplete")] + public RealtimeGenerateContentSetupComplete? SetupComplete { get; set; } + + [JsonPropertyName("serverContent")] + public RealtimeGenerateContentServerContent? ServerContent { get; set; } + + [JsonPropertyName("usageMetadata")] + public RealtimeUsageMetaData? UsageMetaData { get; set; } +} + + +internal class RealtimeGenerateContentSetupComplete { } + +internal class RealtimeGenerateContentServerContent +{ + [JsonPropertyName("turnComplete")] + public bool? TurnComplete { get; set; } + + [JsonPropertyName("generationComplete")] + public bool? GenerationComplete { get; set; } + + [JsonPropertyName("interrupted")] + public bool? Interrupted { get; set; } + + [JsonPropertyName("modelTurn")] + public Content? ModelTurn { get; set; } +} + +internal class RealtimeUsageMetaData +{ + [JsonPropertyName("promptTokenCount")] + public int? PromptTokenCount { get; set; } + + [JsonPropertyName("responseTokenCount")] + public int? ResponseTokenCount { get; set; } + + [JsonPropertyName("totalTokenCount")] + public int? TotalTokenCount { get; set; } + + [JsonPropertyName("promptTokensDetails")] + public List? PromptTokensDetails { get; set; } + + [JsonPropertyName("responseTokensDetails")] + public List? ResponseTokensDetails { get; set; } +} + + +internal class RealtimeTokenDetail +{ + [JsonPropertyName("modality")] + public string? Modality { get; set; } + + [JsonPropertyName("tokenCount")] + public int? TokenCount { get; set; } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs index d901f981..fe3480c7 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -1,6 +1,7 @@ using BotSharp.Abstraction.Options; using BotSharp.Abstraction.Realtime.Models.Session; using BotSharp.Core.Session; +using BotSharp.Plugin.GoogleAI.Models.Realtime; using GenerativeAI; using GenerativeAI.Core; using GenerativeAI.Live; @@ -9,6 +10,7 @@ using GenerativeAI.Types; using GenerativeAI.Types.Converters; using Google.Ai.Generativelanguage.V1Beta2; using Google.Api; +using System; using System.Threading; namespace BotSharp.Plugin.GoogleAi.Providers.Realtime; @@ -29,6 +31,17 @@ public class GoogleRealTimeProvider : IRealTimeCompletion private readonly BotSharpOptions _botsharpOptions; private readonly GoogleAiSettings _settings; + private const string DEFAULT_MIME_TYPE = "audio/pcm;rate=16000"; + + private readonly JsonSerializerOptions _jsonOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + PropertyNameCaseInsensitive = true, + Converters = { new JsonStringEnumConverter(), new DateOnlyJsonConverter(), new TimeOnlyJsonConverter() }, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + UnknownTypeHandling = JsonUnknownTypeHandling.JsonElement + }; + public GoogleRealTimeProvider( IServiceProvider services, GoogleAiSettings settings, @@ -47,25 +60,26 @@ public class GoogleRealTimeProvider : IRealTimeCompletion } private RealtimeHubConnection _conn; - private Action _onModelReady; - private Action _onModelAudioDeltaReceived; - private Action _onModelAudioResponseDone; - private Action _onModelAudioTranscriptDone; - private Action> _onModelResponseDone; - private Action _onConversationItemCreated; - private Action _onInputAudioTranscriptionCompleted; - private Action _onUserInterrupted; + private Func _onModelReady; + private Func _onModelAudioDeltaReceived; + private Func _onModelAudioResponseDone; + private Func _onModelAudioTranscriptDone; + private Func, Task> _onModelResponseDone; + private Func _onConversationItemCreated; + private Func _onInputAudioTranscriptionDone; + private Func _onUserInterrupted; - public async Task Connect(RealtimeHubConnection conn, - Action onModelReady, - Action onModelAudioDeltaReceived, - Action onModelAudioResponseDone, - Action onModelAudioTranscriptDone, - Action> onModelResponseDone, - Action onConversationItemCreated, - Action onInputAudioTranscriptionCompleted, - Action onUserInterrupted) + public async Task Connect( + RealtimeHubConnection conn, + Func onModelReady, + Func onModelAudioDeltaReceived, + Func onModelAudioResponseDone, + Func onModelAudioTranscriptDone, + Func, Task> onModelResponseDone, + Func onConversationItemCreated, + Func onInputAudioTranscriptionDone, + Func onInterruptionDetected) { _conn = conn; _onModelReady = onModelReady; @@ -74,8 +88,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion _onModelAudioTranscriptDone = onModelAudioTranscriptDone; _onModelResponseDone = onModelResponseDone; _onConversationItemCreated = onConversationItemCreated; - _onInputAudioTranscriptionCompleted = onInputAudioTranscriptionCompleted; - _onUserInterrupted = onUserInterrupted; + _onInputAudioTranscriptionDone = onInputAudioTranscriptionDone; + _onUserInterrupted = onInterruptionDetected; var settingsService = _services.GetRequiredService(); var realtimeModelSettings = _services.GetRequiredService(); @@ -83,81 +97,114 @@ public class GoogleRealTimeProvider : IRealTimeCompletion _model = realtimeModelSettings.Model; var modelSettings = settingsService.GetSetting(Provider, _model); - //if (_session != null) - //{ - // _session.Dispose(); - //} + if (_session != null) + { + _session.Dispose(); + } - //_session = new LlmRealtimeSession(_services, new ChatSessionOptions - //{ - // JsonOptions = new JsonSerializerOptions + _session = new LlmRealtimeSession(_services, new ChatSessionOptions + { + JsonOptions = _jsonOptions + }); + + await _session.ConnectAsync( + uri: new Uri($"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent?key={modelSettings.ApiKey}"), + cancellationToken: CancellationToken.None); + + await onModelReady(); + + _ = ReceiveMessage( + conn, + onModelReady, + onModelAudioDeltaReceived, + onModelAudioResponseDone, + onModelAudioTranscriptDone, + onModelResponseDone, + onConversationItemCreated, + onInputAudioTranscriptionDone, + onInterruptionDetected); + + + //var client = ProviderHelper.GetGeminiClient(Provider, _model, _services); + //_chatClient = client.CreateGenerativeModel(_model); + //_client = _chatClient.CreateMultiModalLiveClient( + // config: new GenerationConfig // { - // PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - // PropertyNameCaseInsensitive = true, - // Converters = { new JsonStringEnumConverter(), new DateOnlyJsonConverter(), new TimeOnlyJsonConverter() }, - // DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - // TypeInfoResolver = TypesSerializerContext.Default, - // UnknownTypeHandling = JsonUnknownTypeHandling.JsonElement, + // ResponseModalities = [Modality.AUDIO], + // }, + // systemInstruction: "You are a helpful assistant.", + // logger: _logger); - // } - //}); + //await AttachEvents(_client); - //await _session.ConnectAsync( - // uri: new Uri($"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent?key={modelSettings.ApiKey}"), - // cancellationToken: CancellationToken.None); - - ////await UpdateSession(conn, true); - - //_ = ReceiveMessage( - // conn, - // onModelReady, - // onModelAudioDeltaReceived, - // onModelAudioResponseDone, - // onModelAudioTranscriptDone, - // onModelResponseDone, - // onConversationItemCreated, - // onInputAudioTranscriptionCompleted, - // onUserInterrupted); - - - var client = ProviderHelper.GetGeminiClient(Provider, _model, _services); - _chatClient = client.CreateGenerativeModel(_model); - _client = _chatClient.CreateMultiModalLiveClient( - config: new GenerationConfig - { - ResponseModalities = [Modality.AUDIO], - }, - systemInstruction: "You are a helpful assistant.", - logger: _logger); - - await AttachEvents(_client); - - await _client.ConnectAsync(false); + //await _client.ConnectAsync(false); } - private async Task ReceiveMessage( RealtimeHubConnection conn, - Action onModelReady, - Action onModelAudioDeltaReceived, - Action onModelAudioResponseDone, - Action onModelAudioTranscriptDone, - Action> onModelResponseDone, - Action onConversationItemCreated, - Action onUserAudioTranscriptionCompleted, - Action onInterruptionDetected) + Func onModelReady, + Func onModelAudioDeltaReceived, + Func onModelAudioResponseDone, + Func onModelAudioTranscriptDone, + Func, Task> onModelResponseDone, + Func onConversationItemCreated, + Func onInputAudioTranscriptionCompleted, + Func onInterruptionDetected) { await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None)) { var receivedText = update?.RawResponse; - Console.WriteLine($"Received text: {receivedText}"); - if (string.IsNullOrEmpty(receivedText)) { continue; } - + Console.WriteLine($"Received text: {receivedText}"); + try + { + var response = JsonSerializer.Deserialize(receivedText, _jsonOptions); + + if (response == null) + { + continue; + } + + if (response.SetupComplete != null) + { + _logger.LogInformation($"Session setup completed."); + } + else if (response.ServerContent != null) + { + if (response.ServerContent.ModelTurn != null) + { + _logger.LogInformation($"Model audio delta received."); + var parts = response.ServerContent.ModelTurn.Parts; + if (!parts.IsNullOrEmpty()) + { + foreach (var part in parts) + { + if (!string.IsNullOrEmpty(part.InlineData?.Data)) + { + await onModelAudioDeltaReceived(part.InlineData.Data, string.Empty); + } + } + } + } + else if (response.ServerContent.GenerationComplete == true) + { + _logger.LogInformation($"Model generation completed."); + } + else if (response.ServerContent.TurnComplete == true) + { + _logger.LogInformation($"Model turn completed."); + } + } + } + catch (Exception ex) + { + _logger.LogError(ex, $"Error when deserializing server response."); + continue; + } } _session.Dispose(); @@ -166,42 +213,42 @@ public class GoogleRealTimeProvider : IRealTimeCompletion public async Task Disconnect() { - //if (_session != null) - //{ - // await _session.Disconnect(); - //} - - if (_client != null) + if (_session != null) { - await _client.DisconnectAsync(); + await _session.Disconnect(); } + + //if (_client != null) + //{ + // await _client.DisconnectAsync(); + //} } public async Task AppenAudioBuffer(string message) { - await _client.SendAudioAsync(Convert.FromBase64String(message)); + //await _client.SendAudioAsync(Convert.FromBase64String(message)); - //await SendEventToModel(new BidiClientPayload - //{ - // RealtimeInput = new() - // { - // MediaChunks = [ new() { Data = message, MimeType = "audio/pcm; rate=16000" } ] - // } - //}); + await SendEventToModel(new BidiClientPayload + { + RealtimeInput = new() + { + MediaChunks = [new() { Data = message, MimeType = DEFAULT_MIME_TYPE }] + } + }); } public async Task AppenAudioBuffer(ArraySegment data, int length) { var buffer = data.AsSpan(0, length).ToArray(); - await _client.SendAudioAsync(buffer, "audio/pcm; rate=16000"); + //await _client.SendAudioAsync(buffer, "audio/pcm;rate=16000"); - //await SendEventToModel(new BidiClientPayload - //{ - // RealtimeInput = new() - // { - // MediaChunks = [new() { Data = Convert.ToBase64String(buffer), MimeType = "audio/pcm; rate=16000" }] - // } - //}); + await SendEventToModel(new BidiClientPayload + { + RealtimeInput = new() + { + MediaChunks = [new() { Data = Convert.ToBase64String(buffer), MimeType = DEFAULT_MIME_TYPE }] + } + }); } public async Task TriggerModelInference(string? instructions = null) @@ -210,22 +257,20 @@ public class GoogleRealTimeProvider : IRealTimeCompletion ? new Content(instructions, AgentRole.User) : null; - await _client.SendClientContentAsync(new BidiGenerateContentClientContent() - { - Turns = content != null ? [content] : null, - TurnComplete = true, - }); - - - - //await SendEventToModel(new BidiClientPayload + //await _client.SendClientContentAsync(new BidiGenerateContentClientContent() //{ - // ClientContent = new() - // { - // Turns = content != null ? [content] : null, - // TurnComplete = true - // } + // Turns = content != null ? [content] : null, + // TurnComplete = true, //}); + + await SendEventToModel(new BidiClientPayload + { + ClientContent = new() + { + Turns = content != null ? [content] : null, + TurnComplete = true + } + }); } public async Task CancelModelResponse() @@ -276,7 +321,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion client.TextChunkReceived += (sender, e) => { - _onInputAudioTranscriptionCompleted(new RoleDialogModel(AgentRole.Assistant, e.Text)); + _onInputAudioTranscriptionDone(new RoleDialogModel(AgentRole.Assistant, e.Text)); }; client.GenerationInterrupted += (sender, e) => @@ -358,9 +403,9 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { //todo Send Audio Chunks to Model, Botsharp RealTime Implementation seems to be incomplete - //if (_session == null) return; + if (_session == null) return; - //await _session.SendEventToModel(message); + await _session.SendEventToModel(message); } public async Task UpdateSession(RealtimeHubConnection conn, bool isInit = false) @@ -414,25 +459,25 @@ public class GoogleRealTimeProvider : IRealTimeCompletion }); } - await _client.SendSetupAsync(new BidiGenerateContentSetup() - { - GenerationConfig = config, - Model = Model.ToModelId(), - SystemInstruction = request.SystemInstruction, - //Tools = request.Tools?.ToArray(), - }); - - //await SendEventToModel(new BidiClientPayload + //await _client.SendSetupAsync(new BidiGenerateContentSetup() //{ - // Setup = new BidiGenerateContentSetup() - // { - // GenerationConfig = config, - // Model = $"models/{_model}", - // SystemInstruction = new Content(agent.Instruction, AgentRole.System), - // //Tools = request.Tools?.ToArray(), - // } + // GenerationConfig = config, + // Model = Model.ToModelId(), + // SystemInstruction = request.SystemInstruction, + // //Tools = request.Tools?.ToArray(), //}); + await SendEventToModel(new BidiClientPayload + { + Setup = new BidiGenerateContentSetup() + { + GenerationConfig = config, + Model = Model.ToModelId(), + SystemInstruction = request.SystemInstruction, + Tools = [] + } + }); + return prompt; } @@ -448,42 +493,42 @@ public class GoogleRealTimeProvider : IRealTimeCompletion Response = JsonNode.Parse(message.Content ?? "{}") }; - await _client.SendToolResponseAsync(new BidiGenerateContentToolResponse() - { - FunctionResponses = [function] - }); - - //await SendEventToModel(new BidiClientPayload + //await _client.SendToolResponseAsync(new BidiGenerateContentToolResponse() //{ - // ToolResponse = new() - // { - // FunctionResponses = [function] - // } + // FunctionResponses = [function] //}); + + await SendEventToModel(new BidiClientPayload + { + ToolResponse = new() + { + FunctionResponses = [function] + } + }); } else if (message.Role == AgentRole.Assistant) { - //await SendEventToModel(new BidiClientPayload - //{ - // ClientContent = new() - // { - // Turns = [new Content(message.Content, AgentRole.Model)], - // TurnComplete = true - // } - //}); + await SendEventToModel(new BidiClientPayload + { + ClientContent = new() + { + Turns = [new Content(message.Content, AgentRole.Model)], + TurnComplete = true + } + }); } else if (message.Role == AgentRole.User) { - await _client.SentTextAsync(message.Content); + //await _client.SentTextAsync(message.Content); - //await SendEventToModel(new BidiClientPayload - //{ - // ClientContent = new() - // { - // Turns = [new Content(message.Content, AgentRole.User)], - // TurnComplete = true - // } - //}); + await SendEventToModel(new BidiClientPayload + { + ClientContent = new() + { + Turns = [new Content(message.Content, AgentRole.User)], + TurnComplete = true + } + }); } else { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 3257e9bf..dcb82164 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -30,14 +30,14 @@ public class RealTimeCompletionProvider : IRealTimeCompletion public async Task Connect( RealtimeHubConnection conn, - Action onModelReady, - Action onModelAudioDeltaReceived, - Action onModelAudioResponseDone, - Action onModelAudioTranscriptDone, - Action> onModelResponseDone, - Action onConversationItemCreated, - Action onInputAudioTranscriptionCompleted, - Action onInterruptionDetected) + Func onModelReady, + Func onModelAudioDeltaReceived, + Func onModelAudioResponseDone, + Func onModelAudioTranscriptDone, + Func, Task> onModelResponseDone, + Func onConversationItemCreated, + Func onInputAudioTranscriptionDone, + Func onInterruptionDetected) { var settingsService = _services.GetRequiredService(); var realtimeModelSettings = _services.GetRequiredService(); @@ -72,7 +72,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion onModelAudioTranscriptDone, onModelResponseDone, onConversationItemCreated, - onInputAudioTranscriptionCompleted, + onInputAudioTranscriptionDone, onInterruptionDetected); } @@ -144,14 +144,14 @@ public class RealTimeCompletionProvider : IRealTimeCompletion private async Task ReceiveMessage( RealtimeHubConnection conn, - Action onModelReady, - Action onModelAudioDeltaReceived, - Action onModelAudioResponseDone, - Action onModelAudioTranscriptDone, - Action> onModelResponseDone, - Action onConversationItemCreated, - Action onUserAudioTranscriptionCompleted, - Action onInterruptionDetected) + Func onModelReady, + Func onModelAudioDeltaReceived, + Func onModelAudioResponseDone, + Func onModelAudioTranscriptDone, + Func, Task> onModelResponseDone, + Func onConversationItemCreated, + Func onInputAudioTranscriptionDone, + Func onInterruptionDetected) { await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None)) { @@ -175,7 +175,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion else if (response.Type == "session.created") { _logger.LogInformation($"{response.Type}: {receivedText}"); - onModelReady(); + await onModelReady(); } else if (response.Type == "session.updated") { @@ -189,7 +189,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion { _logger.LogInformation($"{response.Type}: {receivedText}"); var data = JsonSerializer.Deserialize(receivedText); - onModelAudioTranscriptDone(data.Transcript); + await onModelAudioTranscriptDone(data.Transcript); } else if (response.Type == "response.audio.delta") { @@ -197,13 +197,13 @@ public class RealTimeCompletionProvider : IRealTimeCompletion if (audio?.Delta != null) { _logger.LogDebug($"{response.Type}: {receivedText}"); - onModelAudioDeltaReceived(audio.Delta, audio.ItemId); + await onModelAudioDeltaReceived(audio.Delta, audio.ItemId); } } else if (response.Type == "response.audio.done") { _logger.LogInformation($"{response.Type}: {receivedText}"); - onModelAudioResponseDone(); + await onModelAudioResponseDone(); } else if (response.Type == "response.done") { @@ -213,14 +213,14 @@ public class RealTimeCompletionProvider : IRealTimeCompletion { if (data.StatusDetails.Type == "incomplete" && data.StatusDetails.Reason == "max_output_tokens") { - onInterruptionDetected(); + await onInterruptionDetected(); await TriggerModelInference("Response user concisely"); } } else { var messages = await OnResponsedDone(conn, receivedText); - onModelResponseDone(messages); + await onModelResponseDone(messages); } } else if (response.Type == "conversation.item.created") @@ -228,7 +228,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion _logger.LogInformation($"{response.Type}: {receivedText}"); var data = JsonSerializer.Deserialize(receivedText); - onConversationItemCreated(receivedText); + await onConversationItemCreated(receivedText); } else if (response.Type == "conversation.item.input_audio_transcription.completed") { @@ -237,14 +237,14 @@ public class RealTimeCompletionProvider : IRealTimeCompletion var message = await OnUserAudioTranscriptionCompleted(conn, receivedText); if (!string.IsNullOrEmpty(message.Content)) { - onUserAudioTranscriptionCompleted(message); + await onInputAudioTranscriptionDone(message); } } else if (response.Type == "input_audio_buffer.speech_started") { _logger.LogInformation($"{response.Type}: {receivedText}"); // Handle user interuption - onInterruptionDetected(); + await onInterruptionDetected(); } else if (response.Type == "input_audio_buffer.speech_stopped") { diff --git a/tests/BotSharp.LLM.Tests/GoogleRealTimeTests.cs b/tests/BotSharp.LLM.Tests/GoogleRealTimeTests.cs index 8c9eac4c..061f0545 100644 --- a/tests/BotSharp.LLM.Tests/GoogleRealTimeTests.cs +++ b/tests/BotSharp.LLM.Tests/GoogleRealTimeTests.cs @@ -40,11 +40,16 @@ namespace BotSharp.Plugin.Google.Core var realTimeCompleter = services.BuildServiceProvider().GetService(); realTimeCompleter.SetModelName(GoogleAIModels.Gemini2FlashExp); bool modelReady = false; - await realTimeCompleter.Connect(new RealtimeHubConnection(), () => { modelReady = true; }, - (s, s1) => { Console.WriteLine(s); }, () => { }, (s) => { Console.WriteLine(s); }, - (list => { Console.WriteLine(list); }), - (s => { Console.WriteLine(s); }), - (model => { Console.WriteLine(model); }), (() => { Console.WriteLine("UserInterrupted"); })); + await realTimeCompleter.Connect( + new RealtimeHubConnection(), + async () => { modelReady = true; }, + async (s, s1) => { Console.WriteLine(s); }, + async () => { }, + async (s) => { Console.WriteLine(s); }, + async list => { Console.WriteLine(list); }, + async s => { Console.WriteLine(s); }, + async model => { Console.WriteLine(model); }, + async () => { Console.WriteLine("UserInterrupted"); }); Thread.Sleep(1000); modelReady.ShouldBeTrue(); From d9fe6da0990917fbb31564b6edba9a207b658a79 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 13 May 2025 17:33:23 -0500 Subject: [PATCH 09/26] temp save --- .../Models/Realtime/RealtimeClientPayload.cs | 18 +++ .../Realtime/RealtimeGenerateContentSetup.cs | 26 ++++ .../Models/Realtime/RealtimeServerResponse.cs | 12 ++ .../Realtime/RealTimeCompletionProvider.cs | 125 +++++++++++++++--- .../Realtime/RealTimeCompletionProvider.cs | 2 +- 5 files changed, 161 insertions(+), 22 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeClientPayload.cs create mode 100644 src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeGenerateContentSetup.cs diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeClientPayload.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeClientPayload.cs new file mode 100644 index 00000000..57921595 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeClientPayload.cs @@ -0,0 +1,18 @@ +using GenerativeAI.Types; + +namespace BotSharp.Plugin.GoogleAI.Models.Realtime; + +internal class RealtimeClientPayload +{ + [JsonPropertyName("setup")] + public RealtimeGenerateContentSetup? Setup { get; set; } + + [JsonPropertyName("clientContent")] + public BidiGenerateContentClientContent? ClientContent { get; set; } + + [JsonPropertyName("realtimeInput")] + public BidiGenerateContentRealtimeInput? RealtimeInput { get; set; } + + [JsonPropertyName("toolResponse")] + public BidiGenerateContentToolResponse? ToolResponse { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeGenerateContentSetup.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeGenerateContentSetup.cs new file mode 100644 index 00000000..c72e8cb6 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeGenerateContentSetup.cs @@ -0,0 +1,26 @@ +using GenerativeAI.Types; + +namespace BotSharp.Plugin.GoogleAI.Models.Realtime; + +internal class RealtimeGenerateContentSetup +{ + [JsonPropertyName("model")] + public string? Model { get; set; } + + [JsonPropertyName("generationConfig")] + public GenerationConfig? GenerationConfig { get; set; } + + [JsonPropertyName("systemInstruction")] + public Content? SystemInstruction { get; set; } + + [JsonPropertyName("tools")] + public Tool[]? Tools { get; set; } + + [JsonPropertyName("inputAudioTranscription")] + public AudioTranscriptionConfig? InputAudioTranscription { get; set; } + + [JsonPropertyName("outputAudioTranscription")] + public AudioTranscriptionConfig? OutputAudioTranscription { get; set; } +} + +internal class AudioTranscriptionConfig { } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs index 25cf59e4..354a572f 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs @@ -30,6 +30,12 @@ internal class RealtimeGenerateContentServerContent [JsonPropertyName("modelTurn")] public Content? ModelTurn { get; set; } + + [JsonPropertyName("inputTranscription")] + public RealtimeGenerateContentTranscription? InputTranscription { get; set; } + + [JsonPropertyName("outputTranscription")] + public RealtimeGenerateContentTranscription? OutputTranscription { get; set; } } internal class RealtimeUsageMetaData @@ -58,4 +64,10 @@ internal class RealtimeTokenDetail [JsonPropertyName("tokenCount")] public int? TokenCount { get; set; } +} + +internal class RealtimeGenerateContentTranscription +{ + [JsonPropertyName("text")] + public string? Text { get; set; } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs index fe3480c7..b267821c 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -107,8 +107,9 @@ public class GoogleRealTimeProvider : IRealTimeCompletion JsonOptions = _jsonOptions }); + var uri = BuildWebsocketUri(modelSettings.ApiKey, "v1beta"); await _session.ConnectAsync( - uri: new Uri($"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent?key={modelSettings.ApiKey}"), + uri: uri, cancellationToken: CancellationToken.None); await onModelReady(); @@ -148,9 +149,12 @@ public class GoogleRealTimeProvider : IRealTimeCompletion Func onModelAudioTranscriptDone, Func, Task> onModelResponseDone, Func onConversationItemCreated, - Func onInputAudioTranscriptionCompleted, + Func onInputAudioTranscriptionDone, Func onInterruptionDetected) { + var inputTranscription = string.Empty; + var outputTranscription = string.Empty; + await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None)) { var receivedText = update?.RawResponse; @@ -163,7 +167,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion try { var response = JsonSerializer.Deserialize(receivedText, _jsonOptions); - if (response == null) { continue; @@ -175,10 +178,29 @@ public class GoogleRealTimeProvider : IRealTimeCompletion } else if (response.ServerContent != null) { + if (response.ServerContent.InputTranscription?.Text != null) + { + outputTranscription = string.Empty; + inputTranscription += response.ServerContent.InputTranscription.Text; + } + + if (response.ServerContent.OutputTranscription?.Text != null) + { + outputTranscription += response.ServerContent.OutputTranscription.Text; + } + if (response.ServerContent.ModelTurn != null) { _logger.LogInformation($"Model audio delta received."); var parts = response.ServerContent.ModelTurn.Parts; + + if (!string.IsNullOrEmpty(inputTranscription)) + { + var message = await OnUserAudioTranscriptionCompleted(conn, inputTranscription); + await onInputAudioTranscriptionDone(message); + inputTranscription = string.Empty; + } + if (!parts.IsNullOrEmpty()) { foreach (var part in parts) @@ -197,13 +219,23 @@ public class GoogleRealTimeProvider : IRealTimeCompletion else if (response.ServerContent.TurnComplete == true) { _logger.LogInformation($"Model turn completed."); + + if (!string.IsNullOrEmpty(outputTranscription)) + { + var messages = await OnResponseDone(conn, outputTranscription, response.UsageMetaData); + await onModelResponseDone(messages); + + // Reset input/output transcription + inputTranscription = string.Empty; + outputTranscription = string.Empty; + } } } } catch (Exception ex) { - _logger.LogError(ex, $"Error when deserializing server response."); - continue; + _logger.LogError(ex, $"Error when deserializing server response. {ex.Message}"); + break; } } @@ -288,7 +320,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion client.Connected += (sender, e) => { _logger.LogInformation("Google Realtime Client connected."); - _onModelReady(); + _onModelReady().ConfigureAwait(false).GetAwaiter().GetResult(); }; client.Disconnected += (sender, e) => @@ -301,7 +333,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion _logger.LogInformation("User message received."); if (e.Payload.SetupComplete != null) { - _onConversationItemCreated(_client.ConnectionId.ToString()); + _onConversationItemCreated(_client.ConnectionId.ToString()).ConfigureAwait(false).GetAwaiter().GetResult(); } if (e.Payload.ServerContent != null) @@ -309,31 +341,31 @@ public class GoogleRealTimeProvider : IRealTimeCompletion if (e.Payload.ServerContent.TurnComplete == true) { var responseDone = await ResponseDone(_conn, e.Payload.ServerContent); - _onModelResponseDone(responseDone); + _onModelResponseDone(responseDone).ConfigureAwait(false).GetAwaiter().GetResult(); } } }; client.AudioChunkReceived += (sender, e) => { - _onModelAudioDeltaReceived(Convert.ToBase64String(e.Buffer), Guid.NewGuid().ToString()); + _onModelAudioDeltaReceived(Convert.ToBase64String(e.Buffer), Guid.NewGuid().ToString()).ConfigureAwait(false).GetAwaiter().GetResult(); }; client.TextChunkReceived += (sender, e) => { - _onInputAudioTranscriptionDone(new RoleDialogModel(AgentRole.Assistant, e.Text)); + _onInputAudioTranscriptionDone(new RoleDialogModel(AgentRole.Assistant, e.Text)).ConfigureAwait(false).GetAwaiter().GetResult(); }; client.GenerationInterrupted += (sender, e) => { _logger.LogInformation("Audio generation interrupted."); - _onUserInterrupted(); + _onUserInterrupted().ConfigureAwait(false).GetAwaiter().GetResult(); }; client.AudioReceiveCompleted += (sender, e) => { _logger.LogInformation("Audio receive completed."); - _onModelAudioResponseDone(); + _onModelAudioResponseDone().ConfigureAwait(false).GetAwaiter().GetResult(); }; client.ErrorOccurred += (sender, e) => @@ -345,6 +377,43 @@ public class GoogleRealTimeProvider : IRealTimeCompletion return Task.CompletedTask; } + private async Task> OnResponseDone(RealtimeHubConnection conn, string text, RealtimeUsageMetaData? useage) + { + var outputs = new List + { + new(AgentRole.Assistant, text) + { + CurrentAgentId = conn.CurrentAgentId, + MessageId = Guid.NewGuid().ToString(), + MessageType = MessageTypeName.Plain + } + }; + + if (useage != null) + { + var contentHooks = _services.GetServices(); + foreach (var hook in contentHooks) + { + await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, text) + { + CurrentAgentId = conn.CurrentAgentId + }, + new TokenStatsModel + { + Provider = Provider, + Model = _model, + Prompt = text, + TextInputTokens = useage.PromptTokensDetails?.FirstOrDefault(x => x.Modality == Modality.TEXT.ToString())?.TokenCount ?? 0, + AudioInputTokens = useage.PromptTokensDetails?.FirstOrDefault(x => x.Modality == Modality.AUDIO.ToString())?.TokenCount ?? 0, + TextOutputTokens = useage.ResponseTokensDetails?.FirstOrDefault(x => x.Modality == Modality.TEXT.ToString())?.TokenCount ?? 0, + AudioOutputTokens = useage.ResponseTokensDetails?.FirstOrDefault(x => x.Modality == Modality.AUDIO.ToString())?.TokenCount ?? 0 + }); + } + } + + return outputs; + } + private async Task> ResponseDone(RealtimeHubConnection conn, BidiGenerateContentServerContent serverContent) { @@ -401,8 +470,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion public async Task SendEventToModel(object message) { - //todo Send Audio Chunks to Model, Botsharp RealTime Implementation seems to be incomplete - if (_session == null) return; await _session.SendEventToModel(message); @@ -419,9 +486,9 @@ public class GoogleRealTimeProvider : IRealTimeCompletion var (prompt, request) = PrepareOptions(agent, []); var config = request.GenerationConfig; - //Output Modality can either be text or audio if (config != null) { + //Output Modality can either be text or audio config.ResponseModalities = [Modality.AUDIO]; var words = new List(); @@ -467,14 +534,16 @@ public class GoogleRealTimeProvider : IRealTimeCompletion // //Tools = request.Tools?.ToArray(), //}); - await SendEventToModel(new BidiClientPayload + await SendEventToModel(new RealtimeClientPayload { - Setup = new BidiGenerateContentSetup() + Setup = new RealtimeGenerateContentSetup() { GenerationConfig = config, Model = Model.ToModelId(), SystemInstruction = request.SystemInstruction, - Tools = [] + Tools = [], + InputAudioTranscription = new(), + OutputAudioTranscription = new() } }); @@ -532,7 +601,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion } else { - throw new NotImplementedException(""); + throw new NotImplementedException($"Unrecognized role {message.Role}."); } } @@ -542,9 +611,9 @@ public class GoogleRealTimeProvider : IRealTimeCompletion } - public async Task OnConversationItemCreated(RealtimeHubConnection conn, string response) + public async Task OnConversationItemCreated(RealtimeHubConnection conn, string text) { - return await Task.FromResult(new RoleDialogModel(AgentRole.User, response)); + return await Task.FromResult(new RoleDialogModel(AgentRole.User, text)); } private (string, GenerateContentRequest) PrepareOptions(Agent agent, @@ -688,4 +757,18 @@ public class GoogleRealTimeProvider : IRealTimeCompletion return prompt; } + + + private async Task OnUserAudioTranscriptionCompleted(RealtimeHubConnection conn, string text) + { + return new RoleDialogModel(AgentRole.User, text) + { + CurrentAgentId = conn.CurrentAgentId + }; + } + + private Uri BuildWebsocketUri(string apiKey, string version = "v1alpha") + { + return new Uri($"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.{version}.GenerativeService.BidiGenerateContent?key={apiKey}"); + } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index dcb82164..f4507477 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -402,7 +402,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion } else { - throw new NotImplementedException(""); + throw new NotImplementedException($"Unrecognized role {message.Role}."); } } From 04eafedb7837330f442a811d06c0a007d882deac Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 14 May 2025 18:14:01 -0500 Subject: [PATCH 10/26] refine transcription --- .../MLTasks/IRealTimeCompletion.cs | 5 +- .../Realtime/Models/RealtimeModelSettings.cs | 7 +- .../Hooks/RealtimeConversationHook.cs | 1 + .../BotSharp.Core/BotSharp.Core.csproj | 3 - .../BotSharp.Core/Functions/GetWeatherFn.cs | 12 +- .../AsyncWebsocketDataResultEnumerator.cs | 1 - .../Session/BotSharpRealtimeSession.cs | 4 +- .../Session/LlmRealtimeSession.cs | 4 +- .../functions/get_location.json | 20 - .../functions/get_weather.json | 7 +- .../ChatStreamMiddleware.cs | 5 +- .../Models/Realtime/RealtimeServerResponse.cs | 21 ++ .../Realtime/RealtimeTranscriptionResponse.cs | 53 +++ .../Realtime/RealTimeCompletionProvider.cs | 356 +++++------------- src/Plugins/BotSharp.Plugin.GoogleAI/Using.cs | 7 +- .../Realtime/RealTimeCompletionProvider.cs | 225 ++++++----- .../appsettings.json | 24 +- 17 files changed, 328 insertions(+), 427 deletions(-) delete mode 100644 src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_location.json create mode 100644 src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs index 315f4f5a..d6057859 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs @@ -1,6 +1,4 @@ using BotSharp.Abstraction.Realtime.Models; -using System; -using static System.Runtime.InteropServices.JavaScript.JSType; namespace BotSharp.Abstraction.MLTasks; @@ -20,6 +18,7 @@ public interface IRealTimeCompletion Func onConversationItemCreated, Func onInputAudioTranscriptionDone, Func onInterruptionDetected); + Task AppenAudioBuffer(string message); Task AppenAudioBuffer(ArraySegment data, int length); @@ -31,6 +30,4 @@ public interface IRealTimeCompletion Task RemoveConversationItem(string itemId); Task TriggerModelInference(string? instructions = null); Task CancelModelResponse(); - Task> OnResponsedDone(RealtimeHubConnection conn, string response); - Task OnConversationItemCreated(RealtimeHubConnection conn, string response); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs index daf8714a..14f5923f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeModelSettings.cs @@ -12,7 +12,12 @@ public class RealtimeModelSettings public string Voice { get; set; } = "alloy"; public float Temperature { get; set; } = 0.8f; public int MaxResponseOutputTokens { get; set; } = 512; - public int ModelResponseTimeout { get; set; } = 30; + public int ModelResponseTimeoutSeconds { get; set; } = 30; + + /// + /// Whether the target event arrives after ModelResponseTimeoutSeconds, e.g., "response.done" + /// + public string? ModelResponseTimeoutEndEvent { get; set; } public AudioTranscription InputAudioTranscription { get; set; } = new(); public ModelTurnDetection TurnDetection { get; set; } = new(); } diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs index 5e1fcfee..aabac186 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs @@ -42,6 +42,7 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook var routing = _services.GetRequiredService(); message.Role = AgentRole.Function; + //message.Role = AgentRole.Assistant; if (message.FunctionName == "route_to_agent") { diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index de2c8909..b3e29b3f 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -211,9 +211,6 @@ PreserveNewest - - PreserveNewest - diff --git a/src/Infrastructure/BotSharp.Core/Functions/GetWeatherFn.cs b/src/Infrastructure/BotSharp.Core/Functions/GetWeatherFn.cs index 759bc68c..09f6e661 100644 --- a/src/Infrastructure/BotSharp.Core/Functions/GetWeatherFn.cs +++ b/src/Infrastructure/BotSharp.Core/Functions/GetWeatherFn.cs @@ -20,14 +20,14 @@ public class GetWeatherFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { - var args = JsonSerializer.Deserialize(message.FunctionArgs, BotSharpOptions.defaultJsonOptions); + //var args = JsonSerializer.Deserialize(message.FunctionArgs, BotSharpOptions.defaultJsonOptions); - var sidecar = _services.GetService(); - var states = GetSideCarStates(); + //var sidecar = _services.GetService(); + //var states = GetSideCarStates(); - var userMessage = $"Please find the information at location {args.City}, {args.State}"; - var response = await sidecar.SendMessage(BuiltInAgentId.Chatbot, userMessage, states: states); - message.Content = $"It is a sunny day {response.Content}."; + //var userMessage = $"Please find the information at location {args.City}, {args.State}"; + //var response = await sidecar.SendMessage(BuiltInAgentId.Chatbot, userMessage, states: states); + message.Content = $"It is a sunny day."; return true; } diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs index 548850e9..f89127e2 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/Websocket/AsyncWebsocketDataResultEnumerator.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Realtime.Models.Session; using System.Buffers; using System.ClientModel; using System.Net.WebSockets; diff --git a/src/Infrastructure/BotSharp.Core/Session/BotSharpRealtimeSession.cs b/src/Infrastructure/BotSharp.Core/Session/BotSharpRealtimeSession.cs index 0c863b7b..7f5f6c15 100644 --- a/src/Infrastructure/BotSharp.Core/Session/BotSharpRealtimeSession.cs +++ b/src/Infrastructure/BotSharp.Core/Session/BotSharpRealtimeSession.cs @@ -55,7 +55,7 @@ public class BotSharpRealtimeSession : IDisposable }; } - public async Task SendEvent(string message) + public async Task SendEventAsync(string message) { if (_websocket.State == WebSocketState.Open) { @@ -64,7 +64,7 @@ public class BotSharpRealtimeSession : IDisposable } } - public async Task Disconnect() + public async Task DisconnectAsync() { if (_websocket.State == WebSocketState.Open) { diff --git a/src/Infrastructure/BotSharp.Core/Session/LlmRealtimeSession.cs b/src/Infrastructure/BotSharp.Core/Session/LlmRealtimeSession.cs index d799480e..2f3259cf 100644 --- a/src/Infrastructure/BotSharp.Core/Session/LlmRealtimeSession.cs +++ b/src/Infrastructure/BotSharp.Core/Session/LlmRealtimeSession.cs @@ -71,7 +71,7 @@ public class LlmRealtimeSession : IDisposable }; } - public async Task SendEventToModel(object message) + public async Task SendEventToModelAsync(object message) { if (_webSocket.State != WebSocketState.Open) { @@ -96,7 +96,7 @@ public class LlmRealtimeSession : IDisposable } } - public async Task Disconnect() + public async Task DisconnectAsync() { if (_webSocket.State == WebSocketState.Open) { diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_location.json b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_location.json deleted file mode 100644 index b0716218..00000000 --- a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_location.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "name": "get_location", - "description": "Get location information for user.", - "parameters": { - "type": "object", - "properties": { - "city": { - "type": "string", - "visibility_expression": "{% if states.channel == 'email' %}visible{% endif %}", - "description": "The location city that user wants to know about." - }, - "county": { - "type": "string", - "visibility_expression": "{% if states.channel != 'email' %}visible{% endif %}", - "description": "The location county that user wants to know about." - } - }, - "required": [ "city", "county" ] - } -} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_weather.json b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_weather.json index bdb679a2..0fd0a459 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_weather.json +++ b/src/Infrastructure/BotSharp.Core/data/agents/01e2fc5c-2c89-4ec7-8470-7688608b496c/functions/get_weather.json @@ -1,19 +1,14 @@ { "name": "get_weather", "description": "Get weather information for user.", - "visibility_expression": "{% if states.channel != 'email' %}visible{% endif %}", "parameters": { "type": "object", "properties": { "city": { "type": "string", "description": "The city where the user wants to get weather information." - }, - "state": { - "type": "string", - "description": "The state where the user wants to get weather information." } }, - "required": [ "city", "state" ] + "required": [ "city" ] } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/ChatStreamMiddleware.cs b/src/Plugins/BotSharp.Plugin.ChatHub/ChatStreamMiddleware.cs index b7d1b21a..ece767f9 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/ChatStreamMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/ChatStreamMiddleware.cs @@ -94,8 +94,7 @@ public class ChatStreamMiddleware } } - - await _session.Disconnect(); + await _session.DisconnectAsync(); _session.Dispose(); } @@ -105,7 +104,7 @@ public class ChatStreamMiddleware { if (_session != null) { - await _session.SendEvent(data); + await _session.SendEventAsync(data); } }); } diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs index 354a572f..4b65bdf8 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs @@ -12,6 +12,9 @@ internal class RealtimeServerResponse [JsonPropertyName("usageMetadata")] public RealtimeUsageMetaData? UsageMetaData { get; set; } + + [JsonPropertyName("toolCall")] + public RealtimeToolCall? ToolCall { get; set; } } @@ -70,4 +73,22 @@ internal class RealtimeGenerateContentTranscription { [JsonPropertyName("text")] public string? Text { get; set; } +} + +internal class RealtimeToolCall +{ + [JsonPropertyName("functionCalls")] + public List? FunctionCalls { get; set; } +} + +internal class RealtimeFunctionCall +{ + [JsonPropertyName("id")] + public string Id { get; set; } + + [JsonPropertyName("name")] + public string Name { get; set; } + + [JsonPropertyName("args")] + public JsonNode? Args { get; set; } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs new file mode 100644 index 00000000..b14c1bde --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs @@ -0,0 +1,53 @@ +using System.IO; + +namespace BotSharp.Plugin.GoogleAI.Models.Realtime; + +internal class RealtimeTranscriptionResponse : IDisposable +{ + public RealtimeTranscriptionResponse() + { + + } + + private MemoryStream _contentStream = new(); + public Stream? ContentStream + { + get + { + return _contentStream != null ? _contentStream : new MemoryStream(); + } + } + + public void Collect(string text) + { + var binary = BinaryData.FromString(text); + var bytes = binary.ToArray(); + + _contentStream.Position = _contentStream.Length; + _contentStream.Write(bytes, 0, bytes.Length); + _contentStream.Position = 0; + } + + public string GetString() + { + if (_contentStream.Length == 0) + { + return string.Empty; + } + + var bytes = _contentStream.ToArray(); + var text = Encoding.UTF8.GetString(bytes, 0, bytes.Length); + return text; + } + + public void Clear() + { + _contentStream.SetLength(0); + _contentStream.Position = 0; + } + + public void Dispose() + { + _contentStream?.Dispose(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs index b267821c..f628a281 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -1,17 +1,10 @@ -using BotSharp.Abstraction.Options; +using System.Threading; using BotSharp.Abstraction.Realtime.Models.Session; using BotSharp.Core.Session; using BotSharp.Plugin.GoogleAI.Models.Realtime; using GenerativeAI; -using GenerativeAI.Core; -using GenerativeAI.Live; -using GenerativeAI.Live.Extensions; using GenerativeAI.Types; using GenerativeAI.Types.Converters; -using Google.Ai.Generativelanguage.V1Beta2; -using Google.Api; -using System; -using System.Threading; namespace BotSharp.Plugin.GoogleAi.Providers.Realtime; @@ -21,18 +14,15 @@ public class GoogleRealTimeProvider : IRealTimeCompletion public string Model => _model; private string _model = GoogleAIModels.Gemini2FlashExp; - private MultiModalLiveClient _client; - private GenerativeModel _chatClient; + private readonly IServiceProvider _services; private readonly ILogger _logger; private List renderedInstructions = []; private LlmRealtimeSession _session; - private readonly BotSharpOptions _botsharpOptions; private readonly GoogleAiSettings _settings; private const string DEFAULT_MIME_TYPE = "audio/pcm;rate=16000"; - private readonly JsonSerializerOptions _jsonOptions = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, @@ -45,11 +35,9 @@ public class GoogleRealTimeProvider : IRealTimeCompletion public GoogleRealTimeProvider( IServiceProvider services, GoogleAiSettings settings, - BotSharpOptions botSharpOptions, ILogger logger) { _settings = settings; - _botsharpOptions = botSharpOptions; _services = services; _logger = logger; } @@ -59,17 +47,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion _model = model; } - private RealtimeHubConnection _conn; - private Func _onModelReady; - private Func _onModelAudioDeltaReceived; - private Func _onModelAudioResponseDone; - private Func _onModelAudioTranscriptDone; - private Func, Task> _onModelResponseDone; - private Func _onConversationItemCreated; - private Func _onInputAudioTranscriptionDone; - private Func _onUserInterrupted; - - public async Task Connect( RealtimeHubConnection conn, Func onModelReady, @@ -81,16 +58,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion Func onInputAudioTranscriptionDone, Func onInterruptionDetected) { - _conn = conn; - _onModelReady = onModelReady; - _onModelAudioDeltaReceived = onModelAudioDeltaReceived; - _onModelAudioResponseDone = onModelAudioResponseDone; - _onModelAudioTranscriptDone = onModelAudioTranscriptDone; - _onModelResponseDone = onModelResponseDone; - _onConversationItemCreated = onConversationItemCreated; - _onInputAudioTranscriptionDone = onInputAudioTranscriptionDone; - _onUserInterrupted = onInterruptionDetected; - var settingsService = _services.GetRequiredService(); var realtimeModelSettings = _services.GetRequiredService(); @@ -108,9 +75,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion }); var uri = BuildWebsocketUri(modelSettings.ApiKey, "v1beta"); - await _session.ConnectAsync( - uri: uri, - cancellationToken: CancellationToken.None); + await _session.ConnectAsync(uri: uri, cancellationToken: CancellationToken.None); await onModelReady(); @@ -124,21 +89,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion onConversationItemCreated, onInputAudioTranscriptionDone, onInterruptionDetected); - - - //var client = ProviderHelper.GetGeminiClient(Provider, _model, _services); - //_chatClient = client.CreateGenerativeModel(_model); - //_client = _chatClient.CreateMultiModalLiveClient( - // config: new GenerationConfig - // { - // ResponseModalities = [Modality.AUDIO], - // }, - // systemInstruction: "You are a helpful assistant.", - // logger: _logger); - - //await AttachEvents(_client); - - //await _client.ConnectAsync(false); } private async Task ReceiveMessage( @@ -152,8 +102,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion Func onInputAudioTranscriptionDone, Func onInterruptionDetected) { - var inputTranscription = string.Empty; - var outputTranscription = string.Empty; + using var inputStream = new RealtimeTranscriptionResponse(); + using var outputStream = new RealtimeTranscriptionResponse(); await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None)) { @@ -176,31 +126,43 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { _logger.LogInformation($"Session setup completed."); } + else if (response.ToolCall != null && !response.ToolCall.FunctionCalls.IsNullOrEmpty()) + { + var functionCall = response.ToolCall.FunctionCalls.First(); + _logger.LogInformation($"Tool call received {functionCall.Name}({functionCall.Args?.ToJsonString(_jsonOptions) ?? string.Empty})."); + + if (functionCall != null) + { + var messages = OnFunctionCall(conn, functionCall); + await onModelResponseDone(messages); + } + } else if (response.ServerContent != null) { if (response.ServerContent.InputTranscription?.Text != null) { - outputTranscription = string.Empty; - inputTranscription += response.ServerContent.InputTranscription.Text; + inputStream.Collect(response.ServerContent.InputTranscription.Text); } if (response.ServerContent.OutputTranscription?.Text != null) { - outputTranscription += response.ServerContent.OutputTranscription.Text; + outputStream.Collect(response.ServerContent.OutputTranscription.Text); } if (response.ServerContent.ModelTurn != null) { _logger.LogInformation($"Model audio delta received."); - var parts = response.ServerContent.ModelTurn.Parts; + // Handle input transcription + var inputTranscription = inputStream.GetString(); if (!string.IsNullOrEmpty(inputTranscription)) { - var message = await OnUserAudioTranscriptionCompleted(conn, inputTranscription); + var message = OnUserAudioTranscriptionCompleted(conn, inputTranscription); await onInputAudioTranscriptionDone(message); - inputTranscription = string.Empty; } + inputStream.Clear(); + var parts = response.ServerContent.ModelTurn.Parts; if (!parts.IsNullOrEmpty()) { foreach (var part in parts) @@ -220,15 +182,14 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { _logger.LogInformation($"Model turn completed."); + var outputTranscription = outputStream.GetString(); if (!string.IsNullOrEmpty(outputTranscription)) { var messages = await OnResponseDone(conn, outputTranscription, response.UsageMetaData); await onModelResponseDone(messages); - - // Reset input/output transcription - inputTranscription = string.Empty; - outputTranscription = string.Empty; } + inputStream.Clear(); + outputStream.Clear(); } } } @@ -247,19 +208,12 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { if (_session != null) { - await _session.Disconnect(); + await _session.DisconnectAsync(); } - - //if (_client != null) - //{ - // await _client.DisconnectAsync(); - //} } public async Task AppenAudioBuffer(string message) { - //await _client.SendAudioAsync(Convert.FromBase64String(message)); - await SendEventToModel(new BidiClientPayload { RealtimeInput = new() @@ -272,8 +226,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion public async Task AppenAudioBuffer(ArraySegment data, int length) { var buffer = data.AsSpan(0, length).ToArray(); - //await _client.SendAudioAsync(buffer, "audio/pcm;rate=16000"); - await SendEventToModel(new BidiClientPayload { RealtimeInput = new() @@ -285,21 +237,13 @@ public class GoogleRealTimeProvider : IRealTimeCompletion public async Task TriggerModelInference(string? instructions = null) { - var content = !string.IsNullOrWhiteSpace(instructions) - ? new Content(instructions, AgentRole.User) - : null; - - //await _client.SendClientContentAsync(new BidiGenerateContentClientContent() - //{ - // Turns = content != null ? [content] : null, - // TurnComplete = true, - //}); + var content = new Content("Please respond to me.", AgentRole.User); await SendEventToModel(new BidiClientPayload { ClientContent = new() { - Turns = content != null ? [content] : null, + Turns = null, TurnComplete = true } }); @@ -315,164 +259,11 @@ public class GoogleRealTimeProvider : IRealTimeCompletion } - private Task AttachEvents(MultiModalLiveClient client) - { - client.Connected += (sender, e) => - { - _logger.LogInformation("Google Realtime Client connected."); - _onModelReady().ConfigureAwait(false).GetAwaiter().GetResult(); - }; - - client.Disconnected += (sender, e) => - { - _logger.LogInformation("Google Realtime Client disconnected."); - }; - - client.MessageReceived += async (sender, e) => - { - _logger.LogInformation("User message received."); - if (e.Payload.SetupComplete != null) - { - _onConversationItemCreated(_client.ConnectionId.ToString()).ConfigureAwait(false).GetAwaiter().GetResult(); - } - - if (e.Payload.ServerContent != null) - { - if (e.Payload.ServerContent.TurnComplete == true) - { - var responseDone = await ResponseDone(_conn, e.Payload.ServerContent); - _onModelResponseDone(responseDone).ConfigureAwait(false).GetAwaiter().GetResult(); - } - } - }; - - client.AudioChunkReceived += (sender, e) => - { - _onModelAudioDeltaReceived(Convert.ToBase64String(e.Buffer), Guid.NewGuid().ToString()).ConfigureAwait(false).GetAwaiter().GetResult(); - }; - - client.TextChunkReceived += (sender, e) => - { - _onInputAudioTranscriptionDone(new RoleDialogModel(AgentRole.Assistant, e.Text)).ConfigureAwait(false).GetAwaiter().GetResult(); - }; - - client.GenerationInterrupted += (sender, e) => - { - _logger.LogInformation("Audio generation interrupted."); - _onUserInterrupted().ConfigureAwait(false).GetAwaiter().GetResult(); - }; - - client.AudioReceiveCompleted += (sender, e) => - { - _logger.LogInformation("Audio receive completed."); - _onModelAudioResponseDone().ConfigureAwait(false).GetAwaiter().GetResult(); - }; - - client.ErrorOccurred += (sender, e) => - { - var ex = e.GetException(); - _logger.LogError(ex, "Error occurred in Google Realtime Client"); - }; - - return Task.CompletedTask; - } - - private async Task> OnResponseDone(RealtimeHubConnection conn, string text, RealtimeUsageMetaData? useage) - { - var outputs = new List - { - new(AgentRole.Assistant, text) - { - CurrentAgentId = conn.CurrentAgentId, - MessageId = Guid.NewGuid().ToString(), - MessageType = MessageTypeName.Plain - } - }; - - if (useage != null) - { - var contentHooks = _services.GetServices(); - foreach (var hook in contentHooks) - { - await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, text) - { - CurrentAgentId = conn.CurrentAgentId - }, - new TokenStatsModel - { - Provider = Provider, - Model = _model, - Prompt = text, - TextInputTokens = useage.PromptTokensDetails?.FirstOrDefault(x => x.Modality == Modality.TEXT.ToString())?.TokenCount ?? 0, - AudioInputTokens = useage.PromptTokensDetails?.FirstOrDefault(x => x.Modality == Modality.AUDIO.ToString())?.TokenCount ?? 0, - TextOutputTokens = useage.ResponseTokensDetails?.FirstOrDefault(x => x.Modality == Modality.TEXT.ToString())?.TokenCount ?? 0, - AudioOutputTokens = useage.ResponseTokensDetails?.FirstOrDefault(x => x.Modality == Modality.AUDIO.ToString())?.TokenCount ?? 0 - }); - } - } - - return outputs; - } - - private async Task> ResponseDone(RealtimeHubConnection conn, - BidiGenerateContentServerContent serverContent) - { - var outputs = new List(); - - var parts = serverContent.ModelTurn?.Parts; - if (parts != null) - { - foreach (var part in parts) - { - var call = part.FunctionCall; - if (call != null) - { - var item = new RoleDialogModel(AgentRole.Assistant, part.Text) - { - CurrentAgentId = conn.CurrentAgentId, - MessageId = call.Id ?? String.Empty, - MessageType = MessageTypeName.FunctionCall - }; - outputs.Add(item); - } - else - { - var item = new RoleDialogModel(AgentRole.Assistant, call.Args?.ToJsonString() ?? string.Empty) - { - CurrentAgentId = conn.CurrentAgentId, - FunctionName = call.Name, - FunctionArgs = call.Args?.ToJsonString() ?? string.Empty, - ToolCallId = call.Id ?? String.Empty, - MessageId = call.Id ?? String.Empty, - MessageType = MessageTypeName.FunctionCall - }; - outputs.Add(item); - } - } - } - - var contentHooks = _services.GetServices().ToList(); - // After chat completion hook - foreach (var hook in contentHooks) - { - await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, "response.done") - { - CurrentAgentId = conn.CurrentAgentId - }, new TokenStatsModel - { - Provider = Provider, - Model = _model, - }); - } - - return outputs; - } - public async Task SendEventToModel(object message) { if (_session == null) return; - await _session.SendEventToModel(message); + await _session.SendEventToModelAsync(message); } public async Task UpdateSession(RealtimeHubConnection conn, bool isInit = false) @@ -500,7 +291,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion config.MaxOutputTokens = realtimeModelSettings.MaxResponseOutputTokens; } - var functions = request.Tools?.SelectMany(s => s.FunctionDeclarations).Select(x => { var fn = new FunctionDef @@ -526,14 +316,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion }); } - //await _client.SendSetupAsync(new BidiGenerateContentSetup() - //{ - // GenerationConfig = config, - // Model = Model.ToModelId(), - // SystemInstruction = request.SystemInstruction, - // //Tools = request.Tools?.ToArray(), - //}); - + var realtimeSetting = _services.GetRequiredService(); await SendEventToModel(new RealtimeClientPayload { Setup = new RealtimeGenerateContentSetup() @@ -541,9 +324,9 @@ public class GoogleRealTimeProvider : IRealTimeCompletion GenerationConfig = config, Model = Model.ToModelId(), SystemInstruction = request.SystemInstruction, - Tools = [], - InputAudioTranscription = new(), - OutputAudioTranscription = new() + Tools = request.Tools?.ToArray(), + InputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null, + OutputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null } }); @@ -552,21 +335,17 @@ public class GoogleRealTimeProvider : IRealTimeCompletion public async Task InsertConversationItem(RoleDialogModel message) { - //if (_client == null) - // throw new Exception("Client is not initialized"); if (message.Role == AgentRole.Function) { var function = new FunctionResponse() { Name = message.FunctionName ?? string.Empty, - Response = JsonNode.Parse(message.Content ?? "{}") + Response = new JsonObject() + { + ["result"] = message.Content ?? string.Empty + } }; - //await _client.SendToolResponseAsync(new BidiGenerateContentToolResponse() - //{ - // FunctionResponses = [function] - //}); - await SendEventToModel(new BidiClientPayload { ToolResponse = new() @@ -588,8 +367,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion } else if (message.Role == AgentRole.User) { - //await _client.SentTextAsync(message.Content); - await SendEventToModel(new BidiClientPayload { ClientContent = new() @@ -605,17 +382,63 @@ public class GoogleRealTimeProvider : IRealTimeCompletion } } - public async Task> OnResponsedDone(RealtimeHubConnection conn, string response) + #region Private methods + private List OnFunctionCall(RealtimeHubConnection conn, RealtimeFunctionCall functionCall) { - return []; + var outputs = new List + { + new(AgentRole.Assistant, string.Empty) + { + CurrentAgentId = conn.CurrentAgentId, + FunctionName = functionCall.Name, + FunctionArgs = functionCall.Args?.ToJsonString(_jsonOptions), + ToolCallId = functionCall.Id, + MessageType = MessageTypeName.FunctionCall + } + }; + + return outputs; } - public async Task OnConversationItemCreated(RealtimeHubConnection conn, string text) + private async Task> OnResponseDone(RealtimeHubConnection conn, string text, RealtimeUsageMetaData? usage) { - return await Task.FromResult(new RoleDialogModel(AgentRole.User, text)); + var outputs = new List + { + new(AgentRole.Assistant, text) + { + CurrentAgentId = conn.CurrentAgentId, + MessageId = Guid.NewGuid().ToString(), + MessageType = MessageTypeName.Plain + } + }; + + if (usage != null) + { + var contentHooks = _services.GetServices(); + foreach (var hook in contentHooks) + { + await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, text) + { + CurrentAgentId = conn.CurrentAgentId + }, + new TokenStatsModel + { + Provider = Provider, + Model = _model, + Prompt = text, + TextInputTokens = usage.PromptTokensDetails?.FirstOrDefault(x => x.Modality == Modality.TEXT.ToString())?.TokenCount ?? 0, + AudioInputTokens = usage.PromptTokensDetails?.FirstOrDefault(x => x.Modality == Modality.AUDIO.ToString())?.TokenCount ?? 0, + TextOutputTokens = usage.ResponseTokensDetails?.FirstOrDefault(x => x.Modality == Modality.TEXT.ToString())?.TokenCount ?? 0, + AudioOutputTokens = usage.ResponseTokensDetails?.FirstOrDefault(x => x.Modality == Modality.AUDIO.ToString())?.TokenCount ?? 0 + }); + } + } + + return outputs; } + private (string, GenerateContentRequest) PrepareOptions(Agent agent, List conversations) { @@ -759,7 +582,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion } - private async Task OnUserAudioTranscriptionCompleted(RealtimeHubConnection conn, string text) + private RoleDialogModel OnUserAudioTranscriptionCompleted(RealtimeHubConnection conn, string text) { return new RoleDialogModel(AgentRole.User, text) { @@ -771,4 +594,5 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { return new Uri($"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.{version}.GenerativeService.BidiGenerateContent?key={apiKey}"); } + #endregion } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Using.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Using.cs index a4e6606e..daff1e0b 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Using.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Using.cs @@ -16,14 +16,15 @@ global using BotSharp.Abstraction.Agents.Constants; global using BotSharp.Abstraction.Agents.Models; global using BotSharp.Abstraction.MLTasks; global using BotSharp.Abstraction.Utilities; -global using BotSharp.Plugin.GoogleAi.Settings; global using BotSharp.Abstraction.Realtime; global using BotSharp.Abstraction.Realtime.Models; global using BotSharp.Core.Infrastructures; -global using BotSharp.Plugin.GoogleAi.Providers.Chat; global using BotSharp.Abstraction.Agents; global using BotSharp.Abstraction.Agents.Enums; global using BotSharp.Abstraction.Conversations; global using BotSharp.Abstraction.Conversations.Enums; global using BotSharp.Abstraction.Functions.Models; -global using BotSharp.Abstraction.Loggers; \ No newline at end of file +global using BotSharp.Abstraction.Loggers; + +global using BotSharp.Plugin.GoogleAi.Settings; +global using BotSharp.Plugin.GoogleAi.Providers.Chat; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index f4507477..fa1bd623 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -40,9 +40,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion Func onInterruptionDetected) { var settingsService = _services.GetRequiredService(); - var realtimeModelSettings = _services.GetRequiredService(); + var realtimeSettings = _services.GetRequiredService(); - _model = realtimeModelSettings.Model; + _model = realtimeSettings.Model; var settings = settingsService.GetSetting(Provider, _model); if (_session != null) @@ -65,6 +65,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion cancellationToken: CancellationToken.None); _ = ReceiveMessage( + _services, conn, onModelReady, onModelAudioDeltaReceived, @@ -80,7 +81,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion { if (_session != null) { - await _session.Disconnect(); + await _session.DisconnectAsync(); _session.Dispose(); } } @@ -143,6 +144,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion } private async Task ReceiveMessage( + IServiceProvider services, RealtimeHubConnection conn, Func onModelReady, Func onModelAudioDeltaReceived, @@ -153,6 +155,9 @@ public class RealTimeCompletionProvider : IRealTimeCompletion Func onInputAudioTranscriptionDone, Func onInterruptionDetected) { + DateTime? startTime = null; + var realtimeSettings = _services.GetRequiredService(); + await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None)) { var receivedText = update?.RawResponse; @@ -163,6 +168,17 @@ public class RealTimeCompletionProvider : IRealTimeCompletion var response = JsonSerializer.Deserialize(receivedText); + if (realtimeSettings?.ModelResponseTimeoutSeconds > 0 + && !string.IsNullOrWhiteSpace(realtimeSettings?.ModelResponseTimeoutEndEvent) + && startTime.HasValue + && (DateTime.UtcNow - startTime.Value).TotalSeconds >= realtimeSettings.ModelResponseTimeoutSeconds + && response.Type != realtimeSettings.ModelResponseTimeoutEndEvent) + { + startTime = null; + await TriggerModelInference("Responsd to user immediately"); + continue; + } + if (response.Type == "error") { _logger.LogError($"{response.Type}: {receivedText}"); @@ -228,6 +244,11 @@ public class RealTimeCompletionProvider : IRealTimeCompletion _logger.LogInformation($"{response.Type}: {receivedText}"); var data = JsonSerializer.Deserialize(receivedText); + if (data?.Item?.Role == "user") + { + startTime = DateTime.UtcNow; + } + await onConversationItemCreated(receivedText); } else if (response.Type == "conversation.item.input_audio_transcription.completed") @@ -263,7 +284,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion { if (_session == null) return; - await _session.SendEventToModel(message); + await _session.SendEventToModelAsync(message); } public async Task UpdateSession(RealtimeHubConnection conn, bool isInit = false) @@ -406,7 +427,101 @@ public class RealTimeCompletionProvider : IRealTimeCompletion } } - protected (string, IEnumerable, ChatCompletionOptions) PrepareOptions(Agent agent, List conversations) + + public void SetModelName(string model) + { + _model = model; + } + + #region Private methods + private async Task> OnResponsedDone(RealtimeHubConnection conn, string response) + { + var outputs = new List(); + + var data = JsonSerializer.Deserialize(response).Body; + if (data.Status != "completed") + { + _logger.LogError(data.StatusDetails.ToString()); + /*if (data.StatusDetails.Type == "incomplete" && data.StatusDetails.Reason == "max_output_tokens") + { + await TriggerModelInference("Response user concisely"); + }*/ + return []; + } + + var prompts = new List(); + var inputTokenDetails = data.Usage?.InputTokenDetails; + var outputTokenDetails = data.Usage?.OutputTokenDetails; + + foreach (var output in data.Outputs) + { + if (output.Type == "function_call") + { + outputs.Add(new RoleDialogModel(AgentRole.Assistant, output.Arguments) + { + CurrentAgentId = conn.CurrentAgentId, + FunctionName = output.Name, + FunctionArgs = output.Arguments, + ToolCallId = output.CallId, + MessageId = output.Id, + MessageType = MessageTypeName.FunctionCall + }); + + prompts.Add($"{output.Name}({output.Arguments})"); + } + else if (output.Type == "message") + { + var content = output.Content.FirstOrDefault()?.Transcript ?? string.Empty; + + outputs.Add(new RoleDialogModel(output.Role, content) + { + CurrentAgentId = conn.CurrentAgentId, + MessageId = output.Id, + MessageType = MessageTypeName.Plain + }); + + prompts.Add(content); + } + } + + + // After chat completion hook + var text = string.Join("\r\n", prompts); + var contentHooks = _services.GetServices(); + + foreach (var hook in contentHooks) + { + await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, text) + { + CurrentAgentId = conn.CurrentAgentId + }, + new TokenStatsModel + { + Provider = Provider, + Model = _model, + Prompt = text, + TextInputTokens = inputTokenDetails?.TextTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0, + CachedTextInputTokens = data.Usage?.InputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0, + AudioInputTokens = inputTokenDetails?.AudioTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0, + CachedAudioInputTokens = inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0, + TextOutputTokens = outputTokenDetails?.TextTokens ?? 0, + AudioOutputTokens = outputTokenDetails?.AudioTokens ?? 0 + }); + } + + return outputs; + } + + private async Task OnUserAudioTranscriptionCompleted(RealtimeHubConnection conn, string response) + { + var data = JsonSerializer.Deserialize(response); + return new RoleDialogModel(AgentRole.User, data.Transcript) + { + CurrentAgentId = conn.CurrentAgentId + }; + } + + private (string, IEnumerable, ChatCompletionOptions) PrepareOptions(Agent agent, List conversations) { var agentService = _services.GetRequiredService(); var state = _services.GetRequiredService(); @@ -588,103 +703,5 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return prompt; } - - public void SetModelName(string model) - { - _model = model; - } - - public async Task> OnResponsedDone(RealtimeHubConnection conn, string response) - { - var outputs = new List(); - - var data = JsonSerializer.Deserialize(response).Body; - if (data.Status != "completed") - { - _logger.LogError(data.StatusDetails.ToString()); - /*if (data.StatusDetails.Type == "incomplete" && data.StatusDetails.Reason == "max_output_tokens") - { - await TriggerModelInference("Response user concisely"); - }*/ - return []; - } - - var contentHooks = _services.GetServices().ToList(); - - var prompts = new List(); - var inputTokenDetails = data.Usage?.InputTokenDetails; - var outputTokenDetails = data.Usage?.OutputTokenDetails; - - foreach (var output in data.Outputs) - { - if (output.Type == "function_call") - { - outputs.Add(new RoleDialogModel(AgentRole.Assistant, output.Arguments) - { - CurrentAgentId = conn.CurrentAgentId, - FunctionName = output.Name, - FunctionArgs = output.Arguments, - ToolCallId = output.CallId, - MessageId = output.Id, - MessageType = MessageTypeName.FunctionCall - }); - - prompts.Add($"{output.Name}({output.Arguments})"); - } - else if (output.Type == "message") - { - var content = output.Content.FirstOrDefault()?.Transcript ?? string.Empty; - - outputs.Add(new RoleDialogModel(output.Role, content) - { - CurrentAgentId = conn.CurrentAgentId, - MessageId = output.Id, - MessageType = MessageTypeName.Plain - }); - - prompts.Add(content); - } - } - - var text = string.Join("\r\n", prompts); - // After chat completion hook - foreach (var hook in contentHooks) - { - await hook.AfterGenerated(new RoleDialogModel(AgentRole.Assistant, text) - { - CurrentAgentId = conn.CurrentAgentId - }, - new TokenStatsModel - { - Provider = Provider, - Model = _model, - Prompt = text, - TextInputTokens = inputTokenDetails?.TextTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0, - CachedTextInputTokens = data.Usage?.InputTokenDetails?.CachedTokenDetails?.TextTokens ?? 0, - AudioInputTokens = inputTokenDetails?.AudioTokens ?? 0 - inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0, - CachedAudioInputTokens = inputTokenDetails?.CachedTokenDetails?.AudioTokens ?? 0, - TextOutputTokens = outputTokenDetails?.TextTokens ?? 0, - AudioOutputTokens = outputTokenDetails?.AudioTokens ?? 0 - }); - } - - return outputs; - } - - private async Task OnUserAudioTranscriptionCompleted(RealtimeHubConnection conn, string response) - { - var data = JsonSerializer.Deserialize(response); - return new RoleDialogModel(AgentRole.User, data.Transcript) - { - CurrentAgentId = conn.CurrentAgentId - }; - } - - public async Task OnConversationItemCreated(RealtimeHubConnection conn, string response) - { - var item = response.JsonContent().Item; - var message = new RoleDialogModel(item.Role, item.Content.FirstOrDefault()?.Transcript); - - return message; - } + #endregion } \ No newline at end of file diff --git a/tests/BotSharp.Test.RealtimeVoice/appsettings.json b/tests/BotSharp.Test.RealtimeVoice/appsettings.json index e0ffcb8c..38a4b2c1 100644 --- a/tests/BotSharp.Test.RealtimeVoice/appsettings.json +++ b/tests/BotSharp.Test.RealtimeVoice/appsettings.json @@ -16,9 +16,15 @@ "Version": "2024-12-17", "ApiKey": "", "Type": "realtime", - "MultiModal": true, - "PromptCost": 0.0025, - "CompletionCost": 0.01 + "RealTime": true, + "Cost": { + "TextInputCost": 0.0006, + "CachedTextInputCost": 0.0003, + "AudioInputCost": 0.01, + "CachedAudioInputCost": 0.0003, + "TextOutputCost": 0.0024, + "AudioOutputCost": 0.02 + } } ] }, @@ -31,9 +37,15 @@ "Version": "20240620", "ApiKey": "", "Type": "realtime", - "MultiModal": true, - "PromptCost": 0.003, - "CompletionCost": 0.015 + "RealTime": true, + "Cost": { + "TextInputCost": 0.0006, + "CachedTextInputCost": 0.0003, + "AudioInputCost": 0.01, + "CachedAudioInputCost": 0.0003, + "TextOutputCost": 0.0024, + "AudioOutputCost": 0.02 + } } ] } From 1de8953404f18c2fef2af889bca0f343816f393c Mon Sep 17 00:00:00 2001 From: "aden.chen" Date: Thu, 15 May 2025 09:57:16 +0800 Subject: [PATCH 11/26] Enhance Twilio hook execution with conditional logic --- .../Controllers/TwilioRecordController.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs index 4e4b1c75..8bc26a8e 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Infrastructures; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.Models; @@ -36,7 +37,11 @@ public class TwilioRecordController : TwilioController convService.SaveStates(); // recording completed - await HookEmitter.Emit(_services, x => x.OnRecordingCompleted(request)); + var emitOptions = new HookEmitOption + { + ShouldExecute = hook => hook.IsMatch(request) + }; + await HookEmitter.Emit(_services, x => x.OnRecordingCompleted(request), emitOptions); } else { From 91cd150862d9a009c22897100d3335323446c105 Mon Sep 17 00:00:00 2001 From: "aden.chen" Date: Thu, 15 May 2025 10:00:28 +0800 Subject: [PATCH 12/26] Revert "Add null-check for `agent` in `StreamingLogHook`" This reverts commit 3dda7afc7c7448c7bfb1551b79d242d044a88b3a. --- src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs index 2bc9c9e6..720f3fd0 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs @@ -293,7 +293,7 @@ public class StreamingLogHook : ConversationHookBase, IContentGeneratingHook, IR var routing = _services.GetRequiredService(); var agentId = routing.Context.GetCurrentAgentId(); var agent = await _agentService.GetAgent(agentId); - if (agent == null) return; + var input = new ContentLogInputModel() { Name = agent.Name, From a2ddbf913cc43ddcca8931b511eb3a6b5797f4fd Mon Sep 17 00:00:00 2001 From: "aden.chen" Date: Thu, 15 May 2025 10:09:31 +0800 Subject: [PATCH 13/26] Add hook execution options to Twilio controllers --- .../Controllers/TwilioInboundController.cs | 9 +++++++-- .../Controllers/TwilioOutboundController.cs | 7 ++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs index 9d7df9a0..1db293f2 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Infrastructures; using BotSharp.Abstraction.Infrastructures.Enums; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.Interfaces; @@ -65,9 +66,13 @@ public class TwilioInboundController : TwilioController if (twilio.MachineDetected(request)) { response = new VoiceResponse(); - + + var emitOptions = new HookEmitOption + { + ShouldExecute = hook => hook.IsMatch(request) + }; await HookEmitter.Emit(_services, - async hook => await hook.OnVoicemailStarting(request)); + async hook => await hook.OnVoicemailStarting(request), emitOptions); var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3"); response.Play(new Uri(url)); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioOutboundController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioOutboundController.cs index 4796c02d..4cb13a61 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioOutboundController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioOutboundController.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Infrastructures; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.Models; @@ -33,8 +34,12 @@ public class TwilioOutboundController : TwilioController { response = new VoiceResponse(); + var emitOptions = new HookEmitOption + { + ShouldExecute = hook => hook.IsMatch(request) + }; await HookEmitter.Emit(_services, - async hook => await hook.OnVoicemailStarting(request)); + async hook => await hook.OnVoicemailStarting(request), emitOptions); var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3"); response.Play(new Uri(url)); From b748ce3d9fa4ff275a9f71921108f7b3ba24e67f Mon Sep 17 00:00:00 2001 From: "nick.yi" Date: Thu, 15 May 2025 10:48:08 +0800 Subject: [PATCH 14/26] optimize AgentHook,IInstructHook --- .../BotSharp.Abstraction/Agents/IAgentHook.cs | 2 + .../Instructs/IInstructHook.cs | 1 + .../Agents/Services/AgentService.LoadAgent.cs | 43 ++++-------- .../Instruct/FileInstructService.Image.cs | 70 +++++++------------ .../Instruct/FileInstructService.Pdf.cs | 15 ++-- .../Instructs/Functions/ExecuteTemplateFn.cs | 15 ++-- .../Services/InstructService.Execute.cs | 12 +--- .../Controllers/InstructModeController.cs | 31 ++++---- 8 files changed, 68 insertions(+), 121 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentHook.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentHook.cs index ae6a4eaf..7ad9858e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentHook.cs @@ -9,6 +9,8 @@ public interface IAgentHook /// string SelfId { get; } Agent Agent { get; } + + bool IsMatch(string id) => string.IsNullOrEmpty(SelfId) || SelfId == id; void SetAgent(Agent agent); /// diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructHook.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructHook.cs index c9fa1cd7..c2b9a5fe 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructHook.cs @@ -5,6 +5,7 @@ namespace BotSharp.Abstraction.Instructs; public interface IInstructHook { string SelfId { get; } + bool IsMatch(string id) => string.IsNullOrEmpty(SelfId) || SelfId == id; Task BeforeCompletion(Agent agent, RoleDialogModel message); Task AfterCompletion(Agent agent, InstructResult result); Task OnResponseGenerated(InstructResponseModel response); diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs index 636bc811..3aa2e422 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Infrastructures; using BotSharp.Abstraction.Routing.Models; using System.Collections.Concurrent; @@ -10,29 +11,16 @@ public partial class AgentService // [SharpCache(10, perInstanceCache: true)] public async Task LoadAgent(string id, bool loadUtility = true) { - if (string.IsNullOrEmpty(id) || id == Guid.Empty.ToString()) + if (string.IsNullOrEmpty(id) || id == Guid.Empty.ToString()) return null; + + var emitOptions = new HookEmitOption { - return null; - } - - var hooks = _services.GetServices(); - - // Before agent is loaded. - foreach (var hook in hooks) - { - if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != id) - { - continue; - } - - hook.OnAgentLoading(ref id); - } + ShouldExecute = hook => hook.IsMatch(id) + }; + HookEmitter.Emit(_services, hook => hook.OnAgentLoading(ref id), emitOptions); var agent = await GetAgent(id); - if (agent == null) - { - return null; - } + if (agent == null) return null; await InheritAgent(agent); OverrideInstructionByChannel(agent); @@ -43,13 +31,7 @@ public partial class AgentService PopulateState(agent.TemplateDict); // After agent is loaded - foreach (var hook in hooks) - { - if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != id) - { - continue; - } - + HookEmitter.Emit(_services, hook => { hook.SetAgent(agent); if (!string.IsNullOrEmpty(agent.Instruction)) @@ -72,13 +54,14 @@ public partial class AgentService hook.OnAgentUtilityLoaded(agent); } - if(!agent.McpTools.IsNullOrEmpty()) + if (!agent.McpTools.IsNullOrEmpty()) { hook.OnAgentMcpToolLoaded(agent); } - + hook.OnAgentLoaded(agent); - } + + }, emitOptions); _logger.LogInformation($"Loaded agent {agent}."); diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs index defd1459..4f5e928e 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs @@ -1,6 +1,7 @@ using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.Instructs; using System.IO; +using BotSharp.Abstraction.Infrastructures; namespace BotSharp.Core.Files.Services; @@ -24,14 +25,11 @@ public partial class FileInstructService } }); - var hooks = _services.GetServices(); - foreach (var hook in hooks) + var emitOptions = new HookEmitOption { - if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != innerAgentId) - { - continue; - } - + ShouldExecute = hook => hook.IsMatch(innerAgentId) + }; + await HookEmitter.Emit(_services, async hook => await hook.OnResponseGenerated(new InstructResponseModel { AgentId = innerAgentId, @@ -41,8 +39,7 @@ public partial class FileInstructService UserMessage = text, SystemInstruction = instruction, CompletionText = message.Content - }); - } + }), emitOptions); return message.Content; } @@ -59,14 +56,11 @@ public partial class FileInstructService Instruction = instruction }, new RoleDialogModel(AgentRole.User, instruction ?? text)); - var hooks = _services.GetServices(); - foreach (var hook in hooks) + var emitOptions = new HookEmitOption { - if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != innerAgentId) - { - continue; - } - + ShouldExecute = hook => hook.IsMatch(innerAgentId) + }; + await HookEmitter.Emit(_services, async hook => await hook.OnResponseGenerated(new InstructResponseModel { AgentId = innerAgentId, @@ -76,8 +70,7 @@ public partial class FileInstructService UserMessage = text, SystemInstruction = instruction, CompletionText = message.Content - }); - } + }), emitOptions); return message; } @@ -104,14 +97,11 @@ public partial class FileInstructService stream.Close(); - var hooks = _services.GetServices(); - foreach (var hook in hooks) + var emitOptions = new HookEmitOption { - if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != innerAgentId) - { - continue; - } - + ShouldExecute = hook => hook.IsMatch(innerAgentId) + }; + await HookEmitter.Emit(_services, async hook => await hook.OnResponseGenerated(new InstructResponseModel { AgentId = innerAgentId, @@ -119,8 +109,7 @@ public partial class FileInstructService Model = completion.Model, UserMessage = string.Empty, CompletionText = message.Content - }); - } + }), emitOptions); return message; } @@ -150,13 +139,11 @@ public partial class FileInstructService stream.Close(); var hooks = _services.GetServices(); - foreach (var hook in hooks) + var emitOptions = new HookEmitOption { - if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != innerAgentId) - { - continue; - } - + ShouldExecute = hook => hook.IsMatch(innerAgentId) + }; + await HookEmitter.Emit(_services, async hook => await hook.OnResponseGenerated(new InstructResponseModel { AgentId = innerAgentId, @@ -166,8 +153,7 @@ public partial class FileInstructService UserMessage = text, SystemInstruction = instruction, CompletionText = message.Content - }); - } + }), emitOptions); return message; } @@ -205,14 +191,11 @@ public partial class FileInstructService imageStream.Close(); maskStream.Close(); - var hooks = _services.GetServices(); - foreach (var hook in hooks) + var emitOptions = new HookEmitOption { - if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != innerAgentId) - { - continue; - } - + ShouldExecute = hook => hook.IsMatch(innerAgentId) + }; + await HookEmitter.Emit(_services, async hook => await hook.OnResponseGenerated(new InstructResponseModel { AgentId = innerAgentId, @@ -222,8 +205,7 @@ public partial class FileInstructService UserMessage = text, SystemInstruction = instruction, CompletionText = message.Content - }); - } + }), emitOptions); return message; } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index 2336363a..12b950b6 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -1,6 +1,7 @@ using BotSharp.Abstraction.Files.Converters; using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.Instructs; +using BotSharp.Abstraction.Infrastructures; namespace BotSharp.Core.Files.Services; @@ -42,14 +43,11 @@ public partial class FileInstructService } }); - var hooks = _services.GetServices(); - foreach (var hook in hooks) + var emitOptions = new HookEmitOption { - if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != innerAgentId) - { - continue; - } - + ShouldExecute = hook => hook.IsMatch(innerAgentId) + }; + await HookEmitter.Emit(_services, async hook => await hook.OnResponseGenerated(new InstructResponseModel { AgentId = innerAgentId, @@ -59,8 +57,7 @@ public partial class FileInstructService UserMessage = text, SystemInstruction = instruction, CompletionText = message.Content - }); - } + }), emitOptions); return message.Content; } diff --git a/src/Infrastructure/BotSharp.Core/Instructs/Functions/ExecuteTemplateFn.cs b/src/Infrastructure/BotSharp.Core/Instructs/Functions/ExecuteTemplateFn.cs index 346218dd..e225b6d3 100644 --- a/src/Infrastructure/BotSharp.Core/Instructs/Functions/ExecuteTemplateFn.cs +++ b/src/Infrastructure/BotSharp.Core/Instructs/Functions/ExecuteTemplateFn.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Functions; +using BotSharp.Abstraction.Infrastructures; using BotSharp.Abstraction.Instructs; using BotSharp.Abstraction.Instructs.Models; @@ -60,14 +61,11 @@ public class ExecuteTemplateFn : IFunctionCallback new(AgentRole.User, text) }); - var hooks = _services.GetServices(); - foreach (var hook in hooks) + var emitOptions = new HookEmitOption { - if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agent.Id) - { - continue; - } - + ShouldExecute = hook => hook.IsMatch(agent.Id) + }; + await HookEmitter.Emit(_services, async hook => await hook.OnResponseGenerated(new InstructResponseModel { AgentId = agent.Id, @@ -76,8 +74,7 @@ public class ExecuteTemplateFn : IFunctionCallback Model = completion.Model, UserMessage = text, CompletionText = response.Content - }); - } + }), emitOptions); return response.Content; } diff --git a/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs b/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs index 71a7488b..3bc069d5 100644 --- a/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs +++ b/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs @@ -23,14 +23,9 @@ public partial class InstructService } // Trigger before completion hooks - var hooks = _services.GetServices(); + var hooks = _services.GetServices().Where(p => p.IsMatch(agentId)); foreach (var hook in hooks) { - if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId) - { - continue; - } - await hook.BeforeCompletion(agent, message); // Interrupted by hook @@ -99,11 +94,6 @@ public partial class InstructService foreach (var hook in hooks) { - if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId) - { - continue; - } - await hook.AfterCompletion(agent, response); await hook.OnResponseGenerated(new InstructResponseModel { diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index ca3ec45a..b5669a5f 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Files.Utilities; +using BotSharp.Abstraction.Infrastructures; using BotSharp.Abstraction.Instructs; using BotSharp.Abstraction.Instructs.Models; using BotSharp.Core.Infrastructures; @@ -58,14 +59,11 @@ public class InstructModeController : ControllerBase var textCompletion = CompletionProvider.GetTextCompletion(_services); var response = await textCompletion.GetCompletion(input.Text, agentId, Guid.NewGuid().ToString()); - var hooks = _services.GetServices(); - foreach (var hook in hooks) + var emitOptions = new HookEmitOption { - if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId) - { - continue; - } - + ShouldExecute = hook => hook.IsMatch(agentId) + }; + await HookEmitter.Emit(_services, async hook => await hook.OnResponseGenerated(new InstructResponseModel { AgentId = agentId, @@ -74,8 +72,8 @@ public class InstructModeController : ControllerBase TemplateName = input.Template, UserMessage = input.Text, CompletionText = response - }); - } + }), emitOptions); + return response; } @@ -103,14 +101,11 @@ public class InstructModeController : ControllerBase } }); - var hooks = _services.GetServices(); - foreach (var hook in hooks) + var emitOptions = new HookEmitOption { - if (!string.IsNullOrEmpty(hook.SelfId) && hook.SelfId != agentId) - { - continue; - } - + ShouldExecute = hook => hook.IsMatch(agentId) + }; + await HookEmitter.Emit(_services, async hook => await hook.OnResponseGenerated(new InstructResponseModel { AgentId = agentId, @@ -120,8 +115,8 @@ public class InstructModeController : ControllerBase UserMessage = input.Text, SystemInstruction = message.RenderedInstruction, CompletionText = message.Content - }); - } + }), emitOptions); + return message.Content; } #endregion From bc40deed5405088fa3e90851e62e8c0293e00bc6 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Wed, 14 May 2025 23:49:36 -0500 Subject: [PATCH 15/26] tool call done; to do session restart --- .../Hooks/RealtimeConversationHook.cs | 22 ++++++------ .../Conversations/ConversationPlugin.cs | 1 + .../Session/LlmRealtimeSession.cs | 1 + .../Services/BotSharpStatsService.cs | 1 - .../BotSharp.Plugin.ChatHub/ChatHubPlugin.cs | 1 - .../Realtime/RealtimeGenerateContentSetup.cs | 11 +++++- .../Models/Realtime/RealtimeServerResponse.cs | 12 +++++++ .../Realtime/RealtimeTranscriptionResponse.cs | 2 +- .../Realtime/RealTimeCompletionProvider.cs | 35 +++++++++++-------- .../Realtime/RealTimeCompletionProvider.cs | 5 ++- .../appsettings.json | 2 -- 11 files changed, 59 insertions(+), 34 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs index aabac186..2f8a03ee 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs @@ -42,7 +42,6 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook var routing = _services.GetRequiredService(); message.Role = AgentRole.Function; - //message.Role = AgentRole.Assistant; if (message.FunctionName == "route_to_agent") { @@ -66,21 +65,24 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook else { // Update session for changed states - var instruction = await hub.Completer.UpdateSession(hub.HubConn); + + // TO DO + //var instruction = await hub.Completer.UpdateSession(hub.HubConn); await hub.Completer.InsertConversationItem(message); if (string.IsNullOrEmpty(message.Content)) { return; } - else if (message.StopCompletion) - { - await hub.Completer.TriggerModelInference($"Say to user: \"{message.Content}\""); - } - else - { - await hub.Completer.TriggerModelInference(instruction); - } + + //if (message.StopCompletion) + //{ + // await hub.Completer.TriggerModelInference($"Say to user: \"{message.Content}\""); + //} + //else + //{ + // await hub.Completer.TriggerModelInference(); + //} } } } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs b/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs index 13ee1de6..bf694b71 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs @@ -41,6 +41,7 @@ public class ConversationPlugin : IBotSharpPlugin return settingService.Bind("GoogleApi"); }); + services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/src/Infrastructure/BotSharp.Core/Session/LlmRealtimeSession.cs b/src/Infrastructure/BotSharp.Core/Session/LlmRealtimeSession.cs index 2f3259cf..60ecee04 100644 --- a/src/Infrastructure/BotSharp.Core/Session/LlmRealtimeSession.cs +++ b/src/Infrastructure/BotSharp.Core/Session/LlmRealtimeSession.cs @@ -106,6 +106,7 @@ public class LlmRealtimeSession : IDisposable public void Dispose() { + _clientEventSemaphore?.Dispose(); _webSocket?.Dispose(); } } diff --git a/src/Infrastructure/BotSharp.Core/Statistics/Services/BotSharpStatsService.cs b/src/Infrastructure/BotSharp.Core/Statistics/Services/BotSharpStatsService.cs index b5320c45..8cbf974f 100644 --- a/src/Infrastructure/BotSharp.Core/Statistics/Services/BotSharpStatsService.cs +++ b/src/Infrastructure/BotSharp.Core/Statistics/Services/BotSharpStatsService.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Infrastructures; using BotSharp.Abstraction.Statistics.Settings; namespace BotSharp.Core.Statistics.Services; diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/ChatHubPlugin.cs b/src/Plugins/BotSharp.Plugin.ChatHub/ChatHubPlugin.cs index 7275a273..725655fc 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/ChatHubPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/ChatHubPlugin.cs @@ -24,7 +24,6 @@ public class ChatHubPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); services.AddScoped(); - services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeGenerateContentSetup.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeGenerateContentSetup.cs index c72e8cb6..c334c7ff 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeGenerateContentSetup.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeGenerateContentSetup.cs @@ -21,6 +21,15 @@ internal class RealtimeGenerateContentSetup [JsonPropertyName("outputAudioTranscription")] public AudioTranscriptionConfig? OutputAudioTranscription { get; set; } + + [JsonPropertyName("sessionResumption")] + public SessionResumptionConfig? SessionResumption { get; set; } } -internal class AudioTranscriptionConfig { } \ No newline at end of file +internal class AudioTranscriptionConfig { } + +internal class SessionResumptionConfig +{ + [JsonPropertyName("handle")] + public string? Handle { get; set; } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs index 4b65bdf8..dd6ff508 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeServerResponse.cs @@ -15,6 +15,9 @@ internal class RealtimeServerResponse [JsonPropertyName("toolCall")] public RealtimeToolCall? ToolCall { get; set; } + + [JsonPropertyName("sessionResumptionUpdate")] + public RealtimeSessionResumptionUpdate? SessionResumptionUpdate { get; set; } } @@ -91,4 +94,13 @@ internal class RealtimeFunctionCall [JsonPropertyName("args")] public JsonNode? Args { get; set; } +} + +internal class RealtimeSessionResumptionUpdate +{ + [JsonPropertyName("newHandle")] + public string? NewHandle { get; set; } + + [JsonPropertyName("resumable")] + public bool? Resumable { get; set; } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs index b14c1bde..5d75ca97 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs @@ -28,7 +28,7 @@ internal class RealtimeTranscriptionResponse : IDisposable _contentStream.Position = 0; } - public string GetString() + public string GetText() { if (_contentStream.Length == 0) { diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs index f628a281..f8f551fe 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -113,7 +113,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion continue; } - Console.WriteLine($"Received text: {receivedText}"); try { var response = JsonSerializer.Deserialize(receivedText, _jsonOptions); @@ -126,10 +125,15 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { _logger.LogInformation($"Session setup completed."); } + else if (response.SessionResumptionUpdate != null) + { + _logger.LogInformation($"Session resumption update => New handle: {response.SessionResumptionUpdate.NewHandle}, Resumable: {response.SessionResumptionUpdate.Resumable}"); + } else if (response.ToolCall != null && !response.ToolCall.FunctionCalls.IsNullOrEmpty()) { - var functionCall = response.ToolCall.FunctionCalls.First(); - _logger.LogInformation($"Tool call received {functionCall.Name}({functionCall.Args?.ToJsonString(_jsonOptions) ?? string.Empty})."); + var functionCall = response.ToolCall.FunctionCalls!.First(); + + _logger.LogInformation($"Tool call received: {functionCall.Name}({functionCall.Args?.ToJsonString(_jsonOptions) ?? string.Empty})."); if (functionCall != null) { @@ -154,7 +158,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion _logger.LogInformation($"Model audio delta received."); // Handle input transcription - var inputTranscription = inputStream.GetString(); + var inputTranscription = inputStream.GetText(); if (!string.IsNullOrEmpty(inputTranscription)) { var message = OnUserAudioTranscriptionCompleted(conn, inputTranscription); @@ -182,7 +186,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { _logger.LogInformation($"Model turn completed."); - var outputTranscription = outputStream.GetString(); + // Handle output transcription + var outputTranscription = outputStream.GetText(); if (!string.IsNullOrEmpty(outputTranscription)) { var messages = await OnResponseDone(conn, outputTranscription, response.UsageMetaData); @@ -237,13 +242,13 @@ public class GoogleRealTimeProvider : IRealTimeCompletion public async Task TriggerModelInference(string? instructions = null) { - var content = new Content("Please respond to me.", AgentRole.User); + var content = new Content(instructions ?? "Please respond to user.", AgentRole.User); await SendEventToModel(new BidiClientPayload { ClientContent = new() { - Turns = null, + Turns = [content], TurnComplete = true } }); @@ -269,9 +274,10 @@ public class GoogleRealTimeProvider : IRealTimeCompletion public async Task UpdateSession(RealtimeHubConnection conn, bool isInit = false) { var convService = _services.GetRequiredService(); - var conv = await convService.GetConversation(conn.ConversationId); - var agentService = _services.GetRequiredService(); + var realtimeSetting = _services.GetRequiredService(); + + var conv = await convService.GetConversation(conn.ConversationId); var agent = await agentService.LoadAgent(conn.CurrentAgentId); var (prompt, request) = PrepareOptions(agent, []); @@ -285,10 +291,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion var words = new List(); HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent))); - var realtimeModelSettings = _services.GetRequiredService(); - - config.Temperature = Math.Max(realtimeModelSettings.Temperature, 0.6f); - config.MaxOutputTokens = realtimeModelSettings.MaxResponseOutputTokens; + config.Temperature = Math.Max(realtimeSetting.Temperature, 0.6f); + config.MaxOutputTokens = realtimeSetting.MaxResponseOutputTokens; } var functions = request.Tools?.SelectMany(s => s.FunctionDeclarations).Select(x => @@ -316,7 +320,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion }); } - var realtimeSetting = _services.GetRequiredService(); await SendEventToModel(new RealtimeClientPayload { Setup = new RealtimeGenerateContentSetup() @@ -326,7 +329,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion SystemInstruction = request.SystemInstruction, Tools = request.Tools?.ToArray(), InputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null, - OutputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null + OutputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null, + SessionResumption = new() } }); @@ -339,6 +343,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { var function = new FunctionResponse() { + Id = message.ToolCallId, Name = message.FunctionName ?? string.Empty, Response = new JsonObject() { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index fa1bd623..29e495ec 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -65,7 +65,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion cancellationToken: CancellationToken.None); _ = ReceiveMessage( - _services, + realtimeSettings, conn, onModelReady, onModelAudioDeltaReceived, @@ -144,7 +144,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion } private async Task ReceiveMessage( - IServiceProvider services, + RealtimeModelSettings realtimeSettings, RealtimeHubConnection conn, Func onModelReady, Func onModelAudioDeltaReceived, @@ -156,7 +156,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion Func onInterruptionDetected) { DateTime? startTime = null; - var realtimeSettings = _services.GetRequiredService(); await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None)) { diff --git a/tests/BotSharp.Test.RealtimeVoice/appsettings.json b/tests/BotSharp.Test.RealtimeVoice/appsettings.json index 38a4b2c1..24b85266 100644 --- a/tests/BotSharp.Test.RealtimeVoice/appsettings.json +++ b/tests/BotSharp.Test.RealtimeVoice/appsettings.json @@ -16,7 +16,6 @@ "Version": "2024-12-17", "ApiKey": "", "Type": "realtime", - "RealTime": true, "Cost": { "TextInputCost": 0.0006, "CachedTextInputCost": 0.0003, @@ -37,7 +36,6 @@ "Version": "20240620", "ApiKey": "", "Type": "realtime", - "RealTime": true, "Cost": { "TextInputCost": 0.0006, "CachedTextInputCost": 0.0003, From 4c4dce852d99d0a1742443eb2b7142bab40cb8b2 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Wed, 14 May 2025 23:54:18 -0500 Subject: [PATCH 16/26] minor change --- .../BotSharp.Core/Functions/GetLocationFn.cs | 25 ------------------- .../BotSharp.Core/Functions/GetWeatherFn.cs | 18 ------------- .../Realtime/RealTimeCompletionProvider.cs | 3 --- 3 files changed, 46 deletions(-) delete mode 100644 src/Infrastructure/BotSharp.Core/Functions/GetLocationFn.cs diff --git a/src/Infrastructure/BotSharp.Core/Functions/GetLocationFn.cs b/src/Infrastructure/BotSharp.Core/Functions/GetLocationFn.cs deleted file mode 100644 index cabfbdbb..00000000 --- a/src/Infrastructure/BotSharp.Core/Functions/GetLocationFn.cs +++ /dev/null @@ -1,25 +0,0 @@ -using BotSharp.Abstraction.Functions; -using BotSharp.Abstraction.Options; - -namespace BotSharp.Core.Functions; - -public class GetLocationFn : IFunctionCallback -{ - private readonly IServiceProvider _services; - - public GetLocationFn(IServiceProvider services) - { - _services = services; - } - - public string Name => "get_location"; - public string Indication => "Finding location"; - - public async Task Execute(RoleDialogModel message) - { - var args = JsonSerializer.Deserialize(message.FunctionArgs, BotSharpOptions.defaultJsonOptions); - - message.Content = $"There are a lot of fun events here in {args.City}"; - return true; - } -} diff --git a/src/Infrastructure/BotSharp.Core/Functions/GetWeatherFn.cs b/src/Infrastructure/BotSharp.Core/Functions/GetWeatherFn.cs index 09f6e661..d203ac8c 100644 --- a/src/Infrastructure/BotSharp.Core/Functions/GetWeatherFn.cs +++ b/src/Infrastructure/BotSharp.Core/Functions/GetWeatherFn.cs @@ -1,7 +1,4 @@ using BotSharp.Abstraction.Functions; -using BotSharp.Abstraction.Models; -using BotSharp.Abstraction.Options; -using BotSharp.Abstraction.SideCar; using System.Text.Json.Serialization; namespace BotSharp.Core.Functions; @@ -21,24 +18,9 @@ public class GetWeatherFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { //var args = JsonSerializer.Deserialize(message.FunctionArgs, BotSharpOptions.defaultJsonOptions); - - //var sidecar = _services.GetService(); - //var states = GetSideCarStates(); - - //var userMessage = $"Please find the information at location {args.City}, {args.State}"; - //var response = await sidecar.SendMessage(BuiltInAgentId.Chatbot, userMessage, states: states); message.Content = $"It is a sunny day."; return true; } - - private List GetSideCarStates() - { - var sideCarStates = new List() - { - new("channel", "email") - }; - return sideCarStates; - } } class Location diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs index f8f551fe..eb6f4e76 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -273,13 +273,10 @@ public class GoogleRealTimeProvider : IRealTimeCompletion public async Task UpdateSession(RealtimeHubConnection conn, bool isInit = false) { - var convService = _services.GetRequiredService(); var agentService = _services.GetRequiredService(); var realtimeSetting = _services.GetRequiredService(); - var conv = await convService.GetConversation(conn.ConversationId); var agent = await agentService.LoadAgent(conn.CurrentAgentId); - var (prompt, request) = PrepareOptions(agent, []); var config = request.GenerationConfig; From 9cdf9179c0829c6994fe23c1a4cbf6131b87a136 Mon Sep 17 00:00:00 2001 From: "nick.yi" Date: Thu, 15 May 2025 15:25:08 +0800 Subject: [PATCH 17/26] optimize hook --- .../BotSharp.Abstraction/Agents/IAgentHook.cs | 9 ++--- .../Conversations/IConversationHook.cs | 4 ++- .../Crontab/ICrontabHook.cs | 4 ++- .../BotSharp.Abstraction/Hooks/IHookBase.cs | 17 +++++++++ .../Instructs/IInstructHook.cs | 5 ++- .../Loggers/IContentGeneratingHook.cs | 3 +- .../Planning/IPlanningHook.cs | 4 ++- .../Realtime/IRealtimeHook.cs | 3 +- .../Routing/IRoutingHook.cs | 3 +- .../Services/CrontabService.cs | 2 +- .../Services/RealtimeHub.cs | 6 ++-- .../Agents/Services/AgentService.LoadAgent.cs | 8 ++--- .../Agents/Services/AgentService.Rendering.cs | 5 ++- .../ConversationService.SendMessage.cs | 10 +++--- .../Instruct/FileInstructService.Image.cs | 31 +++------------- .../Instruct/FileInstructService.Pdf.cs | 6 +--- .../Infrastructures/HookEmitter.cs | 9 ++--- .../Instructs/Functions/ExecuteTemplateFn.cs | 6 +--- .../Routing/Reasoning/InstructExecutor.cs | 5 ++- .../BotSharp.Core/Routing/RoutingContext.cs | 20 +++++------ .../Routing/RoutingService.InstructLoop.cs | 5 ++- .../Controllers/ConversationController.cs | 3 +- .../Controllers/InstructModeController.cs | 12 ++----- .../Realtime/RealTimeCompletionProvider.cs | 4 +-- .../Realtime/RealTimeCompletionProvider.cs | 4 +-- .../Functions/SqlGenerationFn.cs | 2 +- .../SqlGeneration/Functions/SqlReviewFn.cs | 3 +- .../TwoStaging/Functions/SummaryPlanFn.cs | 5 +-- .../Hooks/SqlDriverPlanningHook.cs | 2 +- .../Interfaces/ISqlDriverHook.cs | 4 ++- .../Controllers/TwilioInboundController.cs | 10 ++---- .../Controllers/TwilioOutboundController.cs | 7 +--- .../Controllers/TwilioRecordController.cs | 6 +--- .../Controllers/TwilioTranscribeController.cs | 3 +- .../Controllers/TwilioVoiceController.cs | 36 ++++++++----------- .../Interfaces/ITwilioCallStatusHook.cs | 4 +-- .../Interfaces/ITwilioSessionHook.cs | 3 +- .../Services/TwilioMessageQueueService.cs | 3 +- .../Services/TwilioService.cs | 9 +++-- 39 files changed, 123 insertions(+), 162 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Hooks/IHookBase.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentHook.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentHook.cs index 7ad9858e..19e29642 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentHook.cs @@ -1,16 +1,11 @@ using BotSharp.Abstraction.Functions.Models; +using BotSharp.Abstraction.Hooks; namespace BotSharp.Abstraction.Agents; -public interface IAgentHook +public interface IAgentHook : IHookBase { - /// - /// Agent Id - /// - string SelfId { get; } Agent Agent { get; } - - bool IsMatch(string id) => string.IsNullOrEmpty(SelfId) || SelfId == id; void SetAgent(Agent agent); /// diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs index b722d3fa..6d967391 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs @@ -1,6 +1,8 @@ +using BotSharp.Abstraction.Hooks; + namespace BotSharp.Abstraction.Conversations; -public interface IConversationHook +public interface IConversationHook : IHookBase { int Priority { get; } Agent Agent { get; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Crontab/ICrontabHook.cs b/src/Infrastructure/BotSharp.Abstraction/Crontab/ICrontabHook.cs index 6ad7e06a..b92fd8f5 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Crontab/ICrontabHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Crontab/ICrontabHook.cs @@ -1,6 +1,8 @@ +using BotSharp.Abstraction.Hooks; + namespace BotSharp.Abstraction.Crontab; -public interface ICrontabHook +public interface ICrontabHook : IHookBase { string[]? Triggers => null; diff --git a/src/Infrastructure/BotSharp.Abstraction/Hooks/IHookBase.cs b/src/Infrastructure/BotSharp.Abstraction/Hooks/IHookBase.cs new file mode 100644 index 00000000..a0ad18e6 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Hooks/IHookBase.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BotSharp.Abstraction.Hooks +{ + public interface IHookBase + { + /// + /// Agent Id + /// + string SelfId => string.Empty; + bool IsMatch(string id) => string.IsNullOrEmpty(SelfId) || SelfId == id; + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructHook.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructHook.cs index c2b9a5fe..23b8bf60 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/IInstructHook.cs @@ -1,11 +1,10 @@ +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.Instructs.Models; namespace BotSharp.Abstraction.Instructs; -public interface IInstructHook +public interface IInstructHook : IHookBase { - string SelfId { get; } - bool IsMatch(string id) => string.IsNullOrEmpty(SelfId) || SelfId == id; Task BeforeCompletion(Agent agent, RoleDialogModel message); Task AfterCompletion(Agent agent, InstructResult result); Task OnResponseGenerated(InstructResponseModel response); diff --git a/src/Infrastructure/BotSharp.Abstraction/Loggers/IContentGeneratingHook.cs b/src/Infrastructure/BotSharp.Abstraction/Loggers/IContentGeneratingHook.cs index 03b69f5f..79f71d70 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Loggers/IContentGeneratingHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Loggers/IContentGeneratingHook.cs @@ -1,11 +1,12 @@ using BotSharp.Abstraction.Functions.Models; +using BotSharp.Abstraction.Hooks; namespace BotSharp.Abstraction.Loggers; /// /// Model content generating hook, it can be used for logging, metrics and tracing. /// -public interface IContentGeneratingHook +public interface IContentGeneratingHook : IHookBase { /// /// Before content generating. diff --git a/src/Infrastructure/BotSharp.Abstraction/Planning/IPlanningHook.cs b/src/Infrastructure/BotSharp.Abstraction/Planning/IPlanningHook.cs index 4dcff7cb..b65c1a7e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Planning/IPlanningHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Planning/IPlanningHook.cs @@ -1,6 +1,8 @@ +using BotSharp.Abstraction.Hooks; + namespace BotSharp.Abstraction.Planning; -public interface IPlanningHook +public interface IPlanningHook : IHookBase { Task GetSummaryAdditionalRequirements(string planner, RoleDialogModel message) => Task.FromResult(string.Empty); diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs index 95608ff0..bd131b93 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs @@ -1,8 +1,9 @@ +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.MLTasks; namespace BotSharp.Abstraction.Realtime; -public interface IRealtimeHook +public interface IRealtimeHook : IHookBase { Task OnModelReady(Agent agent, IRealTimeCompletion completer); string[] OnModelTranscriptPrompt(Agent agent); diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingHook.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingHook.cs index c60bf4f1..b893c8c1 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingHook.cs @@ -1,8 +1,9 @@ using BotSharp.Abstraction.Functions.Models; +using BotSharp.Abstraction.Hooks; namespace BotSharp.Abstraction.Routing; -public interface IRoutingHook +public interface IRoutingHook : IHookBase { /// /// Routing instruction is received from Router diff --git a/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs b/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs index 7ff105c9..255faa71 100644 --- a/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs +++ b/src/Infrastructure/BotSharp.Core.Crontab/Services/CrontabService.cs @@ -125,6 +125,6 @@ public class CrontabService : ICrontabService, ITaskFeeder await hook.OnCronTriggered(item); await hook.OnTaskExecuted(item); } - }); + }, item.AgentId); } } diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 60729758..cc9fa0c1 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -49,7 +49,8 @@ public class RealtimeHub : IRealtimeHub // Not TriggerModelInference, waiting for user utter. var instruction = await _completer.UpdateSession(_conn, isInit: true); var data = _conn.OnModelReady(); - await HookEmitter.Emit(_services, async hook => await hook.OnModelReady(agent, _completer)); + await HookEmitter.Emit(_services, async hook => await hook.OnModelReady(agent, _completer), + agent.Id); await (init?.Invoke(data) ?? Task.CompletedTask); }, onModelAudioDeltaReceived: async (audioDeltaData, itemId) => @@ -92,7 +93,8 @@ public class RealtimeHub : IRealtimeHub if (message.FunctionName == "route_to_agent") { var instruction = JsonSerializer.Deserialize(message.FunctionArgs, BotSharpOptions.defaultJsonOptions); - await HookEmitter.Emit(_services, async hook => await hook.OnRoutingInstructionReceived(instruction, message)); + await HookEmitter.Emit(_services, async hook => await hook.OnRoutingInstructionReceived(instruction, message), + agent.Id); } await routing.InvokeFunction(message.FunctionName, message); diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs index 3aa2e422..37e3d445 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs @@ -13,11 +13,7 @@ public partial class AgentService { if (string.IsNullOrEmpty(id) || id == Guid.Empty.ToString()) return null; - var emitOptions = new HookEmitOption - { - ShouldExecute = hook => hook.IsMatch(id) - }; - HookEmitter.Emit(_services, hook => hook.OnAgentLoading(ref id), emitOptions); + HookEmitter.Emit(_services, hook => hook.OnAgentLoading(ref id), id); var agent = await GetAgent(id); if (agent == null) return null; @@ -61,7 +57,7 @@ public partial class AgentService hook.OnAgentLoaded(agent); - }, emitOptions); + }, id); _logger.LogInformation($"Loaded agent {agent}."); diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs index ef36f4d7..f862482c 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.Rendering.cs @@ -132,9 +132,8 @@ public partial class AgentService agent.TemplateDict[TemplateRenderConstant.RENDER_AGENT] = agent; var content = render.Render(template, agent.TemplateDict); - HookEmitter.Emit(_services, async hook => - await hook.OnRenderingTemplate(agent, templateName, content) - ).Wait(); + HookEmitter.Emit(_services, async hook => await hook.OnRenderingTemplate(agent, templateName, content), + agent.Id).Wait(); return content; } diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs index 5ec1ae76..b2b97f4f 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs @@ -158,16 +158,14 @@ public partial class ConversationService // Emit conversation ending hook if (response.Instruction.ConversationEnd) { - await HookEmitter.Emit(_services, async hook => - await hook.OnConversationEnding(response) - ); + await HookEmitter.Emit(_services, async hook => await hook.OnConversationEnding(response), + response.CurrentAgentId); response.FunctionName = "conversation_end"; } } - await HookEmitter.Emit(_services, async hook => - await hook.OnResponseGenerated(response) - ); + await HookEmitter.Emit(_services, async hook => await hook.OnResponseGenerated(response), + response.CurrentAgentId); await onResponseReceived(response); diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs index 4f5e928e..dd2e02d4 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs @@ -25,10 +25,6 @@ public partial class FileInstructService } }); - var emitOptions = new HookEmitOption - { - ShouldExecute = hook => hook.IsMatch(innerAgentId) - }; await HookEmitter.Emit(_services, async hook => await hook.OnResponseGenerated(new InstructResponseModel { @@ -39,7 +35,7 @@ public partial class FileInstructService UserMessage = text, SystemInstruction = instruction, CompletionText = message.Content - }), emitOptions); + }), innerAgentId); return message.Content; } @@ -56,10 +52,6 @@ public partial class FileInstructService Instruction = instruction }, new RoleDialogModel(AgentRole.User, instruction ?? text)); - var emitOptions = new HookEmitOption - { - ShouldExecute = hook => hook.IsMatch(innerAgentId) - }; await HookEmitter.Emit(_services, async hook => await hook.OnResponseGenerated(new InstructResponseModel { @@ -70,7 +62,7 @@ public partial class FileInstructService UserMessage = text, SystemInstruction = instruction, CompletionText = message.Content - }), emitOptions); + }), innerAgentId); return message; } @@ -97,10 +89,6 @@ public partial class FileInstructService stream.Close(); - var emitOptions = new HookEmitOption - { - ShouldExecute = hook => hook.IsMatch(innerAgentId) - }; await HookEmitter.Emit(_services, async hook => await hook.OnResponseGenerated(new InstructResponseModel { @@ -109,7 +97,7 @@ public partial class FileInstructService Model = completion.Model, UserMessage = string.Empty, CompletionText = message.Content - }), emitOptions); + }), innerAgentId); return message; } @@ -138,11 +126,6 @@ public partial class FileInstructService stream.Close(); - var hooks = _services.GetServices(); - var emitOptions = new HookEmitOption - { - ShouldExecute = hook => hook.IsMatch(innerAgentId) - }; await HookEmitter.Emit(_services, async hook => await hook.OnResponseGenerated(new InstructResponseModel { @@ -153,7 +136,7 @@ public partial class FileInstructService UserMessage = text, SystemInstruction = instruction, CompletionText = message.Content - }), emitOptions); + }), innerAgentId); return message; } @@ -191,10 +174,6 @@ public partial class FileInstructService imageStream.Close(); maskStream.Close(); - var emitOptions = new HookEmitOption - { - ShouldExecute = hook => hook.IsMatch(innerAgentId) - }; await HookEmitter.Emit(_services, async hook => await hook.OnResponseGenerated(new InstructResponseModel { @@ -205,7 +184,7 @@ public partial class FileInstructService UserMessage = text, SystemInstruction = instruction, CompletionText = message.Content - }), emitOptions); + }), innerAgentId); return message; } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index 12b950b6..cd703cc7 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -43,10 +43,6 @@ public partial class FileInstructService } }); - var emitOptions = new HookEmitOption - { - ShouldExecute = hook => hook.IsMatch(innerAgentId) - }; await HookEmitter.Emit(_services, async hook => await hook.OnResponseGenerated(new InstructResponseModel { @@ -57,7 +53,7 @@ public partial class FileInstructService UserMessage = text, SystemInstruction = instruction, CompletionText = message.Content - }), emitOptions); + }), innerAgentId); return message.Content; } diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/HookEmitter.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/HookEmitter.cs index a4baf81c..95d55596 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/HookEmitter.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/HookEmitter.cs @@ -1,14 +1,15 @@ +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.Infrastructures; namespace BotSharp.Core.Infrastructures; public static class HookEmitter { - public static HookEmittedResult Emit(IServiceProvider services, Action action, HookEmitOption? option = null) + public static HookEmittedResult Emit(IServiceProvider services, Action action, string agentId, HookEmitOption? option = null) where T : IHookBase { var logger = services.GetRequiredService>(); var result = new HookEmittedResult(); - var hooks = services.GetServices(); + var hooks = services.GetServices().Where(p => p.IsMatch(agentId)); option = option ?? new(); foreach (var hook in hooks) @@ -35,11 +36,11 @@ public static class HookEmitter return result; } - public static async Task Emit(IServiceProvider services, Func action, HookEmitOption? option = null) + public static async Task Emit(IServiceProvider services, Func action, string agentId, HookEmitOption? option = null) where T : IHookBase { var logger = services.GetRequiredService>(); var result = new HookEmittedResult(); - var hooks = services.GetServices(); + var hooks = services.GetServices().Where(p => p.IsMatch(agentId)); option = option ?? new(); foreach (var hook in hooks) diff --git a/src/Infrastructure/BotSharp.Core/Instructs/Functions/ExecuteTemplateFn.cs b/src/Infrastructure/BotSharp.Core/Instructs/Functions/ExecuteTemplateFn.cs index e225b6d3..7d8bcb46 100644 --- a/src/Infrastructure/BotSharp.Core/Instructs/Functions/ExecuteTemplateFn.cs +++ b/src/Infrastructure/BotSharp.Core/Instructs/Functions/ExecuteTemplateFn.cs @@ -61,10 +61,6 @@ public class ExecuteTemplateFn : IFunctionCallback new(AgentRole.User, text) }); - var emitOptions = new HookEmitOption - { - ShouldExecute = hook => hook.IsMatch(agent.Id) - }; await HookEmitter.Emit(_services, async hook => await hook.OnResponseGenerated(new InstructResponseModel { @@ -74,7 +70,7 @@ public class ExecuteTemplateFn : IFunctionCallback Model = completion.Model, UserMessage = text, CompletionText = response.Content - }), emitOptions); + }), agent.Id); return response.Content; } diff --git a/src/Infrastructure/BotSharp.Core/Routing/Reasoning/InstructExecutor.cs b/src/Infrastructure/BotSharp.Core/Routing/Reasoning/InstructExecutor.cs index e911437d..c93e133c 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Reasoning/InstructExecutor.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Reasoning/InstructExecutor.cs @@ -25,9 +25,8 @@ public class InstructExecutor : IExecutor { inst.OriginalAgent = goalAgent; // Emit hook - await HookEmitter.Emit(_services, async hook => - await hook.OnRoutingInstructionRevised(inst, message) - ); + await HookEmitter.Emit(_services, async hook => await hook.OnRoutingInstructionRevised(inst, message), + message.CurrentAgentId); } message.FunctionArgs = JsonSerializer.Serialize(inst); diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingContext.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingContext.cs index 1c4ca518..b422a423 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingContext.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingContext.cs @@ -99,9 +99,8 @@ public class RoutingContext : IRoutingContext var preAgentId = _stack.Count == 0 ? agentId : _stack.Peek(); _stack.Push(agentId); - HookEmitter.Emit(_services, async hook => - await hook.OnAgentEnqueued(agentId, preAgentId, reason: reason) - ).Wait(); + HookEmitter.Emit(_services, async hook => await hook.OnAgentEnqueued(agentId, preAgentId, reason: reason), + agentId).Wait(); UpdateLazyRoutingAgent(updateLazyRouting); } @@ -120,9 +119,8 @@ public class RoutingContext : IRoutingContext var agentId = _stack.Pop(); var currentAgentId = GetCurrentAgentId(); - HookEmitter.Emit(_services, async hook => - await hook.OnAgentDequeued(agentId, currentAgentId, reason: reason) - ).Wait(); + HookEmitter.Emit(_services, async hook => await hook.OnAgentDequeued(agentId, currentAgentId, reason: reason), + agentId).Wait(); if (string.IsNullOrEmpty(currentAgentId)) { @@ -203,9 +201,8 @@ public class RoutingContext : IRoutingContext _stack.Pop(); _stack.Push(agentId); - HookEmitter.Emit(_services, async hook => - await hook.OnAgentReplaced(fromAgent, toAgent, reason: reason) - ).Wait(); + HookEmitter.Emit(_services, async hook => await hook.OnAgentReplaced(fromAgent, toAgent, reason: reason), + agentId).Wait(); } UpdateLazyRoutingAgent(updateLazyRouting); @@ -220,9 +217,8 @@ public class RoutingContext : IRoutingContext var agentId = GetCurrentAgentId(); _stack.Clear(); - HookEmitter.Emit(_services, async hook => - await hook.OnAgentQueueEmptied(agentId, reason: reason) - ).Wait(); + HookEmitter.Emit(_services, async hook => await hook.OnAgentQueueEmptied(agentId, reason: reason), + agentId).Wait(); } public void SetMessageId(string conversationId, string messageId) diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InstructLoop.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InstructLoop.cs index 364e7198..128debad 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InstructLoop.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InstructLoop.cs @@ -51,9 +51,8 @@ public partial class RoutingService int loopCount = 1; while (true) { - await HookEmitter.Emit(_services, async hook => - await hook.OnRoutingInstructionReceived(inst, message) - ); + await HookEmitter.Emit(_services, async hook => await hook.OnRoutingInstructionReceived(inst, message), + agent.Id); // Save states states.SaveStateByArgs(inst.Arguments); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 5e180f22..897c34e7 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -345,7 +345,8 @@ public class ConversationController : ControllerBase }; await HookEmitter.Emit(_services, async hook => - await hook.OnNotificationGenerated(inputMsg) + await hook.OnNotificationGenerated(inputMsg), + routing.Context.GetCurrentAgentId() ); return response; diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index b5669a5f..430a4563 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -59,10 +59,6 @@ public class InstructModeController : ControllerBase var textCompletion = CompletionProvider.GetTextCompletion(_services); var response = await textCompletion.GetCompletion(input.Text, agentId, Guid.NewGuid().ToString()); - var emitOptions = new HookEmitOption - { - ShouldExecute = hook => hook.IsMatch(agentId) - }; await HookEmitter.Emit(_services, async hook => await hook.OnResponseGenerated(new InstructResponseModel { @@ -72,7 +68,7 @@ public class InstructModeController : ControllerBase TemplateName = input.Template, UserMessage = input.Text, CompletionText = response - }), emitOptions); + }), agentId); return response; } @@ -101,10 +97,6 @@ public class InstructModeController : ControllerBase } }); - var emitOptions = new HookEmitOption - { - ShouldExecute = hook => hook.IsMatch(agentId) - }; await HookEmitter.Emit(_services, async hook => await hook.OnResponseGenerated(new InstructResponseModel { @@ -115,7 +107,7 @@ public class InstructModeController : ControllerBase UserMessage = input.Text, SystemInstruction = message.RenderedInstruction, CompletionText = message.Content - }), emitOptions); + }), agentId); return message.Content; } diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs index 98896ac3..fad14730 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -255,7 +255,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion config.ResponseModalities = new List([Modality.AUDIO]); var words = new List(); - HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent))); + HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)), agent.Id); var realtimeModelSettings = _services.GetRequiredService(); @@ -278,7 +278,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion }).ToArray(); await HookEmitter.Emit(_services, - async hook => { await hook.OnSessionUpdated(agent, prompt, functions, isInit); }); + async hook => { await hook.OnSessionUpdated(agent, prompt, functions, isInit); }, agent.Id); if (_settings.Gemini.UseGoogleSearch) { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index b32c0e1c..23cc5b77 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -319,7 +319,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion if (realtimeModelSettings.InputAudioTranscribe) { var words = new List(); - HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent))); + HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)), agent.Id); sessionUpdate.session.InputAudioTranscription = new InputAudioTranscription { @@ -332,7 +332,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion await HookEmitter.Emit(_services, async hook => { await hook.OnSessionUpdated(agent, instruction, functions, isInit); - }); + }, agent.Id); await SendEventToModel(sessionUpdate); await Task.Delay(300); diff --git a/src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Functions/SqlGenerationFn.cs b/src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Functions/SqlGenerationFn.cs index 4965a67c..11fd2cf7 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Functions/SqlGenerationFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Functions/SqlGenerationFn.cs @@ -90,7 +90,7 @@ public class SqlGenerationFn : IFunctionCallback { var requirement = await x.GetSummaryAdditionalRequirements(nameof(SqlGenerationPlanner), message); additionalRequirements.Add(requirement); - }); + }, message.CurrentAgentId); var globalKnowledges = new List(); foreach (var hook in knowledgeHooks) diff --git a/src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Functions/SqlReviewFn.cs b/src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Functions/SqlReviewFn.cs index 6996fb6d..4255dd7f 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Functions/SqlReviewFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/SqlGeneration/Functions/SqlReviewFn.cs @@ -30,7 +30,8 @@ public class SqlReviewFn : IFunctionCallback if (args != null && !args.IsSqlTemplate && args.ContainsSqlStatements) { await HookEmitter.Emit(_services, async hook => - await hook.OnSourceCodeGenerated(nameof(SqlGenerationPlanner), message, "sql") + await hook.OnSourceCodeGenerated(nameof(SqlGenerationPlanner), message, "sql"), + message.CurrentAgentId ); } return true; diff --git a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Functions/SummaryPlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Functions/SummaryPlanFn.cs index 465f1df2..4212d0c2 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Functions/SummaryPlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/Functions/SummaryPlanFn.cs @@ -68,7 +68,8 @@ public class SummaryPlanFn : IFunctionCallback message.Content = summary.Content; await HookEmitter.Emit(_services, async hook => - await hook.OnPlanningCompleted(nameof(TwoStageTaskPlanner), message) + await hook.OnPlanningCompleted(nameof(TwoStageTaskPlanner), message), + message.CurrentAgentId ); return true; @@ -88,7 +89,7 @@ public class SummaryPlanFn : IFunctionCallback { var requirement = await x.GetSummaryAdditionalRequirements(nameof(TwoStageTaskPlanner), message); additionalRequirements.Add(requirement); - }); + }, message.CurrentAgentId); var globalKnowledges = new List(); foreach (var hook in knowledgeHooks) diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs index 168929ea..99a7744f 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs @@ -30,7 +30,7 @@ public class SqlDriverPlanningHook : IPlanningHook await HookEmitter.Emit(_services, async (hook) => { await hook.SqlGenerated(msg); - }); + }, msg.CurrentAgentId); var settings = _services.GetRequiredService(); if (!settings.ExecuteSqlSelectAutonomous) diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Interfaces/ISqlDriverHook.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Interfaces/ISqlDriverHook.cs index b4871e66..63cff60b 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Interfaces/ISqlDriverHook.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Interfaces/ISqlDriverHook.cs @@ -1,6 +1,8 @@ +using BotSharp.Abstraction.Hooks; + namespace BotSharp.Plugin.SqlDriver.Interfaces; -public interface ISqlDriverHook +public interface ISqlDriverHook : IHookBase { // Get database type string GetDatabaseType(RoleDialogModel message); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs index 1db293f2..511abb45 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs @@ -56,7 +56,7 @@ public class TwilioInboundController : TwilioController await HookEmitter.Emit(_services, async hook => { await hook.OnSessionCreating(request, instruction); - }); + }, request.AgentId); var (agent, conversationId) = await InitConversation(request); request.ConversationId = conversationId.Id; @@ -67,12 +67,8 @@ public class TwilioInboundController : TwilioController { response = new VoiceResponse(); - var emitOptions = new HookEmitOption - { - ShouldExecute = hook => hook.IsMatch(request) - }; await HookEmitter.Emit(_services, - async hook => await hook.OnVoicemailStarting(request), emitOptions); + async hook => await hook.OnVoicemailStarting(request), request.AgentId); var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3"); response.Play(new Uri(url)); @@ -123,7 +119,7 @@ public class TwilioInboundController : TwilioController await HookEmitter.Emit(_services, async hook => { await hook.OnSessionCreated(request); - }); + }, request.AgentId); return TwiML(response); } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioOutboundController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioOutboundController.cs index 4cb13a61..5543af38 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioOutboundController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioOutboundController.cs @@ -33,13 +33,8 @@ public class TwilioOutboundController : TwilioController if (twilio.MachineDetected(request)) { response = new VoiceResponse(); - - var emitOptions = new HookEmitOption - { - ShouldExecute = hook => hook.IsMatch(request) - }; await HookEmitter.Emit(_services, - async hook => await hook.OnVoicemailStarting(request), emitOptions); + async hook => await hook.OnVoicemailStarting(request), request.AgentId); var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3"); response.Play(new Uri(url)); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs index 8bc26a8e..3d13dc73 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs @@ -37,11 +37,7 @@ public class TwilioRecordController : TwilioController convService.SaveStates(); // recording completed - var emitOptions = new HookEmitOption - { - ShouldExecute = hook => hook.IsMatch(request) - }; - await HookEmitter.Emit(_services, x => x.OnRecordingCompleted(request), emitOptions); + await HookEmitter.Emit(_services, x => x.OnRecordingCompleted(request), request.AgentId); } else { diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioTranscribeController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioTranscribeController.cs index 1fa020c0..778aa29d 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioTranscribeController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioTranscribeController.cs @@ -49,7 +49,8 @@ public class TwilioTranscribeController : TwilioController // transcription completed transcript.Language = request.LanguageCode; - await HookEmitter.Emit(_services, async x => await x.OnTranscribeCompleted(message, transcript)); + await HookEmitter.Emit(_services, async x => await x.OnTranscribeCompleted(message, transcript), + request.AgentId); } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 048d75f3..786f116c 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -65,7 +65,7 @@ public class TwilioVoiceController : TwilioController await HookEmitter.Emit(_services, async hook => { await hook.OnSessionCreating(request, instruction); - }); + }, request.AgentId); var twilio = _services.GetRequiredService(); if (string.IsNullOrWhiteSpace(request.Intent)) @@ -98,7 +98,7 @@ public class TwilioVoiceController : TwilioController await HookEmitter.Emit(_services, async hook => { await hook.OnSessionCreated(request); - }); + }, request.AgentId); return TwiML(response); } @@ -151,7 +151,7 @@ public class TwilioVoiceController : TwilioController await HookEmitter.Emit(_services, async hook => { await hook.OnReceivedUserMessage(request); - }); + }, request.AgentId); } else { @@ -161,7 +161,7 @@ public class TwilioVoiceController : TwilioController await HookEmitter.Emit(_services, async hook => { await hook.OnAgentHangUp(request); - }); + }, request.AgentId); response = twilio.HangUp(string.Empty); } @@ -185,7 +185,7 @@ public class TwilioVoiceController : TwilioController await HookEmitter.Emit(_services, async hook => { await hook.OnWaitingUserResponse(request, instruction); - }); + }, request.AgentId); response = twilio.ReturnInstructions(instruction); } @@ -223,7 +223,7 @@ public class TwilioVoiceController : TwilioController { request.AIResponseErrorMessage = $"AI response timeout: AIResponseWaitTime greater than {request.AIResponseWaitTime}, please check internal error log!"; await hook.OnAgentHangUp(request); - }); + }, request.AgentId); response = twilio.HangUp($"twilio/error.mp3"); } @@ -238,7 +238,7 @@ public class TwilioVoiceController : TwilioController await HookEmitter.Emit(_services, async hook => { await hook.OnAgentTransferring(request, _settings); - }); + }, request.AgentId); response = twilio.DialCsrAgent($"twilio/voice/speeches/{request.ConversationId}/{reply.SpeechFileName}"); } @@ -249,7 +249,7 @@ public class TwilioVoiceController : TwilioController await HookEmitter.Emit(_services, async hook => { await hook.OnAgentHangUp(request); - }); + }, request.AgentId); } else { @@ -274,7 +274,7 @@ public class TwilioVoiceController : TwilioController await HookEmitter.Emit(_services, async hook => { await hook.OnAgentResponsing(request, instruction); - }); + }, request.AgentId); response = twilio.ReturnInstructions(instruction); } @@ -343,41 +343,35 @@ public class TwilioVoiceController : TwilioController { var twilio = _services.GetRequiredService(); - // Define the options with the predicate - var emitOptions = new HookEmitOption - { - ShouldExecute = hook => hook.IsMatch(request) - }; - switch (request.CallStatus) { case "completed": if (twilio.MachineDetected(request)) { // voicemail - await HookEmitter.Emit(_services, hook => hook.OnVoicemailLeft(request), emitOptions); + await HookEmitter.Emit(_services, hook => hook.OnVoicemailLeft(request), request.AgentId); } else { // phone call completed - await HookEmitter.Emit(_services, hook => hook.OnUserDisconnected(request), emitOptions); + await HookEmitter.Emit(_services, hook => hook.OnUserDisconnected(request), request.AgentId); } break; case "busy": - await HookEmitter.Emit(_services, hook => hook.OnCallBusyStatus(request), emitOptions); + await HookEmitter.Emit(_services, hook => hook.OnCallBusyStatus(request), request.AgentId); break; case "no-answer": - await HookEmitter.Emit(_services, hook => hook.OnCallNoAnswerStatus(request), emitOptions); + await HookEmitter.Emit(_services, hook => hook.OnCallNoAnswerStatus(request), request.AgentId); break; case "canceled": - await HookEmitter.Emit(_services, hook => hook.OnCallCanceledStatus(request), emitOptions); + await HookEmitter.Emit(_services, hook => hook.OnCallCanceledStatus(request), request.AgentId); break; case "failed": - await HookEmitter.Emit(_services, hook => hook.OnCallFailedStatus(request), emitOptions); + await HookEmitter.Emit(_services, hook => hook.OnCallFailedStatus(request), request.AgentId); break; default: _logger.LogError($"Unknown call status: {request.CallStatus}, {request.CallSid}"); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs index ae461595..e7ce0932 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs @@ -1,11 +1,11 @@ +using BotSharp.Abstraction.Hooks; using BotSharp.Plugin.Twilio.Models; using Task = System.Threading.Tasks.Task; namespace BotSharp.Plugin.Twilio.Interfaces; -public interface ITwilioCallStatusHook +public interface ITwilioCallStatusHook : IHookBase { - bool IsMatch(ConversationalVoiceRequest request) => true; Task OnVoicemailLeft(ConversationalVoiceRequest request) => Task.CompletedTask; Task OnUserDisconnected(ConversationalVoiceRequest request) => Task.CompletedTask; Task OnRecordingCompleted(ConversationalVoiceRequest request) => Task.CompletedTask; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs index 40525475..00837b6a 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioSessionHook.cs @@ -1,10 +1,11 @@ +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.Realtime.Models; using BotSharp.Plugin.Twilio.Models; using Task = System.Threading.Tasks.Task; namespace BotSharp.Plugin.Twilio.Interfaces; -public interface ITwilioSessionHook +public interface ITwilioSessionHook : IHookBase { /// /// Before session creating diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs index 0ca10949..84f88b5b 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs @@ -157,7 +157,8 @@ public class TwilioMessageQueueService : BackgroundService var agentService = sp.GetRequiredService(); var agent = agentService.GetAgent(agentId).Result; var extraWords = new List(); - HookEmitter.Emit(sp, hook => extraWords.AddRange(hook.OnModelTranscriptPrompt(agent))); + HookEmitter.Emit(sp, hook => extraWords.AddRange(hook.OnModelTranscriptPrompt(agent)), + agentId); var phrases = reply.Content.Split(',', StringSplitOptions.RemoveEmptyEntries); int capcity = 100; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index 7bc95382..e0618aaf 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -235,7 +235,8 @@ public class TwilioService if (_settings.TranscribeEnabled) { var words = new List(); - HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent))); + HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)), + agent.Id); var hints = string.Join(", ", words); var start = new Start(); start.Transcription( @@ -323,10 +324,8 @@ public class TwilioService ActionOnEmptyResult = true }; - await HookEmitter.Emit(_services, async hook => - { - await hook.OnWaitingAgentResponse(request, instruction); - }); + await HookEmitter.Emit(_services, async hook => await hook.OnWaitingAgentResponse(request, instruction), + request.AgentId); response = ReturnInstructions(instruction); } From b64f22ef8edb5a5877caebcb79426301dd7e6f30 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 15 May 2025 15:46:42 -0500 Subject: [PATCH 18/26] temp save --- .../Conversations/Models/Conversation.cs | 13 ++ .../Realtime/Models/RealtimeHubConnection.cs | 1 + .../Hooks/RealtimeConversationHook.cs | 6 +- .../Services/RealtimeHub.cs | 2 + .../Services/ConversationStorage.cs | 7 +- .../Realtime/RealTimeCompletionProvider.cs | 121 +++++++++++------- .../Models/DialogMongoElement.cs | 6 + .../MongoRepository.Conversation.cs | 1 - 8 files changed, 105 insertions(+), 52 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs index ac1efb04..5ffdf187 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs @@ -47,15 +47,19 @@ public class DialogElement public string Content { get; set; } = default!; [JsonPropertyName("secondary_content")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? SecondaryContent { get; set; } [JsonPropertyName("rich_content")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? RichContent { get; set; } [JsonPropertyName("secondary_rich_content")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? SecondaryRichContent { get; set; } [JsonPropertyName("payload")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? Payload { get; set; } public DialogElement() @@ -95,8 +99,17 @@ public class DialogMetaData public string MessageType { get; set; } = default!; [JsonPropertyName("function_name")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? FunctionName { get; set; } + [JsonPropertyName("function_args")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FunctionArgs { get; set; } + + [JsonPropertyName("tool_call_id")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ToolCallId { get; set; } + [JsonPropertyName("sender_id")] public string? SenderId { get; set; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs index c0dac6d5..4d66ac5b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs @@ -10,6 +10,7 @@ public class RealtimeHubConnection public string KeypadInputBuffer { get; set; } = string.Empty; public string CurrentAgentId { get; set; } = null!; public string ConversationId { get; set; } = null!; + public string? PrevSessionId { get; set; } public Func OnModelReady { get; set; } = () => string.Empty; public Func OnModelMessageReceived { get; set; } = null!; public Func OnModelAudioResponseDone { get; set; } = null!; diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs index 2f8a03ee..32cdc113 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs @@ -65,9 +65,7 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook else { // Update session for changed states - - // TO DO - //var instruction = await hub.Completer.UpdateSession(hub.HubConn); + var instruction = await hub.Completer.UpdateSession(hub.HubConn); await hub.Completer.InsertConversationItem(message); if (string.IsNullOrEmpty(message.Content)) @@ -81,7 +79,7 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook //} //else //{ - // await hub.Completer.TriggerModelInference(); + // await hub.Completer.TriggerModelInference(instruction); //} } } diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index 93363878..104cd22c 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -96,6 +96,8 @@ public class RealtimeHub : IRealtimeHub } await routing.InvokeFunction(message.FunctionName, message); + dialogs.Add(message); + storage.Append(_conn.ConversationId, message); } else { diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs index 49852eb2..eeceeb09 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs @@ -40,6 +40,8 @@ public class ConversationStorage : IConversationStorage MessageId = dialog.MessageId, MessageType = dialog.MessageType, FunctionName = dialog.FunctionName, + FunctionArgs = dialog.FunctionArgs, + ToolCallId = dialog.ToolCallId, CreatedTime = dialog.CreatedAt }; @@ -109,7 +111,6 @@ public class ConversationStorage : IConversationStorage var currentAgentId = meta.AgentId; var messageId = meta.MessageId; var messageType = meta.MessageType; - var function = meta.FunctionName; var senderId = role == AgentRole.Function ? currentAgentId : meta.SenderId; var createdAt = meta.CreatedTime; var richContent = !string.IsNullOrEmpty(dialog.RichContent) ? @@ -124,7 +125,9 @@ public class ConversationStorage : IConversationStorage MessageType = messageType, CreatedAt = createdAt, SenderId = senderId, - FunctionName = function, + FunctionName = meta.FunctionName, + FunctionArgs = meta.FunctionArgs, + ToolCallId = meta.ToolCallId, RichContent = richContent, SecondaryContent = secondaryContent, SecondaryRichContent = secondaryRichContent, diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs index eb6f4e76..1cd32215 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -32,6 +32,20 @@ public class GoogleRealTimeProvider : IRealTimeCompletion UnknownTypeHandling = JsonUnknownTypeHandling.JsonElement }; + private RealtimeTranscriptionResponse _inputStream = new(); + private RealtimeTranscriptionResponse _outputStream = new(); + + + private RealtimeHubConnection _conn; + private Func _onModelReady; + private Func _onModelAudioDeltaReceived; + private Func _onModelAudioResponseDone; + private Func _onModelAudioTranscriptDone; + private Func, Task> _onModelResponseDone; + private Func _onConversationItemCreated; + private Func _onInputAudioTranscriptionDone; + private Func _onInterruptionDetected; + public GoogleRealTimeProvider( IServiceProvider services, GoogleAiSettings settings, @@ -58,17 +72,26 @@ public class GoogleRealTimeProvider : IRealTimeCompletion Func onInputAudioTranscriptionDone, Func onInterruptionDetected) { + _conn = conn; + _onModelReady = onModelReady; + _onModelAudioDeltaReceived = onModelAudioDeltaReceived; + _onModelAudioResponseDone = onModelAudioResponseDone; + _onModelAudioTranscriptDone = onModelAudioTranscriptDone; + _onModelResponseDone = onModelResponseDone; + _onConversationItemCreated = onConversationItemCreated; + _onInputAudioTranscriptionDone = onInputAudioTranscriptionDone; + _onInterruptionDetected = onInterruptionDetected; + var settingsService = _services.GetRequiredService(); var realtimeModelSettings = _services.GetRequiredService(); _model = realtimeModelSettings.Model; var modelSettings = settingsService.GetSetting(Provider, _model); - if (_session != null) - { - _session.Dispose(); - } + Reset(); + _inputStream = new(); + _outputStream = new(); _session = new LlmRealtimeSession(_services, new ChatSessionOptions { JsonOptions = _jsonOptions @@ -79,32 +102,11 @@ public class GoogleRealTimeProvider : IRealTimeCompletion await onModelReady(); - _ = ReceiveMessage( - conn, - onModelReady, - onModelAudioDeltaReceived, - onModelAudioResponseDone, - onModelAudioTranscriptDone, - onModelResponseDone, - onConversationItemCreated, - onInputAudioTranscriptionDone, - onInterruptionDetected); + _ = ReceiveMessage(); } - private async Task ReceiveMessage( - RealtimeHubConnection conn, - Func onModelReady, - Func onModelAudioDeltaReceived, - Func onModelAudioResponseDone, - Func onModelAudioTranscriptDone, - Func, Task> onModelResponseDone, - Func onConversationItemCreated, - Func onInputAudioTranscriptionDone, - Func onInterruptionDetected) + private async Task ReceiveMessage() { - using var inputStream = new RealtimeTranscriptionResponse(); - using var outputStream = new RealtimeTranscriptionResponse(); - await foreach (ChatSessionUpdate update in _session.ReceiveUpdatesAsync(CancellationToken.None)) { var receivedText = update?.RawResponse; @@ -128,6 +130,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion else if (response.SessionResumptionUpdate != null) { _logger.LogInformation($"Session resumption update => New handle: {response.SessionResumptionUpdate.NewHandle}, Resumable: {response.SessionResumptionUpdate.Resumable}"); + _conn.PrevSessionId = response.SessionResumptionUpdate?.NewHandle; } else if (response.ToolCall != null && !response.ToolCall.FunctionCalls.IsNullOrEmpty()) { @@ -137,20 +140,20 @@ public class GoogleRealTimeProvider : IRealTimeCompletion if (functionCall != null) { - var messages = OnFunctionCall(conn, functionCall); - await onModelResponseDone(messages); + var messages = OnFunctionCall(_conn, functionCall); + await _onModelResponseDone(messages); } } else if (response.ServerContent != null) { if (response.ServerContent.InputTranscription?.Text != null) { - inputStream.Collect(response.ServerContent.InputTranscription.Text); + _inputStream.Collect(response.ServerContent.InputTranscription.Text); } if (response.ServerContent.OutputTranscription?.Text != null) { - outputStream.Collect(response.ServerContent.OutputTranscription.Text); + _outputStream.Collect(response.ServerContent.OutputTranscription.Text); } if (response.ServerContent.ModelTurn != null) @@ -158,13 +161,13 @@ public class GoogleRealTimeProvider : IRealTimeCompletion _logger.LogInformation($"Model audio delta received."); // Handle input transcription - var inputTranscription = inputStream.GetText(); + var inputTranscription = _inputStream.GetText(); if (!string.IsNullOrEmpty(inputTranscription)) { - var message = OnUserAudioTranscriptionCompleted(conn, inputTranscription); - await onInputAudioTranscriptionDone(message); + var message = OnUserAudioTranscriptionCompleted(_conn, inputTranscription); + await _onInputAudioTranscriptionDone(message); } - inputStream.Clear(); + _inputStream.Clear(); var parts = response.ServerContent.ModelTurn.Parts; if (!parts.IsNullOrEmpty()) @@ -173,7 +176,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { if (!string.IsNullOrEmpty(part.InlineData?.Data)) { - await onModelAudioDeltaReceived(part.InlineData.Data, string.Empty); + await _onModelAudioDeltaReceived(part.InlineData.Data, string.Empty); } } } @@ -187,14 +190,14 @@ public class GoogleRealTimeProvider : IRealTimeCompletion _logger.LogInformation($"Model turn completed."); // Handle output transcription - var outputTranscription = outputStream.GetText(); + var outputTranscription = _outputStream.GetText(); if (!string.IsNullOrEmpty(outputTranscription)) { - var messages = await OnResponseDone(conn, outputTranscription, response.UsageMetaData); - await onModelResponseDone(messages); + var messages = await OnResponseDone(_conn, outputTranscription, response.UsageMetaData); + await _onModelResponseDone(messages); } - inputStream.Clear(); - outputStream.Clear(); + _inputStream.Clear(); + _outputStream.Clear(); } } } @@ -205,6 +208,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion } } + _inputStream.Dispose(); + _outputStream.Dispose(); _session.Dispose(); } @@ -213,7 +218,10 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { if (_session != null) { + _inputStream?.Dispose(); + _outputStream?.Dispose(); await _session.DisconnectAsync(); + _session.Dispose(); } } @@ -242,8 +250,9 @@ public class GoogleRealTimeProvider : IRealTimeCompletion public async Task TriggerModelInference(string? instructions = null) { - var content = new Content(instructions ?? "Please respond to user.", AgentRole.User); + if (string.IsNullOrWhiteSpace(instructions)) return; + var content = new Content(instructions, AgentRole.User); await SendEventToModel(new BidiClientPayload { ClientContent = new() @@ -273,13 +282,18 @@ public class GoogleRealTimeProvider : IRealTimeCompletion public async Task UpdateSession(RealtimeHubConnection conn, bool isInit = false) { + if (!isInit) + { + return null; + } + var agentService = _services.GetRequiredService(); var realtimeSetting = _services.GetRequiredService(); var agent = await agentService.LoadAgent(conn.CurrentAgentId); var (prompt, request) = PrepareOptions(agent, []); - var config = request.GenerationConfig; + var config = request.GenerationConfig ?? new(); if (config != null) { //Output Modality can either be text or audio @@ -317,7 +331,8 @@ public class GoogleRealTimeProvider : IRealTimeCompletion }); } - await SendEventToModel(new RealtimeClientPayload + + var payload = new RealtimeClientPayload { Setup = new RealtimeGenerateContentSetup() { @@ -328,8 +343,14 @@ public class GoogleRealTimeProvider : IRealTimeCompletion InputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null, OutputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null, SessionResumption = new() + { + Handle = _conn.PrevSessionId + } } - }); + }; + + Console.WriteLine($"Setup payload: {JsonSerializer.Serialize(payload, _jsonOptions)}"); + await SendEventToModel(payload); return prompt; } @@ -596,5 +617,15 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { return new Uri($"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.{version}.GenerativeService.BidiGenerateContent?key={apiKey}"); } + + private void Reset() + { + _inputStream?.Clear(); + _outputStream?.Clear(); + + _inputStream?.Dispose(); + _outputStream?.Dispose(); + _session?.Dispose(); + } #endregion } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs index 030a93ab..b67fc53c 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Models/DialogMongoElement.cs @@ -47,6 +47,8 @@ public class DialogMetaDataMongoElement public string MessageId { get; set; } = default!; public string MessageType { get; set; } = default!; public string? FunctionName { get; set; } + public string? FunctionArgs { get; set; } + public string? ToolCallId { get; set; } public string? SenderId { get; set; } public DateTime CreateTime { get; set; } @@ -59,6 +61,8 @@ public class DialogMetaDataMongoElement MessageId = meta.MessageId, MessageType = meta.MessageType, FunctionName = meta.FunctionName, + FunctionArgs = meta.FunctionArgs, + ToolCallId = meta.ToolCallId, SenderId = meta.SenderId, CreatedTime = meta.CreateTime, }; @@ -73,6 +77,8 @@ public class DialogMetaDataMongoElement MessageId = meta.MessageId, MessageType = meta.MessageType, FunctionName = meta.FunctionName, + FunctionArgs = meta.FunctionArgs, + ToolCallId = meta.ToolCallId, SenderId = meta.SenderId, CreateTime = meta.CreatedTime, }; diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index 722c7730..02f5a695 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -1,6 +1,5 @@ using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Repositories.Filters; -using MongoDB.Driver; using System.Text.Json; namespace BotSharp.Plugin.MongoStorage.Repository; From 350c479486d052af4d2c4f36d06edc34f257e708 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 15 May 2025 17:46:36 -0500 Subject: [PATCH 19/26] sync and refine --- .../MCP/Services/IMcpService.cs | 2 +- .../Routing/Executor/IFunctionExecutor.cs | 3 +- .../MCP/BotSharpMCPExtensions.cs | 8 +- .../MCP/Helpers/AiFunctionHelper.cs | 20 +++-- .../MCP/Hooks/MCPToolAgentHook.cs | 22 +++-- .../MCP/Managers/McpClientManager.cs | 80 ++++++++++++------- .../BotSharp.Core/MCP/Services/McpService.cs | 19 +++-- .../BotSharp.Core/MCP/Settings/MCPSettings.cs | 1 - .../Routing/Executor/DummyFunctionExecutor.cs | 10 +-- .../Executor/FunctionCallbackExecutor.cs | 9 ++- .../Executor/FunctionExecutorFactory.cs | 38 ++++----- .../Routing/Executor/MCPToolExecutor.cs | 24 +++--- .../Routing/RoutingService.InvokeFunction.cs | 14 +--- .../Controllers/McpController.cs | 4 +- src/WebStarter/appsettings.json | 15 ++-- 15 files changed, 138 insertions(+), 131 deletions(-) rename src/Infrastructure/{BotSharp.Core => BotSharp.Abstraction}/Routing/Executor/IFunctionExecutor.cs (77%) diff --git a/src/Infrastructure/BotSharp.Abstraction/MCP/Services/IMcpService.cs b/src/Infrastructure/BotSharp.Abstraction/MCP/Services/IMcpService.cs index 71952dac..32564a3f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MCP/Services/IMcpService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MCP/Services/IMcpService.cs @@ -2,5 +2,5 @@ namespace BotSharp.Abstraction.MCP.Services; public interface IMcpService { - IEnumerable GetServerConfigs() => []; + Task> GetServerConfigsAsync() => Task.FromResult>([]); } diff --git a/src/Infrastructure/BotSharp.Core/Routing/Executor/IFunctionExecutor.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Executor/IFunctionExecutor.cs similarity index 77% rename from src/Infrastructure/BotSharp.Core/Routing/Executor/IFunctionExecutor.cs rename to src/Infrastructure/BotSharp.Abstraction/Routing/Executor/IFunctionExecutor.cs index 4ba2e69f..e1c604ad 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Executor/IFunctionExecutor.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Executor/IFunctionExecutor.cs @@ -1,8 +1,7 @@ -namespace BotSharp.Core.Routing.Executor; +namespace BotSharp.Abstraction.Routing.Executor; public interface IFunctionExecutor { public Task ExecuteAsync(RoleDialogModel message); - public Task GetIndicatorAsync(RoleDialogModel message); } diff --git a/src/Infrastructure/BotSharp.Core/MCP/BotSharpMCPExtensions.cs b/src/Infrastructure/BotSharp.Core/MCP/BotSharpMCPExtensions.cs index b3c4e6de..8eeee7b3 100644 --- a/src/Infrastructure/BotSharp.Core/MCP/BotSharpMCPExtensions.cs +++ b/src/Infrastructure/BotSharp.Core/MCP/BotSharpMCPExtensions.cs @@ -18,15 +18,11 @@ public static class BotSharpMcpExtensions { var settings = config.GetSection("MCP").Get(); services.AddScoped(provider => settings); + services.AddScoped(); if (settings != null && settings.Enabled && !settings.McpServerConfigs.IsNullOrEmpty()) { - services.AddScoped(); - - var clientManager = new McpClientManager(settings); - services.AddScoped(provider => clientManager); - - // Register hooks + services.AddScoped(); services.AddScoped(); } return services; diff --git a/src/Infrastructure/BotSharp.Core/MCP/Helpers/AiFunctionHelper.cs b/src/Infrastructure/BotSharp.Core/MCP/Helpers/AiFunctionHelper.cs index d6f1cb5b..b9db3ce7 100644 --- a/src/Infrastructure/BotSharp.Core/MCP/Helpers/AiFunctionHelper.cs +++ b/src/Infrastructure/BotSharp.Core/MCP/Helpers/AiFunctionHelper.cs @@ -1,19 +1,25 @@ -using System.Text.Json; using ModelContextProtocol.Client; namespace BotSharp.Core.MCP.Helpers; internal static class AiFunctionHelper { - public static FunctionDef MapToFunctionDef(McpClientTool tool) + public static FunctionDef? MapToFunctionDef(McpClientTool tool) { if (tool == null) { - throw new ArgumentNullException(nameof(tool)); + return null; } - var properties = tool.JsonSchema.GetProperty("properties"); - var required = tool.JsonSchema.GetProperty("required"); + if (!tool.JsonSchema.TryGetProperty("properties", out var properties)) + { + properties = JsonDocument.Parse("{}").RootElement; + } + + if (!tool.JsonSchema.TryGetProperty("required", out var required)) + { + required = JsonDocument.Parse("[]").RootElement; + } var funDef = new FunctionDef { @@ -23,8 +29,8 @@ internal static class AiFunctionHelper Parameters = new FunctionParametersDef { Type = "object", - Properties = JsonDocument.Parse(properties.GetRawText()), - Required = JsonSerializer.Deserialize>(required.GetRawText()) + Properties = JsonDocument.Parse(properties.GetRawText() ?? "{}"), + Required = JsonSerializer.Deserialize>(required.GetRawText() ?? "[]") ?? [] } }; diff --git a/src/Infrastructure/BotSharp.Core/MCP/Hooks/MCPToolAgentHook.cs b/src/Infrastructure/BotSharp.Core/MCP/Hooks/MCPToolAgentHook.cs index 08c38b6b..743bf4c0 100644 --- a/src/Infrastructure/BotSharp.Core/MCP/Hooks/MCPToolAgentHook.cs +++ b/src/Infrastructure/BotSharp.Core/MCP/Hooks/MCPToolAgentHook.cs @@ -41,18 +41,26 @@ public class McpToolAgentHook : AgentHookBase return functionDefs; } - var mcpClientManager = _services.GetRequiredService(); - var mcps = agent.McpTools.Where(x => !x.Disabled); + var mcpClientManager = _services.GetService(); + if (mcpClientManager == null) + { + return functionDefs; + } + + var mcps = agent.McpTools?.Where(x => !x.Disabled) ?? []; foreach (var item in mcps) { var mcpClient = await mcpClientManager.GetMcpClientAsync(item.ServerId); - if (mcpClient != null) + if (mcpClient == null) continue; + + var tools = await mcpClient.ListToolsAsync(); + var toolNames = item.Functions.Select(x => x.Name).ToList(); + var targetTools = tools.Where(x => toolNames.Contains(x.Name, StringComparer.OrdinalIgnoreCase)); + foreach (var tool in targetTools) { - var tools = await mcpClient.ListToolsAsync(); - var toolnames = item.Functions.Select(x => x.Name).ToList(); - foreach (var tool in tools.Where(x => toolnames.Contains(x.Name, StringComparer.OrdinalIgnoreCase))) + var funDef = AiFunctionHelper.MapToFunctionDef(tool); + if (funDef != null) { - var funDef = AiFunctionHelper.MapToFunctionDef(tool); functionDefs.Add(funDef); } } diff --git a/src/Infrastructure/BotSharp.Core/MCP/Managers/McpClientManager.cs b/src/Infrastructure/BotSharp.Core/MCP/Managers/McpClientManager.cs index a4a89d89..50b798eb 100644 --- a/src/Infrastructure/BotSharp.Core/MCP/Managers/McpClientManager.cs +++ b/src/Infrastructure/BotSharp.Core/MCP/Managers/McpClientManager.cs @@ -6,45 +6,63 @@ namespace BotSharp.Core.MCP.Managers; public class McpClientManager : IDisposable { - private readonly McpSettings _mcpSettings; + private readonly IServiceProvider _services; + private readonly ILogger _logger; - public McpClientManager(McpSettings mcpSettings) + public McpClientManager( + IServiceProvider services, + ILogger logger) { - _mcpSettings = mcpSettings; + _services = services; + _logger = logger; } - public async Task GetMcpClientAsync(string serverId) + public async Task GetMcpClientAsync(string serverId) { - var config = _mcpSettings.McpServerConfigs.Where(x => x.Id == serverId).FirstOrDefault(); - - IClientTransport transport; - if (config.SseConfig != null) + try { - transport = new SseClientTransport(new SseClientTransportOptions + var settings = _services.GetRequiredService(); + var config = settings.McpServerConfigs.Where(x => x.Id == serverId).FirstOrDefault(); + if (config == null) { - Name = config.Name, - Endpoint = new Uri(config.SseConfig.EndPoint), - AdditionalHeaders = config.SseConfig.AdditionalHeaders, - ConnectionTimeout = config.SseConfig.ConnectionTimeout - }); - } - else if (config.StdioConfig != null) - { - transport = new StdioClientTransport(new StdioClientTransportOptions - { - Name = config.Name, - Command = config.StdioConfig.Command, - Arguments = config.StdioConfig.Arguments, - EnvironmentVariables = config.StdioConfig.EnvironmentVariables, - ShutdownTimeout = config.StdioConfig.ShutdownTimeout - }); - } - else - { - throw new ArgumentNullException("Invalid MCP server configuration!"); - } + return null; + } - return await McpClientFactory.CreateAsync(transport, _mcpSettings.McpClientOptions); + IClientTransport? transport = null; + if (config.SseConfig != null) + { + transport = new SseClientTransport(new SseClientTransportOptions + { + Name = config.Name, + Endpoint = new Uri(config.SseConfig.EndPoint), + AdditionalHeaders = config.SseConfig.AdditionalHeaders, + ConnectionTimeout = config.SseConfig.ConnectionTimeout + }); + } + else if (config.StdioConfig != null) + { + transport = new StdioClientTransport(new StdioClientTransportOptions + { + Name = config.Name, + Command = config.StdioConfig.Command, + Arguments = config.StdioConfig.Arguments, + EnvironmentVariables = config.StdioConfig.EnvironmentVariables, + ShutdownTimeout = config.StdioConfig.ShutdownTimeout + }); + } + + if (transport == null) + { + return null; + } + + return await McpClientFactory.CreateAsync(transport, settings.McpClientOptions); + } + catch (Exception ex) + { + _logger.LogWarning(ex, $"Error when loading mcp client {serverId}"); + return null; + } } public void Dispose() diff --git a/src/Infrastructure/BotSharp.Core/MCP/Services/McpService.cs b/src/Infrastructure/BotSharp.Core/MCP/Services/McpService.cs index 3bff4442..27f5326c 100644 --- a/src/Infrastructure/BotSharp.Core/MCP/Services/McpService.cs +++ b/src/Infrastructure/BotSharp.Core/MCP/Services/McpService.cs @@ -1,6 +1,5 @@ using BotSharp.Core.MCP.Managers; using BotSharp.Core.MCP.Settings; -using Microsoft.Extensions.Logging; using ModelContextProtocol.Client; namespace BotSharp.Core.MCP.Services; @@ -9,35 +8,35 @@ public class McpService : IMcpService { private readonly IServiceProvider _services; private readonly ILogger _logger; - private readonly McpClientManager _mcpClientManager; public McpService( IServiceProvider services, - ILogger logger, - McpClientManager mcpClient) + ILogger logger) { _services = services; _logger = logger; - _mcpClientManager = mcpClient; } - public IEnumerable GetServerConfigs() + public async Task> GetServerConfigsAsync() { + var clientManager = _services.GetService(); + if (clientManager == null) return []; + var options = new List(); var settings = _services.GetRequiredService(); var configs = settings?.McpServerConfigs ?? []; foreach (var config in configs) { - var tools = _mcpClientManager.GetMcpClientAsync(config.Id) - .Result.ListToolsAsync() - .Result.Select(x=> x.Name); + var client = await clientManager.GetMcpClientAsync(config.Id); + if (client == null) continue; + var tools = await client.ListToolsAsync(); options.Add(new McpServerOptionModel { Id = config.Id, Name = config.Name, - Tools = tools + Tools = tools.Select(x => x.Name) }); } diff --git a/src/Infrastructure/BotSharp.Core/MCP/Settings/MCPSettings.cs b/src/Infrastructure/BotSharp.Core/MCP/Settings/MCPSettings.cs index 2867712f..33723057 100644 --- a/src/Infrastructure/BotSharp.Core/MCP/Settings/MCPSettings.cs +++ b/src/Infrastructure/BotSharp.Core/MCP/Settings/MCPSettings.cs @@ -7,5 +7,4 @@ public class McpSettings public bool Enabled { get; set; } = true; public McpClientOptions McpClientOptions { get; set; } public List McpServerConfigs { get; set; } = []; - } diff --git a/src/Infrastructure/BotSharp.Core/Routing/Executor/DummyFunctionExecutor.cs b/src/Infrastructure/BotSharp.Core/Routing/Executor/DummyFunctionExecutor.cs index d075e185..91e0bc97 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Executor/DummyFunctionExecutor.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Executor/DummyFunctionExecutor.cs @@ -1,19 +1,19 @@ +using BotSharp.Abstraction.Routing.Executor; using BotSharp.Abstraction.Templating; namespace BotSharp.Core.Routing.Executor; public class DummyFunctionExecutor: IFunctionExecutor { - private FunctionDef functionDef; private readonly IServiceProvider _services; + private readonly FunctionDef _functionDef; - public DummyFunctionExecutor(FunctionDef function, IServiceProvider services) + public DummyFunctionExecutor(IServiceProvider services, FunctionDef functionDef) { - functionDef = function; _services = services; + _functionDef = functionDef; } - public async Task ExecuteAsync(RoleDialogModel message) { var render = _services.GetRequiredService(); @@ -25,7 +25,7 @@ public class DummyFunctionExecutor: IFunctionExecutor dict[item.Key] = item.Value; } - var text = render.Render(functionDef.Output, dict); + var text = render.Render(_functionDef.Output!, dict); message.Content = text; return true; } diff --git a/src/Infrastructure/BotSharp.Core/Routing/Executor/FunctionCallbackExecutor.cs b/src/Infrastructure/BotSharp.Core/Routing/Executor/FunctionCallbackExecutor.cs index 939f4fb3..4b208374 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Executor/FunctionCallbackExecutor.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Executor/FunctionCallbackExecutor.cs @@ -1,23 +1,24 @@ +using BotSharp.Abstraction.Routing.Executor; using BotSharp.Abstraction.Functions; namespace BotSharp.Core.Routing.Executor; public class FunctionCallbackExecutor : IFunctionExecutor { - IFunctionCallback functionCallback; + private readonly IFunctionCallback _functionCallback; public FunctionCallbackExecutor(IFunctionCallback functionCallback) { - this.functionCallback = functionCallback; + _functionCallback = functionCallback; } public async Task ExecuteAsync(RoleDialogModel message) { - return await functionCallback.Execute(message); + return await _functionCallback.Execute(message); } public async Task GetIndicatorAsync(RoleDialogModel message) { - return await functionCallback.GetIndication(message); + return await _functionCallback.GetIndication(message); } } diff --git a/src/Infrastructure/BotSharp.Core/Routing/Executor/FunctionExecutorFactory.cs b/src/Infrastructure/BotSharp.Core/Routing/Executor/FunctionExecutorFactory.cs index 3fb89509..8a4a5486 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Executor/FunctionExecutorFactory.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Executor/FunctionExecutorFactory.cs @@ -1,41 +1,31 @@ using BotSharp.Abstraction.Functions; +using BotSharp.Abstraction.Routing.Executor; namespace BotSharp.Core.Routing.Executor; internal class FunctionExecutorFactory { - public static IFunctionExecutor Create(string functionName, Agent agent, IFunctionCallback functioncall, IServiceProvider serviceProvider) + public static IFunctionExecutor? Create(IServiceProvider services, string functionName, Agent agent) { - if(functioncall != null) + var functionCall = services.GetServices().FirstOrDefault(x => x.Name == functionName); + if (functionCall != null) { - return new FunctionCallbackExecutor(functioncall); + return new FunctionCallbackExecutor(functionCall); } - var funDef = agent?.Functions?.FirstOrDefault(x => x.Name == functionName); - if (funDef != null) + var functions = (agent?.Functions ?? []).Concat(agent?.SecondaryFunctions ?? []); + var funcDef = functions.FirstOrDefault(x => x.Name == functionName); + if (!string.IsNullOrWhiteSpace(funcDef?.Output)) { - if (!string.IsNullOrWhiteSpace(funDef?.Output)) - { - return new DummyFunctionExecutor(funDef,serviceProvider); - } + return new DummyFunctionExecutor(services, funcDef); } - else + + var mcpServerId = agent?.McpTools?.Where(x => x.Functions.Any(y => y.Name == funcDef?.Name))?.FirstOrDefault()?.ServerId; + if (!string.IsNullOrWhiteSpace(mcpServerId)) { - funDef = agent?.SecondaryFunctions?.FirstOrDefault(x => x.Name == functionName); - if (funDef != null) - { - if (!string.IsNullOrWhiteSpace(funDef?.Output)) - { - return new DummyFunctionExecutor(funDef, serviceProvider); - } - else - { - var mcpServerId = agent?.McpTools?.Where(x => x.Functions.Any(y => y.Name == funDef.Name)) - .FirstOrDefault().ServerId; - return new MCPToolExecutor(mcpServerId, functionName, serviceProvider); - } - } + return new McpToolExecutor(services, mcpServerId, functionName); } + return null; } } diff --git a/src/Infrastructure/BotSharp.Core/Routing/Executor/MCPToolExecutor.cs b/src/Infrastructure/BotSharp.Core/Routing/Executor/MCPToolExecutor.cs index f7625b48..c452e806 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Executor/MCPToolExecutor.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Executor/MCPToolExecutor.cs @@ -1,21 +1,20 @@ +using BotSharp.Abstraction.Routing.Executor; using BotSharp.Core.MCP.Managers; using ModelContextProtocol.Client; namespace BotSharp.Core.Routing.Executor; -public class MCPToolExecutor: IFunctionExecutor +public class McpToolExecutor: IFunctionExecutor { - private readonly McpClientManager _clientManager; - private string mcpServer; - private string funcName; private readonly IServiceProvider _services; + private readonly string _mcpServerId; + private readonly string _functionName; - public MCPToolExecutor(string mcpserver, string functionName, IServiceProvider services) + public McpToolExecutor(IServiceProvider services, string mcpServerId, string functionName) { _services = services; - this.mcpServer = mcpserver; - this.funcName = functionName; - _clientManager = services.GetRequiredService(); + _mcpServerId = mcpServerId; + _functionName = functionName; } public async Task ExecuteAsync(RoleDialogModel message) @@ -23,12 +22,13 @@ public class MCPToolExecutor: IFunctionExecutor try { // Convert arguments to dictionary format expected by mcpdotnet - Dictionary argDict = JsonToDictionary(message.FunctionArgs); + Dictionary argDict = JsonToDictionary(message.FunctionArgs); - var client = await _clientManager.GetMcpClientAsync(mcpServer); + var clientManager = _services.GetRequiredService(); + var client = await clientManager.GetMcpClientAsync(_mcpServerId); // Call the tool through mcpdotnet - var result = await client.CallToolAsync(funcName, !argDict.IsNullOrEmpty() ? argDict : []); + var result = await client.CallToolAsync(_functionName, !argDict.IsNullOrEmpty() ? argDict : []); // Extract the text content from the result var json = string.Join("\n", result.Content.Where(c => c.Type == "text").Select(c => c.Text)); @@ -39,7 +39,7 @@ public class MCPToolExecutor: IFunctionExecutor } catch (Exception ex) { - message.Content = $"Error when calling tool {funcName} of MCP server {mcpServer}. {ex.Message}"; + message.Content = $"Error when calling tool {_functionName} of MCP server {_mcpServerId}. {ex.Message}"; return false; } } diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs index 126453e8..63bd3b20 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs @@ -1,5 +1,3 @@ -using BotSharp.Abstraction.Functions; -using BotSharp.Abstraction.Templating; using BotSharp.Core.Routing.Executor; namespace BotSharp.Core.Routing; @@ -8,14 +6,11 @@ public partial class RoutingService { public async Task InvokeFunction(string name, RoleDialogModel message) { - var function = _services.GetServices().FirstOrDefault(x => x.Name == name); - var currentAgentId = message.CurrentAgentId; var agentService = _services.GetRequiredService(); var agent = await agentService.GetAgent(currentAgentId); - IFunctionExecutor funcExecutor = FunctionExecutorFactory.Create(name, agent, function, _services); - + var funcExecutor = FunctionExecutorFactory.Create(_services, name, agent); if (funcExecutor == null) { message.StopCompletion = true; @@ -24,17 +19,14 @@ public partial class RoutingService return false; } - // Clone message var clonedMessage = RoleDialogModel.From(message); clonedMessage.FunctionName = name; - var hooks = _services - .GetRequiredService() - .HooksOrderByPriority; + var hooks = _services.GetRequiredService() + .HooksOrderByPriority; var progressService = _services.GetService(); - clonedMessage.Indication = await funcExecutor.GetIndicatorAsync(message); if (progressService?.OnFunctionExecuting != null) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/McpController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/McpController.cs index 7b74a37e..6519d010 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/McpController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/McpController.cs @@ -13,9 +13,9 @@ public class McpController : ControllerBase } [HttpGet("/mcp/server-configs")] - public IEnumerable GetMcpServerConfigs() + public async Task> GetMcpServerConfigs() { var mcp = _services.GetRequiredService(); - return mcp.GetServerConfigs(); + return await mcp.GetServerConfigsAsync(); } } diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index b48d2963..a1d4ed53 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -268,13 +268,13 @@ } }, "McpServerConfigs": [ - { - "Id": "PizzaServer", - "Name": "PizzaServer", - "SseConfig": { - "Endpoint": "http://localhost:58905/sse" - } - } + //{ + // "Id": "PizzaServer", + // "Name": "PizzaServer", + // "SseConfig": { + // "Endpoint": "http://localhost:58905/sse" + // } + //} ] }, @@ -502,7 +502,6 @@ "BotSharp.Core.SideCar", "BotSharp.Core.Crontab", "BotSharp.Core.Realtime", - "BotSharp.Core.MCP", "BotSharp.Logger", "BotSharp.Plugin.MongoStorage", "BotSharp.Plugin.Dashboard", From 37f860d897ffbc8ee768dbfbcb580f7c2123aa19 Mon Sep 17 00:00:00 2001 From: "nick.yi" Date: Fri, 16 May 2025 09:19:55 +0800 Subject: [PATCH 20/26] optimize hook --- .../Conversations/ConversationHookProvider.cs | 20 -------------- .../Hooks/HookProvider.cs | 25 +++++++++++++++++ .../BotSharp.Abstraction/Hooks/IHookBase.cs | 2 +- .../Services/RealtimeHub.cs | 8 +++--- .../ConversationService.SendMessage.cs | 5 ++-- .../ConversationService.UpdateBreakpoint.cs | 5 ++-- .../Services/ConversationService.cs | 3 ++- .../Services/ConversationStateService.cs | 27 ++++++++++--------- .../Infrastructures/HookEmitter.cs | 4 +-- .../Services/InstructService.Execute.cs | 3 ++- .../Functions/HumanInterventionNeededFn.cs | 5 ++-- .../Routing/RoutingService.InvokeFunction.cs | 6 ++--- .../Providers/ChatCompletionProvider.cs | 3 ++- .../Providers/Chat/ChatCompletionProvider.cs | 5 ++-- .../Providers/Text/TextCompletionProvider.cs | 3 ++- .../BotSharp.Plugin.ChatHub/ChatHubPlugin.cs | 1 - .../BotSharp.Plugin.ChatHub/SignalRHub.cs | 3 ++- .../Providers/Chat/ChatCompletionProvider.cs | 5 ++-- .../Providers/Text/TextCompletionProvider.cs | 3 ++- .../Chat/GeminiChatCompletionProvider.cs | 5 ++-- .../Chat/PalmChatCompletionProvider.cs | 3 ++- .../Realtime/RealTimeCompletionProvider.cs | 3 ++- .../Text/GeminiTextCompletionProvider.cs | 3 ++- .../Text/PalmTextCompletionProvider.cs | 3 ++- .../Providers/Chat/ChatCompletionProvider.cs | 5 ++-- .../Realtime/RealTimeCompletionProvider.cs | 3 ++- .../Providers/Text/TextCompletionProvider.cs | 3 ++- .../Hooks/TwilioConversationHook.cs | 3 ++- .../TwilioStreamMiddleware.cs | 7 ++--- 29 files changed, 98 insertions(+), 76 deletions(-) delete mode 100644 src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookProvider.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Hooks/HookProvider.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookProvider.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookProvider.cs deleted file mode 100644 index 00393da4..00000000 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookProvider.cs +++ /dev/null @@ -1,20 +0,0 @@ -namespace BotSharp.Abstraction.Conversations; - -public class ConversationHookProvider -{ - public IEnumerable Hooks { get; } - - private readonly Lazy> _hooksOrderByPriority; - - public IEnumerable HooksOrderByPriority - => _hooksOrderByPriority.Value; - - public ConversationHookProvider(IEnumerable conversationHooks) - { - Hooks = conversationHooks; - _hooksOrderByPriority = new Lazy>(() => - { - return conversationHooks.OrderBy(hook => hook.Priority).ToArray(); - }); - } -} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Hooks/HookProvider.cs b/src/Infrastructure/BotSharp.Abstraction/Hooks/HookProvider.cs new file mode 100644 index 00000000..cedc7c2a --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Hooks/HookProvider.cs @@ -0,0 +1,25 @@ +using BotSharp.Abstraction.Conversations; +using Microsoft.Extensions.DependencyInjection; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BotSharp.Abstraction.Hooks +{ + public static class HookProvider + { + public static List GetHooks(this IServiceProvider services, string agentId) where T : IHookBase + { + var hooks = services.GetServices().Where(p => p.IsMatch(agentId)); + return hooks.ToList(); + } + + public static List GetHooksOrderByPriority(this IServiceProvider services, string agentId) where T: IConversationHook + { + var hooks = services.GetServices().Where(p => p.IsMatch(agentId)); + return hooks.OrderBy(p => p.Priority).ToList(); + } + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Hooks/IHookBase.cs b/src/Infrastructure/BotSharp.Abstraction/Hooks/IHookBase.cs index a0ad18e6..834577e9 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Hooks/IHookBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Hooks/IHookBase.cs @@ -12,6 +12,6 @@ namespace BotSharp.Abstraction.Hooks /// Agent Id /// string SelfId => string.Empty; - bool IsMatch(string id) => string.IsNullOrEmpty(SelfId) || SelfId == id; + bool IsMatch(string agentId) => string.IsNullOrEmpty(SelfId) || SelfId == agentId; } } diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index cc9fa0c1..97ac91c2 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Functions.Models; +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.Options; using BotSharp.Core.Infrastructures; @@ -23,7 +24,6 @@ public class RealtimeHub : IRealtimeHub public async Task ConnectToModel(Func? responseToUser = null, Func? init = null) { - var hookProvider = _services.GetService(); var convService = _services.GetRequiredService(); convService.SetConversationId(_conn.ConversationId, []); var conversation = await convService.GetConversation(_conn.ConversationId); @@ -105,7 +105,8 @@ public class RealtimeHub : IRealtimeHub dialogs.Add(message); storage.Append(_conn.ConversationId, message); - foreach (var hook in hookProvider?.HooksOrderByPriority ?? []) + var hooks = _services.GetHooksOrderByPriority(_conn.CurrentAgentId); + foreach (var hook in hooks) { hook.SetAgent(agent) .SetConversation(conversation); @@ -126,7 +127,8 @@ public class RealtimeHub : IRealtimeHub storage.Append(_conn.ConversationId, message); routing.Context.SetMessageId(_conn.ConversationId, message.MessageId); - foreach (var hook in hookProvider?.HooksOrderByPriority ?? []) + var hooks = _services.GetHooksOrderByPriority(_conn.CurrentAgentId); + foreach (var hook in hooks) { hook.SetAgent(agent) .SetConversation(conversation); diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs index b2b97f4f..1227d3dc 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.SendMessage.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.Infrastructures.Enums; using BotSharp.Abstraction.Messaging; using BotSharp.Abstraction.Messaging.Models.RichContent; @@ -29,7 +30,6 @@ public partial class ConversationService var dialogs = conv.GetDialogHistory(); var statistics = _services.GetRequiredService(); - var hookProvider = _services.GetRequiredService(); RoleDialogModel response = message; bool stopCompletion = false; @@ -44,7 +44,8 @@ public partial class ConversationService message.Payload = replyMessage.Payload; } - foreach (var hook in hookProvider.HooksOrderByPriority) + var hooks = _services.GetHooksOrderByPriority(message.CurrentAgentId); + foreach (var hook in hooks) { hook.SetAgent(agent) .SetConversation(conversation); diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.UpdateBreakpoint.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.UpdateBreakpoint.cs index 203fa885..25cf32d1 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.UpdateBreakpoint.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.UpdateBreakpoint.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.Infrastructures.Enums; namespace BotSharp.Core.Conversations.Services; @@ -31,9 +32,7 @@ public partial class ConversationService states.CleanStates(excludedStates); } - var hooks = _services - .GetRequiredService() - .HooksOrderByPriority; + var hooks = _services.GetHooksOrderByPriority(routingCtx.GetCurrentAgentId()); // Before executing functions foreach (var hook in hooks) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index dc98d440..6eaeb167 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Conversations.Enums; +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.Models; namespace BotSharp.Core.Conversations.Services; @@ -116,7 +117,7 @@ public partial class ConversationService : IConversationService db.CreateNewConversation(record); - var hooks = _services.GetServices(); + var hooks = _services.GetHooks(record.AgentId); foreach (var hook in hooks) { diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs index 853b1561..40c84c1c 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs @@ -15,6 +15,7 @@ ******************************************************************************/ using BotSharp.Abstraction.Conversations.Enums; +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.Options; using BotSharp.Abstraction.SideCar; @@ -28,6 +29,7 @@ public class ConversationStateService : IConversationStateService private readonly ILogger _logger; private readonly IServiceProvider _services; private readonly IBotSharpRepository _db; + private readonly IRoutingContext _routingContext; private readonly IConversationSideCar? _sidecar; private string _conversationId; /// @@ -42,10 +44,12 @@ public class ConversationStateService : IConversationStateService public ConversationStateService( IServiceProvider services, IBotSharpRepository db, + IRoutingContext routingContext, ILogger logger) { _services = services; _db = db; + _routingContext = routingContext; _logger = logger; _curStates = new ConversationState(); _historyStates = new ConversationState(); @@ -87,7 +91,6 @@ public class ConversationStateService : IConversationStateService } _logger.LogDebug($"[STATE] {name} = {value}"); - var routingCtx = _services.GetRequiredService(); var isNoChange = ContainsState(name) && preValue == currentValue @@ -98,7 +101,7 @@ public class ConversationStateService : IConversationStateService && prevLeafNode?.Active == curActive && pair?.Readonly == readOnly; - var hooks = _services.GetServices(); + var hooks = _services.GetHooks(_routingContext.GetCurrentAgentId()); if (!ContainsState(name) || preValue != currentValue || prevLeafNode?.ActiveRounds != curActiveRounds) { foreach (var hook in hooks) @@ -106,7 +109,7 @@ public class ConversationStateService : IConversationStateService hook.OnStateChanged(new StateChangeModel { ConversationId = _conversationId, - MessageId = routingCtx.MessageId, + MessageId = _routingContext.MessageId, Name = name, BeforeValue = preValue, BeforeActiveRounds = prevLeafNode?.ActiveRounds, @@ -129,7 +132,7 @@ public class ConversationStateService : IConversationStateService var newValue = new StateValue { Data = currentValue, - MessageId = routingCtx.MessageId, + MessageId = _routingContext.MessageId, Active = curActive, ActiveRounds = curActiveRounds, DataType = valueType, @@ -171,8 +174,8 @@ public class ConversationStateService : IConversationStateService return endNodes; } - var routingCtx = _services.GetRequiredService(); - var curMsgId = routingCtx.MessageId; + var _routingContext = _services.GetRequiredService(); + var curMsgId = _routingContext.MessageId; var dialogs = _db.GetConversationDialogs(conversationId); var userDialogs = dialogs.Where(x => x.MetaData?.Role == AgentRole.User) .GroupBy(x => x.MetaData?.MessageId) @@ -225,7 +228,7 @@ public class ConversationStateService : IConversationStateService } _logger.LogInformation($"Loaded conversation states: {conversationId}"); - var hooks = _services.GetServices(); + var hooks = _services.GetHooks(_routingContext.GetCurrentAgentId()); foreach (var hook in hooks) { hook.OnStateLoaded(_curStates).Wait(); @@ -277,7 +280,6 @@ public class ConversationStateService : IConversationStateService { if (!ContainsState(name)) return false; - var routingCtx = _services.GetRequiredService(); var value = _curStates[name]; var leafNode = value?.Values?.LastOrDefault(); if (value == null || !value.Versioning || leafNode == null) return false; @@ -285,7 +287,7 @@ public class ConversationStateService : IConversationStateService _curStates[name].Values.Add(new StateValue { Data = leafNode.Data, - MessageId = routingCtx.MessageId, + MessageId = _routingContext.MessageId, Active = false, ActiveRounds = leafNode.ActiveRounds, DataType = leafNode.DataType, @@ -293,13 +295,13 @@ public class ConversationStateService : IConversationStateService UpdateTime = DateTime.UtcNow }); - var hooks = _services.GetServices(); + var hooks = _services.GetHooks(_routingContext.GetCurrentAgentId()); foreach (var hook in hooks) { hook.OnStateChanged(new StateChangeModel { ConversationId = _conversationId, - MessageId = routingCtx.MessageId, + MessageId = _routingContext.MessageId, Name = name, BeforeValue = leafNode.Data, BeforeActiveRounds = leafNode.ActiveRounds, @@ -316,8 +318,7 @@ public class ConversationStateService : IConversationStateService public void CleanStates(params string[] excludedStates) { - var routingCtx = _services.GetRequiredService(); - var curMsgId = routingCtx.MessageId; + var curMsgId = _routingContext.MessageId; var utcNow = DateTime.UtcNow; foreach (var key in _curStates.Keys) diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/HookEmitter.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/HookEmitter.cs index 95d55596..3615faf5 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/HookEmitter.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/HookEmitter.cs @@ -9,7 +9,7 @@ public static class HookEmitter { var logger = services.GetRequiredService>(); var result = new HookEmittedResult(); - var hooks = services.GetServices().Where(p => p.IsMatch(agentId)); + var hooks = services.GetHooks(agentId); option = option ?? new(); foreach (var hook in hooks) @@ -40,7 +40,7 @@ public static class HookEmitter { var logger = services.GetRequiredService>(); var result = new HookEmittedResult(); - var hooks = services.GetServices().Where(p => p.IsMatch(agentId)); + var hooks = services.GetHooks(agentId); option = option ?? new(); foreach (var hook in hooks) diff --git a/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs b/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs index 3bc069d5..a6422de6 100644 --- a/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs +++ b/src/Infrastructure/BotSharp.Core/Instructs/Services/InstructService.Execute.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.Instructs; using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.MLTasks; @@ -23,7 +24,7 @@ public partial class InstructService } // Trigger before completion hooks - var hooks = _services.GetServices().Where(p => p.IsMatch(agentId)); + var hooks = _services.GetHooks(agentId); foreach (var hook in hooks) { await hook.BeforeCompletion(agent, message); diff --git a/src/Infrastructure/BotSharp.Core/Routing/Functions/HumanInterventionNeededFn.cs b/src/Infrastructure/BotSharp.Core/Routing/Functions/HumanInterventionNeededFn.cs index 719dd805..66cd4484 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Functions/HumanInterventionNeededFn.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Functions/HumanInterventionNeededFn.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Functions; +using BotSharp.Abstraction.Hooks; namespace BotSharp.Core.Routing.Functions; @@ -15,9 +16,7 @@ public class HumanInterventionNeededFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { - var hooks = _services - .GetRequiredService() - .HooksOrderByPriority; + var hooks = _services.GetHooksOrderByPriority(message.CurrentAgentId); foreach (var hook in hooks) { diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs index 8c37536c..81f7cc26 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Functions; +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.Templating; namespace BotSharp.Core.Routing; @@ -28,10 +29,6 @@ public partial class RoutingService var clonedMessage = RoleDialogModel.From(message); clonedMessage.FunctionName = name; - var hooks = _services - .GetRequiredService() - .HooksOrderByPriority; - var progressService = _services.GetService(); // Before executing functions @@ -51,6 +48,7 @@ public partial class RoutingService var agentService = _services.GetRequiredService(); var agent = await agentService.GetAgent(clonedMessage.CurrentAgentId); + var hooks = _services.GetHooksOrderByPriority(clonedMessage.CurrentAgentId); foreach (var hook in hooks) { hook.SetAgent(agent); diff --git a/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs index 6c6c2da2..285e8abd 100644 --- a/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs @@ -1,5 +1,6 @@ using Anthropic.SDK.Common; using BotSharp.Abstraction.Conversations; +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.MLTasks.Settings; using System.Text.Json.Nodes; using System.Text.Json.Serialization; @@ -29,7 +30,7 @@ public class ChatCompletionProvider : IChatCompletion public async Task GetChatCompletions(Agent agent, List conversations) { - var contentHooks = _services.GetServices().ToList(); + var contentHooks = _services.GetHooks(agent.Id); // Before chat completion hook foreach (var hook in contentHooks) diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index 35fd0d2d..92b1a6b5 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -1,5 +1,6 @@ using Azure; using BotSharp.Abstraction.Files.Utilities; +using BotSharp.Abstraction.Hooks; using OpenAI.Chat; using System.ClientModel; @@ -29,7 +30,7 @@ public class ChatCompletionProvider : IChatCompletion public async Task GetChatCompletions(Agent agent, List conversations) { - var contentHooks = _services.GetServices().ToList(); + var contentHooks = _services.GetHooks(agent.Id); // Before chat completion hook foreach (var hook in contentHooks) @@ -128,7 +129,7 @@ public class ChatCompletionProvider : IChatCompletion Func onMessageReceived, Func onFunctionExecuting) { - var hooks = _services.GetServices().ToList(); + var hooks = _services.GetHooks(agent.Id); // Before chat completion hook foreach (var hook in hooks) diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Text/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Text/TextCompletionProvider.cs index f3bb7d1f..538bbfaa 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Text/TextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Text/TextCompletionProvider.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.MLTasks.Settings; using System.Net.Http; using System.Net.Mime; @@ -36,7 +37,7 @@ public class TextCompletionProvider : ITextCompletion public async Task GetCompletion(string text, string agentId, string messageId) { - var contentHooks = _services.GetServices().ToList(); + var contentHooks = _services.GetHooks(agentId); // Before chat completion hook var agent = new Agent() diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/ChatHubPlugin.cs b/src/Plugins/BotSharp.Plugin.ChatHub/ChatHubPlugin.cs index 7275a273..725655fc 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/ChatHubPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/ChatHubPlugin.cs @@ -24,7 +24,6 @@ public class ChatHubPlugin : IBotSharpPlugin services.AddScoped(); services.AddScoped(); services.AddScoped(); - services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/SignalRHub.cs b/src/Plugins/BotSharp.Plugin.ChatHub/SignalRHub.cs index 9ef3318e..12b595f2 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/SignalRHub.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/SignalRHub.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Hooks; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.SignalR; @@ -26,7 +27,6 @@ public class SignalRHub : Hub { _logger.LogInformation($"SignalR Hub: {_user.FirstName} {_user.LastName} ({Context.User.Identity.Name}) connected in {Context.ConnectionId}"); - var hooks = _services.GetServices(); var convService = _services.GetRequiredService(); _context.HttpContext.Request.Query.TryGetValue("conversationId", out var conversationId); @@ -38,6 +38,7 @@ public class SignalRHub : Hub var conv = await convService.GetConversation(conversationId); if (conv != null) { + var hooks = _services.GetHooks(conv.AgentId); foreach (var hook in hooks) { // Check if user connected with agent is the first time. diff --git a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs index 955d0ad2..85304054 100644 --- a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Chat/ChatCompletionProvider.cs @@ -2,6 +2,7 @@ using Microsoft.Extensions.Logging; using OpenAI.Chat; using BotSharp.Abstraction.Files; using BotSharp.Plugin.DeepSeek.Providers; +using BotSharp.Abstraction.Hooks; namespace BotSharp.Plugin.DeepSeekAI.Providers.Chat; @@ -25,7 +26,7 @@ public class ChatCompletionProvider : IChatCompletion public async Task GetChatCompletions(Agent agent, List conversations) { - var contentHooks = _services.GetServices().ToList(); + var contentHooks = _services.GetHooks(agent.Id); // Before chat completion hook foreach (var hook in contentHooks) @@ -95,7 +96,7 @@ public class ChatCompletionProvider : IChatCompletion public async Task GetChatCompletionsAsync(Agent agent, List conversations, Func onMessageReceived, Func onFunctionExecuting) { - var hooks = _services.GetServices().ToList(); + var hooks = _services.GetHooks(agent.Id); // Before chat completion hook foreach (var hook in hooks) diff --git a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Text/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Text/TextCompletionProvider.cs index b9494aff..28ad97f7 100644 --- a/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Text/TextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.DeepSeekAI/Providers/Text/TextCompletionProvider.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Hooks; using Microsoft.Extensions.Logging; using OpenAI.Chat; @@ -22,7 +23,7 @@ public class TextCompletionProvider : ITextCompletion public async Task GetCompletion(string text, string agentId, string messageId) { - var contentHooks = _services.GetServices().ToList(); + var contentHooks = _services.GetHooks(agentId); var state = _services.GetRequiredService(); // Before chat completion hook diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs index 4b3a72a8..95dda6c1 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs @@ -2,6 +2,7 @@ using System.Text.Json.Nodes; using BotSharp.Abstraction.Agents; using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Conversations; +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.Loggers; using GenerativeAI; using GenerativeAI.Core; @@ -33,7 +34,7 @@ public class GeminiChatCompletionProvider : IChatCompletion public async Task GetChatCompletions(Agent agent, List conversations) { - var contentHooks = _services.GetServices().ToList(); + var contentHooks = _services.GetHooks(agent.Id); // Before chat completion hook foreach (var hook in contentHooks) @@ -91,7 +92,7 @@ public class GeminiChatCompletionProvider : IChatCompletion public async Task GetChatCompletionsAsync(Agent agent, List conversations, Func onMessageReceived, Func onFunctionExecuting) { - var hooks = _services.GetServices().ToList(); + var hooks = _services.GetHooks(agent.Id); // Before chat completion hook foreach (var hook in hooks) diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs index de7e316f..dde9fa57 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs @@ -5,6 +5,7 @@ using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Routing; using LLMSharp.Google.Palm; using LLMSharp.Google.Palm.DiscussService; +using BotSharp.Abstraction.Hooks; namespace BotSharp.Plugin.GoogleAi.Providers.Chat; @@ -29,7 +30,7 @@ public class PalmChatCompletionProvider : IChatCompletion public async Task GetChatCompletions(Agent agent, List conversations) { - var contentHooks = _services.GetServices().ToList(); + var contentHooks = _services.GetHooks(agent.Id); // Before chat completion hook foreach (var hook in contentHooks) diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs index fad14730..3f4b8753 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Hooks; using GenerativeAI; using GenerativeAI.Core; using GenerativeAI.Live; @@ -216,7 +217,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion } } - var contentHooks = _services.GetServices().ToList(); + var contentHooks = _services.GetHooks(conn.CurrentAgentId); // After chat completion hook foreach (var hook in contentHooks) { diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs index 7568fddc..abc12c3e 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/GeminiTextCompletionProvider.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Conversations; +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.Loggers; using GenerativeAI; using GenerativeAI.Core; @@ -32,7 +33,7 @@ public class GeminiTextCompletionProvider : ITextCompletion public async Task GetCompletion(string text, string agentId, string messageId) { - var contentHooks = _services.GetServices().ToList(); + var contentHooks = _services.GetHooks(agentId); // Before completion hook var agent = new Agent() diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs index ea8ecd5e..b75116d9 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Conversations; +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.Loggers; namespace BotSharp.Plugin.GoogleAi.Providers.Text; @@ -27,7 +28,7 @@ public class PalmTextCompletionProvider : ITextCompletion public async Task GetCompletion(string text, string agentId, string messageId) { - var contentHooks = _services.GetServices().ToList(); + var contentHooks = _services.GetHooks(agentId); // Before completion hook var agent = new Agent() { Id = agentId }; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 9d542b19..95d4b909 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Hooks; using OpenAI.Chat; namespace BotSharp.Plugin.OpenAI.Providers.Chat; @@ -32,7 +33,7 @@ public class ChatCompletionProvider : IChatCompletion public async Task GetChatCompletions(Agent agent, List conversations) { - var contentHooks = _services.GetServices().ToList(); + var contentHooks = _services.GetHooks(agent.Id); // Before chat completion hook foreach (var hook in contentHooks) @@ -105,7 +106,7 @@ public class ChatCompletionProvider : IChatCompletion Func onMessageReceived, Func onFunctionExecuting) { - var hooks = _services.GetServices().ToList(); + var hooks = _services.GetHooks(agent.Id); // Before chat completion hook foreach (var hook in hooks) diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 23cc5b77..61e7dddc 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Hooks; using BotSharp.Plugin.OpenAI.Models.Realtime; using OpenAI.Chat; @@ -609,7 +610,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion return []; } - var contentHooks = _services.GetServices().ToList(); + var contentHooks = _services.GetHooks(conn.CurrentAgentId); var prompts = new List(); var inputTokenDetails = data.Usage?.InputTokenDetails; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Text/TextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Text/TextCompletionProvider.cs index bf0252ea..fd3cb5a1 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Text/TextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Text/TextCompletionProvider.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Hooks; using System.Net.Http; using System.Net.Mime; @@ -25,7 +26,7 @@ public class TextCompletionProvider : ITextCompletion public async Task GetCompletion(string text, string agentId, string messageId) { - var contentHooks = _services.GetServices().ToList(); + var contentHooks = _services.GetHooks(agentId); // Before chat completion hook var agent = new Agent() diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Hooks/TwilioConversationHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/Hooks/TwilioConversationHook.cs index 0e0682f3..bc647140 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Hooks/TwilioConversationHook.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Hooks/TwilioConversationHook.cs @@ -3,6 +3,7 @@ using Task = System.Threading.Tasks.Task; using Twilio.Rest.Api.V2010.Account; using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.Models; +using BotSharp.Abstraction.Hooks; namespace BotSharp.Plugin.Twilio.Hooks; @@ -23,7 +24,7 @@ public class TwilioConversationHook : ConversationHookBase, IConversationHook public override async Task OnFunctionExecuted(RoleDialogModel message) { - var hooks = _services.GetServices(); + var hooks = _services.GetHooks(message.CurrentAgentId); var routing = _services.GetRequiredService(); var conversationId = routing.Context.ConversationId; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs index 589b2db2..ea4d5c50 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Hooks; using BotSharp.Abstraction.MLTasks; using BotSharp.Abstraction.Realtime; using BotSharp.Abstraction.Realtime.Models; @@ -63,7 +64,7 @@ public class TwilioStreamMiddleware // load conversation and state var convService = services.GetRequiredService(); convService.SetConversationId(conversationId, []); - var hooks = services.GetServices(); + var hooks = services.GetHooks(agentId); foreach (var hook in hooks) { await hook.OnStreamingStarted(conn); @@ -229,7 +230,6 @@ public class TwilioStreamMiddleware private async Task HandleUserDtmfReceived(IServiceProvider _services, RealtimeHubConnection conn, IRealTimeCompletion completer, string data) { var routing = _services.GetRequiredService(); - var hookProvider = _services.GetRequiredService(); var agentService = _services.GetRequiredService(); var agent = await agentService.GetAgent(conn.CurrentAgentId); var dialogs = routing.Context.GetDialogs(); @@ -245,7 +245,8 @@ public class TwilioStreamMiddleware var storage = _services.GetRequiredService(); storage.Append(conn.ConversationId, message); - foreach (var hook in hookProvider.HooksOrderByPriority) + var hooks = _services.GetHooksOrderByPriority(conn.CurrentAgentId); + foreach (var hook in hooks) { hook.SetAgent(agent) .SetConversation(conversation); From d10f0a635af423c4798d2353f8332c60dbe8badd Mon Sep 17 00:00:00 2001 From: "nick.yi" Date: Fri, 16 May 2025 09:32:05 +0800 Subject: [PATCH 21/26] hotfix error --- .../Conversations/Services/ConversationStateService.cs | 1 - .../BotSharp.Core/Routing/RoutingService.InvokeFunction.cs | 2 -- 2 files changed, 3 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs index 40c84c1c..c41d3110 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs @@ -174,7 +174,6 @@ public class ConversationStateService : IConversationStateService return endNodes; } - var _routingContext = _services.GetRequiredService(); var curMsgId = _routingContext.MessageId; var dialogs = _db.GetConversationDialogs(conversationId); var userDialogs = dialogs.Where(x => x.MetaData?.Role == AgentRole.User) diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs index 9d5bfc5d..697e2aa6 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs @@ -39,8 +39,6 @@ public partial class RoutingService await progressService.OnFunctionExecuting(clonedMessage); } - var agentService = _services.GetRequiredService(); - var agent = await agentService.GetAgent(clonedMessage.CurrentAgentId); var hooks = _services.GetHooksOrderByPriority(clonedMessage.CurrentAgentId); foreach (var hook in hooks) { From 0c2ba423d8c243487a5cd43739283aa75dcc38bb Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Thu, 15 May 2025 22:59:57 -0500 Subject: [PATCH 22/26] refine --- .../Realtime/Models/RealtimeHubConnection.cs | 1 - .../Hooks/RealtimeConversationHook.cs | 16 +- .../{ => Demo}/Functions/GetWeatherFn.cs | 11 +- .../MCP/Helpers/AiFunctionHelper.cs | 15 +- .../Realtime/RealtimeTranscriptionResponse.cs | 2 +- .../Chat/PalmChatCompletionProvider.cs | 5 +- .../Realtime/RealTimeCompletionProvider.cs | 45 ++---- .../Text/PalmTextCompletionProvider.cs | 5 +- .../Realtime/RealTimeCompletionProvider.cs | 138 +++++++++--------- 9 files changed, 105 insertions(+), 133 deletions(-) rename src/Infrastructure/BotSharp.Core/{ => Demo}/Functions/GetWeatherFn.cs (69%) diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs index 4d66ac5b..c0dac6d5 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs @@ -10,7 +10,6 @@ public class RealtimeHubConnection public string KeypadInputBuffer { get; set; } = string.Empty; public string CurrentAgentId { get; set; } = null!; public string ConversationId { get; set; } = null!; - public string? PrevSessionId { get; set; } public Func OnModelReady { get; set; } = () => string.Empty; public Func OnModelMessageReceived { get; set; } = null!; public Func OnModelAudioResponseDone { get; set; } = null!; diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs index 32cdc113..4eb4c64e 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Hooks/RealtimeConversationHook.cs @@ -73,14 +73,14 @@ public class RealtimeConversationHook : ConversationHookBase, IConversationHook return; } - //if (message.StopCompletion) - //{ - // await hub.Completer.TriggerModelInference($"Say to user: \"{message.Content}\""); - //} - //else - //{ - // await hub.Completer.TriggerModelInference(instruction); - //} + if (message.StopCompletion) + { + await hub.Completer.TriggerModelInference($"Say to user: \"{message.Content}\""); + } + else + { + await hub.Completer.TriggerModelInference(instruction); + } } } } diff --git a/src/Infrastructure/BotSharp.Core/Functions/GetWeatherFn.cs b/src/Infrastructure/BotSharp.Core/Demo/Functions/GetWeatherFn.cs similarity index 69% rename from src/Infrastructure/BotSharp.Core/Functions/GetWeatherFn.cs rename to src/Infrastructure/BotSharp.Core/Demo/Functions/GetWeatherFn.cs index d203ac8c..8dbdbfaf 100644 --- a/src/Infrastructure/BotSharp.Core/Functions/GetWeatherFn.cs +++ b/src/Infrastructure/BotSharp.Core/Demo/Functions/GetWeatherFn.cs @@ -1,7 +1,7 @@ using BotSharp.Abstraction.Functions; using System.Text.Json.Serialization; -namespace BotSharp.Core.Functions; +namespace BotSharp.Core.Demo.Functions; public class GetWeatherFn : IFunctionCallback { @@ -19,6 +19,7 @@ public class GetWeatherFn : IFunctionCallback { //var args = JsonSerializer.Deserialize(message.FunctionArgs, BotSharpOptions.defaultJsonOptions); message.Content = $"It is a sunny day."; + //message.StopCompletion = true; return true; } } @@ -27,12 +28,4 @@ class Location { [JsonPropertyName("city")] public string? City { get; set; } - - [JsonPropertyName("state")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? State { get; set; } - - [JsonPropertyName("county")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? County { get; set; } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/MCP/Helpers/AiFunctionHelper.cs b/src/Infrastructure/BotSharp.Core/MCP/Helpers/AiFunctionHelper.cs index b9db3ce7..3e15cd64 100644 --- a/src/Infrastructure/BotSharp.Core/MCP/Helpers/AiFunctionHelper.cs +++ b/src/Infrastructure/BotSharp.Core/MCP/Helpers/AiFunctionHelper.cs @@ -11,14 +11,17 @@ internal static class AiFunctionHelper return null; } - if (!tool.JsonSchema.TryGetProperty("properties", out var properties)) + var properties = "{}"; + var required = "[]"; + + if (tool.JsonSchema.TryGetProperty("properties", out var p)) { - properties = JsonDocument.Parse("{}").RootElement; + properties = p.GetRawText(); } - if (!tool.JsonSchema.TryGetProperty("required", out var required)) + if (tool.JsonSchema.TryGetProperty("required", out var r)) { - required = JsonDocument.Parse("[]").RootElement; + required = r.GetRawText(); } var funDef = new FunctionDef @@ -29,8 +32,8 @@ internal static class AiFunctionHelper Parameters = new FunctionParametersDef { Type = "object", - Properties = JsonDocument.Parse(properties.GetRawText() ?? "{}"), - Required = JsonSerializer.Deserialize>(required.GetRawText() ?? "[]") ?? [] + Properties = JsonDocument.Parse(properties), + Required = JsonSerializer.Deserialize>(required) ?? [] } }; diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs index 5d75ca97..189252fa 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Models/Realtime/RealtimeTranscriptionResponse.cs @@ -42,8 +42,8 @@ internal class RealtimeTranscriptionResponse : IDisposable public void Clear() { - _contentStream.SetLength(0); _contentStream.Position = 0; + _contentStream.SetLength(0); } public void Dispose() diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs index de7e316f..7ec9268b 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/PalmChatCompletionProvider.cs @@ -1,13 +1,10 @@ -using BotSharp.Abstraction.Agents; -using BotSharp.Abstraction.Agents.Enums; -using BotSharp.Abstraction.Loggers; -using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Routing; using LLMSharp.Google.Palm; using LLMSharp.Google.Palm.DiscussService; namespace BotSharp.Plugin.GoogleAi.Providers.Chat; +[Obsolete] public class PalmChatCompletionProvider : IChatCompletion { private readonly IServiceProvider _services; diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs index b3182b9a..b8d33d90 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -89,7 +89,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion var modelSettings = settingsService.GetSetting(Provider, _model); Reset(); - _inputStream = new(); _outputStream = new(); _session = new LlmRealtimeSession(_services, new ChatSessionOptions @@ -99,9 +98,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion var uri = BuildWebsocketUri(modelSettings.ApiKey, "v1beta"); await _session.ConnectAsync(uri: uri, cancellationToken: CancellationToken.None); - await onModelReady(); - _ = ReceiveMessage(); } @@ -130,7 +127,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion else if (response.SessionResumptionUpdate != null) { _logger.LogInformation($"Session resumption update => New handle: {response.SessionResumptionUpdate.NewHandle}, Resumable: {response.SessionResumptionUpdate.Resumable}"); - _conn.PrevSessionId = response.SessionResumptionUpdate?.NewHandle; } else if (response.ToolCall != null && !response.ToolCall.FunctionCalls.IsNullOrEmpty()) { @@ -227,7 +223,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion public async Task AppenAudioBuffer(string message) { - await SendEventToModel(new BidiClientPayload + await SendEventToModel(new RealtimeClientPayload { RealtimeInput = new() { @@ -239,7 +235,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion public async Task AppenAudioBuffer(ArraySegment data, int length) { var buffer = data.AsSpan(0, length).ToArray(); - await SendEventToModel(new BidiClientPayload + await SendEventToModel(new RealtimeClientPayload { RealtimeInput = new() { @@ -253,7 +249,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion if (string.IsNullOrWhiteSpace(instructions)) return; var content = new Content(instructions, AgentRole.User); - await SendEventToModel(new BidiClientPayload + await SendEventToModel(new RealtimeClientPayload { ClientContent = new() { @@ -284,7 +280,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { if (!isInit) { - return null; + return string.Empty; } var agentService = _services.GetRequiredService(); @@ -294,18 +290,14 @@ public class GoogleRealTimeProvider : IRealTimeCompletion var (prompt, request) = PrepareOptions(agent, []); var config = request.GenerationConfig ?? new(); - if (config != null) - { - //Output Modality can either be text or audio - config.ResponseModalities = [Modality.AUDIO]; + //Output Modality can either be text or audio + config.ResponseModalities = [Modality.AUDIO]; + config.Temperature = Math.Max(realtimeSetting.Temperature, 0.6f); + config.MaxOutputTokens = realtimeSetting.MaxResponseOutputTokens; - var words = new List(); - HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)), agent.Id); + var words = new List(); + HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent)), agent.Id); - config.Temperature = Math.Max(realtimeSetting.Temperature, 0.6f); - config.MaxOutputTokens = realtimeSetting.MaxResponseOutputTokens; - } - var functions = request.Tools?.SelectMany(s => s.FunctionDeclarations).Select(x => { var fn = new FunctionDef @@ -331,7 +323,6 @@ public class GoogleRealTimeProvider : IRealTimeCompletion }); } - var payload = new RealtimeClientPayload { Setup = new RealtimeGenerateContentSetup() @@ -341,15 +332,11 @@ public class GoogleRealTimeProvider : IRealTimeCompletion SystemInstruction = request.SystemInstruction, Tools = request.Tools?.ToArray(), InputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null, - OutputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null, - SessionResumption = new() - { - Handle = _conn.PrevSessionId - } + OutputAudioTranscription = realtimeSetting.InputAudioTranscribe ? new() : null } }; - Console.WriteLine($"Setup payload: {JsonSerializer.Serialize(payload, _jsonOptions)}"); + _logger.LogInformation($"Setup payload: {JsonSerializer.Serialize(payload, _jsonOptions)}"); await SendEventToModel(payload); return prompt; @@ -369,7 +356,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion } }; - await SendEventToModel(new BidiClientPayload + await SendEventToModel(new RealtimeClientPayload { ToolResponse = new() { @@ -379,7 +366,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion } else if (message.Role == AgentRole.Assistant) { - await SendEventToModel(new BidiClientPayload + await SendEventToModel(new RealtimeClientPayload { ClientContent = new() { @@ -390,7 +377,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion } else if (message.Role == AgentRole.User) { - await SendEventToModel(new BidiClientPayload + await SendEventToModel(new RealtimeClientPayload { ClientContent = new() { @@ -613,7 +600,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion }; } - private Uri BuildWebsocketUri(string apiKey, string version = "v1alpha") + private Uri BuildWebsocketUri(string apiKey, string version = "v1beta") { return new Uri($"wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.{version}.GenerativeService.BidiGenerateContent?key={apiKey}"); } diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs index ea8ecd5e..ba206a73 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Text/PalmTextCompletionProvider.cs @@ -1,9 +1,6 @@ -using BotSharp.Abstraction.Agents.Enums; -using BotSharp.Abstraction.Conversations; -using BotSharp.Abstraction.Loggers; - namespace BotSharp.Plugin.GoogleAi.Providers.Text; +[Obsolete] public class PalmTextCompletionProvider : ITextCompletion { private readonly IServiceProvider _services; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs index 22daf3ff..9b0b14c4 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -45,11 +45,7 @@ public class RealTimeCompletionProvider : IRealTimeCompletion _model = realtimeSettings.Model; var settings = settingsService.GetSetting(Provider, _model); - if (_session != null) - { - _session.Dispose(); - } - + _session?.Dispose(); _session = new LlmRealtimeSession(_services, new ChatSessionOptions { JsonOptions = _botsharpOptions.JsonSerializerOptions @@ -77,72 +73,6 @@ public class RealTimeCompletionProvider : IRealTimeCompletion onInterruptionDetected); } - public async Task Disconnect() - { - if (_session != null) - { - await _session.DisconnectAsync(); - _session.Dispose(); - } - } - - public async Task AppenAudioBuffer(string message) - { - var audioAppend = new - { - type = "input_audio_buffer.append", - audio = message - }; - - await SendEventToModel(audioAppend); - } - - public async Task AppenAudioBuffer(ArraySegment data, int length) - { - var message = Convert.ToBase64String(data.AsSpan(0, length).ToArray()); - await AppenAudioBuffer(message); - } - - public async Task TriggerModelInference(string? instructions = null) - { - // Triggering model inference - if (!string.IsNullOrEmpty(instructions)) - { - await SendEventToModel(new - { - type = "response.create", - response = new - { - instructions - } - }); - } - else - { - await SendEventToModel(new - { - type = "response.create" - }); - } - } - - public async Task CancelModelResponse() - { - await SendEventToModel(new - { - type = "response.cancel" - }); - } - - public async Task RemoveConversationItem(string itemId) - { - await SendEventToModel(new - { - type = "conversation.item.delete", - item_id = itemId - }); - } - private async Task ReceiveMessage( RealtimeModelSettings realtimeSettings, RealtimeHubConnection conn, @@ -279,6 +209,72 @@ public class RealTimeCompletionProvider : IRealTimeCompletion _session.Dispose(); } + public async Task Disconnect() + { + if (_session != null) + { + await _session.DisconnectAsync(); + _session.Dispose(); + } + } + + public async Task AppenAudioBuffer(string message) + { + var audioAppend = new + { + type = "input_audio_buffer.append", + audio = message + }; + + await SendEventToModel(audioAppend); + } + + public async Task AppenAudioBuffer(ArraySegment data, int length) + { + var message = Convert.ToBase64String(data.AsSpan(0, length).ToArray()); + await AppenAudioBuffer(message); + } + + public async Task TriggerModelInference(string? instructions = null) + { + // Triggering model inference + if (!string.IsNullOrEmpty(instructions)) + { + await SendEventToModel(new + { + type = "response.create", + response = new + { + instructions + } + }); + } + else + { + await SendEventToModel(new + { + type = "response.create" + }); + } + } + + public async Task CancelModelResponse() + { + await SendEventToModel(new + { + type = "response.cancel" + }); + } + + public async Task RemoveConversationItem(string itemId) + { + await SendEventToModel(new + { + type = "conversation.item.delete", + item_id = itemId + }); + } + public async Task SendEventToModel(object message) { if (_session == null) return; From d5d03834f1c1018029181a5bcead96fd42ab1e6d Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Thu, 15 May 2025 23:02:38 -0500 Subject: [PATCH 23/26] minor change --- .../BotSharp.Core/Demo/Functions/GetWeatherFn.cs | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Demo/Functions/GetWeatherFn.cs b/src/Infrastructure/BotSharp.Core/Demo/Functions/GetWeatherFn.cs index 8dbdbfaf..a78ad4ec 100644 --- a/src/Infrastructure/BotSharp.Core/Demo/Functions/GetWeatherFn.cs +++ b/src/Infrastructure/BotSharp.Core/Demo/Functions/GetWeatherFn.cs @@ -1,5 +1,4 @@ using BotSharp.Abstraction.Functions; -using System.Text.Json.Serialization; namespace BotSharp.Core.Demo.Functions; @@ -17,15 +16,8 @@ public class GetWeatherFn : IFunctionCallback public async Task Execute(RoleDialogModel message) { - //var args = JsonSerializer.Deserialize(message.FunctionArgs, BotSharpOptions.defaultJsonOptions); - message.Content = $"It is a sunny day."; + message.Content = $"It is a sunny day!"; //message.StopCompletion = true; return true; } -} - -class Location -{ - [JsonPropertyName("city")] - public string? City { get; set; } } \ No newline at end of file From d91a55232bea7352818a28a4e0ddc637c2b31f6d Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 16 May 2025 10:49:09 -0500 Subject: [PATCH 24/26] refine tool call and response --- .../Providers/Chat/ChatCompletionProvider.cs | 4 ++-- .../Providers/Chat/GeminiChatCompletionProvider.cs | 2 ++ .../Providers/Realtime/RealTimeCompletionProvider.cs | 2 ++ .../Providers/Chat/ChatCompletionProvider.cs | 5 ++--- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index 92b1a6b5..e2d9a161 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -305,10 +305,10 @@ public class ChatCompletionProvider : IChatCompletion { messages.Add(new AssistantChatMessage(new List { - ChatToolCall.CreateFunctionToolCall(message.FunctionName, message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? string.Empty)) + ChatToolCall.CreateFunctionToolCall(message.ToolCallId ?? message.FunctionName, message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? "{}")) })); - messages.Add(new ToolChatMessage(message.FunctionName, message.Content)); + messages.Add(new ToolChatMessage(message.ToolCallId ?? message.FunctionName, message.Content)); } else if (message.Role == AgentRole.User) { diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs index 95dda6c1..60853362 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Chat/GeminiChatCompletionProvider.cs @@ -271,6 +271,7 @@ public class GeminiChatCompletionProvider : IChatCompletion { FunctionCall = new FunctionCall { + Id = message.ToolCallId, Name = message.FunctionName, Args = JsonNode.Parse(message.FunctionArgs ?? "{}") } @@ -282,6 +283,7 @@ public class GeminiChatCompletionProvider : IChatCompletion { FunctionResponse = new FunctionResponse { + Id = message.ToolCallId, Name = message.FunctionName, Response = new JsonObject() { diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs index f3fb11d7..5df8ee13 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/Realtime/RealTimeCompletionProvider.cs @@ -512,6 +512,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { FunctionCall = new FunctionCall { + Id = message.ToolCallId, Name = message.FunctionName, Args = JsonNode.Parse(message.FunctionArgs ?? "{}") } @@ -523,6 +524,7 @@ public class GoogleRealTimeProvider : IRealTimeCompletion { FunctionResponse = new FunctionResponse { + Id = message.ToolCallId, Name = message.FunctionName ?? string.Empty, Response = new JsonObject() { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 95d4b909..c0a9c0d4 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Hooks; using OpenAI.Chat; @@ -273,10 +272,10 @@ public class ChatCompletionProvider : IChatCompletion { messages.Add(new AssistantChatMessage(new List { - ChatToolCall.CreateFunctionToolCall(message.FunctionName, message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? string.Empty)) + ChatToolCall.CreateFunctionToolCall(message.ToolCallId ?? message.FunctionName, message.FunctionName, BinaryData.FromString(message.FunctionArgs ?? "{}")) })); - messages.Add(new ToolChatMessage(message.FunctionName, message.Content)); + messages.Add(new ToolChatMessage(message.ToolCallId ?? message.FunctionName, message.Content)); } else if (message.Role == AgentRole.User) { From b4dda5e63a5997faa8aa76e8162d8ddabd2a0c81 Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Fri, 16 May 2025 21:16:49 -0400 Subject: [PATCH 25/26] Update to Microsoft.Extensions.AI stable release --- Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 01fc6c90..e5f4fcd6 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -60,7 +60,7 @@ - + From 968bac623d4ae5144bd56638345b50691a13481c Mon Sep 17 00:00:00 2001 From: Haiping Date: Mon, 19 May 2025 09:48:16 -0500 Subject: [PATCH 26/26] Update RoutingService.InstructLoop.cs hotfix GetNextStepPrompt error(can't find naive reasoner template) --- .../BotSharp.Core/Routing/RoutingService.InstructLoop.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InstructLoop.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InstructLoop.cs index 128debad..757919cd 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InstructLoop.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InstructLoop.cs @@ -15,7 +15,7 @@ public partial class RoutingService var convService = _services.GetRequiredService(); var storage = _services.GetRequiredService(); - _router = await agentService.GetAgent(message.CurrentAgentId); + _router = agent; var states = _services.GetRequiredService(); var executor = _services.GetRequiredService();