using System.Net.Security; using Microsoft.EntityFrameworkCore; using Npgsql; using Serilog; using Serilog.Events; using Serilog.Sinks.OpenSearch; using StackExchange.Redis; using Scalar.AspNetCore; using w4c_workflows.Data; using w4c_workflows.Middleware; using w4c_workflows.Services; using w4c_workflows.Services.Audit; using w4c_workflows.Services.Credentials; using w4c_workflows.Services.Execution; using w4c_workflows.Services.Messaging; using w4c_workflows.Services.Nodes; using w4c_workflows.Services.Nodes.Binary; using w4c_workflows.Services.Nodes.Connectors; using w4c_workflows.Services.Nodes.Executors; using w4c_workflows.Services.Nodes.Interpolation; using w4c_workflows.Services.Quota; using w4c_workflows.Services.Runs; using w4c_workflows.Services.Security; using w4c_workflows.Services.Triggers; // --------------------------------------------------------------------------- // w4c-workflows-api — control plane for the custom workflows engine. // // Two entrypoints share this image: // - default : HTTP control plane (API, trigger scheduler, Mermaid, keys) // - --worker : per-tenant execution worker (pull/exec/result over Redis) // --------------------------------------------------------------------------- var runWorker = args.Any(a => a == "--worker"); var builder = WebApplication.CreateBuilder(args); var openSearchUrl = builder.Configuration["OpenSearch:Url"] ?? "http://localhost:9200"; var appName = builder.Configuration["APP_NAME"] ?? (runWorker ? "workflows-worker" : "workflows-api"); Log.Logger = new LoggerConfiguration() .MinimumLevel.Information() .MinimumLevel.Override("Microsoft.AspNetCore", LogEventLevel.Warning) // EF Core logs the raw SQL ("Executed DbCommand ...") at Information for // EVERY query. The background pollers (run lifecycle, trigger scheduler, // handler consumer) fire those several times a second and flood the console // / OpenSearch. Keep our own Information logs, but drop the EF command log. .MinimumLevel.Override("Microsoft.EntityFrameworkCore.Database.Command", LogEventLevel.Warning) .Enrich.FromLogContext() .Enrich.WithProperty("App", appName) .WriteTo.Console() .WriteTo.OpenSearch(new OpenSearchSinkOptions(new Uri(openSearchUrl)) { IndexFormat = "w4c-workflows-{0:yyyy.MM.dd}", AutoRegisterTemplate = false, BatchAction = OpenOpType.Create, EmitEventFailure = EmitEventFailureHandling.WriteToSelfLog, BufferBaseFilename = Path.Combine(Path.GetTempPath(), "serilog-opensearch-buffer"), }) .CreateLogger(); builder.Host.UseSerilog(); builder.Services.AddControllers(); builder.Services.AddOpenApi(); builder.Services.AddHttpClient(); // HTTP nodes must not follow redirects at the transport layer: the executor // re-checks every hop against the egress policy and follows redirects itself. // Every egress client pins the vetted address at connect time (EgressPinning) so // a name that flips between public and private cannot be raced (DNS rebinding). builder.Services.AddHttpClient(HttpRequestNodeExecutor.TypeName) .ConfigurePrimaryHttpMessageHandler(sp => EgressPinning.CreateHandler( sp.GetRequiredService(), sp.GetRequiredService())); // Opt-in named client for nodes that set ignoreSslIssues; still redirect-vetted // and address-pinned at connect time. builder.Services.AddHttpClient(HttpRequestNodeExecutor.InsecureClientName) .ConfigurePrimaryHttpMessageHandler(sp => EgressPinning.CreateHandler( sp.GetRequiredService(), sp.GetRequiredService(), allowInsecureTls: true)); builder.Services.AddHttpContextAccessor(); // Credential probe client: caller-supplied URL + decrypted credential, so it // must not follow redirects and must connect to a vetted address (a public host // could otherwise bounce the secret to an internal address after the check). builder.Services.AddHttpClient("credential-test") .ConfigurePrimaryHttpMessageHandler(sp => EgressPinning.CreateHandler( sp.GetRequiredService(), sp.GetRequiredService())); // Credential vault encryption keys (used by ICredentialCipher). builder.Services.AddDataProtection(); // Lite mode flag: when true, the engine uses SQLite + in-process channels // instead of PostgreSQL + Redis. Intended for self-hosted single-container // deployments where external infrastructure is not available. var liteMode = builder.Configuration.GetValue("UseLiteMode"); // A transient failure in ONE background service (worker loop, trigger scheduler, // run lifecycle, handler consumer) must never take the whole control plane down. // Without this, HostOptions.BackgroundServiceExceptionBehavior defaults to // StopHost and a single early exception prevents the API from starting. builder.Services.Configure(options => options.BackgroundServiceExceptionBehavior = BackgroundServiceExceptionBehavior.Ignore); if (liteMode) { // Lite mode: SQLite. The connection string should be something like // "Data Source=/app/data/workflows.db". var sqliteCs = builder.Configuration.GetConnectionString("DefaultConnection") ?? "Data Source=workflows.db"; builder.Services.AddDbContext(options => options.UseSqlite(sqliteCs)); Log.Logger.Information("Lite mode: using SQLite ({ConnectionString})", sqliteCs); } else { builder.Services.AddDbContext(options => options.UseNpgsql(builder.Configuration.GetConnectionString("DefaultConnection"))); // NpgsqlDataSource for lightweight queries outside EF (e.g. login resolution). builder.Services.AddSingleton(_ => NpgsqlDataSource.Create(builder.Configuration.GetConnectionString("DefaultConnection")!)); builder.Services.AddSingleton(_ => ConnectionMultiplexer.Connect( builder.Configuration["Redis:ConnectionString"] ?? "localhost:6379,abortConnect=false")); } builder.Services.AddScoped(); // Transport + lease. if (liteMode) { // Lite mode: in-process channels replace Redis Streams. builder.Services.AddSingleton(); builder.Services.AddSingleton(sp => sp.GetRequiredService()); builder.Services.AddSingleton(sp => sp.GetRequiredService()); builder.Services.AddSingleton(); builder.Services.AddSingleton(); Log.Logger.Information("Lite mode: using in-memory transport + trigger state (no Redis)"); } else { // Full mode: Redis Streams. builder.Services.AddSingleton(); builder.Services.AddSingleton(sp => sp.GetRequiredService()); builder.Services.AddSingleton(sp => sp.GetRequiredService()); builder.Services.AddSingleton(); builder.Services.AddSingleton(); } // YAML compile pipeline (step 4). builder.Services.AddSingleton(); builder.Services.AddSingleton(); // Workflows:LowerLegacyScripts (default off) opts function-mode script YAML into // compile-time lowering onto the node kernel (S1 migration switch). builder.Services.AddSingleton(sp => new WorkflowCompiler( sp.GetRequiredService(), sp.GetRequiredService(), builder.Configuration.GetValue("Workflows:LowerLegacyScripts", false))); // Node catalog: the declarative blueprint registry backing GET /api/nodes and // the authoring UI. Core/connector blueprints ship embedded; extra connector // packs can be dropped into Nodes:ConnectorDirectory without a rebuild. var blueprints = NodeBlueprintCatalog.LoadEmbedded().ToList(); var connectorDirectory = builder.Configuration["Nodes:ConnectorDirectory"]; if (!string.IsNullOrWhiteSpace(connectorDirectory)) blueprints.AddRange(NodeBlueprintCatalog.LoadDirectory(connectorDirectory)); var nodeCatalog = new NodeBlueprintCatalog(blueprints); builder.Services.AddSingleton(nodeCatalog); builder.Services.AddSingleton(); // Node execution kernel: parameter interpolation, the in-process executors and // the edge-driven runner that turns a validated node graph into items. builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); // Item-shaping transforms (n8n Limit/Sort/Remove Duplicates/Aggregate parity). builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); // SSRF/egress policy for outbound node HTTP; configuration lives under Nodes:Egress. var egressOptions = new EgressPolicyOptions(); builder.Configuration.GetSection(EgressPolicyOptions.SectionName).Bind(egressOptions); builder.Services.AddSingleton(egressOptions); builder.Services.AddSingleton(new EgressPolicy(egressOptions)); builder.Services.AddSingleton(); builder.Services.AddSingleton(); // Per-run node resource quotas (outbound request budget, response size cap). var quotaOptions = new NodeQuotaOptions(); builder.Configuration.GetSection(NodeQuotaOptions.SectionName).Bind(quotaOptions); builder.Services.AddSingleton(quotaOptions); builder.Services.AddSingleton(new NodeRequestBudget(quotaOptions)); // Binary store: content-addressed payload storage behind IBinaryStore, used by // the HTTP node's file responses and binary/multipart uploads. var binaryStoreOptions = new BinaryStoreOptions(); builder.Configuration.GetSection(BinaryStoreOptions.SectionName).Bind(binaryStoreOptions); builder.Services.AddSingleton(binaryStoreOptions); builder.Services.AddSingleton(); // Node permissions: operator allow/deny policy over which node types may run. // Enforced in the palette endpoint, the compiler and the run kernel. var permissionOptions = new NodePermissionOptions(); builder.Configuration.GetSection(NodePermissionOptions.SectionName).Bind(permissionOptions); builder.Services.AddSingleton(permissionOptions); builder.Services.AddSingleton(new NodePermissionPolicy(permissionOptions)); // Action audit trail (OpenSearch w4c-actions-*). Opt-in: without an enabled // sink the null sink drops entries, so no audit traffic is produced. var auditOptions = new ActionAuditOptions(); builder.Configuration.GetSection(ActionAuditOptions.SectionName).Bind(auditOptions); builder.Services.AddSingleton(auditOptions); if (auditOptions.Enabled && !string.IsNullOrWhiteSpace(auditOptions.Url)) { builder.Services.AddHttpClient(OpenSearchActionAuditSink.ClientName) .ConfigurePrimaryHttpMessageHandler(() => new SocketsHttpHandler { AllowAutoRedirect = false }); builder.Services.AddSingleton(); } else { builder.Services.AddSingleton(NullActionAuditSink.Instance); } builder.Services.AddSingleton(sp => new HttpRequestNodeExecutor( sp.GetRequiredService(), sp.GetRequiredService(), sp.GetRequiredService(), sp.GetRequiredService(), sp.GetRequiredService())); // Declarative REST connectors: one executor per blueprint carrying a connector // spec (Telegram, Discord, runtime connector packs), so integrations are catalog // data rather than bespoke C#. Registered once per type (highest version). foreach (var connectorBlueprint in nodeCatalog.All .Where(b => b.Connector != null) .GroupBy(b => b.Type, StringComparer.Ordinal) .Select(versions => versions.Last())) { var blueprint = connectorBlueprint; builder.Services.AddSingleton(sp => new RestConnectorExecutor( blueprint, sp.GetRequiredService(), sp.GetRequiredService(), sp.GetRequiredService(), sp.GetRequiredService())); } builder.Services.AddSingleton(); builder.Services.AddSingleton(); // Credential vault: encryption + built-in HTTP credential types (injection and // extraction strategies) + the tenant-scoped CRUD/resolver. builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); // DB-level node workflow execution: reconstructs the graph from the compiled // tasks+edges, runs it and records a TaskRun per node. Executed by the worker // (GraphJobExecutor) for `graph.run` jobs, not inline on the lifecycle loop. builder.Services.AddSingleton(); // Worker half of node-mode execution: consumes `graph.run` jobs off the same // tenant job streams as `task.run`, so node graphs get uniform leases/timeouts // and scale across --worker replicas (S2). builder.Services.AddSingleton(); // Realtime SSE hub — notifies subscribed clients when a workflow file changes server-side. builder.Services.AddSingleton(); // Workflow source: factory-based — each request creates a tenant-scoped source. // WorkflowSource:CopiesRoot is REQUIRED; the factory throws if it is empty, // preventing any request from serving cross-tenant workflows. // When Forgejo:AdminToken is configured, the factory creates Forgejo-backed // sources with per-tenant repos; otherwise it falls back to filesystem-only. // Single async git entry point (status/state + file-controller reads), with a // short-TTL state cache so a workflow list does not shell out to git per request. builder.Services.AddSingleton(); // Pooled client for the Forgejo admin API (avoids per-call socket churn). builder.Services.AddHttpClient(ForgejoWorkflowRepoService.AdminClientName); builder.Services.AddSingleton(); builder.Services.AddScoped(); builder.Services.AddScoped(sp => { var http = sp.GetRequiredService(); var factory = sp.GetRequiredService(); // Check if a Forgejo-resolved source was already prepared by the middleware. if (http.HttpContext?.Items["WorkflowSource"] is IWorkflowSource forgejoSrc) return forgejoSrc; var tenantId = http.HttpContext?.Items["TenantId"] as string ?? throw new InvalidOperationException("TenantId not resolved by auth middleware"); return factory.Create(tenantId); }); builder.Services.AddScoped(); // Serializes concurrent syncs of the same (tenant, repo) so overlapping requests // cannot insert the same deterministic ids and collide on the primary key. builder.Services.AddSingleton(); // Mermaid diagrams (step 6). builder.Services.AddSingleton(); // Rich HTML workflow preview (backend-rendered, embeddable in an iframe). builder.Services.AddSingleton(); builder.Services.AddSingleton(); // Runtime registry + worker executors (step 7). // The executors map each language to its execution strategy; the registry // (which also backs GET /api/languages) reports per-runtime availability. builder.Services.AddSingleton(new SubprocessScriptExecutor("shell", "sh", builder.Configuration)); builder.Services.AddSingleton(new SubprocessScriptExecutor("python", "python3", builder.Configuration)); builder.Services.AddSingleton(new SubprocessScriptExecutor("javascript", "node", builder.Configuration)); builder.Services.AddSingleton(new TypeScriptExecutor(builder.Configuration)); builder.Services.AddSingleton(new CSharpScriptExecutor(builder.Configuration)); // W9: `agent` step type — invokes an LLM agent via chatapi over HTTP (D3), keeping // workflows-api decoupled from BotSharp. builder.Services.AddSingleton(sp => new AgentScriptExecutor( sp.GetRequiredService(), builder.Configuration, sp.GetRequiredService>())); builder.Services.AddSingleton(); // Workflow runtime registry (which workflow-api instance runs a tenant's // workflows): self-info provider (GET /api/about) + the per-tenant runtime store. builder.Services.AddSingleton(); builder.Services.AddScoped(); // S8: remote (SSH) task execution via the w4c-webapi server-console exec endpoint. // Used only when a task carries a `server` reference; local subprocess stays the default. builder.Services.AddSingleton(); // Trigger engine (step 8): the scheduler + handler consumer run only in the // control-plane entrypoint, never in the worker. They share the run launcher — // the seam that creates pending runs for the run lifecycle engine (step 9). builder.Services.AddSingleton(TimeProvider.System); builder.Services.AddScoped(); // Per-tenant workflow execution quota: counts every top-level run (manual, // resume, cron/interval, webhook, handler event) and blocks new ones once the // configured monthly limit is reached. The counter is decoupled from run history. var workflowQuotaOptions = new WorkflowQuotaOptions(); builder.Configuration.GetSection(WorkflowQuotaOptions.SectionName).Bind(workflowQuotaOptions); builder.Services.AddSingleton(workflowQuotaOptions); builder.Services.AddScoped(); // Run lifecycle engine (step 9): dispatch pending runs + consume task.results + // advance the nextId chain + durable checkpoint/resume. builder.Services.AddSingleton(); builder.Services.AddScoped(); builder.Services.AddScoped(); if (runWorker) { // Dedicated worker process (scaled/separate deployment). builder.Services.AddHostedService(); } else { // Control-plane entrypoint. In single-node / self-hosted deployments one // process does both: it accounts, dispatches, AND runs the worker loop so // dispatched jobs actually execute. This closes the gap where the API // created run + task.run jobs but nothing consumed them, leaving every run // 'running' forever. A dedicated --worker can still be used for scale-out. builder.Services.AddHostedService(); builder.Services.AddHostedService(); builder.Services.AddHostedService(); builder.Services.AddHostedService(); } builder.Services.AddCors(options => { options.AddDefaultPolicy(policy => { var origins = builder.Configuration.GetSection("Cors:Origins").Get(); if (origins is { Length: > 0 }) { // Production: restrict to explicitly configured origins. policy.WithOrigins(origins) .AllowAnyMethod() .AllowAnyHeader() .AllowCredentials(); } else { // Dev fallback: no origins configured → allow all. policy.AllowAnyOrigin() .AllowAnyMethod() .AllowAnyHeader(); } }); }); var app = builder.Build(); app.UseCors(); // Global exception handler: catches unhandled exceptions from any controller // or middleware and returns a structured JSON error instead of a raw 500. app.UseExceptionHandler(handler => { handler.Run(async context => { context.Response.StatusCode = StatusCodes.Status500InternalServerError; context.Response.ContentType = "application/json"; var ex = context.Features.Get(); var message = ex?.Error.Message ?? "An unexpected error occurred."; var json = System.Text.Json.JsonSerializer.Serialize(new { error = message }); await context.Response.WriteAsync(json); }); }); app.UseMiddleware(); // Resolve the Forgejo-backed workflow source asynchronously after auth. // The auth middleware sets TenantId; this middleware resolves the user's Forgejo // login, clones/pulls the repo, and stores the IWorkflowSource in HttpContext.Items // so the DI registration can pick it up synchronously. app.Use(async (context, next) => { var tenantId = context.Items["TenantId"] as string; if (!string.IsNullOrEmpty(tenantId)) { // Scope all workflow queries to the tenant's CURRENT workflow repo, so a // workflow compiled from a previously-selected repo is unloaded/disabled. var repoName = (context.RequestServices.GetRequiredService()["WorkflowSource:WorkflowRepoName"] ?? string.Empty).Trim(); if (string.IsNullOrWhiteSpace(repoName)) repoName = "workflows"; try { var store = context.RequestServices.GetRequiredService(); repoName = await store.GetNameAsync(tenantId, context.RequestAborted); } catch (Exception ex) { var l = context.RequestServices.GetRequiredService().CreateLogger("WorkflowSourceMiddleware"); l.LogDebug(ex, "Could not resolve workflow repo for tenant {TenantId}", tenantId); } var db = context.RequestServices.GetRequiredService(); db.CurrentRepo = repoName; context.Items["WorkflowRepo"] = repoName; var factory = context.RequestServices.GetRequiredService(); if (factory.IsForgejoBacked) { try { // ResolveAsync caches clone/pull + login per (tenant, repo) for a // short TTL, so this no longer runs git on every authenticated request. var (source, login) = await factory.ResolveAsync(tenantId, repoName, context.RequestAborted); context.Items["WorkflowSource"] = source; // Store the resolved Forgejo login for controllers that need // to resolve repo paths (WorkflowFilesController, etc.). if (!string.IsNullOrEmpty(login)) context.Items["ForgejoLogin"] = login; } catch (Exception ex) { var logger = context.RequestServices.GetRequiredService() .CreateLogger("WorkflowSourceMiddleware"); logger.LogWarning(ex, "Forgejo-backed source failed for tenant {TenantId}, falling back to filesystem", tenantId); } } } await next(); }); // Ensure the database schema exists before serving traffic: the recorded EF // migrations plus the additive columns/tables that intentionally have no // migration. Applies to both PostgreSQL (full mode) and SQLite (self-hosted Lite // mode) — see WorkflowsSchema. Idempotent, so it is safe on every startup against // freshly-created and long-lived databases alike. try { using var scope = app.Services.CreateScope(); var db = scope.ServiceProvider.GetRequiredService(); await WorkflowsSchema.ApplyAsync(db, liteMode); } catch (Exception ex) { // Serving traffic with an unusable/partial schema produces silent, hard-to- // trace failures in every request. Fail fast so the orchestrator restarts or // surfaces the real error instead. Log.Logger.Fatal(ex, "Failed to apply EF Core migrations on startup; refusing to start"); throw; } // Scalar API reference + OpenAPI document are exposed under a dedicated // per-service prefix (/api/scalar/workflows/...) so Caddy can route them on the // public origin without colliding with the webapi Scalar reference. The OpenAPI // document is served at the SAME prefix (not the default /openapi/v1.json) so // Scalar's relative spec + asset resolution stays consistent behind the proxy. app.MapOpenApi("/api/scalar/workflows/openapi/{documentName}.json"); app.MapScalarApiReference("/api/scalar/workflows", options => { options.OpenApiRoutePattern = "/api/scalar/workflows/openapi/{documentName}.json"; }); app.MapControllers(); if (runWorker) { Log.Logger.Information("w4c-workflows-worker entrypoint selected; worker loop running for tenant {TenantId}", builder.Configuration["Workflows:TenantId"] ?? Environment.GetEnvironmentVariable("WF_TENANT_ID") ?? "default"); } app.Run(); Log.CloseAndFlush();