using Microsoft.EntityFrameworkCore; using w4c_workflows.Data; using w4c_workflows.Models; using w4c_workflows.Services.Messaging; using w4c_workflows.Services.Nodes; namespace w4c_workflows.Services.Execution; /// /// Executes a graph.run job: loads the run and its compiled node graph, /// drives the whole graph through , then acks /// the job. This is the worker half of S2 — node-mode execution no longer runs /// inline on the control-plane lifecycle loop, so a waiting node cannot stall /// dispatch/result/timeout handling for other tenants and node runs scale across /// --worker replicas like script tasks. /// /// Delivery is at-least-once: a cancelled or crashed execution leaves the job /// unacked so it is re-claimed after the idle threshold. Idempotency is safe /// because the graph job carries only a run reference and a run is only executed /// while it is pending/running; a terminal run is skipped. /// public sealed class GraphJobExecutor { private readonly IServiceScopeFactory _scopeFactory; private readonly IJobQueue _jobs; private readonly NodeWorkflowRunner _runner; private readonly ILogger _logger; public GraphJobExecutor( IServiceScopeFactory scopeFactory, IJobQueue jobs, NodeWorkflowRunner runner, ILogger logger) { _scopeFactory = scopeFactory; _jobs = jobs; _runner = runner; _logger = logger; } /// /// Runs one graph job. A malformed or cross-tenant message is dead-lettered; /// a stale/terminal run is acked without re-running; a cancellation leaves /// the job pending for redelivery. /// public async Task ExecuteAsync(string tenant, StreamMessage message, CancellationToken ct) { GraphRunInvocation invocation; try { invocation = GraphRunInvocation.FromFields(message.Fields); } catch (Exception ex) { _logger.LogWarning("Dead-lettering malformed graph job {MessageId}: {Error}", message.Id, ex.Message); await _jobs.DeadLetterAsync(tenant, message.Id, message.Fields, "malformed", ct); return; } if (!string.Equals(invocation.TenantId, tenant, StringComparison.Ordinal)) { _logger.LogWarning("Graph job {MessageId} tenant {JobTenant} != stream tenant {StreamTenant}; dead-lettering", message.Id, invocation.TenantId, tenant); await _jobs.DeadLetterAsync(tenant, message.Id, message.Fields, "tenant_mismatch", ct); return; } if (!Guid.TryParse(invocation.RunId, out var runId)) { _logger.LogWarning("Dead-lettering graph job {MessageId}: run id '{RunId}' is malformed", message.Id, invocation.RunId); await _jobs.DeadLetterAsync(tenant, message.Id, message.Fields, "malformed_run_id", ct); return; } using var scope = _scopeFactory.CreateScope(); var db = scope.ServiceProvider.GetRequiredService(); var run = await db.WorkflowRuns .Include(r => r.Workflow).ThenInclude(w => w.Tasks) .Include(r => r.Workflow).ThenInclude(w => w.TaskEdges) .FirstOrDefaultAsync(r => r.Id == runId && r.TenantId == tenant, ct); if (run == null) { // The run row was deleted (or belongs to another tenant). Nothing to // execute; ack so the job does not redeliver forever. _logger.LogWarning("Graph job {MessageId} references missing run {RunId}; acking", message.Id, runId); await _jobs.AckAsync(tenant, message.Id, ct); return; } // A run the control plane already timed out (or that finished) must not // be resurrected by a late/duplicate graph job. if (run.Status is not (RunStatus.Pending or RunStatus.Running)) { _logger.LogDebug("Graph job {MessageId} run {RunId} is already {Status}; skipping", message.Id, runId, run.Status); await _jobs.AckAsync(tenant, message.Id, ct); return; } if (run.Status == RunStatus.Pending) { run.Status = RunStatus.Running; run.StartedAt ??= DateTime.UtcNow; await db.SaveChangesAsync(ct); } try { // The runner owns the terminal status and records one TaskRun per // executed node; it absorbs non-cancellation failures internally. await _runner.RunAsync(db, run, run.Workflow, invocation.WorkingDir, ct); } catch (OperationCanceledException) when (ct.IsCancellationRequested) { // Shutdown: do not ack — the job stays pending for redelivery and the // control-plane timeout sweep recovers the run if this worker never // comes back. throw; } catch (Exception ex) { // Safety net: the runner should already have failed the run, but if // anything escapes we must not leave it hanging in `running`. _logger.LogError(ex, "Graph run {RunId} threw outside the runner; failing the run", runId); await RunTerminalWriter.TrySetTerminalAsync( db, run, RunStatus.Failed, ex.Message, output: null, ct); } await _jobs.AckAsync(tenant, message.Id, ct); } }