w4c-workflows-api/Services/Execution/GraphJobExecutor.cs

using Microsoft.EntityFrameworkCore;
using w4c_workflows.Data;
using w4c_workflows.Models;
using w4c_workflows.Services.Messaging;
using w4c_workflows.Services.Nodes;

namespace w4c_workflows.Services.Execution;

/// <summary>
/// Executes a <c>graph.run</c> job: loads the run and its compiled node graph,
/// drives the whole graph through <see cref="NodeWorkflowRunner"/>, then acks
/// the job. This is the worker half of S2 — node-mode execution no longer runs
/// inline on the control-plane lifecycle loop, so a waiting node cannot stall
/// dispatch/result/timeout handling for other tenants and node runs scale across
/// <c>--worker</c> replicas like script tasks.
///
/// Delivery is at-least-once: a cancelled or crashed execution leaves the job
/// unacked so it is re-claimed after the idle threshold. Idempotency is safe
/// because the graph job carries only a run reference and a run is only executed
/// while it is <c>pending</c>/<c>running</c>; a terminal run is skipped.
/// </summary>
public sealed class GraphJobExecutor
{
    private readonly IServiceScopeFactory _scopeFactory;
    private readonly IJobQueue _jobs;
    private readonly NodeWorkflowRunner _runner;
    private readonly ILogger<GraphJobExecutor> _logger;

    public GraphJobExecutor(
        IServiceScopeFactory scopeFactory,
        IJobQueue jobs,
        NodeWorkflowRunner runner,
        ILogger<GraphJobExecutor> logger)
    {
        _scopeFactory = scopeFactory;
        _jobs = jobs;
        _runner = runner;
        _logger = logger;
    }

    /// <summary>
    /// Runs one graph job. A malformed or cross-tenant message is dead-lettered;
    /// a stale/terminal run is acked without re-running; a cancellation leaves
    /// the job pending for redelivery.
    /// </summary>
    public async Task ExecuteAsync(string tenant, StreamMessage message, CancellationToken ct)
    {
        GraphRunInvocation invocation;
        try
        {
            invocation = GraphRunInvocation.FromFields(message.Fields);
        }
        catch (Exception ex)
        {
            _logger.LogWarning("Dead-lettering malformed graph job {MessageId}: {Error}", message.Id, ex.Message);
            await _jobs.DeadLetterAsync(tenant, message.Id, message.Fields, "malformed", ct);
            return;
        }

        if (!string.Equals(invocation.TenantId, tenant, StringComparison.Ordinal))
        {
            _logger.LogWarning("Graph job {MessageId} tenant {JobTenant} != stream tenant {StreamTenant}; dead-lettering",
                message.Id, invocation.TenantId, tenant);
            await _jobs.DeadLetterAsync(tenant, message.Id, message.Fields, "tenant_mismatch", ct);
            return;
        }

        if (!Guid.TryParse(invocation.RunId, out var runId))
        {
            _logger.LogWarning("Dead-lettering graph job {MessageId}: run id '{RunId}' is malformed",
                message.Id, invocation.RunId);
            await _jobs.DeadLetterAsync(tenant, message.Id, message.Fields, "malformed_run_id", ct);
            return;
        }

        using var scope = _scopeFactory.CreateScope();
        var db = scope.ServiceProvider.GetRequiredService<WorkflowsDbContext>();

        var run = await db.WorkflowRuns
            .Include(r => r.Workflow).ThenInclude(w => w.Tasks)
            .Include(r => r.Workflow).ThenInclude(w => w.TaskEdges)
            .FirstOrDefaultAsync(r => r.Id == runId && r.TenantId == tenant, ct);

        if (run == null)
        {
            // The run row was deleted (or belongs to another tenant). Nothing to
            // execute; ack so the job does not redeliver forever.
            _logger.LogWarning("Graph job {MessageId} references missing run {RunId}; acking", message.Id, runId);
            await _jobs.AckAsync(tenant, message.Id, ct);
            return;
        }

        // A run the control plane already timed out (or that finished) must not
        // be resurrected by a late/duplicate graph job.
        if (run.Status is not (RunStatus.Pending or RunStatus.Running))
        {
            _logger.LogDebug("Graph job {MessageId} run {RunId} is already {Status}; skipping",
                message.Id, runId, run.Status);
            await _jobs.AckAsync(tenant, message.Id, ct);
            return;
        }

        if (run.Status == RunStatus.Pending)
        {
            run.Status = RunStatus.Running;
            run.StartedAt ??= DateTime.UtcNow;
            await db.SaveChangesAsync(ct);
        }

        try
        {
            // The runner owns the terminal status and records one TaskRun per
            // executed node; it absorbs non-cancellation failures internally.
            await _runner.RunAsync(db, run, run.Workflow, invocation.WorkingDir, ct);
        }
        catch (OperationCanceledException) when (ct.IsCancellationRequested)
        {
            // Shutdown: do not ack — the job stays pending for redelivery and the
            // control-plane timeout sweep recovers the run if this worker never
            // comes back.
            throw;
        }
        catch (Exception ex)
        {
            // Safety net: the runner should already have failed the run, but if
            // anything escapes we must not leave it hanging in `running`.
            _logger.LogError(ex, "Graph run {RunId} threw outside the runner; failing the run", runId);
            await RunTerminalWriter.TrySetTerminalAsync(
                db, run, RunStatus.Failed, ex.Message, output: null, ct);
        }

        await _jobs.AckAsync(tenant, message.Id, ct);
    }
}