From be5bdae501997401bd248f7642eeabacd1994d8a Mon Sep 17 00:00:00 2001 From: Sipke Schoorstra Date: Sat, 5 Sep 2026 10:30:06 -0700 Subject: [PATCH] release: anchor recovery workflow validation (#8025) --- .agents/skills/elsa-release/SKILL.md | 2 +- .../skills/elsa-release/references/runbook.md | 24 +++--- .../elsa-release/scripts/release_train.py | 53 +++++++++++-- .../elsa-release/tests/test_release_train.py | 75 ++++++++++++++++++- 4 files changed, 136 insertions(+), 18 deletions(-) diff --git a/.agents/skills/elsa-release/SKILL.md b/.agents/skills/elsa-release/SKILL.md index 8d579bf44..2e31bbf55 100644 --- a/.agents/skills/elsa-release/SKILL.md +++ b/.agents/skills/elsa-release/SKILL.md @@ -32,7 +32,7 @@ Use the checkpoint helper in the runbook. It reports the next phase from GitHub - A green upload job is insufficient: verify the expected package inventory, versions, source commits, dependencies, and actual feed content. Verify Studio npm artifacts and `latest`/`next` according to release kind. - Bind exact source commits and reviewed manifests/notes before publication. Preserve existing tags; matching releases are reusable, conflicting releases require investigation. -- If the original run has exactly one configured NuGet publishing failure, use only the runbook's validated `record-recovery` receipt plus the downloaded machine-readable recovery evidence from the reviewed workflow source. Preserve the failed run and tag; reject any other failed job, retag, rebuild, artifact mismatch, forged evidence, or blanket skip, and then perform normal package verification against the recovery run. +- If the original run has exactly one configured NuGet publishing failure, use only the runbook's validated `record-recovery` receipt plus the downloaded machine-readable recovery evidence from the reviewed workflow source. Independently anchor the recovery workflow tree to an approved commit in the live canonical default-branch history. Preserve the failed run and tag; reject any other failed job, unknown or renamed recovery job, retag, rebuild, artifact mismatch, forged evidence, or blanket skip, and then perform normal package verification against the recovery run. - Resume by inspecting live GitHub state and recorded package/post evidence. Do not recreate tags or repost after an uncertain result. Retain run and message IDs; wait on concrete jobs with bounded polling and backoff. - Assess advisory findings during preflight. Record severity, relevant usage, and disposition. Existing warnings are not automatically blockers or automatically accepted forever. Never insert an unvalidated dependency upgrade into a release to suppress a warning; a new material unresolved risk requires a concrete scope decision. - After package verification, invoke [Elsa Release Announcements](../elsa-release-announcements/SKILL.md). Default to publishing now on Discord, LinkedIn, and X. Draft-only is an explicit override, not completion of a request to announce. diff --git a/.agents/skills/elsa-release/references/runbook.md b/.agents/skills/elsa-release/references/runbook.md index ef136abc8..ba6684457 100644 --- a/.agents/skills/elsa-release/references/runbook.md +++ b/.agents/skills/elsa-release/references/runbook.md @@ -136,9 +136,10 @@ If the original release run has exactly one infrastructure failure in its configured `Publish to nuget.org` job, preserve that failed run, its immutable tag, and its artifact. Do not retag, recreate the release, or rerun the package build. Correct the workflow's explicit recovery path and dispatch it against the -reviewed workflow source SHA. The recovery may run from an infrastructure repair -commit rather than the release tag; record that SHA and verify it against the -live run. It must consume the original configured artifact and run only the +reviewed workflow source SHA. Record an `approved_source_commit` from the +canonical default branch as an independently reviewed source anchor. The recovery +may run from an infrastructure repair commit rather than the release tag; record +that SHA and verify its full tree against the approved anchor. It must consume the original configured artifact and run only the NuGet publication; a successful `Build packages` job is a rebuild, not recovery. The recovery workflow must upload a machine-readable `recovery-receipt.json` in the configured `elsa-template-recovery-evidence` artifact. Download the artifact @@ -151,6 +152,7 @@ operator receipt includes: "version": "3.8.0", "tag": "3.8.0", "source_commit": "", + "approved_source_commit": "", "original_release_run": { "id": 33977531328, "failed_jobs": ["Publish to nuget.org"] @@ -187,7 +189,9 @@ operator receipt includes: The downloaded `recovery-receipt.json` must contain the same version, recovery run ID and workflow SHA, the original release run ID/source commit, the original -artifact ID, name, run ID, digest and size, and the exact NuGet target/package IDs. This +artifact ID, name, run ID, digest and size, and the exact NuGet target/package IDs. The +operator-provided approved source commit is checked against the repository's live +default-branch history and the recovery commit's full tree. This machine-readable evidence is produced by the reviewed workflow and is checked against GitHub's live artifact metadata; a self-authored operator receipt is not evidence by itself. @@ -228,12 +232,12 @@ python3 /scripts/release_train.py --state /state.json record-recover ``` The command checks the live tag/source, original release run and jobs, exact -artifact ID/digest/size, recovery workflow SHA and dispatch event, the successful -NuGet job, evidence artifact metadata, archive hash and contents, and the exact -target package/version. It accepts only the sole configured NuGet failure with every -other required job successful; it rejects a different failed job, retag, rebuild, -artifact mismatch, forged/missing evidence, unreviewed workflow source, or target -mismatch. +artifact ID/digest/size, approved source ancestry and tree, recovery workflow SHA +and dispatch event, the successful NuGet job, evidence artifact metadata, archive +hash and contents, and the exact target package/version. It accepts only the sole +configured NuGet failure with every other expected job successful; it rejects +unknown failed jobs, duplicate or renamed jobs, retag, rebuild, artifact mismatch, +forged/missing evidence, unreviewed workflow source, or target mismatch. The checkpoint retains both run IDs and the original failure history, then requires normal package verification against the recovery run. A failed or ambiguous recovery remains `repair-pipeline`. diff --git a/.agents/skills/elsa-release/scripts/release_train.py b/.agents/skills/elsa-release/scripts/release_train.py index 8a33535a9..913c48512 100644 --- a/.agents/skills/elsa-release/scripts/release_train.py +++ b/.agents/skills/elsa-release/scripts/release_train.py @@ -15,7 +15,7 @@ import sys import tempfile import zipfile from datetime import datetime, timezone -from urllib.parse import urlparse +from urllib.parse import quote, urlparse from release_support import parse_version @@ -391,6 +391,39 @@ def workflow_artifacts(cfg, run_id): return [artifact for page in pages for artifact in page.get('artifacts', [])] +def indexed_workflow_jobs(cfg, run_id, phase): + indexed = {} + for job in workflow_jobs(cfg, run_id): + name = job.get('name') + if not isinstance(name, str) or not name: + raise ValueError(f'{phase} recovery run contains a job without a name') + if name in indexed: + raise ValueError(f'{phase} recovery run contains duplicate job {name!r}') + indexed[name] = job + return indexed + + +def validate_recovery_source(cfg, receipt, recovery_sha): + approved_sha = commit_sha(receipt.get('approved_source_commit'), 'approved_source_commit') + repository = gh('api', f"repos/{cfg['github']}") + if repository.get('full_name') != cfg['github']: + raise ValueError('Recovery source repository does not match the release profile') + default_branch = repository.get('default_branch') + if not isinstance(default_branch, str) or not default_branch: + raise ValueError('Recovery source repository has no canonical default branch') + comparison = gh('api', f"repos/{cfg['github']}/compare/{approved_sha}...{quote(default_branch, safe='')}") + if comparison.get('status') not in {'ahead', 'identical'}: + raise ValueError('Approved recovery source is not in the canonical default-branch history') + approved_commit = gh('api', f"repos/{cfg['github']}/commits/{approved_sha}") + recovery_commit = gh('api', f"repos/{cfg['github']}/commits/{recovery_sha}") + if approved_commit.get('sha') != approved_sha or recovery_commit.get('sha') != recovery_sha: + raise ValueError('Recovery source commit lookup returned a different SHA') + approved_tree = approved_commit.get('commit', {}).get('tree', {}).get('sha') + recovery_tree = recovery_commit.get('commit', {}).get('tree', {}).get('sha') + if not approved_tree or not recovery_tree or approved_tree != recovery_tree: + raise ValueError('Recovery workflow tree differs from the approved default-branch source tree') + + def read_recovery_evidence_archive(path, expected_digest=None): path = Path(path) if not path.is_file(): @@ -535,12 +568,16 @@ def validate_recovery_receipt(state, name, receipt, expected_original_run_id=Non raise ValueError('Original recovery run used a different workflow') target_job = recovery_job_names(cfg) - original_jobs = {job.get('name'): job for job in workflow_jobs(cfg, original_id)} required_jobs = set(cfg.get('required_jobs', [])) + original_jobs = indexed_workflow_jobs(cfg, original_id, 'Original') if not required_jobs <= original_jobs.keys(): raise ValueError('Original recovery run is missing configured required jobs') + unknown_jobs = set(original_jobs) - required_jobs + if any(original_jobs[job].get('conclusion') not in {'success', 'skipped'} for job in unknown_jobs): + raise ValueError('Original recovery run contains an unknown non-success job') failed_jobs = original.get('failed_jobs') - if failed_jobs != [target_job] or original_jobs[target_job].get('conclusion') != 'failure': + failed_live = [job for job, value in original_jobs.items() if value.get('conclusion') == 'failure'] + if failed_jobs != [target_job] or failed_live != [target_job] or original_jobs[target_job].get('conclusion') != 'failure': raise ValueError('Recovery is allowed only for the failed NuGet publishing job') if any(original_jobs[job].get('conclusion') != 'success' for job in required_jobs if job != target_job): raise ValueError('Recovery cannot bypass a failed or skipped build/feed job') @@ -565,11 +602,15 @@ def validate_recovery_receipt(state, name, receipt, expected_original_run_id=Non raise ValueError('Recovery run source commit does not match the reviewed recovery workflow SHA') if recovery_remote.get('path') and recovery_remote['path'].lstrip('/') != expected_workflow: raise ValueError('Recovery run used a different workflow') - recovery_jobs = {job.get('name'): job for job in workflow_jobs(cfg, recovery_id)} + validate_recovery_source(cfg, receipt, recovery_sha) + required_jobs = set(cfg.get('required_jobs', [])) + recovery_jobs = indexed_workflow_jobs(cfg, recovery_id, 'Recovery') + if set(recovery_jobs) != required_jobs: + raise ValueError('Recovery run must contain exactly the configured required jobs') if recovery_jobs.get(target_job, {}).get('conclusion') != 'success': raise ValueError('Recovery NuGet publishing job did not succeed') - if recovery_jobs.get('Build packages', {}).get('conclusion') == 'success': - raise ValueError('Recovery run rebuilt packages instead of publishing the original artifact') + if any(recovery_jobs[job].get('conclusion') != 'skipped' for job in required_jobs if job != target_job): + raise ValueError('Recovery run performed non-NuGet work instead of publishing the original artifact') matching_evidence = [artifact for artifact in workflow_artifacts(cfg, recovery_id) if artifact.get('id') == evidence_id] if len(matching_evidence) != 1: raise ValueError('Recovery receipt must identify exactly one workflow evidence artifact') diff --git a/.agents/skills/elsa-release/tests/test_release_train.py b/.agents/skills/elsa-release/tests/test_release_train.py index 01b2bac15..3558dfad5 100644 --- a/.agents/skills/elsa-release/tests/test_release_train.py +++ b/.agents/skills/elsa-release/tests/test_release_train.py @@ -96,6 +96,7 @@ class TrainTests(unittest.TestCase): 'version': '3.9.0', 'tag': '3.9.0', 'source_commit': 'a' * 40, + 'approved_source_commit': 'b' * 40, 'original_release_run': { 'id': 33977531328, 'failed_jobs': ['Publish to nuget.org'], @@ -174,6 +175,14 @@ class TrainTests(unittest.TestCase): return {'object': {'type': 'tag', 'sha': 'tag-object'}} if '/git/tags/' in url: return {'object': {'type': 'commit', 'sha': 'a' * 40}} + if url == 'repos/elsa-workflows/elsa-templates': + return {'full_name': 'elsa-workflows/elsa-templates', 'default_branch': 'main'} + if '/compare/' in url: + approved = url.split('/compare/', 1)[1].split('...', 1)[0] + return {'status': 'identical' if approved == 'b' * 40 else 'behind'} + if '/commits/' in url: + sha = url.rsplit('/', 1)[-1] + return {'sha': sha, 'commit': {'tree': {'sha': 'reviewed-tree'}}} if '/workflows/' in url: return [{'workflow_runs': [{ 'id': 33977531328, @@ -269,7 +278,7 @@ class TrainTests(unittest.TestCase): return value with patch.object(train, 'gh', side_effect=rebuilt): - with self.assertRaisesRegex(ValueError, 'rebuilt packages'): + with self.assertRaisesRegex(ValueError, 'non-NuGet work'): train.validate_recovery_receipt(self.state, 'templates', receipt, evidence=evidence) def test_nuget_recovery_rejects_unreviewed_workflow_sha(self): @@ -289,6 +298,70 @@ class TrainTests(unittest.TestCase): with self.assertRaisesRegex(ValueError, 'does not bind the original artifact payload'): train.validate_recovery_receipt(self.state, 'templates', receipt, evidence=evidence) + def test_nuget_recovery_rejects_unanchored_approved_source(self): + self.prepare_template_recovery() + receipt = self.template_recovery_receipt() + receipt['approved_source_commit'] = 'c' * 40 + with patch.object(train, 'gh', side_effect=self.recovery_github): + with self.assertRaisesRegex(ValueError, 'canonical default-branch history'): + train.validate_recovery_receipt(self.state, 'templates', receipt, evidence=self.template_recovery_evidence()) + + def test_nuget_recovery_rejects_approved_tree_mismatch(self): + self.prepare_template_recovery() + receipt = self.template_recovery_receipt() + + def mismatched(*args): + value = self.recovery_github(*args) + if '/commits/e' + 'e' * 39 in args[-1]: + value['commit']['tree']['sha'] = 'different-tree' + return value + + with patch.object(train, 'gh', side_effect=mismatched): + with self.assertRaisesRegex(ValueError, 'tree differs'): + train.validate_recovery_receipt(self.state, 'templates', receipt, evidence=self.template_recovery_evidence()) + + def test_nuget_recovery_rejects_extra_failed_original_job(self): + self.prepare_template_recovery() + receipt = self.template_recovery_receipt() + + def extra_failure(*args): + value = self.recovery_github(*args) + if '/actions/runs/33977531328/jobs?' in args[-1]: + value[0]['jobs'].append({'name': 'Upload diagnostics', 'conclusion': 'failure'}) + return value + + with patch.object(train, 'gh', side_effect=extra_failure): + with self.assertRaisesRegex(ValueError, 'unknown non-success job'): + train.validate_recovery_receipt(self.state, 'templates', receipt, evidence=self.template_recovery_evidence()) + + def test_nuget_recovery_rejects_renamed_rebuild_job(self): + self.prepare_template_recovery() + receipt = self.template_recovery_receipt() + + def renamed(*args): + value = self.recovery_github(*args) + if '/actions/runs/33977531329/jobs?' in args[-1]: + value[0]['jobs'][0]['name'] = 'Build packages (rebuild)' + return value + + with patch.object(train, 'gh', side_effect=renamed): + with self.assertRaisesRegex(ValueError, 'exactly the configured required jobs'): + train.validate_recovery_receipt(self.state, 'templates', receipt, evidence=self.template_recovery_evidence()) + + def test_nuget_recovery_rejects_duplicate_job_names(self): + self.prepare_template_recovery() + receipt = self.template_recovery_receipt() + + def duplicate(*args): + value = self.recovery_github(*args) + if '/actions/runs/33977531329/jobs?' in args[-1]: + value[0]['jobs'].append({'name': 'Build packages', 'conclusion': 'skipped'}) + return value + + with patch.object(train, 'gh', side_effect=duplicate): + with self.assertRaisesRegex(ValueError, 'duplicate job'): + train.validate_recovery_receipt(self.state, 'templates', receipt, evidence=self.template_recovery_evidence()) + def test_nuget_recovery_rejects_tampered_evidence_archive(self): self.prepare_template_recovery() receipt_value, evidence_archive = self.template_recovery_files()