diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b901aafb6..3b9305c36 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,12 @@ jobs: name: Classify changes runs-on: ubuntu-latest timeout-minutes: 3 + # In the merge queue, reads the queued PR's verified-tree status and the CI run behind it. + permissions: + contents: read + actions: read + pull-requests: read + statuses: read outputs: scope: ${{ steps.classify.outputs.scope }} steps: @@ -37,6 +43,10 @@ jobs: id: classify env: CI_BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }} + # In the merge queue, a tree the pull request's CI already passed skips the suites. + CI_EVENT: ${{ github.event_name }} + CI_HEAD_REF: ${{ github.event.merge_group.head_ref }} + GITHUB_TOKEN: ${{ github.token }} run: bun tools/ci/policy/src/classify.ts commit-messages: @@ -260,6 +270,10 @@ jobs: if: always() runs-on: ubuntu-latest timeout-minutes: 3 + # Records the verified tree as a commit status once the gate has passed. + permissions: + contents: read + statuses: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: @@ -269,3 +283,14 @@ jobs: env: CI_NEEDS: ${{ toJSON(needs) }} run: bun tools/ci/policy/src/gate.ts + # Lets the merge queue skip the suites when it tests this exact tree. Forks cannot write it, + # and the queue accepts it only from this run. Recording is an optimization: a failure + # leaves the queue to run every check, so it must not fail the gate. + - name: Record the verified tree + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository + continue-on-error: true + env: + CI_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + CI_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GITHUB_TOKEN: ${{ github.token }} + run: bun tools/ci/policy/src/verified-tree/record.ts diff --git a/packages/vue/tsdown.config.ts b/packages/vue/tsdown.config.ts index 693e89a35..afa5716bf 100644 --- a/packages/vue/tsdown.config.ts +++ b/packages/vue/tsdown.config.ts @@ -39,8 +39,7 @@ export default defineConfig({ format: ['esm'], dts: { vue: true, - sourcemap: true, - resolver: 'tsc' + sourcemap: true }, sourcemap: true, hash: false, diff --git a/tools/AGENTS.md b/tools/AGENTS.md index 026917210..ba449bfe7 100644 --- a/tools/AGENTS.md +++ b/tools/AGENTS.md @@ -22,6 +22,7 @@ Private tooling lives under `tools///{src,tests}`; Steiger enforce - `commitlint.config.ts` enforces commit structure in the **Commit messages** job; the separate **PR title** workflow validates titles. Preserve the `Release vX.Y.Z` exception and product casing when changing rules; the known AI co-author check does not rewrite base history. Gate policy lives in `tools/ci/policy/src/policy.ts`. - `tools/ci/policy` runs right after Bun is set up, before any install, so it imports only Node built-ins; `oxlint.json` switches `open-pencil/prefer-es-toolkit` off there. Other CI tools install their workspace first, as `pr-review-guidance.yml` does through `.github/actions/setup-bun`. - Required checks must also run on `merge_group`, the merge queue's event; read base and head from `merge_group.base_sha`/`head_sha` there (`ci.yml`, `pr-title.yml`). +- A passing pull request run records the tree it tested as the `CI verified tree` status on the PR head; a merge queue commit with that exact tree is classified `verified` and runs only the always-on checks. Any other tree, including a group with other PRs ahead, gets the full run (`tools/ci/policy/src/verified-tree/`). - App and docs production workflows run on `v*` tags or `workflow_dispatch`, not ordinary `master` pushes. `build.yml` checks the tooling out under `.pipeline/` and runs `tools/release/release-packages` from there. ## Releases diff --git a/tools/checks/package-quality/src/checks/attw.ts b/tools/checks/package-quality/src/checks/attw.ts index 63b2713d4..e962f1068 100644 --- a/tools/checks/package-quality/src/checks/attw.ts +++ b/tools/checks/package-quality/src/checks/attw.ts @@ -1,10 +1,12 @@ +import { availableParallelism } from 'node:os' + import { runCommand } from '@open-pencil/package-artifacts-tools' import { publicPackageDirs } from '../packages' import { runPackageChecks } from './run' -// ATTW packs distinct package directories; bound concurrent TypeScript analyses. -const TYPE_CHECK_CONCURRENCY = 2 +// ATTW packs distinct package directories, and each analysis keeps one core busy. +const TYPE_CHECK_CONCURRENCY = availableParallelism() export async function checkTypes(root: string): Promise { await runPackageChecks( diff --git a/tools/checks/package-quality/src/smoke/pack.ts b/tools/checks/package-quality/src/smoke/pack.ts index 82c0196dd..e62d2c68b 100644 --- a/tools/checks/package-quality/src/smoke/pack.ts +++ b/tools/checks/package-quality/src/smoke/pack.ts @@ -1,6 +1,9 @@ import { mkdir } from 'node:fs/promises' +import { availableParallelism } from 'node:os' import { isAbsolute, join } from 'node:path' +import { mapAsync } from 'es-toolkit' + import { parseNpmPack, runCommand, @@ -34,24 +37,36 @@ export async function packPublicPackages( packageManager: 'bun' | 'npm' ): Promise { await mkdir(outputDirectory, { recursive: true }) - const tarballs: string[] = [] - for (const pkg of packages) { - const command = - packageManager === 'bun' - ? ['bun', 'pm', 'pack', '--ignore-scripts', '--destination', outputDirectory, '--quiet'] - : ['npm', 'pack', '--json', '--ignore-scripts', '--pack-destination', outputDirectory] - const result = await runCommand({ - command: command[0] ?? packageManager, - args: command.slice(1), - cwd: join(root, pkg.directory), - timeoutMs: 60_000 - }) - const tarball = - packageManager === 'bun' - ? tarballFromOutput(result.stdout, outputDirectory) - : npmTarballFromOutput(result.stdout, outputDirectory) - tarballs.push(tarball) + const command = + packageManager === 'bun' + ? ['bun', 'pm', 'pack', '--ignore-scripts', '--destination', outputDirectory, '--quiet'] + : ['npm', 'pack', '--json', '--ignore-scripts', '--pack-destination', outputDirectory] + // Packing reads only its own package, so packages pack side by side; tarballs keep their order. + // Every pack settles before a failure is reported, so none writes into a removed directory. + const outcomes = await mapAsync( + packages, + async (pkg) => { + try { + const result = await runCommand({ + command: command[0] ?? packageManager, + args: command.slice(1), + cwd: join(root, pkg.directory), + timeoutMs: 60_000 + }) + return packageManager === 'bun' + ? tarballFromOutput(result.stdout, outputDirectory) + : npmTarballFromOutput(result.stdout, outputDirectory) + } catch (error) { + return error instanceof Error ? error : new Error(String(error)) + } + }, + { concurrency: availableParallelism() } + ) + const failures = outcomes.filter((outcome) => outcome instanceof Error) + if (failures.length > 0) { + throw new AggregateError(failures, failures.map(({ message }) => message).join('\n')) } + const tarballs = outcomes.filter((outcome) => typeof outcome === 'string') const inspections = await Promise.all(tarballs.map(inspectTarball)) const diagnostics = inspections.flatMap(({ diagnostics }) => diagnostics) if (diagnostics.length > 0) { diff --git a/tools/ci/policy/src/classify.ts b/tools/ci/policy/src/classify.ts index ec212aa0e..f432376fe 100644 --- a/tools/ci/policy/src/classify.ts +++ b/tools/ci/policy/src/classify.ts @@ -2,6 +2,7 @@ import { execFileSync } from 'node:child_process' import { appendFile } from 'node:fs/promises' import { classifyPaths } from './policy' +import { isVerifiedTree } from './verified-tree/status' const base = process.env.CI_BASE_SHA const output = process.env.GITHUB_OUTPUT @@ -15,6 +16,25 @@ const paths = execFileSync('git', ['diff', '--no-renames', '--name-only', '-z', .toString('utf8') .split('\0') .filter(Boolean) -const scope = classifyPaths(paths) +const scope = (await queuedTreeVerified()) ? 'verified' : classifyPaths(paths) await appendFile(output, `scope=${scope}\n`) console.log(`Selected ${scope} checks for ${paths.length} changed paths`) + +/** In the merge queue, whether the queued commit's tree already passed its pull request's CI. */ +async function queuedTreeVerified(): Promise { + const { CI_EVENT, CI_HEAD_REF, GITHUB_REPOSITORY, GITHUB_TOKEN } = process.env + if (CI_EVENT !== 'merge_group' || !CI_HEAD_REF || !GITHUB_REPOSITORY || !GITHUB_TOKEN) { + return false + } + const tree = execFileSync('git', ['rev-parse', 'HEAD^{tree}']).toString('utf8').trim() + try { + const access = { repository: GITHUB_REPOSITORY, token: GITHUB_TOKEN } + const verified = await isVerifiedTree(access, CI_HEAD_REF, tree) + if (verified) console.log(`Tree ${tree} already passed its pull request's CI`) + return verified + } catch (error) { + // A failed lookup only costs the full run. + console.log(`Could not look up a verified tree: ${String(error)}`) + return false + } +} diff --git a/tools/ci/policy/src/policy.ts b/tools/ci/policy/src/policy.ts index 998811c63..5e8d3e201 100644 --- a/tools/ci/policy/src/policy.ts +++ b/tools/ci/policy/src/policy.ts @@ -1,10 +1,11 @@ const ROOT_DOCS = new Set(['README.md', 'CONTRIBUTING.md', 'AGENTS.md', 'CHANGELOG.md', 'LICENSE']) const DOC_ASSET = /\.(?:md|png|jpe?g|gif|webp|svg|ico|pdf|woff2?|ttf)$/i -export type ChangeScope = 'docs' | 'code' +/** `verified`: a merge queue commit whose exact tree already passed its pull request's CI. */ +export type ChangeScope = 'docs' | 'code' | 'verified' /** Root docs, package READMEs, and every AGENTS.md guide are docs-only; unknown paths, executable docs, and runtime prompt Markdown require the code checks. */ -export function classifyPaths(paths: readonly string[]): ChangeScope { +export function classifyPaths(paths: readonly string[]): Exclude { if (paths.length === 0) return 'code' return paths.every((path) => { if (ROOT_DOCS.has(path) || /^packages\/[^/]+\/README\.md$/.test(path)) return true @@ -28,6 +29,12 @@ export const CODE_JOBS = [ ] as const export const DOCS_JOB = 'documentation' export const ALWAYS_JOBS = ['commit-messages'] as const +/** The jobs each scope runs besides `ALWAYS_JOBS`; a verified tree runs none of them. */ +const SELECTED_JOBS = new Map([ + ['docs', [DOCS_JOB]], + ['code', CODE_JOBS], + ['verified', []] +] satisfies [ChangeScope, readonly string[]][]) type JobResult = 'success' | 'failure' | 'cancelled' | 'skipped' export interface JobStatus { @@ -40,19 +47,17 @@ export function gateErrors(needs: Partial>): string[] const detection = needs.changes if (detection?.result !== 'success') return ['Change detection did not succeed'] const scope = detection.outputs?.scope - if (scope !== 'docs' && scope !== 'code') return ['Invalid or missing change scope'] - const required: readonly string[] = [ - ...ALWAYS_JOBS, - ...(scope === 'docs' ? [DOCS_JOB] : CODE_JOBS) - ] - const excluded: readonly string[] = scope === 'docs' ? CODE_JOBS : [DOCS_JOB] + const selected = scope ? SELECTED_JOBS.get(scope) : undefined + if (!selected) return ['Invalid or missing change scope'] + const required: readonly string[] = [...ALWAYS_JOBS, ...selected] + const excluded = [...CODE_JOBS, DOCS_JOB].filter((job) => !required.includes(job)) const errors = required .filter((job) => needs[job]?.result !== 'success') .map((job) => `${job} did not succeed`) - // Unexpected execution is also a policy failure: docs must not run the full suites. + // Unexpected execution is also a policy failure: docs must not run the full suites, and a + // verified tree runs nothing it already passed. for (const job of excluded) { - if (needs[job]?.result !== 'skipped') - errors.push(`${job} was not skipped for ${scope}-only routing`) + if (needs[job]?.result !== 'skipped') errors.push(`${job} was not skipped for ${scope} routing`) } return errors } diff --git a/tools/ci/policy/src/verified-tree/record.ts b/tools/ci/policy/src/verified-tree/record.ts new file mode 100644 index 000000000..696ac1d03 --- /dev/null +++ b/tools/ci/policy/src/verified-tree/record.ts @@ -0,0 +1,13 @@ +import { execFileSync } from 'node:child_process' + +import { recordVerifiedTree } from './status' + +// Runs in the CI result job after the gate passed on a pull request, from its merge checkout. +const { CI_HEAD_SHA, CI_RUN_URL, GITHUB_REPOSITORY, GITHUB_TOKEN } = process.env +if (!CI_HEAD_SHA || !CI_RUN_URL || !GITHUB_REPOSITORY || !GITHUB_TOKEN) { + throw new Error('Missing pull request head, run URL, or token') +} +const tree = execFileSync('git', ['rev-parse', 'HEAD^{tree}']).toString('utf8').trim() +const access = { repository: GITHUB_REPOSITORY, token: GITHUB_TOKEN } +await recordVerifiedTree(access, CI_HEAD_SHA, tree, CI_RUN_URL) +console.log(`Recorded that ${CI_HEAD_SHA} passed CI on tree ${tree}`) diff --git a/tools/ci/policy/src/verified-tree/status.ts b/tools/ci/policy/src/verified-tree/status.ts new file mode 100644 index 000000000..ee9f6f957 --- /dev/null +++ b/tools/ci/policy/src/verified-tree/status.ts @@ -0,0 +1,114 @@ +/** + * A pull request's CI records the tree it verified as a commit status on the PR head. When the + * merge queue later tests a commit with exactly that tree, nothing has changed since the PR's + * CI passed, so the queue can rely on it instead of running every check again. + */ +export const VERIFIED_TREE_CONTEXT = 'CI verified tree' + +const SHA = /^[a-f0-9]{40}$/ +/** The workflow whose passing pull request runs may vouch for a tree. */ +const CI_WORKFLOW = '.github/workflows/ci.yml' +const ACTIONS_BOT = 'github-actions[bot]' + +export interface GitHubAccess { + repository: string + token: string + fetch?: typeof fetch +} + +async function github( + access: GitHubAccess, + path: string, + init: RequestInit = {} +): Promise { + const response = await (access.fetch ?? fetch)( + `https://api.github.com/repos/${access.repository}${path}`, + { + ...init, + headers: { + accept: 'application/vnd.github+json', + authorization: `Bearer ${access.token}`, + 'x-github-api-version': '2022-11-28', + ...init.headers + } + } + ) + if (!response.ok) throw new Error(`GitHub ${path} answered ${response.status}`) + const body: unknown = await response.json() + return body +} + +function field(value: unknown, key: string): unknown { + return typeof value === 'object' && value !== null ? Reflect.get(value, key) : undefined +} + +/** The pull request a merge queue branch tests, `gh-readonly-queue//pr--`. */ +export function queuedPullRequest(headRef: string): number | undefined { + const match = /^(?:refs\/heads\/)?gh-readonly-queue\/.+\/pr-(\d+)-[a-f0-9]{40}$/.exec(headRef) + return match ? Number(match[1]) : undefined +} + +/** Whether the queued pull request's own CI already passed on exactly `tree`. */ +export async function isVerifiedTree( + access: GitHubAccess, + headRef: string, + tree: string +): Promise { + const pullRequest = queuedPullRequest(headRef) + if (pullRequest === undefined || !SHA.test(tree)) return false + const head = field(field(await github(access, `/pulls/${pullRequest}`), 'head'), 'sha') + if (typeof head !== 'string' || !SHA.test(head)) return false + const statuses = await github(access, `/commits/${head}/statuses?per_page=100`) + if (!Array.isArray(statuses)) return false + // Newest first: only the latest record for this context counts. + const latest = statuses.find((status) => field(status, 'context') === VERIFIED_TREE_CONTEXT) + if (field(latest, 'state') !== 'success' || field(latest, 'description') !== tree) return false + return vouchedByRun(access, latest, head) +} + +/** + * A record counts only when GitHub shows it came from a passing CI run on this pull request's + * head: Actions wrote it, and its target is that run. Anyone with write access can post a status, + * and another pull request's CI could post one here; neither is such a run. + */ +async function vouchedByRun(access: GitHubAccess, status: unknown, head: string): Promise { + if (field(field(status, 'creator'), 'login') !== ACTIONS_BOT) return false + const target = field(status, 'target_url') + const runId = typeof target === 'string' ? runIdFromURL(access.repository, target) : undefined + if (runId === undefined) return false + const run = await github(access, `/actions/runs/${runId}`) + return ( + field(run, 'path') === CI_WORKFLOW && + field(run, 'event') === 'pull_request' && + field(run, 'conclusion') === 'success' && + field(run, 'head_sha') === head + ) +} + +/** The run id in `https://github.com//actions/runs/`, for this repository only. */ +function runIdFromURL(repository: string, url: string): string | undefined { + const prefix = `https://github.com/${repository}/actions/runs/` + if (!url.startsWith(prefix)) return undefined + const [id] = url.slice(prefix.length).split('/') + return id && /^\d+$/.test(id) ? id : undefined +} + +/** Records on the PR head that the CI run `runURL` passed on `tree`. */ +export async function recordVerifiedTree( + access: GitHubAccess, + head: string, + tree: string, + runURL: string +): Promise { + if (!SHA.test(head) || !SHA.test(tree)) throw new Error('Invalid head or tree SHA') + if (runIdFromURL(access.repository, runURL) === undefined) throw new Error('Invalid run URL') + await github(access, `/statuses/${head}`, { + method: 'POST', + body: JSON.stringify({ + state: 'success', + context: VERIFIED_TREE_CONTEXT, + description: tree, + target_url: runURL + }) + }) +} diff --git a/tools/ci/policy/tests/policy.test.ts b/tools/ci/policy/tests/policy.test.ts index 8a665590a..d2aab0104 100644 --- a/tools/ci/policy/tests/policy.test.ts +++ b/tools/ci/policy/tests/policy.test.ts @@ -75,7 +75,7 @@ function results(scope: ChangeScope): Record { } } -test.each(['docs', 'code'] as const)( +test.each(['docs', 'code', 'verified'] as const)( 'accepts only appropriate successful checks for %s', (scope) => { expect(gateErrors(results(scope))).toEqual([]) @@ -118,3 +118,12 @@ test('docs routing rejects unexpected execution of test suites', () => { expect(gateErrors({ ...results('docs'), [job]: { result: 'success' } })).not.toEqual([]) } }) + +test('a verified tree runs only the checks every change runs', () => { + for (const job of [...CODE_JOBS, DOCS_JOB]) { + expect(gateErrors({ ...results('verified'), [job]: { result: 'success' } })).not.toEqual([]) + } + for (const job of ALWAYS_JOBS) { + expect(gateErrors({ ...results('verified'), [job]: { result: 'skipped' } })).not.toEqual([]) + } +}) diff --git a/tools/ci/policy/tests/verified-tree/status.test.ts b/tools/ci/policy/tests/verified-tree/status.test.ts new file mode 100644 index 000000000..676417a58 --- /dev/null +++ b/tools/ci/policy/tests/verified-tree/status.test.ts @@ -0,0 +1,112 @@ +import { expect, test } from 'bun:test' + +import { + isVerifiedTree, + queuedPullRequest, + recordVerifiedTree, + VERIFIED_TREE_CONTEXT +} from '#ci/verified-tree/status' + +const HEAD = 'a'.repeat(40) +const OTHER_HEAD = 'e'.repeat(40) +const TREE = 'b'.repeat(40) +const OTHER_TREE = 'c'.repeat(40) +const QUEUE_REF = `refs/heads/gh-readonly-queue/master/pr-923-${'d'.repeat(40)}` +const RUN_URL = 'https://github.com/o/r/actions/runs/42' + +interface Run { + path: string + event: string + conclusion: string + head_sha: string +} +const PASSING_RUN: Run = { + path: '.github/workflows/ci.yml', + event: 'pull_request', + conclusion: 'success', + head_sha: HEAD +} + +/** A GitHub API stand-in: the PR's head, the statuses on it (newest first), and run 42. */ +function github(statuses: unknown[], run: Run = PASSING_RUN, requests: Request[] = []) { + const respond = async (input: RequestInfo | URL, init?: RequestInit) => { + const request = new Request(input, init) + requests.push(request) + const path = new URL(request.url).pathname + if (path.endsWith('/pulls/923')) return Response.json({ head: { sha: HEAD } }) + if (path.endsWith(`/commits/${HEAD}/statuses`)) return Response.json(statuses) + if (path.endsWith('/actions/runs/42')) return Response.json(run) + if (path.endsWith(`/statuses/${HEAD}`)) return Response.json({}, { status: 201 }) + return new Response('not found', { status: 404 }) + } + return Object.assign(respond, { preconnect: fetch.preconnect }) +} + +const access = (fetcher: typeof fetch) => ({ repository: 'o/r', token: 't', fetch: fetcher }) +function status(description: string, overrides: Record = {}) { + return { + context: VERIFIED_TREE_CONTEXT, + state: 'success', + description, + target_url: RUN_URL, + creator: { login: 'github-actions[bot]' }, + ...overrides + } +} + +test('reads the pull request from a merge queue branch', () => { + expect(queuedPullRequest(QUEUE_REF)).toBe(923) + expect(queuedPullRequest('refs/heads/master')).toBeUndefined() + expect(queuedPullRequest('gh-readonly-queue/master/pr-x-123')).toBeUndefined() +}) + +test('a queued tree is verified only by the latest record of exactly that tree', async () => { + const verified = (statuses: unknown[], ref = QUEUE_REF) => + isVerifiedTree(access(github(statuses)), ref, TREE) + + expect(await verified([status(TREE)])).toBe(true) + // Master moved since the PR's CI, so the queue tests a different tree. + expect(await verified([status(OTHER_TREE)])).toBe(false) + // A newer record for another tree supersedes an older match. + expect(await verified([status(OTHER_TREE), status(TREE)])).toBe(false) + expect(await verified([status(TREE, { state: 'failure' })])).toBe(false) + expect(await verified([])).toBe(false) + expect(await verified([status(TREE)], 'refs/heads/feature')).toBe(false) +}) + +test('a record counts only from a passing CI run on the same pull request head', async () => { + const verified = (record: unknown, run?: Run) => + isVerifiedTree(access(github([record], run)), QUEUE_REF, TREE) + + // Anyone with write access can post a status with their own token. + expect(await verified(status(TREE, { creator: { login: 'someone' } }))).toBe(false) + expect(await verified(status(TREE, { target_url: undefined }))).toBe(false) + expect( + await verified(status(TREE, { target_url: 'https://github.com/x/r/actions/runs/42' })) + ).toBe(false) + // Another pull request's CI wrote it, or the run did not pass, or it was another workflow. + expect(await verified(status(TREE), { ...PASSING_RUN, head_sha: OTHER_HEAD })).toBe(false) + expect(await verified(status(TREE), { ...PASSING_RUN, conclusion: 'failure' })).toBe(false) + expect(await verified(status(TREE), { ...PASSING_RUN, event: 'push' })).toBe(false) + expect(await verified(status(TREE), { ...PASSING_RUN, path: '.github/workflows/x.yml' })).toBe( + false + ) +}) + +test('records the tested tree and its run on the pull request head', async () => { + const requests: Request[] = [] + await recordVerifiedTree(access(github([], PASSING_RUN, requests)), HEAD, TREE, RUN_URL) + const [request] = requests + expect(request?.method).toBe('POST') + expect(new URL(request?.url ?? 'https://invalid').pathname).toBe(`/repos/o/r/statuses/${HEAD}`) + expect(await request?.json()).toEqual({ + state: 'success', + context: VERIFIED_TREE_CONTEXT, + description: TREE, + target_url: RUN_URL + }) + await expect(recordVerifiedTree(access(github([])), 'not-a-sha', TREE, RUN_URL)).rejects.toThrow() + await expect( + recordVerifiedTree(access(github([])), HEAD, TREE, 'https://example.com/run') + ).rejects.toThrow() +}) diff --git a/tools/release/package-artifacts/src/build.ts b/tools/release/package-artifacts/src/build.ts index 7e9235771..ec9fd347f 100644 --- a/tools/release/package-artifacts/src/build.ts +++ b/tools/release/package-artifacts/src/build.ts @@ -1,8 +1,12 @@ import { join } from 'node:path' import type { WorkspacePackage } from './manifest/types' -import { runCommand } from './process' -import { discoverPublicPackages, orderPackagesByDependencies } from './workspace/catalog' +import { CommandError, runCommand } from './process' +import { + discoverPublicPackages, + groupPackagesByDependencyLevel, + orderPackagesByDependencies +} from './workspace/catalog' import { resolveWorkspaceRoot } from './workspace/root' export interface BuildPublicPackagesOptions { @@ -11,23 +15,57 @@ export interface BuildPublicPackagesOptions { timeoutMs?: number } +/** + * Builds public packages level by level: a level's packages depend only on earlier levels, so + * they build at the same time. Each build's output is printed whole when it finishes, unless + * the caller captures it, so parallel builds do not interleave. + */ export async function buildPublicPackages( root: string, options: BuildPublicPackagesOptions = {} ): Promise { - const packages = orderPackagesByDependencies(await discoverPublicPackages(root)) - for (const pkg of packages) { - if (!pkg.manifest.scripts?.build) continue - options.log?.(`Building ${pkg.manifest.name}`) - await runCommand({ - command: 'bun', - args: ['run', 'build'], - cwd: join(root, pkg.directory), - output: options.output ?? 'inherit', - timeoutMs: options.timeoutMs ?? 180_000 - }) + const packages = await discoverPublicPackages(root) + for (const level of groupPackagesByDependencyLevel(packages)) { + // Every build in the level settles before a failure is reported, so none keeps writing + // output after the caller sees the error. + const results = await Promise.allSettled( + level + .filter((pkg) => pkg.manifest.scripts?.build) + .map(async (pkg) => { + options.log?.(`Building ${pkg.manifest.name}`) + const print = (output: { stdout: string; stderr: string }) => { + if (options.output === 'capture') return + process.stdout.write(output.stdout) + process.stderr.write(output.stderr) + } + try { + print( + await runCommand({ + command: 'bun', + args: ['run', 'build'], + cwd: join(root, pkg.directory), + output: 'capture', + timeoutMs: options.timeoutMs ?? 180_000 + }) + ) + } catch (error) { + if (!(error instanceof CommandError)) throw error + print(error) + throw new Error(`Building ${pkg.manifest.name} failed: ${error.message}`, { + cause: error + }) + } + }) + ) + const failures = results.flatMap((result) => + result.status === 'rejected' ? [result.reason] : [] + ) + if (failures.length === 1) throw failures[0] + if (failures.length > 1) { + throw new AggregateError(failures, `${failures.length} package builds failed`) + } } - return packages + return orderPackagesByDependencies(packages) } if (import.meta.main) { diff --git a/tools/release/package-artifacts/src/workspace/catalog.ts b/tools/release/package-artifacts/src/workspace/catalog.ts index 30e945954..540ce1cae 100644 --- a/tools/release/package-artifacts/src/workspace/catalog.ts +++ b/tools/release/package-artifacts/src/workspace/catalog.ts @@ -57,3 +57,21 @@ export function orderPackagesByDependencies(packages: WorkspacePackage[]): Works for (const pkg of packages) visit(pkg, []) return ordered } + +/** + * Packages grouped into levels: each level depends only on earlier ones, so a level's packages + * can build at the same time. + */ +export function groupPackagesByDependencyLevel(packages: WorkspacePackage[]): WorkspacePackage[][] { + const levelByName = new Map() + const levels: WorkspacePackage[][] = [] + for (const pkg of orderPackagesByDependencies(packages)) { + const dependencyLevels = DEPENDENCY_FIELDS.flatMap((field) => + Object.keys(pkg.manifest[field] ?? {}).flatMap((name) => levelByName.get(name) ?? []) + ) + const level = dependencyLevels.length > 0 ? Math.max(...dependencyLevels) + 1 : 0 + levelByName.set(pkg.manifest.name, level) + ;(levels[level] ??= []).push(pkg) + } + return levels +} diff --git a/tools/release/package-artifacts/tests/catalog.test.ts b/tools/release/package-artifacts/tests/catalog.test.ts index f354b4734..80da82e39 100644 --- a/tools/release/package-artifacts/tests/catalog.test.ts +++ b/tools/release/package-artifacts/tests/catalog.test.ts @@ -6,6 +6,7 @@ import { join } from 'node:path' import type { WorkspacePackage } from '#package-artifacts/manifest/types' import { discoverPublicPackages, + groupPackagesByDependencyLevel, orderPackagesByDependencies } from '#package-artifacts/workspace/catalog' @@ -68,3 +69,22 @@ describe('orderPackagesByDependencies', () => { ).toThrow('Workspace dependency cycle: one -> two -> one') }) }) + +describe('groupPackagesByDependencyLevel', () => { + test('puts each package one level after its deepest internal dependency', () => { + const levels = groupPackagesByDependencyLevel([ + pkg('cli', { app: 'workspace:*', base: 'workspace:*' }), + pkg('app', { left: 'workspace:*', right: 'workspace:*' }), + pkg('left', { base: 'workspace:*', external: '^1.0.0' }), + pkg('right', { base: 'workspace:*' }), + pkg('base'), + pkg('standalone') + ]) + expect(levels.map((level) => level.map(({ manifest }) => manifest.name))).toEqual([ + ['base', 'standalone'], + ['left', 'right'], + ['app'], + ['cli'] + ]) + }) +})