Merge pull request #689 from open-pencil/conditional-ci

ci: validate docs-only PRs without running test suites

.github/workflows/ci.yml

@@ -3,10 +3,6 @@ name: CI
 on:
   pull_request:
     branches: [master]
-    paths-ignore:
-      - 'packages/docs/**'
-      - 'openspec/**'
-      - '*.md'
 
 permissions:
   contents: read
@@ -17,8 +13,52 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  changes:
+    name: Classify changes
+    runs-on: ubuntu-latest
+    timeout-minutes: 3
+    outputs:
+      scope: ${{ steps.classify.outputs.scope }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+        with:
+          persist-credentials: false
+
+      - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
+
+      - name: Fetch comparison base
+        env:
+          CI_BASE_SHA: ${{ github.event.pull_request.base.sha }}
+        run: git fetch --no-tags --depth=1 origin "$CI_BASE_SHA"
+
+      - name: Select validation scope
+        id: classify
+        env:
+          CI_BASE_SHA: ${{ github.event.pull_request.base.sha }}
+        run: bun tools/ci/src/classify.ts
+
+  documentation:
+    name: Documentation
+    needs: changes
+    if: needs.changes.outputs.scope == 'docs'
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+        with:
+          persist-credentials: false
+      - uses: ./.github/actions/setup-bun
+      - name: Build documentation type dependencies
+        run: bun run build:packages
+      - name: Validate documentation and generated references
+        run: bun run check:docs
+      - name: Build documentation and check examples
+        run: bun run docs:build
+
   source-quality:
     name: Code quality
+    needs: changes
+    if: needs.changes.outputs.scope == 'code'
     timeout-minutes: 10
     runs-on: ubuntu-latest
     steps:
@@ -45,6 +85,8 @@ jobs:
 
   package-quality:
     name: Package integrity
+    needs: changes
+    if: needs.changes.outputs.scope == 'code'
     timeout-minutes: 10
     runs-on: ubuntu-latest
     steps:
@@ -68,6 +110,8 @@ jobs:
 
   repository-quality:
     name: Repository hygiene
+    needs: changes
+    if: needs.changes.outputs.scope == 'code'
     timeout-minutes: 10
     runs-on: ubuntu-latest
     steps:
@@ -97,6 +141,8 @@ jobs:
 
   storybook:
     name: Component workshop
+    needs: changes
+    if: needs.changes.outputs.scope == 'code'
     timeout-minutes: 10
     runs-on: ubuntu-latest
     steps:
@@ -114,6 +160,8 @@ jobs:
 
   native-test-contracts:
     name: Native app contracts
+    needs: changes
+    if: needs.changes.outputs.scope == 'code'
     timeout-minutes: 8
     runs-on: ubuntu-24.04
     container:
@@ -142,6 +190,8 @@ jobs:
         run: cargo check --manifest-path desktop/Cargo.toml --features native-test
 
   unit-tests:
+    needs: changes
+    if: needs.changes.outputs.scope == 'code'
     timeout-minutes: 10
     runs-on: ubuntu-latest
     strategy:
@@ -172,3 +222,19 @@ jobs:
           bun test "${test_files[@]}"
         env:
           BUN_HEAVY_TESTS: 'false'
+
+  result:
+    name: CI result
+    needs: [changes, documentation, source-quality, package-quality, repository-quality, storybook, native-test-contracts, unit-tests]
+    if: always()
+    runs-on: ubuntu-latest
+    timeout-minutes: 3
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+        with:
+          persist-credentials: false
+      - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
+      - name: Require successful checks for the selected scope
+        env:
+          CI_NEEDS: ${{ toJSON(needs) }}
+        run: bun tools/ci/src/gate.ts

AGENTS.md

@@ -86,6 +86,8 @@ For releases, update versions in the root and publishable package manifests plus
 
 App/docs production workflows run on `v*` tags or `workflow_dispatch`, not ordinary `master` pushes. `ci.yml` and `heavy-tests.yml` define validation gates.
 
+PR CI always classifies changed paths through `tools/ci/`. Docs-only changes run documentation integrity/reference checks and the docs build, not engine, browser, Storybook, or native suites. Runtime prompt Markdown, executable examples, configuration, and unknown paths require code validation. The aggregate `CI result` gate requires successful classification and every applicable job; failures, cancellations, and unexpected skips cannot pass. Do not restore workflow-level path filtering on required CI.
+
 ## Documentation
 
 - `CHANGELOG.md` — curated user-facing changes by version; `Unreleased` stays first.

tools/ci/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "@open-pencil/ci-tools",
+  "private": true,
+  "type": "module",
+  "imports": {
+    "#ci/*": "./src/*.ts"
+  },
+  "scripts": {
+    "test": "bun test tests"
+  }
+}

tools/ci/src/classify.ts

@@ -0,0 +1,20 @@
+import { execFileSync } from 'node:child_process'
+import { appendFile } from 'node:fs/promises'
+
+import { classifyPaths } from './policy'
+
+const base = process.env.CI_BASE_SHA
+const output = process.env.GITHUB_OUTPUT
+if (!base || !/^[a-f0-9]{40}$/.test(base) || !output)
+  throw new Error('Missing CI base SHA or output file')
+
+// Disable rename detection so both the old and new paths participate in routing.
+const paths = execFileSync('git', ['diff', '--no-renames', '--name-only', '-z', base, 'HEAD'], {
+  maxBuffer: 32 * 1024 * 1024
+})
+  .toString('utf8')
+  .split('\0')
+  .filter(Boolean)
+const scope = classifyPaths(paths)
+await appendFile(output, `scope=${scope}\n`)
+console.log(`Selected ${scope} checks for ${paths.length} changed paths`)

tools/ci/src/gate.ts

@@ -0,0 +1,10 @@
+import { gateErrors, type JobStatus } from './policy'
+
+const input = process.env.CI_NEEDS
+if (!input) throw new Error('Missing CI job results')
+// GitHub serializes the needs context. Malformed data or unexpected statuses fail closed.
+const needs: Record<string, JobStatus> = JSON.parse(input)
+const errors = gateErrors(needs)
+for (const error of errors) console.error(error)
+if (errors.length > 0) process.exitCode = 1
+else console.log('All checks required for this change succeeded')

tools/ci/src/policy.ts

@@ -0,0 +1,53 @@
+const ROOT_DOCS = new Set(['README.md', 'CONTRIBUTING.md', 'AGENTS.md', 'CHANGELOG.md', 'LICENSE'])
+const DOC_ASSET = /\.(?:md|png|jpe?g|gif|webp|svg|ico|pdf|woff2?|ttf)$/i
+
+export type ChangeScope = 'docs' | 'code'
+
+/** Unknown paths, executable docs, and runtime prompt Markdown require the code checks. */
+export function classifyPaths(paths: readonly string[]): ChangeScope {
+  if (paths.length === 0) return 'code'
+  return paths.every((path) => {
+    if (ROOT_DOCS.has(path)) return true
+    if (path.startsWith('packages/docs/')) return DOC_ASSET.test(path)
+    if (path.startsWith('openspec/')) return path.endsWith('.md')
+    if (path.startsWith('skills/')) return path.endsWith('.md') || path.endsWith('/LICENSE.txt')
+    return false
+  })
+    ? 'docs'
+    : 'code'
+}
+
+export const CODE_JOBS = [
+  'source-quality',
+  'package-quality',
+  'repository-quality',
+  'storybook',
+  'native-test-contracts',
+  'unit-tests'
+] as const
+export const DOCS_JOB = 'documentation'
+
+type JobResult = 'success' | 'failure' | 'cancelled' | 'skipped'
+export interface JobStatus {
+  result: JobResult
+  outputs?: Record<string, string>
+}
+
+/** The sole required gate accepts only the successful checks selected by successful detection. */
+export function gateErrors(needs: Partial<Record<string, JobStatus>>): string[] {
+  const detection = needs.changes
+  if (detection?.result !== 'success') return ['Change detection did not succeed']
+  const scope = detection.outputs?.scope
+  if (scope !== 'docs' && scope !== 'code') return ['Invalid or missing change scope']
+  const required: readonly string[] = scope === 'docs' ? [DOCS_JOB] : CODE_JOBS
+  const excluded: readonly string[] = scope === 'docs' ? CODE_JOBS : [DOCS_JOB]
+  const errors = required
+    .filter((job) => needs[job]?.result !== 'success')
+    .map((job) => `${job} did not succeed`)
+  // Unexpected execution is also a policy failure: docs must not run the full suites.
+  for (const job of excluded) {
+    if (needs[job]?.result !== 'skipped')
+      errors.push(`${job} was not skipped for ${scope}-only routing`)
+  }
+  return errors
+}

tools/ci/tests/classify.test.ts

@@ -0,0 +1,47 @@
+import { afterEach, expect, test } from 'bun:test'
+import { mkdtemp, readFile, rename, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const roots: string[] = []
+afterEach(async () => {
+  await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })))
+})
+
+function git(cwd: string, ...args: string[]): string {
+  const result = Bun.spawnSync(
+    ['git', '-c', 'user.name=CI Test', '-c', 'user.email=ci@example.invalid', ...args],
+    { cwd }
+  )
+  if (result.exitCode !== 0) throw new Error(result.stderr.toString())
+  return result.stdout.toString().trim()
+}
+
+test('classifier reads real Git deletions and renames without losing the original path', async () => {
+  const root = await mkdtemp(join(tmpdir(), 'open-pencil-ci-'))
+  roots.push(root)
+  git(root, 'init')
+  await writeFile(join(root, 'runtime.ts'), 'export const version = 1\n')
+  git(root, 'add', '.')
+  git(root, 'commit', '-m', 'base')
+  const base = git(root, 'rev-parse', 'HEAD')
+  await rename(join(root, 'runtime.ts'), join(root, 'README.md'))
+  git(root, 'add', '.')
+  git(root, 'commit', '-m', 'move')
+  const output = join(root, 'output')
+  const command = fileURLToPath(import.meta.resolve('#ci/classify'))
+  const result = Bun.spawnSync([process.execPath, command], {
+    cwd: root,
+    env: { ...process.env, CI_BASE_SHA: base, GITHUB_OUTPUT: output }
+  })
+  expect(result.exitCode).toBe(0)
+  expect(await readFile(output, 'utf8')).toBe('scope=code\n')
+
+  const failed = Bun.spawnSync([process.execPath, command], {
+    cwd: root,
+    env: { ...process.env, CI_BASE_SHA: 'invalid', GITHUB_OUTPUT: output }
+  })
+  expect(failed.exitCode).not.toBe(0)
+  expect(await readFile(output, 'utf8')).toBe('scope=code\n')
+})

tools/ci/tests/policy.test.ts

@@ -0,0 +1,102 @@
+import { expect, test } from 'bun:test'
+
+import {
+  classifyPaths,
+  CODE_JOBS,
+  DOCS_JOB,
+  gateErrors,
+  type ChangeScope,
+  type JobStatus
+} from '#ci/policy'
+
+const documentation = [
+  'README.md',
+  'AGENTS.md',
+  'CHANGELOG.md',
+  'packages/docs/programmable/sdk/api/components/bindable-value.md',
+  'packages/docs/public/logo.svg',
+  'skills/open-pencil/SKILL.md',
+  'skills/open-pencil/references/design-authoring.md',
+  'openspec/proposal.md'
+]
+const code = [
+  'src/app/ai/chat/system-prompt.md',
+  'packages/core/src/design-jsx/reference/authoring.md',
+  'packages/docs/.vitepress/config.ts',
+  'packages/docs/demo.vue',
+  'skills/open-pencil/scripts/create.ts',
+  'package.json',
+  'bun.lock',
+  '.github/workflows/ci.yml',
+  'new-domain/instructions.md',
+  'src/editor.ts'
+]
+
+test.each(documentation)('docs-only path: %s', (path) => {
+  expect(classifyPaths([path])).toBe('docs')
+})
+test.each(code)('code or unknown path: %s', (path) => {
+  expect(classifyPaths([path])).toBe('code')
+  expect(classifyPaths([...documentation, path])).toBe('code')
+})
+test('empty diffs fail safe to code checks', () => {
+  expect(classifyPaths([])).toBe('code')
+})
+test('both sides of a rename affect classification', () => {
+  expect(classifyPaths(['src/prompt.md', 'packages/docs/prompt.md'])).toBe('code')
+  expect(classifyPaths(['packages/docs/old.md', 'packages/docs/new.md'])).toBe('docs')
+})
+
+function results(scope: ChangeScope): Record<string, JobStatus> {
+  return {
+    changes: { result: 'success', outputs: { scope } },
+    [DOCS_JOB]: { result: scope === 'docs' ? 'success' : 'skipped' },
+    ...Object.fromEntries(
+      CODE_JOBS.map((job) => [job, { result: scope === 'code' ? 'success' : 'skipped' }])
+    )
+  }
+}
+
+test.each(['docs', 'code'] as const)(
+  'accepts only appropriate successful checks for %s',
+  (scope) => {
+    expect(gateErrors(results(scope))).toEqual([])
+  }
+)
+
+test.each(['failure', 'cancelled', 'skipped'] as const)(
+  'rejects %s detection and required checks',
+  (result) => {
+    for (const scope of ['docs', 'code'] as const) {
+      expect(gateErrors({ ...results(scope), changes: { result } })).not.toEqual([])
+      for (const job of scope === 'docs' ? [DOCS_JOB] : CODE_JOBS) {
+        expect(gateErrors({ ...results(scope), [job]: { result } })).not.toEqual([])
+      }
+    }
+  }
+)
+
+test('missing jobs or outputs cannot pass', () => {
+  expect(gateErrors({})).not.toEqual([])
+  expect(gateErrors({ ...results('docs'), changes: { result: 'success' } })).not.toEqual([])
+  expect(
+    gateErrors({
+      ...results('docs'),
+      changes: { result: 'success', outputs: { scope: 'unknown' } }
+    })
+  ).not.toEqual([])
+  for (const scope of ['docs', 'code'] as const) {
+    for (const job of scope === 'docs' ? [DOCS_JOB] : CODE_JOBS) {
+      const incomplete = Object.fromEntries(
+        Object.entries(results(scope)).filter(([name]) => name !== job)
+      )
+      expect(gateErrors(incomplete)).not.toEqual([])
+    }
+  }
+})
+
+test('docs routing rejects unexpected execution of test suites', () => {
+  for (const job of CODE_JOBS) {
+    expect(gateErrors({ ...results('docs'), [job]: { result: 'success' } })).not.toEqual([])
+  }
+})

tsconfig.json

@@ -21,6 +21,7 @@
     "noUnusedParameters": true,
     "noFallthroughCasesInSwitch": true,
     "paths": {
+      "#ci/*": ["./tools/ci/src/*"],
       "@/*": [
         "./src/*"
       ],