From 38bee364e8104f8328b2c8e8f3f6cfce8ee272c3 Mon Sep 17 00:00:00 2001 From: Danila Poyarkov Date: Sat, 29 Aug 2026 14:42:44 +0300 Subject: [PATCH] refactor(tools): organize internal CLI workflows (#600) * refactor(tools): organize internal CLI workflows - Group visual-oracle commands behind a Citty CLI and domain folders\n- Add imported-document visual comparison manifests and pixel-diff outputs\n- Consolidate package-quality checks behind stable check, smoke, and verify commands\n- Remove obsolete visual root aliases and compatibility scripts * fix(tools): validate visual oracle target scope - Validate manifest records, required strings, and optional field types\n- Reject explicit OpenPencil nodes outside the configured page * fix(tools): validate packaged runtime exports --- bun.lock | 16 ++ package.json | 9 +- packages/docs/development/roadmap.md | 2 +- scripts/export-fixture-visuals.ts | 2 - scripts/visual-bisect.ts | 2 - scripts/visual-compare.ts | 2 - .../visual-comparison-report.json | 2 +- tools/package-quality/package.json | 8 +- .../src/{check => checks}/attw.ts | 0 .../src/{check => checks}/metadata.ts | 3 +- .../src/{check => checks}/publint.ts | 0 tools/package-quality/src/cli.ts | 16 ++ tools/package-quality/src/commands/index.ts | 33 +++ tools/package-quality/src/commands/run.ts | 11 + tools/package-quality/tests/cli.test.ts | 14 ++ tools/visual-oracles/package.json | 11 +- tools/visual-oracles/src/cli.ts | 31 +++ tools/visual-oracles/src/commands/analyze.ts | 19 ++ tools/visual-oracles/src/commands/compare.ts | 19 ++ .../visual-oracles/src/commands/operation.ts | 16 ++ tools/visual-oracles/src/document/manifest.ts | 85 ++++++++ .../src/{ => image}/pixel-image.ts | 0 .../src/{ => operations}/analyze-pattern.ts | 2 +- .../analyze-text-decoration.ts | 2 +- .../src/{ => operations}/bisect.ts | 2 +- .../src/operations/compare-document.ts | 200 ++++++++++++++++++ .../compare-node.ts} | 4 +- .../src/{ => operations}/export-fixtures.ts | 0 .../src/{ => operations}/update-report.ts | 0 tools/visual-oracles/tests/cli.test.ts | 14 ++ .../tests/document/manifest.test.ts | 57 +++++ 31 files changed, 562 insertions(+), 20 deletions(-) delete mode 100644 scripts/export-fixture-visuals.ts delete mode 100644 scripts/visual-bisect.ts delete mode 100644 scripts/visual-compare.ts rename tools/package-quality/src/{check => checks}/attw.ts (100%) rename tools/package-quality/src/{check => checks}/metadata.ts (96%) rename tools/package-quality/src/{check => checks}/publint.ts (100%) create mode 100644 tools/package-quality/src/cli.ts create mode 100644 tools/package-quality/src/commands/index.ts create mode 100644 tools/package-quality/src/commands/run.ts create mode 100644 tools/package-quality/tests/cli.test.ts create mode 100644 tools/visual-oracles/src/cli.ts create mode 100644 tools/visual-oracles/src/commands/analyze.ts create mode 100644 tools/visual-oracles/src/commands/compare.ts create mode 100644 tools/visual-oracles/src/commands/operation.ts create mode 100644 tools/visual-oracles/src/document/manifest.ts rename tools/visual-oracles/src/{ => image}/pixel-image.ts (100%) rename tools/visual-oracles/src/{ => operations}/analyze-pattern.ts (99%) rename tools/visual-oracles/src/{ => operations}/analyze-text-decoration.ts (98%) rename tools/visual-oracles/src/{ => operations}/bisect.ts (99%) create mode 100644 tools/visual-oracles/src/operations/compare-document.ts rename tools/visual-oracles/src/{compare.ts => operations/compare-node.ts} (98%) rename tools/visual-oracles/src/{ => operations}/export-fixtures.ts (100%) rename tools/visual-oracles/src/{ => operations}/update-report.ts (100%) create mode 100644 tools/visual-oracles/tests/cli.test.ts create mode 100644 tools/visual-oracles/tests/document/manifest.test.ts diff --git a/bun.lock b/bun.lock index 727ea1baa..44711623b 100644 --- a/bun.lock +++ b/bun.lock @@ -368,6 +368,18 @@ "tools/docs": { "name": "@open-pencil/docs-tools", }, + "tools/package-quality": { + "name": "@open-pencil/package-quality-tools", + "dependencies": { + "citty": "^0.1.6", + }, + }, + "tools/visual-oracles": { + "name": "@open-pencil/visual-oracle-tools", + "dependencies": { + "citty": "^0.1.6", + }, + }, }, "overrides": { "@codemirror/view": "6.43.9", @@ -1127,10 +1139,14 @@ "@open-pencil/mcp": ["@open-pencil/mcp@workspace:packages/mcp"], + "@open-pencil/package-quality-tools": ["@open-pencil/package-quality-tools@workspace:tools/package-quality"], + "@open-pencil/pen": ["@open-pencil/pen@workspace:packages/pen"], "@open-pencil/scene-graph": ["@open-pencil/scene-graph@workspace:packages/scene-graph"], + "@open-pencil/visual-oracle-tools": ["@open-pencil/visual-oracle-tools@workspace:tools/visual-oracles"], + "@open-pencil/vue": ["@open-pencil/vue@workspace:packages/vue"], "@openrouter/ai-sdk-provider": ["@openrouter/ai-sdk-provider@3.0.0", "", { "peerDependencies": { "ai": "^7.0.0", "zod": "^3.25.76 || ^4.1.8" } }, "sha512-m9XTSWoODH2RM5OsZpaGiN7QRR8cdP5paBWq699Tu3JVmGPBKT8xF8XwV0ZBVVsjikD/JgWfak4VSsTR4wAVbg=="], diff --git a/package.json b/package.json index d477d07af..f0eef7053 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,9 @@ "packages/mcp", "packages/harness", "packages/docs", - "tools/docs" + "tools/docs", + "tools/package-quality", + "tools/visual-oracles" ], "type": "module", "scripts": { @@ -40,7 +42,7 @@ "check:monorepo": "sherif --ignore-rule root-package-dependencies", "check:native-test": "tsc --noEmit -p tests/e2e/native/tsconfig.json", "check:i18n": "bun tools/i18n/src/check-locales.ts", - "check:packages": "bun tools/package-quality/src/check/metadata.ts && bun tools/package-quality/src/check/publint.ts && bun tools/package-quality/src/check/attw.ts", + "check:packages": "bun tools/package-quality/src/cli.ts check", "check:arch": "steiger .", "check:vue": "vue-tsc --noEmit -p tsconfig.json && vue-tsc --noEmit -p packages/vue/tsconfig.json", "test": "playwright test --project=openpencil", @@ -55,14 +57,13 @@ "test:type-shapes": "bun tools/type-shapes/src/index.ts", "test:tools": "bun tools/test.ts && bun --filter @open-pencil/harness test", "test:dupes": "jscpd packages/scene-graph/src packages/pen/src packages/core/src packages/cli/src packages/dom-css/src src --min-lines 5 --min-tokens 50 --format typescript --threshold 0", - "test:packages": "bun tools/package-quality/src/check/metadata.ts && bun tools/package-quality/src/check/publint.ts && bun tools/package-quality/src/check/attw.ts && bun tools/package-quality/src/smoke.ts", + "test:packages": "bun tools/package-quality/src/cli.ts verify", "build:packages": "bun --filter @open-pencil/scene-graph build && bun --filter @open-pencil/pen build && bun --filter @open-pencil/kiwi build && bun --filter @open-pencil/fig build && bun --filter @open-pencil/core build && bun --filter @open-pencil/dom-css build && bun --filter @open-pencil/vue build && bun --filter @open-pencil/mcp build && bun --filter @open-pencil/cli build && bun --filter @open-pencil/harness build", "open-pencil": "bun packages/cli/src/index.ts", "docs:dev": "bun --filter @open-pencil/docs dev", "docs:build": "bun --filter @open-pencil/docs build", "docs:build:production": "bun --filter @open-pencil/docs build:production", "docs:preview": "bun --filter @open-pencil/docs preview", - "visual-compare": "bun tools/visual-oracles/src/compare.ts", "generate:tauri-menu": "bun tools/tauri-menu/src/generate.ts" }, "dependencies": { diff --git a/packages/docs/development/roadmap.md b/packages/docs/development/roadmap.md index dbaf3e795..69f9aa749 100644 --- a/packages/docs/development/roadmap.md +++ b/packages/docs/development/roadmap.md @@ -31,7 +31,7 @@ v0.14.0 established several foundations that earlier versions of this roadmap tr ### Figma fidelity - Preserve and round-trip more Figma metadata safely. -- Add visual regression coverage for full multi-page `.fig` documents. `tools/visual-oracles/src/export-fixtures.ts` exports current smoke fixture pages to `/tmp` for manual comparison without committing large images; `tests/fixtures/figma-oracles/visual-comparison-report.json` records the current Figma-vs-OpenPencil oracle diff findings. +- Add visual regression coverage for full multi-page `.fig` documents. `bun tools/visual-oracles/src/cli.ts export-fixtures` exports current smoke fixture pages to `/tmp` for manual comparison without committing large images; `tests/fixtures/figma-oracles/visual-comparison-report.json` records the current Figma-vs-OpenPencil oracle diff findings. - Close high-impact renderer gaps: remaining mask edge cases, blend isolation, pattern fills, and broader variable-font fixtures. - Improve boolean operation editing/export now that imported Figma `BOOLEAN_OPERATION` nodes remain boolean operations. diff --git a/scripts/export-fixture-visuals.ts b/scripts/export-fixture-visuals.ts deleted file mode 100644 index 39c2cec5b..000000000 --- a/scripts/export-fixture-visuals.ts +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env bun -import '../tools/visual-oracles/src/export-fixtures' diff --git a/scripts/visual-bisect.ts b/scripts/visual-bisect.ts deleted file mode 100644 index 0f0165daa..000000000 --- a/scripts/visual-bisect.ts +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env bun -import '../tools/visual-oracles/src/bisect' diff --git a/scripts/visual-compare.ts b/scripts/visual-compare.ts deleted file mode 100644 index ff1d42f7a..000000000 --- a/scripts/visual-compare.ts +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env bun -import '../tools/visual-oracles/src/compare' diff --git a/tests/fixtures/figma-oracles/visual-comparison-report.json b/tests/fixtures/figma-oracles/visual-comparison-report.json index cba7af2ab..098f1eb62 100644 --- a/tests/fixtures/figma-oracles/visual-comparison-report.json +++ b/tests/fixtures/figma-oracles/visual-comparison-report.json @@ -1,6 +1,6 @@ { "source": { - "tool": "tools/visual-oracles/src/compare.ts", + "tool": "tools/visual-oracles/src/cli.ts compare node", "captured": "2026-05-22", "outputRoot": "/tmp/open-pencil-oracles" }, diff --git a/tools/package-quality/package.json b/tools/package-quality/package.json index a81073e2d..3fe7d6ecd 100644 --- a/tools/package-quality/package.json +++ b/tools/package-quality/package.json @@ -1,5 +1,11 @@ { "name": "@open-pencil/package-quality-tools", "private": true, - "type": "module" + "type": "module", + "scripts": { + "test": "bun test tests" + }, + "dependencies": { + "citty": "^0.1.6" + } } diff --git a/tools/package-quality/src/check/attw.ts b/tools/package-quality/src/checks/attw.ts similarity index 100% rename from tools/package-quality/src/check/attw.ts rename to tools/package-quality/src/checks/attw.ts diff --git a/tools/package-quality/src/check/metadata.ts b/tools/package-quality/src/checks/metadata.ts similarity index 96% rename from tools/package-quality/src/check/metadata.ts rename to tools/package-quality/src/checks/metadata.ts index 41fc3a30d..f04f73e0a 100644 --- a/tools/package-quality/src/check/metadata.ts +++ b/tools/package-quality/src/checks/metadata.ts @@ -44,6 +44,7 @@ function checkIncludedRuntimePath( ): void { checkRuntimePath(packageName, field, value) const normalized = value.replace(/^\.\//, '') + if (normalized === 'package.json') return const topLevelDir = normalized.split('/')[0] if (topLevelDir && !files.includes(topLevelDir)) { errors.push( @@ -85,7 +86,7 @@ function walkExports( if (key === 'types') { checkIncludedTypePath(packageName, `exports.${path.join('.')}`, value, files) } else { - checkRuntimePath(packageName, `exports.${path.join('.')}`, value) + checkIncludedRuntimePath(packageName, `exports.${path.join('.')}`, value, files) } return } diff --git a/tools/package-quality/src/check/publint.ts b/tools/package-quality/src/checks/publint.ts similarity index 100% rename from tools/package-quality/src/check/publint.ts rename to tools/package-quality/src/checks/publint.ts diff --git a/tools/package-quality/src/cli.ts b/tools/package-quality/src/cli.ts new file mode 100644 index 000000000..9d8b6c193 --- /dev/null +++ b/tools/package-quality/src/cli.ts @@ -0,0 +1,16 @@ +#!/usr/bin/env bun + +import { defineCommand, runMain } from 'citty' + +import { checkCommand, smokeCommand, verifyCommand } from './commands' + +const main = defineCommand({ + meta: { name: 'package-quality', description: 'Validate OpenPencil public packages' }, + subCommands: { + check: checkCommand, + smoke: smokeCommand, + verify: verifyCommand + } +}) + +await runMain(main) diff --git a/tools/package-quality/src/commands/index.ts b/tools/package-quality/src/commands/index.ts new file mode 100644 index 000000000..3e3080b8c --- /dev/null +++ b/tools/package-quality/src/commands/index.ts @@ -0,0 +1,33 @@ +import { defineCommand } from 'citty' + +import { runPackageQualityCommand } from './run' + +export const PACKAGE_CHECK_ENTRYPOINTS = [ + 'tools/package-quality/src/checks/metadata.ts', + 'tools/package-quality/src/checks/publint.ts', + 'tools/package-quality/src/checks/attw.ts' +] + +export const checkCommand = defineCommand({ + meta: { name: 'check', description: 'Check public package metadata and declarations' }, + async run() { + await runPackageQualityCommand(PACKAGE_CHECK_ENTRYPOINTS) + } +}) + +export const smokeCommand = defineCommand({ + meta: { name: 'smoke', description: 'Smoke-test built public packages' }, + async run() { + await runPackageQualityCommand(['tools/package-quality/src/smoke.ts']) + } +}) + +export const verifyCommand = defineCommand({ + meta: { name: 'verify', description: 'Run package checks and built-package smoke tests' }, + async run() { + await runPackageQualityCommand([ + ...PACKAGE_CHECK_ENTRYPOINTS, + 'tools/package-quality/src/smoke.ts' + ]) + } +}) diff --git a/tools/package-quality/src/commands/run.ts b/tools/package-quality/src/commands/run.ts new file mode 100644 index 000000000..29b99edc6 --- /dev/null +++ b/tools/package-quality/src/commands/run.ts @@ -0,0 +1,11 @@ +export async function runPackageQualityCommand(entrypoints: string[]): Promise { + for (const entrypoint of entrypoints) { + const process = Bun.spawn(['bun', entrypoint], { + stdin: 'inherit', + stdout: 'inherit', + stderr: 'inherit' + }) + const exitCode = await process.exited + if (exitCode !== 0) throw new Error(`${entrypoint} exited with code ${exitCode}`) + } +} diff --git a/tools/package-quality/tests/cli.test.ts b/tools/package-quality/tests/cli.test.ts new file mode 100644 index 000000000..73edaab8f --- /dev/null +++ b/tools/package-quality/tests/cli.test.ts @@ -0,0 +1,14 @@ +import { describe, expect, test } from 'bun:test' + +import { renderUsage } from 'citty' + +import { checkCommand, smokeCommand, verifyCommand } from '../src/commands' + +describe('package quality commands', () => { + test('exposes stable check, smoke, and verify workflows', async () => { + expect(checkCommand.meta?.name).toBe('check') + expect(smokeCommand.meta?.name).toBe('smoke') + expect(verifyCommand.meta?.name).toBe('verify') + expect(await renderUsage(verifyCommand)).toContain('built-package smoke tests') + }) +}) diff --git a/tools/visual-oracles/package.json b/tools/visual-oracles/package.json index 18abfc624..ac311d8a4 100644 --- a/tools/visual-oracles/package.json +++ b/tools/visual-oracles/package.json @@ -1,5 +1,14 @@ { "name": "@open-pencil/visual-oracle-tools", "private": true, - "type": "module" + "type": "module", + "imports": { + "#visual/manifest": "./src/document/manifest.ts" + }, + "scripts": { + "test": "bun test tests" + }, + "dependencies": { + "citty": "^0.1.6" + } } diff --git a/tools/visual-oracles/src/cli.ts b/tools/visual-oracles/src/cli.ts new file mode 100644 index 000000000..16ea41b3f --- /dev/null +++ b/tools/visual-oracles/src/cli.ts @@ -0,0 +1,31 @@ +import { defineCommand, runMain } from 'citty' + +import { operationCommand } from './commands/operation' + +const main = defineCommand({ + meta: { + name: 'visual-oracles', + description: 'Figma and OpenPencil visual comparison tools' + }, + subCommands: { + compare: () => import('./commands/compare').then((module) => module.default), + bisect: operationCommand( + 'bisect', + 'Bisect page children to isolate visual differences', + 'tools/visual-oracles/src/operations/bisect.ts' + ), + 'export-fixtures': operationCommand( + 'export-fixtures', + 'Export configured OpenPencil fixture images', + 'tools/visual-oracles/src/operations/export-fixtures.ts' + ), + analyze: () => import('./commands/analyze').then((module) => module.default), + 'update-report': operationCommand( + 'update-report', + 'Update the visual comparison report', + 'tools/visual-oracles/src/operations/update-report.ts' + ) + } +}) + +await runMain(main) diff --git a/tools/visual-oracles/src/commands/analyze.ts b/tools/visual-oracles/src/commands/analyze.ts new file mode 100644 index 000000000..516645b2b --- /dev/null +++ b/tools/visual-oracles/src/commands/analyze.ts @@ -0,0 +1,19 @@ +import { defineCommand } from 'citty' + +import { operationCommand } from './operation' + +export default defineCommand({ + meta: { name: 'analyze', description: 'Analyze specialized Figma visual oracles' }, + subCommands: { + pattern: operationCommand( + 'pattern', + 'Analyze pattern-paint oracle captures', + 'tools/visual-oracles/src/operations/analyze-pattern.ts' + ), + 'text-decoration': operationCommand( + 'text-decoration', + 'Analyze text-decoration oracle captures', + 'tools/visual-oracles/src/operations/analyze-text-decoration.ts' + ) + } +}) diff --git a/tools/visual-oracles/src/commands/compare.ts b/tools/visual-oracles/src/commands/compare.ts new file mode 100644 index 000000000..30e77100a --- /dev/null +++ b/tools/visual-oracles/src/commands/compare.ts @@ -0,0 +1,19 @@ +import { defineCommand } from 'citty' + +import { operationCommand } from './operation' + +export default defineCommand({ + meta: { name: 'compare', description: 'Compare Figma and OpenPencil rendering' }, + subCommands: { + node: operationCommand( + 'node', + 'Compare one Figma node or clipboard selection', + 'tools/visual-oracles/src/operations/compare-node.ts' + ), + document: operationCommand( + 'document', + 'Compare exact imported-document targets from a manifest', + 'tools/visual-oracles/src/operations/compare-document.ts' + ) + } +}) diff --git a/tools/visual-oracles/src/commands/operation.ts b/tools/visual-oracles/src/commands/operation.ts new file mode 100644 index 000000000..ac65aa428 --- /dev/null +++ b/tools/visual-oracles/src/commands/operation.ts @@ -0,0 +1,16 @@ +import { defineCommand } from 'citty' + +export function operationCommand(name: string, description: string, entrypoint: string) { + return defineCommand({ + meta: { name, description }, + async run({ rawArgs }) { + const process = Bun.spawn(['bun', entrypoint, ...rawArgs], { + stdin: 'inherit', + stdout: 'inherit', + stderr: 'inherit' + }) + const exitCode = await process.exited + if (exitCode !== 0) throw new Error(`${name} exited with code ${exitCode}`) + } + }) +} diff --git a/tools/visual-oracles/src/document/manifest.ts b/tools/visual-oracles/src/document/manifest.ts new file mode 100644 index 000000000..b79bf47ca --- /dev/null +++ b/tools/visual-oracles/src/document/manifest.ts @@ -0,0 +1,85 @@ +import { readFileSync } from 'node:fs' + +export interface VisualOracleTarget { + page: string + node: string + figmaNodeId: string + openPencilNodeId?: string + scale?: number + fuzz?: string + maximumDifferentPercent?: number + expectedWidth?: number + expectedHeight?: number + minimumPageRoots?: number +} + +export interface VisualOracleManifest { + document: string + appURL: string + output?: string + targets: VisualOracleTarget[] +} + +const OPTIONAL_STRING_FIELDS = ['openPencilNodeId', 'fuzz'] as const +const OPTIONAL_NUMBER_FIELDS = [ + 'scale', + 'maximumDifferentPercent', + 'expectedWidth', + 'expectedHeight', + 'minimumPageRoots' +] as const + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value) +} + +function requiredString(record: Record, field: string, context: string): string { + const value = record[field] + if (typeof value !== 'string' || value.trim() === '') { + throw new Error(`${context} requires a non-empty ${field} string`) + } + return value +} + +function parseTarget(value: unknown, index: number): VisualOracleTarget { + if (!isRecord(value)) throw new Error(`Visual oracle target ${index} must be an object`) + const target: VisualOracleTarget = { + page: requiredString(value, 'page', `Visual oracle target ${index}`), + node: requiredString(value, 'node', `Visual oracle target ${index}`), + figmaNodeId: requiredString(value, 'figmaNodeId', `Visual oracle target ${index}`) + } + for (const field of OPTIONAL_STRING_FIELDS) { + const fieldValue = value[field] + if (fieldValue === undefined) continue + if (typeof fieldValue !== 'string') throw new TypeError(`${field} must be a string`) + target[field] = fieldValue + } + for (const field of OPTIONAL_NUMBER_FIELDS) { + const fieldValue = value[field] + if (fieldValue === undefined) continue + if (typeof fieldValue !== 'number' || !Number.isFinite(fieldValue)) { + throw new TypeError(`${field} must be a finite number`) + } + target[field] = fieldValue + } + return target +} + +export function readVisualOracleManifest(path: string): VisualOracleManifest { + const parsed: unknown = JSON.parse(readFileSync(path, 'utf8')) + if (!isRecord(parsed)) throw new TypeError('Visual oracle manifest must be an object') + const targets = parsed.targets + if (!Array.isArray(targets)) { + throw new TypeError('Visual oracle manifest requires a targets array') + } + const manifest: VisualOracleManifest = { + document: requiredString(parsed, 'document', 'Visual oracle manifest'), + appURL: requiredString(parsed, 'appURL', 'Visual oracle manifest'), + targets: targets.map(parseTarget) + } + if (parsed.output !== undefined) { + if (typeof parsed.output !== 'string') throw new TypeError('output must be a string') + manifest.output = parsed.output + } + return manifest +} diff --git a/tools/visual-oracles/src/pixel-image.ts b/tools/visual-oracles/src/image/pixel-image.ts similarity index 100% rename from tools/visual-oracles/src/pixel-image.ts rename to tools/visual-oracles/src/image/pixel-image.ts diff --git a/tools/visual-oracles/src/analyze-pattern.ts b/tools/visual-oracles/src/operations/analyze-pattern.ts similarity index 99% rename from tools/visual-oracles/src/analyze-pattern.ts rename to tools/visual-oracles/src/operations/analyze-pattern.ts index bf09908a9..359a8a866 100644 --- a/tools/visual-oracles/src/analyze-pattern.ts +++ b/tools/visual-oracles/src/operations/analyze-pattern.ts @@ -4,7 +4,7 @@ import { parseArgs } from 'node:util' import { initCanvasKit } from '@open-pencil/core/io' -import type { PixelImage } from './pixel-image' +import type { PixelImage } from '../image/pixel-image' interface Component { count: number diff --git a/tools/visual-oracles/src/analyze-text-decoration.ts b/tools/visual-oracles/src/operations/analyze-text-decoration.ts similarity index 98% rename from tools/visual-oracles/src/analyze-text-decoration.ts rename to tools/visual-oracles/src/operations/analyze-text-decoration.ts index e8e333459..4f8d9f570 100644 --- a/tools/visual-oracles/src/analyze-text-decoration.ts +++ b/tools/visual-oracles/src/operations/analyze-text-decoration.ts @@ -4,7 +4,7 @@ import { parseArgs } from 'node:util' import { initCanvasKit } from '@open-pencil/core/io' -import type { PixelImage } from './pixel-image' +import type { PixelImage } from '../image/pixel-image' interface ChannelSummary { rows: Array<{ y: number; count: number; minX: number; maxX: number }> diff --git a/tools/visual-oracles/src/bisect.ts b/tools/visual-oracles/src/operations/bisect.ts similarity index 99% rename from tools/visual-oracles/src/bisect.ts rename to tools/visual-oracles/src/operations/bisect.ts index 1545a11a7..b7f7cf102 100644 --- a/tools/visual-oracles/src/bisect.ts +++ b/tools/visual-oracles/src/operations/bisect.ts @@ -48,7 +48,7 @@ const { values, positionals } = parseArgs({ const figPath = positionals[0] if (!figPath || !values.page || !values['figma-page-id']) { console.error(`Usage: - bun scripts/visual-bisect.ts --page Primitives --figma-page-id 1:22 [options] + bun tools/visual-oracles/src/cli.ts bisect --page Primitives --figma-page-id 1:22 [options] Options: --output DIR Output directory (default: /tmp/open-pencil-visual-bisect) diff --git a/tools/visual-oracles/src/operations/compare-document.ts b/tools/visual-oracles/src/operations/compare-document.ts new file mode 100644 index 000000000..5ba85625b --- /dev/null +++ b/tools/visual-oracles/src/operations/compare-document.ts @@ -0,0 +1,200 @@ +#!/usr/bin/env bun + +import { mkdirSync } from 'node:fs' +import { basename, resolve } from 'node:path' +import { parseArgs } from 'node:util' + +import { chromium, type Page } from '@playwright/test' +import { $ } from 'bun' + +import { readVisualOracleManifest, type VisualOracleTarget } from '../document/manifest' + +const { values, positionals } = parseArgs({ + allowPositionals: true, + options: { + output: { type: 'string', short: 'o' }, + headed: { type: 'boolean', default: false }, + 'skip-figma-export': { type: 'boolean', default: false } + } +}) +const manifestPath = positionals[0] +if (!manifestPath) { + console.error( + 'Usage: bun tools/visual-oracles/src/cli.ts compare document manifest.json [--output DIR]' + ) + process.exit(1) +} +const manifest = readVisualOracleManifest(manifestPath) +const outputDir = resolve(values.output ?? manifest.output ?? '/tmp/open-pencil-document-oracle') +mkdirSync(outputDir, { recursive: true }) +const bytes = await Bun.file(resolve(manifest.document)).bytes() +const browser = await chromium.launch({ headless: !values.headed }) +const context = await browser.newContext({ + viewport: { width: 1280, height: 800 }, + deviceScaleFactor: 1, + ignoreHTTPSErrors: true +}) +const page = await context.newPage() +const documentPath = '/__visual-oracle-document.fig' +await page.route(`**${documentPath}`, (route) => + route.fulfill({ status: 200, contentType: 'application/octet-stream', body: bytes }) +) + +let failed = false +try { + await page.goto( + `${manifest.appURL}${manifest.appURL.includes('?') ? '&' : '?'}test&no-chrome&no-rulers&navigation-benchmark` + ) + await page.locator('[data-test-id="canvas-element"][data-ready="1"]').waitFor({ timeout: 30_000 }) + await page.evaluate((path) => window.openPencil?.openFile?.(path), documentPath) + await page.waitForFunction( + () => window.openPencil?.getStore?.().state.preparation == null, + undefined, + { + timeout: 120_000 + } + ) + + for (const target of manifest.targets) { + const result = await compareTarget(page, target) + console.log(`${result.passed ? 'PASS' : 'FAIL'} ${target.page} / ${target.node}`) + console.log(` ${result.differentPercent.toFixed(3)}% pixels differ (${result.fuzz} fuzz)`) + if (!result.passed) failed = true + } +} finally { + await browser.close() +} +if (failed) process.exit(1) + +async function compareTarget(page: Page, target: VisualOracleTarget) { + console.log(`Comparing ${target.page} / ${target.node}`) + const stem = `${sanitize(target.page)}--${sanitize(target.node)}` + const targetDir = `${outputDir}/${stem}` + mkdirSync(targetDir, { recursive: true }) + const figmaPath = `${targetDir}/figma.png` + const openPencilPath = `${targetDir}/openpencil.png` + const normalizedPath = `${targetDir}/openpencil-normalized.png` + const diffPath = `${targetDir}/diff.png` + const heatmapPath = `${targetDir}/heatmap.png` + const scale = target.scale ?? 1 + if (!values['skip-figma-export']) { + await $`figma-use export node ${target.figmaNodeId} --output ${figmaPath} --scale ${String(scale)}`.quiet() + } + + const metadata = await captureOpenPencilTarget(page, target, openPencilPath, scale) + if (metadata.pageRoots < (target.minimumPageRoots ?? 1)) { + throw new Error( + `${target.page} has ${metadata.pageRoots} roots; expected at least ${target.minimumPageRoots ?? 1}` + ) + } + if (target.expectedWidth !== undefined && metadata.width !== target.expectedWidth) { + throw new Error(`${target.node} width ${metadata.width}; expected ${target.expectedWidth}`) + } + if (target.expectedHeight !== undefined && metadata.height !== target.expectedHeight) { + throw new Error(`${target.node} height ${metadata.height}; expected ${target.expectedHeight}`) + } + + const figmaSize = await imageSize(figmaPath) + const openPencilSize = await imageSize(openPencilPath) + const comparePath = figmaSize === openPencilSize ? openPencilPath : normalizedPath + if (comparePath === normalizedPath) { + await $`magick ${openPencilPath} -background none -gravity northwest -extent ${figmaSize} ${normalizedPath}`.quiet() + } + const fuzz = target.fuzz ?? '2%' + const difference = + await $`magick compare -metric AE -fuzz ${fuzz} ${figmaPath} ${comparePath} null:` + .quiet() + .nothrow() + const pixels = Number.parseInt(difference.stderr.toString().trim(), 10) || 0 + const [width, height] = figmaSize.split('x').map(Number) + const total = width * height + const differentPercent = total > 0 ? (pixels / total) * 100 : 100 + await $`magick compare -highlight-color red -lowlight-color '#FFFFFF22' ${figmaPath} ${comparePath} ${diffPath}` + .quiet() + .nothrow() + await $`magick ${diffPath} -filter point -resize 400% ${heatmapPath}`.quiet() + const maximum = target.maximumDifferentPercent ?? 10 + const metrics = { + target, + figmaSize, + openPencilSize, + metadata, + fuzz, + differentPixels: pixels, + differentPercent, + maximumDifferentPercent: maximum, + passed: differentPercent <= maximum + } + await Bun.write(`${targetDir}/metrics.json`, `${JSON.stringify(metrics, null, 2)}\n`) + return metrics +} + +async function captureOpenPencilTarget( + page: Page, + target: VisualOracleTarget, + outputPath: string, + scale: number +) { + const metadata = await page.evaluate( + async ({ target, scale }) => { + const store = window.openPencil?.getStore?.() + if (!store) throw new Error('OpenPencil store unavailable') + const pageNode = store.graph.getPages().find((candidate) => candidate.name === target.page) + if (!pageNode) throw new Error(`OpenPencil page missing: ${target.page}`) + await store.switchPage(pageNode.id) + const pageRoots = store.graph.getChildren(pageNode.id) + const node = target.openPencilNodeId + ? store.graph.getNode(target.openPencilNodeId) + : store.graph + .getAllNodes() + .find( + (candidate) => candidate.name === target.node && candidate.parentId === pageNode.id + ) + if (!node || node.parentId !== pageNode.id) { + throw new Error(`OpenPencil node missing from page: ${target.page} / ${target.node}`) + } + const position = store.graph.getAbsolutePosition(node.id) + const viewportWidth = 1280 + const viewportHeight = 800 + store.state.zoom = scale + store.state.panX = viewportWidth / 2 - (position.x + node.width / 2) * scale + store.state.panY = viewportHeight / 2 - (position.y + node.height / 2) * scale + store.requestRepaint() + await new Promise((resolveFrame) => { + requestAnimationFrame(() => { + requestAnimationFrame(() => resolveFrame()) + }) + }) + await window.openPencil?.test?.navigation?.waitForSettlement() + return { + pageId: pageNode.id, + nodeId: node.id, + pageRoots: pageRoots.length, + width: node.width, + height: node.height, + clip: { + x: viewportWidth / 2 - (node.width * scale) / 2, + y: viewportHeight / 2 - (node.height * scale) / 2, + width: node.width * scale, + height: node.height * scale + } + } + }, + { target, scale } + ) + await page.screenshot({ path: outputPath, clip: metadata.clip }) + return metadata +} + +async function imageSize(path: string): Promise { + return (await $`identify -format '%wx%h' ${path}`.quiet()).text().trim() +} + +function sanitize(value: string): string { + return value + .replaceAll(/[^A-Za-z0-9]+/g, '-') + .replaceAll(/^-|-$/g, '') + .toLowerCase() +} + +console.log(`Compared ${manifest.targets.length} targets from ${basename(manifest.document)}`) diff --git a/tools/visual-oracles/src/compare.ts b/tools/visual-oracles/src/operations/compare-node.ts similarity index 98% rename from tools/visual-oracles/src/compare.ts rename to tools/visual-oracles/src/operations/compare-node.ts index 95dfee146..b5ddcf2f4 100644 --- a/tools/visual-oracles/src/compare.ts +++ b/tools/visual-oracles/src/operations/compare-node.ts @@ -3,10 +3,10 @@ * Visual comparison pipeline: Figma vs OpenPencil renderer. * * Copy an element in Figma, then run: - * bun scripts/visual-compare.ts [--scale 2] [--output /tmp/visual-compare] + * bun tools/visual-oracles/src/cli.ts compare node [--scale 2] [--output /tmp/visual-compare] * * Or pass a node ID directly (skips clipboard): - * bun scripts/visual-compare.ts --node 1:23 [--scale 2] + * bun tools/visual-oracles/src/cli.ts compare node --node 1:23 [--scale 2] * * Outputs: * figma.png — exported from real Figma diff --git a/tools/visual-oracles/src/export-fixtures.ts b/tools/visual-oracles/src/operations/export-fixtures.ts similarity index 100% rename from tools/visual-oracles/src/export-fixtures.ts rename to tools/visual-oracles/src/operations/export-fixtures.ts diff --git a/tools/visual-oracles/src/update-report.ts b/tools/visual-oracles/src/operations/update-report.ts similarity index 100% rename from tools/visual-oracles/src/update-report.ts rename to tools/visual-oracles/src/operations/update-report.ts diff --git a/tools/visual-oracles/tests/cli.test.ts b/tools/visual-oracles/tests/cli.test.ts new file mode 100644 index 000000000..9071487fb --- /dev/null +++ b/tools/visual-oracles/tests/cli.test.ts @@ -0,0 +1,14 @@ +import { describe, expect, test } from 'bun:test' + +import { renderUsage } from 'citty' + +import compareCommand from '../src/commands/compare' + +describe('visual oracle CLI', () => { + test('groups node and document comparison under one command', async () => { + const usage = await renderUsage(compareCommand) + + expect(usage).toContain('node') + expect(usage).toContain('document') + }) +}) diff --git a/tools/visual-oracles/tests/document/manifest.test.ts b/tools/visual-oracles/tests/document/manifest.test.ts new file mode 100644 index 000000000..c61b7cb81 --- /dev/null +++ b/tools/visual-oracles/tests/document/manifest.test.ts @@ -0,0 +1,57 @@ +import { afterEach, describe, expect, test } from 'bun:test' +import { rmSync, writeFileSync } from 'node:fs' + +import { readVisualOracleManifest } from '#visual/manifest' + +const path = '/tmp/open-pencil-visual-oracle-manifest-test.json' + +afterEach(() => rmSync(path, { force: true })) + +describe('document visual oracle manifest', () => { + test('reads document targets and visual thresholds', () => { + writeFileSync( + path, + JSON.stringify({ + document: '/tmp/design.fig', + appURL: 'http://localhost:1420/', + targets: [ + { + page: 'Foundations', + node: 'Effects', + figmaNodeId: '1:2', + expectedWidth: 2080, + minimumPageRoots: 6 + } + ] + }) + ) + + expect(readVisualOracleManifest(path).targets[0]).toMatchObject({ + page: 'Foundations', + expectedWidth: 2080, + minimumPageRoots: 6 + }) + }) + + test('rejects malformed required and optional fields', () => { + writeFileSync( + path, + JSON.stringify({ + document: '/tmp/design.fig', + appURL: 42, + targets: [null] + }) + ) + expect(() => readVisualOracleManifest(path)).toThrow('appURL') + + writeFileSync( + path, + JSON.stringify({ + document: '/tmp/design.fig', + appURL: 'http://localhost:1420/', + targets: [{ page: 'Foundations', node: 'Effects', figmaNodeId: '1:2', scale: '2' }] + }) + ) + expect(() => readVisualOracleManifest(path)).toThrow('scale') + }) +})