diff --git a/packages/pen-ai-skills/src/__tests__/detectors-typography.test.ts b/packages/pen-ai-skills/src/__tests__/detectors-typography.test.ts index ad652740c..7a2853803 100644 --- a/packages/pen-ai-skills/src/__tests__/detectors-typography.test.ts +++ b/packages/pen-ai-skills/src/__tests__/detectors-typography.test.ts @@ -52,15 +52,32 @@ describe('detectTextBgContrast', () => { expect(detectTextBgContrast(root, emptyDoc)).toHaveLength(0); }); - it('flags light-gray text on white (ratio ~3.9 < AA 4.5)', () => { - const root = frame('page', [text('t1', solid('#888888'))], solid('#FFFFFF')); + it('flags very-low-contrast gray on white (ratio ~2.1 < default 2.5)', () => { + // 2026-05-10 calibration — was #888888 (ratio 3.95) when threshold was + // WCAG AA 4.5; corpus replay showed the strict threshold flagging + // industry-standard caption patterns. Lowered to 2.5 normal / 2.0 large + // so only genuinely-broken contrast trips. #B0B0B0 is the lighter + // boundary still failing 2.5. + const root = frame('page', [text('t1', solid('#B0B0B0'))], solid('#FFFFFF')); const issues = detectTextBgContrast(root, emptyDoc); expect(issues).toHaveLength(1); expect(issues[0].nodeId).toBe('t1'); expect(issues[0].category).toBe('text-bg-contrast'); expect(issues[0].severity).toBe('info'); expect(issues[0].suggestedValue).toBeNull(); - expect(issues[0].reason).toMatch(/below WCAG AA/); + expect(issues[0].reason).toMatch(/below 2\.5:1/); + }); + + it('does NOT flag Tailwind slate-400 captions on white (ratio ~2.56 — intentional tertiary text)', () => { + // The 2026-05-08 corpus replay was 43% noise because WCAG-AA strict + // flagged this pattern. New threshold tolerates it. + const root = frame('page', [text('t1', solid('#94A3B8'))], solid('#FFFFFF')); + expect(detectTextBgContrast(root, emptyDoc)).toHaveLength(0); + }); + + it('does NOT flag Tailwind blue-600 chips on blue-100 (ratio ~4.24 — chip pattern)', () => { + const root = frame('page', [text('t1', solid('#2563EB'))], solid('#DBEAFE')); + expect(detectTextBgContrast(root, emptyDoc)).toHaveLength(0); }); it('flags white text on white bg (ratio 1.0 — invisible)', () => { @@ -85,22 +102,31 @@ describe('detectTextBgContrast', () => { expect(detectTextBgContrast(root, emptyDoc)).toHaveLength(1); }); - it('uses the LARGE-text threshold (3.0) for fontSize >= 24', () => { - // Ratio ~3.5 — fails normal 4.5 but passes large 3.0 - const root = frame('page', [text('t1', solid('#787878'), 32)], solid('#FFFFFF')); + it('uses the LARGE-text threshold (2.0) for fontSize >= 24', () => { + // #B0B0B0 on white = ratio ~2.13 — fails normal 2.5 but passes large 2.0 + const root = frame('page', [text('t1', solid('#B0B0B0'), 32)], solid('#FFFFFF')); expect(detectTextBgContrast(root, emptyDoc)).toHaveLength(0); }); - it('uses the LARGE-text threshold (3.0) for fontSize >= 19 + bold weight', () => { - const root = frame('page', [text('t1', solid('#787878'), 20, 700)], solid('#FFFFFF')); + it('uses the LARGE-text threshold (2.0) for fontSize >= 19 + bold weight', () => { + const root = frame('page', [text('t1', solid('#B0B0B0'), 20, 700)], solid('#FFFFFF')); expect(detectTextBgContrast(root, emptyDoc)).toHaveLength(0); }); it('still flags >=19px non-bold text (large rule needs 700+ weight)', () => { - const root = frame('page', [text('t1', solid('#888888'), 20, 400)], solid('#FFFFFF')); + // Non-bold large text uses the NORMAL threshold (2.5); 2.13 < 2.5 → flag. + const root = frame('page', [text('t1', solid('#B0B0B0'), 20, 400)], solid('#FFFFFF')); expect(detectTextBgContrast(root, emptyDoc)).toHaveLength(1); }); + it('honors caller-supplied opts.normalThreshold to enforce stricter audits', () => { + // 2.56:1 (slate-400) is silenced by default 2.5 but should re-fire when + // a stricter audit asks for WCAG-AA 4.5. + const root = frame('page', [text('t1', solid('#94A3B8'))], solid('#FFFFFF')); + expect(detectTextBgContrast(root, emptyDoc)).toHaveLength(0); + expect(detectTextBgContrast(root, emptyDoc, { normalThreshold: 4.5 })).toHaveLength(1); + }); + it('walks ancestor chain to find first non-transparent bg', () => { // Outer page has cream; inner section has no fill (transparent); // the text's effective bg should still resolve to cream. @@ -123,13 +149,13 @@ describe('detectTextBgContrast', () => { it('resolves $variable refs through doc.variables / theme', () => { const doc = docWithVars({ - 'color-text': { type: 'color', value: '#888888' }, + 'color-text': { type: 'color', value: '#B0B0B0' }, 'color-bg': { type: 'color', value: '#FFFFFF' }, }); const root = frame('page', [text('t1', solid('$color-text'))], solid('$color-bg')); const issues = detectTextBgContrast(root, doc); expect(issues).toHaveLength(1); - expect(issues[0].reason).toMatch(/text=#888888 on bg=#FFFFFF/); + expect(issues[0].reason).toMatch(/text=#B0B0B0 on bg=#FFFFFF/); }); it('skips text whose color ref does not resolve (no false positive)', () => { diff --git a/packages/pen-ai-skills/src/diagnostics/detectors-typography.ts b/packages/pen-ai-skills/src/diagnostics/detectors-typography.ts index 998e6a796..ee1de8891 100644 --- a/packages/pen-ai-skills/src/diagnostics/detectors-typography.ts +++ b/packages/pen-ai-skills/src/diagnostics/detectors-typography.ts @@ -3,10 +3,35 @@ import { resolveColorRef, getDefaultTheme } from '@zseven-w/pen-core'; import type { Issue } from './types'; import { colorContrast } from './color-utils'; -/** WCAG 2.x AA threshold for normal-size text. */ -const WCAG_AA_NORMAL = 4.5; -/** WCAG 2.x AA threshold for large text (>= 18pt or >= 14pt bold). */ -const WCAG_AA_LARGE = 3.0; +/** + * Default contrast thresholds — looser than WCAG 2.x AA on purpose. + * + * 2026-05-10 calibration against `2026-05-08-rank4-gpt55` corpus (104 real + * GPT-5.5 dashboard outputs, 95 applied successfully): + * + * - WCAG-AA strict (4.5/3.0) → 41 hits, ~43% of designs flagged. + * Most hits were industry-standard Tailwind palettes used as + * intentional tertiary text (slate-400 captions on white = 2.56:1, + * blue-600 chips on blue-100 = 4.24:1, emerald-500 deltas on white + * = 2.54:1). These pass design review at Linear / Vercel / Notion / + * GitHub and feel like false positives to the user. + * - 2.5/2.0 → expected ~5 hits. Catches the user's reported pain + * (white-on-cream = 1.10:1, white-on-white = 1.0:1, white-on-near- + * white) without flagging muted-caption patterns. + * + * This detector reports physical readability, not WCAG compliance. + * Callers can override via opts.normalThreshold / opts.largeThreshold + * if they want stricter audits without re-implementing the walk. + */ +const DEFAULT_NORMAL_THRESHOLD = 2.5; +const DEFAULT_LARGE_THRESHOLD = 2.0; + +export interface DetectTextBgContrastOptions { + /** Contrast ratio below which normal-size text is flagged. Default 2.5. */ + normalThreshold?: number; + /** Contrast ratio below which large text (>=24px or >=19px bold) is flagged. Default 2.0. */ + largeThreshold?: number; +} /** * Pull the first solid color out of a fill array. Gradients get reduced to @@ -73,10 +98,16 @@ function isLargeText(node: PenNode): boolean { * audit panel and chat status line so the user / agent can decide, * without silently rewriting their fills. */ -export function detectTextBgContrast(root: PenNode, doc: PenDocument): Issue[] { +export function detectTextBgContrast( + root: PenNode, + doc: PenDocument, + opts: DetectTextBgContrastOptions = {}, +): Issue[] { const issues: Issue[] = []; const variables = doc.variables ?? {}; const theme = getDefaultTheme(doc.themes); + const normalThreshold = opts.normalThreshold ?? DEFAULT_NORMAL_THRESHOLD; + const largeThreshold = opts.largeThreshold ?? DEFAULT_LARGE_THRESHOLD; walk(root, []); return issues; @@ -117,7 +148,7 @@ export function detectTextBgContrast(root: PenNode, doc: PenDocument): Issue[] { const ratio = colorContrast(textColor, bgColor); if (!Number.isFinite(ratio)) return; // either color failed to parse - const threshold = isLargeText(node) ? WCAG_AA_LARGE : WCAG_AA_NORMAL; + const threshold = isLargeText(node) ? largeThreshold : normalThreshold; if (ratio >= threshold) return; issues.push({ @@ -131,7 +162,7 @@ export function detectTextBgContrast(root: PenNode, doc: PenDocument): Issue[] { // No suggestedValue: the right replacement depends on the design // system + theme + intent, which only the user/agent can decide. suggestedValue: null, - reason: `text/bg contrast ${ratio.toFixed(2)}:1 below WCAG AA ${threshold}:1 (text=${textColor} on bg=${bgColor})`, + reason: `text/bg contrast ${ratio.toFixed(2)}:1 below ${threshold}:1 (text=${textColor} on bg=${bgColor})`, }); } } diff --git a/scripts/ab-corpus/inspect-contrast-hits.ts b/scripts/ab-corpus/inspect-contrast-hits.ts new file mode 100644 index 000000000..ae5410148 --- /dev/null +++ b/scripts/ab-corpus/inspect-contrast-hits.ts @@ -0,0 +1,56 @@ +/** + * For a given run, list every text-bg-contrast hit with the failing + * text color, resolved bg color, and ratio. Helps eyeball whether the + * detector is catching real contrast violations or over-firing because + * the apply-to-fresh-doc harness has no page-level fill. + */ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { detectTextBgContrast, parseModelOutput, type Issue } from '@zseven-w/pen-ai-skills'; +import { applyToFreshDoc } from './apply'; + +interface JsonlRow { + promptId: string; + category: string; + difficulty: string; + variant: string; + rawOutput: string; +} + +async function main(): Promise { + const runId = process.argv[2]; + const limit = Number(process.argv[3] ?? 50); + if (!runId) { + console.error('usage: bun run scripts/ab-corpus/inspect-contrast-hits.ts [limit]'); + process.exit(1); + } + const path = join(import.meta.dir, 'runs', runId, 'scores.jsonl'); + const lines = readFileSync(path, 'utf-8').split('\n').filter(Boolean); + const rows: JsonlRow[] = lines.map((l) => JSON.parse(l)); + + let shown = 0; + for (const r of rows) { + if (shown >= limit) break; + const parsed = parseModelOutput(r.rawOutput); + if (parsed.kind === 'garbage') continue; + const result = await applyToFreshDoc(parsed); + if (!result.ok || !result.doc) continue; + const root = result.doc.children?.[0] ?? null; + if (!root) continue; + const issues: Issue[] = detectTextBgContrast(root, result.doc); + if (issues.length === 0) continue; + console.log( + `\n=== ${r.promptId} [${r.category}/${r.difficulty}/${r.variant}] (${issues.length} hits) ===`, + ); + for (const issue of issues) { + // reason looks like: "text/bg contrast 3.05:1 below WCAG AA 4.5:1 (text=#... on bg=#...)" + console.log(` ${issue.nodeId.padEnd(50)} ${issue.reason}`); + } + shown++; + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/scripts/ab-corpus/replay-detectors.ts b/scripts/ab-corpus/replay-detectors.ts new file mode 100644 index 000000000..b1b6eed48 --- /dev/null +++ b/scripts/ab-corpus/replay-detectors.ts @@ -0,0 +1,113 @@ +/** + * Replay an existing scores.jsonl through the CURRENT detector set. + * + * 2026-05-10 — used to validate the post-2026-05-08 detector additions + * (detectEdgeSectionPadding + detectTextBgContrast) against real GPT-5.5 + * output without burning fresh API tokens. Reads each row's rawOutput, + * re-applies it to a fresh doc, runs detectAllIssues with today's 13 + * detectors, and reports per-category counts so we can spot detectors + * that now over-fire (false-positive prone) or stay silent. + * + * Usage: bun run scripts/ab-corpus/replay-detectors.ts + */ +import { readFileSync, existsSync } from 'node:fs'; +import { join } from 'node:path'; +import { detectAllIssues, parseModelOutput, type Issue } from '@zseven-w/pen-ai-skills'; +import { applyToFreshDoc } from './apply'; + +interface JsonlRow { + promptId: string; + category: string; + difficulty: string; + model: string; + variant: string; + rawOutput: string; + issues: Issue[]; +} + +async function main(): Promise { + const runId = process.argv[2]; + if (!runId) { + console.error('usage: bun run scripts/ab-corpus/replay-detectors.ts '); + process.exit(1); + } + const path = join(import.meta.dir, 'runs', runId, 'scores.jsonl'); + if (!existsSync(path)) { + console.error(`scores.jsonl not found: ${path}`); + process.exit(1); + } + + const lines = readFileSync(path, 'utf-8').split('\n').filter(Boolean); + const rows: JsonlRow[] = lines.map((l) => JSON.parse(l)); + console.log(`replay: ${rows.length} rows from ${runId}`); + + let appliedOk = 0; + let appliedErr = 0; + const oldByCategory = new Map(); + const newByCategory = new Map(); + const sampleByCategory = new Map(); + + for (const r of rows) { + for (const issue of r.issues ?? []) { + oldByCategory.set(issue.category, (oldByCategory.get(issue.category) ?? 0) + 1); + } + } + + for (const r of rows) { + const parsed = parseModelOutput(r.rawOutput); + if (parsed.kind === 'garbage') { + appliedErr++; + continue; + } + const result = await applyToFreshDoc(parsed); + if (!result.ok || !result.doc) { + appliedErr++; + continue; + } + appliedOk++; + + const root = result.doc.children?.[0] ?? null; + if (!root) continue; + const issues = detectAllIssues(root, result.doc); + for (const issue of issues) { + newByCategory.set(issue.category, (newByCategory.get(issue.category) ?? 0) + 1); + const samples = sampleByCategory.get(issue.category) ?? []; + if (samples.length < 3) { + samples.push(r); + sampleByCategory.set(issue.category, samples); + } + } + } + + console.log(`\napplied: ok=${appliedOk} err=${appliedErr}`); + const allCategories = new Set([...oldByCategory.keys(), ...newByCategory.keys()]); + console.log('\nper-category counts (old → new):'); + console.log('category old new delta'); + console.log('--------------------------------- ----- ----- -------'); + const sorted = Array.from(allCategories).sort(); + for (const cat of sorted) { + const o = oldByCategory.get(cat) ?? 0; + const n = newByCategory.get(cat) ?? 0; + const delta = n - o; + const sign = delta > 0 ? '+' : ''; + console.log( + `${cat.padEnd(33)} ${String(o).padStart(5)} ${String(n).padStart(5)} ${sign}${String(delta).padStart(5)}`, + ); + } + + const newCats = Array.from(newByCategory.keys()).filter((c) => !oldByCategory.has(c)); + if (newCats.length > 0) { + console.log('\nfirst-time-firing detectors (worth spot-checking):'); + for (const cat of newCats) { + console.log(`\n ${cat} (${newByCategory.get(cat)} hits)`); + for (const sample of sampleByCategory.get(cat) ?? []) { + console.log(` - ${sample.promptId} [${sample.category}/${sample.difficulty}]`); + } + } + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +});