fix(ai): retune contrast thresholds to 2.5/2.0 — kill 35/41 false positives
Replayed the 2026-05-08-rank4-gpt55 corpus (104 GPT-5.5 dashboard
outputs, 95 applied) through the new detectTextBgContrast and got
41 hits — 43% of designs flagged. Sampling showed almost all of them
were industry-standard Tailwind palettes used as intentional tertiary
text:
- #94A3B8 (slate-400) caption on #FFFFFF, ratio 2.56 ← Linear/Vercel/Notion
- #2563EB (blue-600) chip on #DBEAFE, ratio 4.24 ← shadcn/ui tag pattern
- #10B981 (emerald-500) delta on #FFFFFF, ratio 2.54 ← stat-positive pattern
- #64748B (slate-500) row text on #F1F5F9, ratio 4.34 ← muted-row pattern
WCAG-AA 4.5:1 is a compliance threshold, not a design-diagnosis
threshold. The user-reported pain point is "white-on-cream" (1.10:1)
and "white-on-white" (1.0:1) — disasters that read as obviously broken
to anyone, not borderline-WCAG cases that production designers ship
on purpose.
Drop default normalThreshold to 2.5 and largeThreshold to 2.0. Open
both as opts so callers needing a stricter audit (e.g. compliance
report) can bring back WCAG-AA without re-implementing the walk.
Replay confirms the new thresholds:
- 41 hits → 6 hits (signal-to-noise from 50% to 0% on the sample)
- All 6 remaining are true positives:
* 3 × slate-400 on slate-100 (caption color used on a non-white
bg — designer mis-paired the palette)
* 3 × white initial on amber-500 avatar (the readability gap the
industry routinely ignores; legitimately worth flagging)
Codex review (a47ef892f72a2d315) confirmed the direction, the
specific numeric pair (2.5 not 3.0 — 3.0 still hits slate-400 at 2.56),
parameterization over a mode-flag, and keeping severity at info-only.
Side-quest: scripts/ab-corpus/replay-detectors.ts +
inspect-contrast-hits.ts ship with this commit so the next detector
calibration doesn't have to rebuild the harness from scratch.
This commit is contained in:
Fini
parent
bf4273f328
commit
761c5202e2
|
|
@ -52,15 +52,32 @@ describe('detectTextBgContrast', () => {
|
|||
expect(detectTextBgContrast(root, emptyDoc)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('flags light-gray text on white (ratio ~3.9 < AA 4.5)', () => {
|
||||
const root = frame('page', [text('t1', solid('#888888'))], solid('#FFFFFF'));
|
||||
it('flags very-low-contrast gray on white (ratio ~2.1 < default 2.5)', () => {
|
||||
// 2026-05-10 calibration — was #888888 (ratio 3.95) when threshold was
|
||||
// WCAG AA 4.5; corpus replay showed the strict threshold flagging
|
||||
// industry-standard caption patterns. Lowered to 2.5 normal / 2.0 large
|
||||
// so only genuinely-broken contrast trips. #B0B0B0 is the lighter
|
||||
// boundary still failing 2.5.
|
||||
const root = frame('page', [text('t1', solid('#B0B0B0'))], solid('#FFFFFF'));
|
||||
const issues = detectTextBgContrast(root, emptyDoc);
|
||||
expect(issues).toHaveLength(1);
|
||||
expect(issues[0].nodeId).toBe('t1');
|
||||
expect(issues[0].category).toBe('text-bg-contrast');
|
||||
expect(issues[0].severity).toBe('info');
|
||||
expect(issues[0].suggestedValue).toBeNull();
|
||||
expect(issues[0].reason).toMatch(/below WCAG AA/);
|
||||
expect(issues[0].reason).toMatch(/below 2\.5:1/);
|
||||
});
|
||||
|
||||
it('does NOT flag Tailwind slate-400 captions on white (ratio ~2.56 — intentional tertiary text)', () => {
|
||||
// The 2026-05-08 corpus replay was 43% noise because WCAG-AA strict
|
||||
// flagged this pattern. New threshold tolerates it.
|
||||
const root = frame('page', [text('t1', solid('#94A3B8'))], solid('#FFFFFF'));
|
||||
expect(detectTextBgContrast(root, emptyDoc)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('does NOT flag Tailwind blue-600 chips on blue-100 (ratio ~4.24 — chip pattern)', () => {
|
||||
const root = frame('page', [text('t1', solid('#2563EB'))], solid('#DBEAFE'));
|
||||
expect(detectTextBgContrast(root, emptyDoc)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('flags white text on white bg (ratio 1.0 — invisible)', () => {
|
||||
|
|
@ -85,22 +102,31 @@ describe('detectTextBgContrast', () => {
|
|||
expect(detectTextBgContrast(root, emptyDoc)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('uses the LARGE-text threshold (3.0) for fontSize >= 24', () => {
|
||||
// Ratio ~3.5 — fails normal 4.5 but passes large 3.0
|
||||
const root = frame('page', [text('t1', solid('#787878'), 32)], solid('#FFFFFF'));
|
||||
it('uses the LARGE-text threshold (2.0) for fontSize >= 24', () => {
|
||||
// #B0B0B0 on white = ratio ~2.13 — fails normal 2.5 but passes large 2.0
|
||||
const root = frame('page', [text('t1', solid('#B0B0B0'), 32)], solid('#FFFFFF'));
|
||||
expect(detectTextBgContrast(root, emptyDoc)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('uses the LARGE-text threshold (3.0) for fontSize >= 19 + bold weight', () => {
|
||||
const root = frame('page', [text('t1', solid('#787878'), 20, 700)], solid('#FFFFFF'));
|
||||
it('uses the LARGE-text threshold (2.0) for fontSize >= 19 + bold weight', () => {
|
||||
const root = frame('page', [text('t1', solid('#B0B0B0'), 20, 700)], solid('#FFFFFF'));
|
||||
expect(detectTextBgContrast(root, emptyDoc)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('still flags >=19px non-bold text (large rule needs 700+ weight)', () => {
|
||||
const root = frame('page', [text('t1', solid('#888888'), 20, 400)], solid('#FFFFFF'));
|
||||
// Non-bold large text uses the NORMAL threshold (2.5); 2.13 < 2.5 → flag.
|
||||
const root = frame('page', [text('t1', solid('#B0B0B0'), 20, 400)], solid('#FFFFFF'));
|
||||
expect(detectTextBgContrast(root, emptyDoc)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('honors caller-supplied opts.normalThreshold to enforce stricter audits', () => {
|
||||
// 2.56:1 (slate-400) is silenced by default 2.5 but should re-fire when
|
||||
// a stricter audit asks for WCAG-AA 4.5.
|
||||
const root = frame('page', [text('t1', solid('#94A3B8'))], solid('#FFFFFF'));
|
||||
expect(detectTextBgContrast(root, emptyDoc)).toHaveLength(0);
|
||||
expect(detectTextBgContrast(root, emptyDoc, { normalThreshold: 4.5 })).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('walks ancestor chain to find first non-transparent bg', () => {
|
||||
// Outer page has cream; inner section has no fill (transparent);
|
||||
// the text's effective bg should still resolve to cream.
|
||||
|
|
@ -123,13 +149,13 @@ describe('detectTextBgContrast', () => {
|
|||
|
||||
it('resolves $variable refs through doc.variables / theme', () => {
|
||||
const doc = docWithVars({
|
||||
'color-text': { type: 'color', value: '#888888' },
|
||||
'color-text': { type: 'color', value: '#B0B0B0' },
|
||||
'color-bg': { type: 'color', value: '#FFFFFF' },
|
||||
});
|
||||
const root = frame('page', [text('t1', solid('$color-text'))], solid('$color-bg'));
|
||||
const issues = detectTextBgContrast(root, doc);
|
||||
expect(issues).toHaveLength(1);
|
||||
expect(issues[0].reason).toMatch(/text=#888888 on bg=#FFFFFF/);
|
||||
expect(issues[0].reason).toMatch(/text=#B0B0B0 on bg=#FFFFFF/);
|
||||
});
|
||||
|
||||
it('skips text whose color ref does not resolve (no false positive)', () => {
|
||||
|
|
|
|||
|
|
@ -3,10 +3,35 @@ import { resolveColorRef, getDefaultTheme } from '@zseven-w/pen-core';
|
|||
import type { Issue } from './types';
|
||||
import { colorContrast } from './color-utils';
|
||||
|
||||
/** WCAG 2.x AA threshold for normal-size text. */
|
||||
const WCAG_AA_NORMAL = 4.5;
|
||||
/** WCAG 2.x AA threshold for large text (>= 18pt or >= 14pt bold). */
|
||||
const WCAG_AA_LARGE = 3.0;
|
||||
/**
|
||||
* Default contrast thresholds — looser than WCAG 2.x AA on purpose.
|
||||
*
|
||||
* 2026-05-10 calibration against `2026-05-08-rank4-gpt55` corpus (104 real
|
||||
* GPT-5.5 dashboard outputs, 95 applied successfully):
|
||||
*
|
||||
* - WCAG-AA strict (4.5/3.0) → 41 hits, ~43% of designs flagged.
|
||||
* Most hits were industry-standard Tailwind palettes used as
|
||||
* intentional tertiary text (slate-400 captions on white = 2.56:1,
|
||||
* blue-600 chips on blue-100 = 4.24:1, emerald-500 deltas on white
|
||||
* = 2.54:1). These pass design review at Linear / Vercel / Notion /
|
||||
* GitHub and feel like false positives to the user.
|
||||
* - 2.5/2.0 → expected ~5 hits. Catches the user's reported pain
|
||||
* (white-on-cream = 1.10:1, white-on-white = 1.0:1, white-on-near-
|
||||
* white) without flagging muted-caption patterns.
|
||||
*
|
||||
* This detector reports physical readability, not WCAG compliance.
|
||||
* Callers can override via opts.normalThreshold / opts.largeThreshold
|
||||
* if they want stricter audits without re-implementing the walk.
|
||||
*/
|
||||
const DEFAULT_NORMAL_THRESHOLD = 2.5;
|
||||
const DEFAULT_LARGE_THRESHOLD = 2.0;
|
||||
|
||||
export interface DetectTextBgContrastOptions {
|
||||
/** Contrast ratio below which normal-size text is flagged. Default 2.5. */
|
||||
normalThreshold?: number;
|
||||
/** Contrast ratio below which large text (>=24px or >=19px bold) is flagged. Default 2.0. */
|
||||
largeThreshold?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pull the first solid color out of a fill array. Gradients get reduced to
|
||||
|
|
@ -73,10 +98,16 @@ function isLargeText(node: PenNode): boolean {
|
|||
* audit panel and chat status line so the user / agent can decide,
|
||||
* without silently rewriting their fills.
|
||||
*/
|
||||
export function detectTextBgContrast(root: PenNode, doc: PenDocument): Issue[] {
|
||||
export function detectTextBgContrast(
|
||||
root: PenNode,
|
||||
doc: PenDocument,
|
||||
opts: DetectTextBgContrastOptions = {},
|
||||
): Issue[] {
|
||||
const issues: Issue[] = [];
|
||||
const variables = doc.variables ?? {};
|
||||
const theme = getDefaultTheme(doc.themes);
|
||||
const normalThreshold = opts.normalThreshold ?? DEFAULT_NORMAL_THRESHOLD;
|
||||
const largeThreshold = opts.largeThreshold ?? DEFAULT_LARGE_THRESHOLD;
|
||||
|
||||
walk(root, []);
|
||||
return issues;
|
||||
|
|
@ -117,7 +148,7 @@ export function detectTextBgContrast(root: PenNode, doc: PenDocument): Issue[] {
|
|||
const ratio = colorContrast(textColor, bgColor);
|
||||
if (!Number.isFinite(ratio)) return; // either color failed to parse
|
||||
|
||||
const threshold = isLargeText(node) ? WCAG_AA_LARGE : WCAG_AA_NORMAL;
|
||||
const threshold = isLargeText(node) ? largeThreshold : normalThreshold;
|
||||
if (ratio >= threshold) return;
|
||||
|
||||
issues.push({
|
||||
|
|
@ -131,7 +162,7 @@ export function detectTextBgContrast(root: PenNode, doc: PenDocument): Issue[] {
|
|||
// No suggestedValue: the right replacement depends on the design
|
||||
// system + theme + intent, which only the user/agent can decide.
|
||||
suggestedValue: null,
|
||||
reason: `text/bg contrast ${ratio.toFixed(2)}:1 below WCAG AA ${threshold}:1 (text=${textColor} on bg=${bgColor})`,
|
||||
reason: `text/bg contrast ${ratio.toFixed(2)}:1 below ${threshold}:1 (text=${textColor} on bg=${bgColor})`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
56
scripts/ab-corpus/inspect-contrast-hits.ts
Normal file
56
scripts/ab-corpus/inspect-contrast-hits.ts
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
/**
|
||||
* For a given run, list every text-bg-contrast hit with the failing
|
||||
* text color, resolved bg color, and ratio. Helps eyeball whether the
|
||||
* detector is catching real contrast violations or over-firing because
|
||||
* the apply-to-fresh-doc harness has no page-level fill.
|
||||
*/
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { detectTextBgContrast, parseModelOutput, type Issue } from '@zseven-w/pen-ai-skills';
|
||||
import { applyToFreshDoc } from './apply';
|
||||
|
||||
interface JsonlRow {
|
||||
promptId: string;
|
||||
category: string;
|
||||
difficulty: string;
|
||||
variant: string;
|
||||
rawOutput: string;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const runId = process.argv[2];
|
||||
const limit = Number(process.argv[3] ?? 50);
|
||||
if (!runId) {
|
||||
console.error('usage: bun run scripts/ab-corpus/inspect-contrast-hits.ts <run-id> [limit]');
|
||||
process.exit(1);
|
||||
}
|
||||
const path = join(import.meta.dir, 'runs', runId, 'scores.jsonl');
|
||||
const lines = readFileSync(path, 'utf-8').split('\n').filter(Boolean);
|
||||
const rows: JsonlRow[] = lines.map((l) => JSON.parse(l));
|
||||
|
||||
let shown = 0;
|
||||
for (const r of rows) {
|
||||
if (shown >= limit) break;
|
||||
const parsed = parseModelOutput(r.rawOutput);
|
||||
if (parsed.kind === 'garbage') continue;
|
||||
const result = await applyToFreshDoc(parsed);
|
||||
if (!result.ok || !result.doc) continue;
|
||||
const root = result.doc.children?.[0] ?? null;
|
||||
if (!root) continue;
|
||||
const issues: Issue[] = detectTextBgContrast(root, result.doc);
|
||||
if (issues.length === 0) continue;
|
||||
console.log(
|
||||
`\n=== ${r.promptId} [${r.category}/${r.difficulty}/${r.variant}] (${issues.length} hits) ===`,
|
||||
);
|
||||
for (const issue of issues) {
|
||||
// reason looks like: "text/bg contrast 3.05:1 below WCAG AA 4.5:1 (text=#... on bg=#...)"
|
||||
console.log(` ${issue.nodeId.padEnd(50)} ${issue.reason}`);
|
||||
}
|
||||
shown++;
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
113
scripts/ab-corpus/replay-detectors.ts
Normal file
113
scripts/ab-corpus/replay-detectors.ts
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
/**
|
||||
* Replay an existing scores.jsonl through the CURRENT detector set.
|
||||
*
|
||||
* 2026-05-10 — used to validate the post-2026-05-08 detector additions
|
||||
* (detectEdgeSectionPadding + detectTextBgContrast) against real GPT-5.5
|
||||
* output without burning fresh API tokens. Reads each row's rawOutput,
|
||||
* re-applies it to a fresh doc, runs detectAllIssues with today's 13
|
||||
* detectors, and reports per-category counts so we can spot detectors
|
||||
* that now over-fire (false-positive prone) or stay silent.
|
||||
*
|
||||
* Usage: bun run scripts/ab-corpus/replay-detectors.ts <run-id>
|
||||
*/
|
||||
import { readFileSync, existsSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { detectAllIssues, parseModelOutput, type Issue } from '@zseven-w/pen-ai-skills';
|
||||
import { applyToFreshDoc } from './apply';
|
||||
|
||||
interface JsonlRow {
|
||||
promptId: string;
|
||||
category: string;
|
||||
difficulty: string;
|
||||
model: string;
|
||||
variant: string;
|
||||
rawOutput: string;
|
||||
issues: Issue[];
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const runId = process.argv[2];
|
||||
if (!runId) {
|
||||
console.error('usage: bun run scripts/ab-corpus/replay-detectors.ts <run-id>');
|
||||
process.exit(1);
|
||||
}
|
||||
const path = join(import.meta.dir, 'runs', runId, 'scores.jsonl');
|
||||
if (!existsSync(path)) {
|
||||
console.error(`scores.jsonl not found: ${path}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const lines = readFileSync(path, 'utf-8').split('\n').filter(Boolean);
|
||||
const rows: JsonlRow[] = lines.map((l) => JSON.parse(l));
|
||||
console.log(`replay: ${rows.length} rows from ${runId}`);
|
||||
|
||||
let appliedOk = 0;
|
||||
let appliedErr = 0;
|
||||
const oldByCategory = new Map<string, number>();
|
||||
const newByCategory = new Map<string, number>();
|
||||
const sampleByCategory = new Map<string, JsonlRow[]>();
|
||||
|
||||
for (const r of rows) {
|
||||
for (const issue of r.issues ?? []) {
|
||||
oldByCategory.set(issue.category, (oldByCategory.get(issue.category) ?? 0) + 1);
|
||||
}
|
||||
}
|
||||
|
||||
for (const r of rows) {
|
||||
const parsed = parseModelOutput(r.rawOutput);
|
||||
if (parsed.kind === 'garbage') {
|
||||
appliedErr++;
|
||||
continue;
|
||||
}
|
||||
const result = await applyToFreshDoc(parsed);
|
||||
if (!result.ok || !result.doc) {
|
||||
appliedErr++;
|
||||
continue;
|
||||
}
|
||||
appliedOk++;
|
||||
|
||||
const root = result.doc.children?.[0] ?? null;
|
||||
if (!root) continue;
|
||||
const issues = detectAllIssues(root, result.doc);
|
||||
for (const issue of issues) {
|
||||
newByCategory.set(issue.category, (newByCategory.get(issue.category) ?? 0) + 1);
|
||||
const samples = sampleByCategory.get(issue.category) ?? [];
|
||||
if (samples.length < 3) {
|
||||
samples.push(r);
|
||||
sampleByCategory.set(issue.category, samples);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\napplied: ok=${appliedOk} err=${appliedErr}`);
|
||||
const allCategories = new Set([...oldByCategory.keys(), ...newByCategory.keys()]);
|
||||
console.log('\nper-category counts (old → new):');
|
||||
console.log('category old new delta');
|
||||
console.log('--------------------------------- ----- ----- -------');
|
||||
const sorted = Array.from(allCategories).sort();
|
||||
for (const cat of sorted) {
|
||||
const o = oldByCategory.get(cat) ?? 0;
|
||||
const n = newByCategory.get(cat) ?? 0;
|
||||
const delta = n - o;
|
||||
const sign = delta > 0 ? '+' : '';
|
||||
console.log(
|
||||
`${cat.padEnd(33)} ${String(o).padStart(5)} ${String(n).padStart(5)} ${sign}${String(delta).padStart(5)}`,
|
||||
);
|
||||
}
|
||||
|
||||
const newCats = Array.from(newByCategory.keys()).filter((c) => !oldByCategory.has(c));
|
||||
if (newCats.length > 0) {
|
||||
console.log('\nfirst-time-firing detectors (worth spot-checking):');
|
||||
for (const cat of newCats) {
|
||||
console.log(`\n ${cat} (${newByCategory.get(cat)} hits)`);
|
||||
for (const sample of sampleByCategory.get(cat) ?? []) {
|
||||
console.log(` - ${sample.promptId} [${sample.category}/${sample.difficulty}]`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
Loading…
Reference in a new issue