chore(ab-corpus): add shape + per-category issue inspectors

Two complement scripts that ride alongside replay-detectors.ts:

- inspect-shape.ts: bucket every applied row's root by (width-bucket /
  height-bucket / aspect-ratio / child-count). Used 2026-05-10 to
  diagnose why detectEdgeSectionPadding scored 0 hits on a 220-row
  mobile subset — turned out 49% of mobile rows produce roots with a
  string-typed width ("fill_container" / "fit_content") because the
  element-tools path emits component fragments, not pages. The
  detector predicate `typeof width === 'number'` correctly skips them
  → 0 hits is the EVAL HARNESS coverage gap, not a detector bug.

- inspect-issue-category.ts: print every issue in a chosen category
  with row id + node id + reason. Generic version of inspect-contrast-
  hits.ts. 2026-05-10 used it to validate
  excessive-frame-effects (4/4 TP — blur 48 cards + OTP slot spread)
  and the two borderline mixed-sibling hits (header padding outlier,
  spacer cornerRadius outlier — known role-aware limitation, 0.2%
  noise rate, accepted).

Together with replay-detectors.ts these three give a fast empirical
loop for tuning a detector against real corpus output without burning
fresh API tokens.
This commit is contained in:
Fini 2026-05-10 14:50:00 +08:00
parent 761c5202e2
commit 31d45b79be
2 changed files with 138 additions and 0 deletions

View file

@ -0,0 +1,59 @@
/**
* Print every issue of a specific category across an entire run, with the
* row id, node id, and reason. Generic version of inspect-contrast-hits
* — used for spot-checking any single detector's true-positive vs
* false-positive rate against a real corpus.
*/
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
import {
detectAllIssues,
parseModelOutput,
type Issue,
type IssueCategory,
} from '@zseven-w/pen-ai-skills';
import { applyToFreshDoc } from './apply';
interface JsonlRow {
promptId: string;
category: string;
difficulty: string;
variant: string;
rawOutput: string;
}
async function main(): Promise<void> {
const runId = process.argv[2];
const wantCat = process.argv[3] as IssueCategory | undefined;
if (!runId || !wantCat) {
console.error(
'usage: bun run scripts/ab-corpus/inspect-issue-category.ts <run-id> <category>',
);
process.exit(1);
}
const path = join(import.meta.dir, 'runs', runId, 'scores.jsonl');
const rows: JsonlRow[] = readFileSync(path, 'utf-8')
.split('\n')
.filter(Boolean)
.map((l) => JSON.parse(l));
for (const r of rows) {
const parsed = parseModelOutput(r.rawOutput);
if (parsed.kind === 'garbage') continue;
const result = await applyToFreshDoc(parsed);
if (!result.ok || !result.doc) continue;
const root = result.doc.children?.[0] ?? null;
if (!root) continue;
const issues: Issue[] = detectAllIssues(root, result.doc).filter((i) => i.category === wantCat);
if (issues.length === 0) continue;
console.log(`\n=== ${r.promptId} [${r.category}/${r.difficulty}/${r.variant}] (${issues.length} hits) ===`);
for (const issue of issues) {
console.log(` ${issue.nodeId.padEnd(50)} ${issue.reason}`);
}
}
}
main().catch((e) => {
console.error(e);
process.exit(1);
});

View file

@ -0,0 +1,79 @@
/**
* Dump the doc shape (root width / height / layout / direct-children
* roles) for every applied row in a category. Used 2026-05-10 to figure
* out why detectEdgeSectionPadding scored 0 hits on the 220-row mobile
* subset of `2026-05-03-ab-v8-v1-default` — needed to see whether the
* model is producing page-shaped roots at all or just fragments that
* don't look like a mobile page.
*/
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
import { parseModelOutput } from '@zseven-w/pen-ai-skills';
import { applyToFreshDoc } from './apply';
interface JsonlRow {
promptId: string;
category: string;
difficulty: string;
variant: string;
rawOutput: string;
}
async function main(): Promise<void> {
const runId = process.argv[2];
const wantCategory = process.argv[3] ?? 'mobile';
if (!runId) {
console.error('usage: bun run scripts/ab-corpus/inspect-shape.ts <run-id> [category]');
process.exit(1);
}
const path = join(import.meta.dir, 'runs', runId, 'scores.jsonl');
const rows: JsonlRow[] = readFileSync(path, 'utf-8')
.split('\n')
.filter(Boolean)
.map((l) => JSON.parse(l));
const buckets = new Map<string, number>();
for (const r of rows) {
if (r.category !== wantCategory) continue;
const parsed = parseModelOutput(r.rawOutput);
if (parsed.kind === 'garbage') continue;
const result = await applyToFreshDoc(parsed);
if (!result.ok || !result.doc) continue;
const root = result.doc.children?.[0] as
| (Record<string, unknown> & { width?: unknown; height?: unknown; children?: unknown[] })
| undefined;
if (!root) continue;
const w = root.width;
const h = root.height;
const childCount = Array.isArray(root.children) ? root.children.length : 0;
const wKey =
typeof w === 'number'
? w >= 320 && w <= 480
? 'mobile-w'
: w > 480 && w <= 800
? 'tablet-w'
: w > 800
? 'desktop-w'
: 'tiny-w'
: `non-numeric:${typeof w}`;
const hKey =
typeof h === 'number'
? h >= 568
? 'tall'
: 'short'
: `non-numeric:${typeof h}`;
const ratioKey =
typeof w === 'number' && typeof h === 'number' && w > 0 && h / w >= 1.5 ? 'aspect-ok' : 'aspect-low';
const key = `${wKey} / ${hKey} / ${ratioKey} / children=${childCount}`;
buckets.set(key, (buckets.get(key) ?? 0) + 1);
}
console.log(`shapes for category=${wantCategory}:`);
const sorted = Array.from(buckets.entries()).sort((a, b) => b[1] - a[1]);
for (const [k, v] of sorted) console.log(` ${String(v).padStart(4)} ${k}`);
}
main().catch((e) => {
console.error(e);
process.exit(1);
});