chore(ab-corpus): add shape + per-category issue inspectors
Two complement scripts that ride alongside replay-detectors.ts:
- inspect-shape.ts: bucket every applied row's root by (width-bucket /
height-bucket / aspect-ratio / child-count). Used 2026-05-10 to
diagnose why detectEdgeSectionPadding scored 0 hits on a 220-row
mobile subset — turned out 49% of mobile rows produce roots with a
string-typed width ("fill_container" / "fit_content") because the
element-tools path emits component fragments, not pages. The
detector predicate `typeof width === 'number'` correctly skips them
→ 0 hits is the EVAL HARNESS coverage gap, not a detector bug.
- inspect-issue-category.ts: print every issue in a chosen category
with row id + node id + reason. Generic version of inspect-contrast-
hits.ts. 2026-05-10 used it to validate
excessive-frame-effects (4/4 TP — blur 48 cards + OTP slot spread)
and the two borderline mixed-sibling hits (header padding outlier,
spacer cornerRadius outlier — known role-aware limitation, 0.2%
noise rate, accepted).
Together with replay-detectors.ts these three give a fast empirical
loop for tuning a detector against real corpus output without burning
fresh API tokens.
This commit is contained in:
parent
761c5202e2
commit
31d45b79be
59
scripts/ab-corpus/inspect-issue-category.ts
Normal file
59
scripts/ab-corpus/inspect-issue-category.ts
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
/**
|
||||
* Print every issue of a specific category across an entire run, with the
|
||||
* row id, node id, and reason. Generic version of inspect-contrast-hits
|
||||
* — used for spot-checking any single detector's true-positive vs
|
||||
* false-positive rate against a real corpus.
|
||||
*/
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import {
|
||||
detectAllIssues,
|
||||
parseModelOutput,
|
||||
type Issue,
|
||||
type IssueCategory,
|
||||
} from '@zseven-w/pen-ai-skills';
|
||||
import { applyToFreshDoc } from './apply';
|
||||
|
||||
interface JsonlRow {
|
||||
promptId: string;
|
||||
category: string;
|
||||
difficulty: string;
|
||||
variant: string;
|
||||
rawOutput: string;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const runId = process.argv[2];
|
||||
const wantCat = process.argv[3] as IssueCategory | undefined;
|
||||
if (!runId || !wantCat) {
|
||||
console.error(
|
||||
'usage: bun run scripts/ab-corpus/inspect-issue-category.ts <run-id> <category>',
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
const path = join(import.meta.dir, 'runs', runId, 'scores.jsonl');
|
||||
const rows: JsonlRow[] = readFileSync(path, 'utf-8')
|
||||
.split('\n')
|
||||
.filter(Boolean)
|
||||
.map((l) => JSON.parse(l));
|
||||
|
||||
for (const r of rows) {
|
||||
const parsed = parseModelOutput(r.rawOutput);
|
||||
if (parsed.kind === 'garbage') continue;
|
||||
const result = await applyToFreshDoc(parsed);
|
||||
if (!result.ok || !result.doc) continue;
|
||||
const root = result.doc.children?.[0] ?? null;
|
||||
if (!root) continue;
|
||||
const issues: Issue[] = detectAllIssues(root, result.doc).filter((i) => i.category === wantCat);
|
||||
if (issues.length === 0) continue;
|
||||
console.log(`\n=== ${r.promptId} [${r.category}/${r.difficulty}/${r.variant}] (${issues.length} hits) ===`);
|
||||
for (const issue of issues) {
|
||||
console.log(` ${issue.nodeId.padEnd(50)} ${issue.reason}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((e) => {
|
||||
console.error(e);
|
||||
process.exit(1);
|
||||
});
|
||||
79
scripts/ab-corpus/inspect-shape.ts
Normal file
79
scripts/ab-corpus/inspect-shape.ts
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
/**
|
||||
* Dump the doc shape (root width / height / layout / direct-children
|
||||
* roles) for every applied row in a category. Used 2026-05-10 to figure
|
||||
* out why detectEdgeSectionPadding scored 0 hits on the 220-row mobile
|
||||
* subset of `2026-05-03-ab-v8-v1-default` — needed to see whether the
|
||||
* model is producing page-shaped roots at all or just fragments that
|
||||
* don't look like a mobile page.
|
||||
*/
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { parseModelOutput } from '@zseven-w/pen-ai-skills';
|
||||
import { applyToFreshDoc } from './apply';
|
||||
|
||||
interface JsonlRow {
|
||||
promptId: string;
|
||||
category: string;
|
||||
difficulty: string;
|
||||
variant: string;
|
||||
rawOutput: string;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const runId = process.argv[2];
|
||||
const wantCategory = process.argv[3] ?? 'mobile';
|
||||
if (!runId) {
|
||||
console.error('usage: bun run scripts/ab-corpus/inspect-shape.ts <run-id> [category]');
|
||||
process.exit(1);
|
||||
}
|
||||
const path = join(import.meta.dir, 'runs', runId, 'scores.jsonl');
|
||||
const rows: JsonlRow[] = readFileSync(path, 'utf-8')
|
||||
.split('\n')
|
||||
.filter(Boolean)
|
||||
.map((l) => JSON.parse(l));
|
||||
|
||||
const buckets = new Map<string, number>();
|
||||
for (const r of rows) {
|
||||
if (r.category !== wantCategory) continue;
|
||||
const parsed = parseModelOutput(r.rawOutput);
|
||||
if (parsed.kind === 'garbage') continue;
|
||||
const result = await applyToFreshDoc(parsed);
|
||||
if (!result.ok || !result.doc) continue;
|
||||
const root = result.doc.children?.[0] as
|
||||
| (Record<string, unknown> & { width?: unknown; height?: unknown; children?: unknown[] })
|
||||
| undefined;
|
||||
if (!root) continue;
|
||||
const w = root.width;
|
||||
const h = root.height;
|
||||
const childCount = Array.isArray(root.children) ? root.children.length : 0;
|
||||
const wKey =
|
||||
typeof w === 'number'
|
||||
? w >= 320 && w <= 480
|
||||
? 'mobile-w'
|
||||
: w > 480 && w <= 800
|
||||
? 'tablet-w'
|
||||
: w > 800
|
||||
? 'desktop-w'
|
||||
: 'tiny-w'
|
||||
: `non-numeric:${typeof w}`;
|
||||
const hKey =
|
||||
typeof h === 'number'
|
||||
? h >= 568
|
||||
? 'tall'
|
||||
: 'short'
|
||||
: `non-numeric:${typeof h}`;
|
||||
const ratioKey =
|
||||
typeof w === 'number' && typeof h === 'number' && w > 0 && h / w >= 1.5 ? 'aspect-ok' : 'aspect-low';
|
||||
const key = `${wKey} / ${hKey} / ${ratioKey} / children=${childCount}`;
|
||||
buckets.set(key, (buckets.get(key) ?? 0) + 1);
|
||||
}
|
||||
|
||||
console.log(`shapes for category=${wantCategory}:`);
|
||||
const sorted = Array.from(buckets.entries()).sort((a, b) => b[1] - a[1]);
|
||||
for (const [k, v] of sorted) console.log(` ${String(v).padStart(4)} ${k}`);
|
||||
}
|
||||
|
||||
main().catch((e) => {
|
||||
console.error(e);
|
||||
process.exit(1);
|
||||
});
|
||||
Loading…
Reference in a new issue