51878c3894
Harness at scripts/ab-corpus/ wires the pen-ai-skills corpus evaluator to
real model endpoints and pen-mcp handlers:
run.ts — CLI entry (--dry-run / --live / --models A,B,C / --only ID)
apply.ts — ApplyFn impl dispatching tool_call → element handler
and batch_design DSL → handleBatchDesign, against a
fresh tmp .op per run (isolated, auto-cleanup)
build-prompt.ts — B variant strips elements.md + appends batch_design
<op_tool> format instruction; T keeps elements + adds
element-tool PRIMARY / batch_design FALLBACK
instruction. Uniform <op_tool> wrapper in both arms
isolates "tool set width" as the only A/B variable.
stub-model.ts — fixture-based offline model for --dry-run
real-model.ts — router by model id (minimax* / gpt-*/o* / glm-5.1 /
glm-* / kimi-*)
clients/
openai-compat.ts — generic chat/completions POST
minimax.ts — api.minimax.io/v1, MINIMAX_API_KEY
codex-cli.ts — spawns `codex exec` (GPT-5.4 via Codex Pro sub)
bailian.ts — coding.dashscope.aliyuncs.com/v1 CP,
DASHSCOPE_BAILIAN_CODING_KEY (hosts glm-4.7, kimi-k2.5)
glm.ts — open.bigmodel.cn/api/coding/paas/v4 official CP,
GLM_OFFICIAL_CODING_KEY
write-report.ts — Report → report.md + report.json in out dir;
4-way routing breakdown table per model
Kept entirely outside packages/ — scripts are a local dev tool, not part
of the published SDK. API keys never hit disk or git.
v1 run results logged separately in openpencil-docs
superpowers/notes/2026-04-20-ab-v1-results.md (5 models × 24 prompts).
78 lines
3.2 KiB
TypeScript
78 lines
3.2 KiB
TypeScript
/**
|
|
* Render an aggregated Report as both a markdown file (human review) and
|
|
* a json file (machine-diffable + fed to downstream plotting). Only
|
|
* writes; computes nothing — aggregation is the scorer's job.
|
|
*/
|
|
|
|
import { writeFileSync } from 'node:fs';
|
|
import { join } from 'node:path';
|
|
import type { Report } from '@zseven-w/pen-ai-skills';
|
|
|
|
export function writeReport(outDir: string, report: Report): { mdPath: string; jsonPath: string } {
|
|
const jsonPath = join(outDir, 'report.json');
|
|
writeFileSync(jsonPath, JSON.stringify(report, null, 2), 'utf-8');
|
|
const mdPath = join(outDir, 'report.md');
|
|
writeFileSync(mdPath, renderMarkdown(report), 'utf-8');
|
|
return { mdPath, jsonPath };
|
|
}
|
|
|
|
function renderMarkdown(r: Report): string {
|
|
const lines: string[] = [];
|
|
lines.push(`# Element Tools A/B Report`, '');
|
|
lines.push(`Generated: ${r.generatedAt}`);
|
|
lines.push(`Total runs: ${r.totalRuns}`, '');
|
|
|
|
lines.push(`## By model`, '');
|
|
lines.push(
|
|
'| Model | N (B) | N (T) | M1 B | M1 T | Δ M1 (pp) | M3 B | M3 T | Δ M3 (pp) | Right tool | Wrong tool | Fallback | Garbage |',
|
|
);
|
|
lines.push(
|
|
'|-------|-------|-------|------|------|-----------|------|------|-----------|------------|------------|----------|---------|',
|
|
);
|
|
for (const m of r.byModel) {
|
|
lines.push(
|
|
`| ${m.model} | ${m.runCountBaseline} | ${m.runCountTreatment} | ${pct(m.m1_baseline)} | ${pct(m.m1_treatment)} | ${signed(m.m1_delta_pp)} | ${pct(m.m3_baseline)} | ${pct(m.m3_treatment)} | ${signed(m.m3_delta_pp)} | ${pct(m.m5_right_tool)} | ${pct(m.m5_wrong_tool)} | ${pct(m.m5_fallback)} | ${pct(m.m5_garbage)} |`,
|
|
);
|
|
}
|
|
lines.push('');
|
|
lines.push(
|
|
'_Right / Wrong / Fallback / Garbage = routing breakdown on the `difficulty: obvious` treatment subset; all four rates sum to 100%. Right = tool named in `expected_tool_if_any`. Wrong = routed to a different `add_*_v0` (schema constraint worked, intent match failed). Fallback = emitted `batch_design` DSL. Garbage = output unparseable — counted here so right-tool never looks rosy when most outputs fail to parse._',
|
|
'',
|
|
);
|
|
|
|
lines.push(`## By category`, '');
|
|
lines.push('| Category | M1 B | M1 T | Δ M1 (pp) |');
|
|
lines.push('|----------|------|------|-----------|');
|
|
for (const c of r.byCategory) {
|
|
lines.push(
|
|
`| ${c.category} | ${pct(c.m1_baseline)} | ${pct(c.m1_treatment)} | ${signed(c.m1_delta_pp)} |`,
|
|
);
|
|
}
|
|
lines.push('');
|
|
|
|
lines.push(`## Tool usage (treatment arm only)`, '');
|
|
if (r.byTool.length === 0) {
|
|
lines.push(
|
|
"_No element tools were invoked in the treatment arm. Check decision-tree prompt phrasing and the weak model's tool-use capabilities._",
|
|
);
|
|
} else {
|
|
lines.push('| Tool | Invocations | Successful |');
|
|
lines.push('|------|-------------|------------|');
|
|
for (const t of r.byTool) {
|
|
lines.push(`| \`${t.tool}\` | ${t.invocations} | ${t.successfulInvocations} |`);
|
|
}
|
|
}
|
|
lines.push('');
|
|
return lines.join('\n');
|
|
}
|
|
|
|
function pct(n: number): string {
|
|
if (Number.isNaN(n)) return '—';
|
|
return `${(n * 100).toFixed(1)}%`;
|
|
}
|
|
function signed(n: number): string {
|
|
if (Number.isNaN(n)) return '—';
|
|
const sign = n > 0 ? '+' : '';
|
|
return `${sign}${n.toFixed(1)}`;
|
|
}
|