51878c3894
Harness at scripts/ab-corpus/ wires the pen-ai-skills corpus evaluator to
real model endpoints and pen-mcp handlers:
run.ts — CLI entry (--dry-run / --live / --models A,B,C / --only ID)
apply.ts — ApplyFn impl dispatching tool_call → element handler
and batch_design DSL → handleBatchDesign, against a
fresh tmp .op per run (isolated, auto-cleanup)
build-prompt.ts — B variant strips elements.md + appends batch_design
<op_tool> format instruction; T keeps elements + adds
element-tool PRIMARY / batch_design FALLBACK
instruction. Uniform <op_tool> wrapper in both arms
isolates "tool set width" as the only A/B variable.
stub-model.ts — fixture-based offline model for --dry-run
real-model.ts — router by model id (minimax* / gpt-*/o* / glm-5.1 /
glm-* / kimi-*)
clients/
openai-compat.ts — generic chat/completions POST
minimax.ts — api.minimax.io/v1, MINIMAX_API_KEY
codex-cli.ts — spawns `codex exec` (GPT-5.4 via Codex Pro sub)
bailian.ts — coding.dashscope.aliyuncs.com/v1 CP,
DASHSCOPE_BAILIAN_CODING_KEY (hosts glm-4.7, kimi-k2.5)
glm.ts — open.bigmodel.cn/api/coding/paas/v4 official CP,
GLM_OFFICIAL_CODING_KEY
write-report.ts — Report → report.md + report.json in out dir;
4-way routing breakdown table per model
Kept entirely outside packages/ — scripts are a local dev tool, not part
of the published SDK. API keys never hit disk or git.
v1 run results logged separately in openpencil-docs
superpowers/notes/2026-04-20-ab-v1-results.md (5 models × 24 prompts).
82 lines
2.9 KiB
TypeScript
82 lines
2.9 KiB
TypeScript
/**
|
|
* Concrete ApplyFn implementation that routes element-tool calls and
|
|
* batch_design DSL through the real pen-mcp handlers. Lives in
|
|
* scripts/ (not a package) to keep the pen-ai-skills ↔ pen-mcp
|
|
* dependency unidirectional — the scorer in packages/pen-ai-skills
|
|
* declares the ApplyFn contract, this file implements it.
|
|
*/
|
|
|
|
import { mkdtempSync, writeFileSync, readFileSync, rmSync } from 'node:fs';
|
|
import { tmpdir } from 'node:os';
|
|
import { join } from 'node:path';
|
|
import type { PenDocument } from '@zseven-w/pen-types';
|
|
import type { ApplyFn, ApplyResult, ParsedOutput } from '@zseven-w/pen-ai-skills';
|
|
import {
|
|
ELEMENT_TOOL_NAMES,
|
|
handleBatchDesign,
|
|
handleElementToolCall,
|
|
invalidateCache,
|
|
} from '@zseven-w/pen-mcp';
|
|
|
|
const EMPTY = JSON.stringify({ version: '1.0.0', children: [] });
|
|
|
|
/**
|
|
* Apply a parsed output to a fresh, disposable .op file. The scorer
|
|
* reads the resulting PenDocument back and does not need the file to
|
|
* persist — we clean up per call.
|
|
*
|
|
* On success: `ok: true` + `doc: PenDocument`.
|
|
* On failure (handler throws OR writes no root): `ok: false` +
|
|
* `error: <message>` + `doc: null`. Throws are caught so a single
|
|
* bad run never halts the whole corpus sweep.
|
|
*/
|
|
export const applyToFreshDoc: ApplyFn = async (parsed: ParsedOutput): Promise<ApplyResult> => {
|
|
if (parsed.kind === 'garbage') {
|
|
return { ok: false, error: `unparseable: ${parsed.reason}`, doc: null };
|
|
}
|
|
const tmpDir = mkdtempSync(join(tmpdir(), 'ab-corpus-apply-'));
|
|
const fp = join(tmpDir, 'doc.op');
|
|
writeFileSync(fp, EMPTY, 'utf-8');
|
|
try {
|
|
if (parsed.kind === 'tool_call') {
|
|
if (!ELEMENT_TOOL_NAMES.has(parsed.name)) {
|
|
return {
|
|
ok: false,
|
|
error: `unknown element tool "${parsed.name}"`,
|
|
doc: null,
|
|
};
|
|
}
|
|
await handleElementToolCall(parsed.name, { ...parsed.arguments, filePath: fp });
|
|
} else {
|
|
const result = await handleBatchDesign({
|
|
operations: parsed.dsl,
|
|
filePath: fp,
|
|
postProcess: false,
|
|
});
|
|
if (result.errors && result.errors.length > 0) {
|
|
const summary = result.errors.map((e) => `${e.line.slice(0, 60)}: ${e.error}`).join('; ');
|
|
return { ok: false, error: `batch_design errors: ${summary}`, doc: null };
|
|
}
|
|
}
|
|
const doc = JSON.parse(readFileSync(fp, 'utf-8')) as PenDocument;
|
|
const topLevel = (doc.children ?? doc.pages?.[0]?.children ?? []) as unknown[];
|
|
if (topLevel.length === 0) {
|
|
return { ok: false, error: 'apply produced no root nodes', doc: null };
|
|
}
|
|
return { ok: true, error: '', doc };
|
|
} catch (err) {
|
|
return {
|
|
ok: false,
|
|
error: err instanceof Error ? err.message : String(err),
|
|
doc: null,
|
|
};
|
|
} finally {
|
|
invalidateCache(fp);
|
|
try {
|
|
rmSync(tmpDir, { recursive: true, force: true });
|
|
} catch {
|
|
// Best-effort cleanup; leaving a tmp file behind is harmless.
|
|
}
|
|
}
|
|
};
|