51878c3894
Harness at scripts/ab-corpus/ wires the pen-ai-skills corpus evaluator to
real model endpoints and pen-mcp handlers:
run.ts — CLI entry (--dry-run / --live / --models A,B,C / --only ID)
apply.ts — ApplyFn impl dispatching tool_call → element handler
and batch_design DSL → handleBatchDesign, against a
fresh tmp .op per run (isolated, auto-cleanup)
build-prompt.ts — B variant strips elements.md + appends batch_design
<op_tool> format instruction; T keeps elements + adds
element-tool PRIMARY / batch_design FALLBACK
instruction. Uniform <op_tool> wrapper in both arms
isolates "tool set width" as the only A/B variable.
stub-model.ts — fixture-based offline model for --dry-run
real-model.ts — router by model id (minimax* / gpt-*/o* / glm-5.1 /
glm-* / kimi-*)
clients/
openai-compat.ts — generic chat/completions POST
minimax.ts — api.minimax.io/v1, MINIMAX_API_KEY
codex-cli.ts — spawns `codex exec` (GPT-5.4 via Codex Pro sub)
bailian.ts — coding.dashscope.aliyuncs.com/v1 CP,
DASHSCOPE_BAILIAN_CODING_KEY (hosts glm-4.7, kimi-k2.5)
glm.ts — open.bigmodel.cn/api/coding/paas/v4 official CP,
GLM_OFFICIAL_CODING_KEY
write-report.ts — Report → report.md + report.json in out dir;
4-way routing breakdown table per model
Kept entirely outside packages/ — scripts are a local dev tool, not part
of the published SDK. API keys never hit disk or git.
v1 run results logged separately in openpencil-docs
superpowers/notes/2026-04-20-ab-v1-results.md (5 models × 24 prompts).
102 lines
3.5 KiB
TypeScript
102 lines
3.5 KiB
TypeScript
/**
|
|
* Codex CLI client — drives `codex exec` as a subprocess to reach
|
|
* GPT-5.4 (the CLI's default) via the user's Codex Pro subscription.
|
|
*
|
|
* Trade-offs:
|
|
* + No per-token API billing (counts against Codex subscription)
|
|
* + Works offline of API key management
|
|
* - Each call incurs Codex's own system-prompt overhead (~20k tokens
|
|
* of codex agent framing) which the model still sees alongside
|
|
* our design-generation system prompt. Interpret A/B deltas
|
|
* against GPT-5.4/codex with that caveat; they're not a clean
|
|
* "raw GPT-5.4 completion" measurement.
|
|
* - `codex exec` cold-start adds a few seconds per call → 48 runs
|
|
* at 10-20s each = 10-15 min wall time for full GPT-5.4 pass.
|
|
*
|
|
* Output shape: we pass `--output-last-message` to write just the
|
|
* assistant's final message to disk, then read + delete. Skips the
|
|
* CLI's pretty-printed session log (which is noisy and harder to
|
|
* parse reliably).
|
|
*/
|
|
|
|
import { spawn } from 'node:child_process';
|
|
import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
|
|
import { tmpdir } from 'node:os';
|
|
import { join } from 'node:path';
|
|
|
|
export interface CallCodexArgs {
|
|
model: string;
|
|
system: string;
|
|
user: string;
|
|
/** Timeout in ms (default 5 minutes — tight since corpus prompts
|
|
* don't need long agentic runs; we want single-turn completions). */
|
|
timeoutMs?: number;
|
|
}
|
|
|
|
export async function callCodex(args: CallCodexArgs): Promise<string> {
|
|
const timeoutMs = args.timeoutMs ?? 5 * 60 * 1000;
|
|
const dir = mkdtempSync(join(tmpdir(), 'ab-codex-'));
|
|
const outPath = join(dir, 'last-message.txt');
|
|
// Codex CLI has its own system prompt (coding agent framing) that
|
|
// we can't override. Fold OUR system prompt into the user-visible
|
|
// prompt so it at least reaches the model. Tag the boundary so the
|
|
// model treats the system block as authoritative context, not as
|
|
// part of the task to answer.
|
|
const prompt = `SYSTEM CONTEXT (treat as authoritative system prompt):\n<<<\n${args.system}\n>>>\n\nUSER REQUEST:\n${args.user}`;
|
|
try {
|
|
return await new Promise<string>((resolve, reject) => {
|
|
const proc = spawn(
|
|
'codex',
|
|
[
|
|
'exec',
|
|
'--skip-git-repo-check',
|
|
'--ephemeral',
|
|
'--sandbox',
|
|
'read-only',
|
|
'-m',
|
|
args.model,
|
|
'--output-last-message',
|
|
outPath,
|
|
prompt,
|
|
],
|
|
{ stdio: ['ignore', 'pipe', 'pipe'] },
|
|
);
|
|
let stderr = '';
|
|
proc.stderr.on('data', (chunk) => {
|
|
stderr += chunk.toString();
|
|
});
|
|
const killTimer = setTimeout(() => {
|
|
proc.kill('SIGKILL');
|
|
reject(new Error(`codex timed out after ${timeoutMs}ms`));
|
|
}, timeoutMs);
|
|
proc.on('error', (err) => {
|
|
clearTimeout(killTimer);
|
|
reject(err);
|
|
});
|
|
proc.on('close', (code) => {
|
|
clearTimeout(killTimer);
|
|
if (code !== 0) {
|
|
reject(new Error(`codex exit ${code}: ${stderr.slice(0, 300)}`));
|
|
return;
|
|
}
|
|
try {
|
|
const out = readFileSync(outPath, 'utf-8');
|
|
if (out.trim().length === 0) {
|
|
reject(new Error('codex produced empty output-last-message'));
|
|
return;
|
|
}
|
|
resolve(out);
|
|
} catch (err) {
|
|
reject(err instanceof Error ? err : new Error(String(err)));
|
|
}
|
|
});
|
|
});
|
|
} finally {
|
|
try {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
} catch {
|
|
// Best-effort cleanup
|
|
}
|
|
}
|
|
}
|