fix(ab-corpus): append each row when ready, not after Promise.all

并行 dispatch 改动(前一 commit)把每个 prompt 内 10 个调用 Promise.all
集合后再批量 append。问题是任何 1 个调用卡满 120s ARK timeout 都会
把其它 9 个已完成 row 的落盘也推迟,破坏了 scores.jsonl 作为 partial-
state-on-crash 的设计。

把 fs.appendFileSync 移进 runOne:每个 row 由 scoreRun 返回后立即
落盘。fs.appendFileSync 是 Node 同步 syscall,单线程 event loop 下
绝不会与另一次 append 交错,所以不需要 mutex。
This commit is contained in:
Fini 2026-04-29 09:50:00 +08:00
parent be04c8beea
commit 577928472f

View file

@ -121,47 +121,73 @@ async function main(): Promise<void> {
// Truncate on start so a re-run into the same dir overwrites.
fs.writeFileSync(scoresPath, '', 'utf-8');
// Per-prompt parallelism: 5 models × 2 variants = 10 calls fan out
// concurrently per prompt. Prompts themselves stay sequential so
// ARK / Bailian rate-limits aren't hammered by 50× concurrent
// dispatch, and the `· prompt.id` progress line still marks one
// prompt at a time. Each provider's client is fetch-based (or a
// codex subprocess) with no shared mutable state, so concurrent
// calls are safe. Cuts wall-clock from sequential 13h → ~1.5h on
// ab-v3 (520 runs) since the slowest single call (~120s ARK
// timeout) sets the per-prompt floor instead of the per-call one.
//
// Each finished row appends to scores.jsonl from inside runOne so
// a crash mid-batch keeps every completed row durable — buffering
// until Promise.all settles would let an ARK 120s timeout hold 9
// already-finished rows hostage. fs.appendFileSync is a sync
// syscall under Node's single-threaded event loop, so concurrent
// runOne completions serialize into non-interleaved writes
// automatically (no mutex needed).
async function runOne(
model: string,
variant: 'B' | 'T',
prompt: (typeof prompts)[number],
): Promise<ScoreRow> {
const call: ModelCall = {
model,
prompt,
variant,
systemPrompt: '<resolved in dispatcher>',
userPrompt: prompt.prompt,
};
let raw: string;
let usage: TokenUsage = { promptTokens: 0, completionTokens: 0 };
try {
const res = args.dryRun ? await stubModelCall(call) : await realModelCall(call);
raw = res.content;
usage = res.usage;
} catch (err) {
// Network / subprocess failure → treat as garbage so the
// run still scores (M1=false, routing='garbage' for obvious
// treatment). Beats aborting a 520-run sweep over one
// transient failure. usage stays 0/0 — aggregate skips zero
// rows when computing token averages so a flaky cell
// doesn't drag the model's average down to ~0.
raw = `__HARNESS_ERROR__: ${err instanceof Error ? err.message : String(err)}`;
}
const parsed = parseModelOutput(raw);
const row = await scoreRun({
prompt,
parsed,
apply: applyToFreshDoc,
model,
variant,
usage,
});
fs.appendFileSync(scoresPath, JSON.stringify(row) + '\n', 'utf-8');
return row;
}
const rows: ScoreRow[] = [];
for (const prompt of prompts) {
const promptCalls: Promise<ScoreRow>[] = [];
for (const model of args.models) {
for (const variant of ['B', 'T'] as const) {
const call: ModelCall = {
model,
prompt,
variant,
systemPrompt: '<resolved in dispatcher>',
userPrompt: prompt.prompt,
};
let raw: string;
let usage: TokenUsage = { promptTokens: 0, completionTokens: 0 };
try {
const res = args.dryRun ? await stubModelCall(call) : await realModelCall(call);
raw = res.content;
usage = res.usage;
} catch (err) {
// Network / subprocess failure → treat as garbage so the
// run still scores (M1=false, routing='garbage' for obvious
// treatment). Beats aborting a 96-run sweep over one
// transient failure. usage stays 0/0 — aggregate skips zero
// rows when computing token averages so a flaky cell
// doesn't drag the model's average down to ~0.
raw = `__HARNESS_ERROR__: ${err instanceof Error ? err.message : String(err)}`;
}
const parsed = parseModelOutput(raw);
const row = await scoreRun({
prompt,
parsed,
apply: applyToFreshDoc,
model,
variant,
usage,
});
rows.push(row);
// Append this row to scores.jsonl immediately — durable
// partial state for crash recovery.
fs.appendFileSync(scoresPath, JSON.stringify(row) + '\n', 'utf-8');
promptCalls.push(runOne(model, variant, prompt));
}
}
const newRows = await Promise.all(promptCalls);
rows.push(...newRows);
process.stderr.write(` · ${prompt.id}\n`);
}