From bd370bd5b8cee9fdbba492cf01840a7265a0c0d7 Mon Sep 17 00:00:00 2001
From: Fini <fini.yang@gmail.com>
Date: Wed, 29 Apr 2026 09:49:55 +0800
Subject: [PATCH] fix(ab-corpus): keep applying composite tags past per-shape
 failures
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Codex stop-time review caught the previous commit (113bd55a) message
overstating apply.ts's behavior — I claimed "scripts/ab-corpus/apply
both catch per-shape and keep running the remaining tags" but the
loop at line 55 had no inner try/catch. A single throw from any
handleElementToolCall (e.g. the heading invalid-level reject 113bd55a
just added) would bubble up through the outer try at line 40 and
return early, dropping every remaining tag in a composite batch on
the floor — gpt-5.4's 13-tag team-people-page response would lose
tags 12-13 instead of just tag 12.

Wraps each handleElementToolCall in its own try/catch + accumulates
failures into a per-shape list. ELEMENT_TOOL_NAMES miss is also a
push-and-continue (was a return). When `failures.length > 0` we
return ok:false with a message listing every failed tag, AND the
partial PenDocument that DID land — so M3 (role coverage) can still
score the 11 tags that worked. M1 stays strict (any failure → false).

Mirrors apps/web/src/services/ai/element-tools-dispatcher::
dispatchElementToolCalls's "collect-errors-keep-going" semantics —
production already worked this way; ab-corpus now does too.

3772 vitest pass, format clean, tsc silent. Existing dry-run + live
sweeps exercise the path; a focused apply.ts unit test would need
pen-mcp setup that the harness's existing build-prompt test sidesteps,
so leaving that as a followup.
---
 scripts/ab-corpus/apply.ts | 44 +++++++++++++++++++++++++++++++-------
 1 file changed, 36 insertions(+), 8 deletions(-)

diff --git a/scripts/ab-corpus/apply.ts b/scripts/ab-corpus/apply.ts
index dc4a34c06..3297cfb08 100644
--- a/scripts/ab-corpus/apply.ts
+++ b/scripts/ab-corpus/apply.ts
@@ -41,18 +41,46 @@ export const applyToFreshDoc: ApplyFn = async (parsed: ParsedOutput): Promise<Ap
     if (parsed.kind === 'tool_calls') {
       // Apply every call into the same fresh doc, in emit order.
       // Composite prompts route here with N≥2 calls; obvious prompts
-      // typically with N=1. Any single call failing aborts the row
-      // (M1=false) — we don't partially apply.
+      // typically with N=1. Per-shape try/catch keeps the batch going
+      // past a single bad tag — matches production
+      // (apps/web/src/services/ai/element-tools-dispatcher::dispatchElementToolCalls)
+      // and prevents one invented arg (e.g. an unknown heading level
+      // on tag 12 of 13, seen on gpt-5.4 in ab-v4) from dropping the
+      // other 12 valid tags on the floor. M1 stays strict: any failure
+      // → ok:false; the partial PenDocument is returned so M3 (role
+      // coverage) can still score against whatever did land.
+      const failures: string[] = [];
       for (let i = 0; i < parsed.calls.length; i += 1) {
         const call = parsed.calls[i];
+        const tagLabel = `call ${i + 1}/${parsed.calls.length} (${call.name})`;
         if (!ELEMENT_TOOL_NAMES.has(call.name)) {
-          return {
-            ok: false,
-            error: `unknown element tool "${call.name}" (call ${i + 1}/${parsed.calls.length})`,
-            doc: null,
-          };
+          failures.push(`${tagLabel}: unknown element tool`);
+          continue;
         }
-        await handleElementToolCall(call.name, { ...call.arguments, filePath: fp });
+        try {
+          await handleElementToolCall(call.name, { ...call.arguments, filePath: fp });
+        } catch (err) {
+          const msg = err instanceof Error ? err.message : String(err);
+          failures.push(`${tagLabel}: ${msg}`);
+        }
+      }
+      if (failures.length > 0) {
+        let partialDoc: PenDocument | null = null;
+        try {
+          const candidate = JSON.parse(readFileSync(fp, 'utf-8')) as PenDocument;
+          const topLevel = (candidate.children ??
+            candidate.pages?.[0]?.children ??
+            []) as unknown[];
+          if (topLevel.length > 0) partialDoc = candidate;
+        } catch {
+          // File unreadable or empty (e.g. every tag failed before
+          // the first element-tool wrote anything) — surface null.
+        }
+        return {
+          ok: false,
+          error: `${failures.length}/${parsed.calls.length} tag(s) failed: ${failures.join('; ')}`,
+          doc: partialDoc,
+        };
       }
     } else {
       const result = await handleBatchDesign({