fix(ab-corpus): allow multi-tool output in T system prompt

Codex stop-time review caught a contradiction: elements.md taught
"emit one <op_tool> per component" while T_TOOL_CALL_INSTRUCTIONS
explicitly forbade it ("Respond with one tag, nothing else" + "Do
not combine multiple tags"). Live models were probably reading the
later, more authoritative trailing instructions and dropping back to
batch_design — which matches ab-v3's 0/25 composite-T multi-tool
runs even though the cookbook had recipes.

Rewrites the T output-format block to:
- declare "one or more <op_tool> tags" up front
- add an explicit COMPOSITE clause showing chained tags for
  multi-component briefs (settings panel / team list / audit feed /
  onboarding) with a 3-tag worked example
- keep the FALLBACK clause for batch_design when no element tool fits
- keep the "no prose between tags" rule

Tests now assert the multi-tool marker is present in every T variant
AND the forbidding phrases ("one tag, nothing else", "Do not combine
multiple tags") are GONE — the regression guard Codex effectively
asked for. Also pulls in an oxfmt auto-format on the measurement
helper that landed unformatted in a3c5bf2d.
This commit is contained in:
Fini 2026-04-29 09:49:44 +08:00
parent 2f31b62ca9
commit b7e6097da5
3 changed files with 43 additions and 13 deletions

View file

@ -8,9 +8,17 @@ import { buildSystemPrompt } from '../build-prompt';
// large enough that structural assertions catch any regression where
// the diet silently no-ops.
const T_PRIMARY_MARKER = 'PRIMARY: when your intent matches an add_*_v0 element tool above';
const T_MULTI_TOOL_MARKER = 'Respond with one or more `<op_tool>` tags';
const T_COMPOSITE_MARKER = 'COMPOSITE (multi-component briefs';
const B_BATCH_MARKER =
'<op_tool>{"name": "batch_design", "arguments": {"operations": "<DSL_STRING>"}}</op_tool>';
// Earlier versions of the T instructions explicitly forbade multi-tool
// output ("Respond with one tag" / "Do not combine multiple tags");
// that wording undercut the multi-tool teaching baked into elements.md.
// Codex stop-time review caught the contradiction — guard against
// regression by asserting the forbidding text is GONE.
const FORBIDDEN_SINGLE_TAG_PHRASE = 'one `<op_tool>` tag, nothing else';
const FORBIDDEN_NO_COMBINE_PHRASE = 'Do not combine multiple tags';
describe('buildSystemPrompt', () => {
it('T + composite returns the largest prompt (cookbook present)', () => {
@ -56,16 +64,25 @@ describe('buildSystemPrompt', () => {
expect(distinct.size).toBe(1);
});
it('every T variant carries the PRIMARY/FALLBACK output-format marker', () => {
it('every T variant carries the multi-tool output-format marker', () => {
for (const difficulty of ['obvious', 'optional', 'composite'] as const) {
const built = buildSystemPrompt('T', { difficulty });
expect(built.system).toContain(T_PRIMARY_MARKER);
expect(built.system).toContain(T_MULTI_TOOL_MARKER);
expect(built.system).toContain(T_COMPOSITE_MARKER);
}
});
it('B variant carries the batch_design marker but not the PRIMARY split', () => {
it('T variant does NOT forbid multi-tool output (regression guard)', () => {
for (const difficulty of ['obvious', 'optional', 'composite'] as const) {
const built = buildSystemPrompt('T', { difficulty });
expect(built.system).not.toContain(FORBIDDEN_SINGLE_TAG_PHRASE);
expect(built.system).not.toContain(FORBIDDEN_NO_COMBINE_PHRASE);
}
});
it('B variant carries the batch_design marker but not the multi-tool split', () => {
const built = buildSystemPrompt('B');
expect(built.system).toContain(B_BATCH_MARKER);
expect(built.system).not.toContain(T_PRIMARY_MARKER);
expect(built.system).not.toContain(T_COMPOSITE_MARKER);
});
});

View file

@ -43,18 +43,23 @@ const B_TOOL_CALL_INSTRUCTIONS = [
const T_TOOL_CALL_INSTRUCTIONS = [
'',
'OUTPUT FORMAT — EMIT AS TOOL CALL:',
'OUTPUT FORMAT — EMIT AS TOOL CALL(S):',
'',
'Respond with one `<op_tool>` tag, nothing else. Choose based on intent:',
'Respond with one or more `<op_tool>` tags, nothing else. The harness reads every `<op_tool>` tag in your output, so chain as many as the brief implies. Choose tools based on intent:',
'',
'PRIMARY: when your intent matches an add_*_v0 element tool above, emit:',
'PRIMARY (single component): emit one tag for the add_*_v0 element tool that fits.',
' <op_tool>{"name": "add_X_v0", "arguments": {...}}</op_tool>',
'',
'FALLBACK: when no element tool fits (heterogeneous layout, composite section, post-hoc styling), emit:',
'COMPOSITE (multi-component briefs — settings panel with N rows, team list with N members, audit feed with N entries, onboarding screen with N steps): emit ONE `<op_tool>` per component under the same parent. Tags appear in render order (top-to-bottom for vertical, left-to-right for horizontal).',
' <op_tool>{"name": "add_section_header_v0", "arguments": {"title": "Notifications"}}</op_tool>',
' <op_tool>{"name": "add_setting_row_v0", "arguments": {...}}</op_tool>',
' <op_tool>{"name": "add_setting_row_v0", "arguments": {...}}</op_tool>',
'',
'FALLBACK: when no element tool fits a given component shape, emit a single batch_design call.',
' <op_tool>{"name": "batch_design", "arguments": {"operations": "<DSL_STRING>"}}</op_tool>',
'The `operations` value is a single string containing the batch_design DSL.',
'',
'Do not combine multiple tags. Do not add prose before or after.',
'Do not add prose before, after, or between tags.',
'',
].join('\n');

View file

@ -13,7 +13,11 @@
import { buildSystemPrompt } from './build-prompt';
const cases: Array<{ label: string; variant: 'B' | 'T'; difficulty?: 'obvious' | 'optional' | 'composite' }> = [
const cases: Array<{
label: string;
variant: 'B' | 'T';
difficulty?: 'obvious' | 'optional' | 'composite';
}> = [
{ label: 'B (any difficulty) ', variant: 'B' },
{ label: 'T + difficulty="obvious" ', variant: 'T', difficulty: 'obvious' },
{ label: 'T + difficulty="optional" ', variant: 'T', difficulty: 'optional' },
@ -30,6 +34,10 @@ process.stdout.write(`---------------------+---------+------------------\n`);
for (const c of cases) {
const built = buildSystemPrompt(c.variant, c.difficulty ? { difficulty: c.difficulty } : {});
const len = built.system.length;
process.stdout.write(`${c.label} | ${String(len).padStart(7)} | ${String(Math.round(len / 4)).padStart(6)}\n`);
process.stdout.write(
`${c.label} | ${String(len).padStart(7)} | ${String(Math.round(len / 4)).padStart(6)}\n`,
);
}
process.stdout.write(`\nObvious-vs-composite savings: ${savings} chars (~${Math.round(savings / 4)} tokens)\n`);
process.stdout.write(
`\nObvious-vs-composite savings: ${savings} chars (~${Math.round(savings / 4)} tokens)\n`,
);