diff --git a/scripts/ab-corpus/__tests__/build-prompt.test.ts b/scripts/ab-corpus/__tests__/build-prompt.test.ts index 32b774f48..e05e7eb90 100644 --- a/scripts/ab-corpus/__tests__/build-prompt.test.ts +++ b/scripts/ab-corpus/__tests__/build-prompt.test.ts @@ -8,9 +8,17 @@ import { buildSystemPrompt } from '../build-prompt'; // large enough that structural assertions catch any regression where // the diet silently no-ops. -const T_PRIMARY_MARKER = 'PRIMARY: when your intent matches an add_*_v0 element tool above'; +const T_MULTI_TOOL_MARKER = 'Respond with one or more `` tags'; +const T_COMPOSITE_MARKER = 'COMPOSITE (multi-component briefs'; const B_BATCH_MARKER = '{"name": "batch_design", "arguments": {"operations": ""}}'; +// Earlier versions of the T instructions explicitly forbade multi-tool +// output ("Respond with one tag" / "Do not combine multiple tags"); +// that wording undercut the multi-tool teaching baked into elements.md. +// Codex stop-time review caught the contradiction — guard against +// regression by asserting the forbidding text is GONE. +const FORBIDDEN_SINGLE_TAG_PHRASE = 'one `` tag, nothing else'; +const FORBIDDEN_NO_COMBINE_PHRASE = 'Do not combine multiple tags'; describe('buildSystemPrompt', () => { it('T + composite returns the largest prompt (cookbook present)', () => { @@ -56,16 +64,25 @@ describe('buildSystemPrompt', () => { expect(distinct.size).toBe(1); }); - it('every T variant carries the PRIMARY/FALLBACK output-format marker', () => { + it('every T variant carries the multi-tool output-format marker', () => { for (const difficulty of ['obvious', 'optional', 'composite'] as const) { const built = buildSystemPrompt('T', { difficulty }); - expect(built.system).toContain(T_PRIMARY_MARKER); + expect(built.system).toContain(T_MULTI_TOOL_MARKER); + expect(built.system).toContain(T_COMPOSITE_MARKER); } }); - it('B variant carries the batch_design marker but not the PRIMARY split', () => { + it('T variant does NOT forbid multi-tool output (regression guard)', () => { + for (const difficulty of ['obvious', 'optional', 'composite'] as const) { + const built = buildSystemPrompt('T', { difficulty }); + expect(built.system).not.toContain(FORBIDDEN_SINGLE_TAG_PHRASE); + expect(built.system).not.toContain(FORBIDDEN_NO_COMBINE_PHRASE); + } + }); + + it('B variant carries the batch_design marker but not the multi-tool split', () => { const built = buildSystemPrompt('B'); expect(built.system).toContain(B_BATCH_MARKER); - expect(built.system).not.toContain(T_PRIMARY_MARKER); + expect(built.system).not.toContain(T_COMPOSITE_MARKER); }); }); diff --git a/scripts/ab-corpus/build-prompt.ts b/scripts/ab-corpus/build-prompt.ts index 992a93115..dfa826f0d 100644 --- a/scripts/ab-corpus/build-prompt.ts +++ b/scripts/ab-corpus/build-prompt.ts @@ -43,18 +43,23 @@ const B_TOOL_CALL_INSTRUCTIONS = [ const T_TOOL_CALL_INSTRUCTIONS = [ '', - 'OUTPUT FORMAT — EMIT AS TOOL CALL:', + 'OUTPUT FORMAT — EMIT AS TOOL CALL(S):', '', - 'Respond with one `` tag, nothing else. Choose based on intent:', + 'Respond with one or more `` tags, nothing else. The harness reads every `` tag in your output, so chain as many as the brief implies. Choose tools based on intent:', '', - 'PRIMARY: when your intent matches an add_*_v0 element tool above, emit:', + 'PRIMARY (single component): emit one tag for the add_*_v0 element tool that fits.', ' {"name": "add_X_v0", "arguments": {...}}', '', - 'FALLBACK: when no element tool fits (heterogeneous layout, composite section, post-hoc styling), emit:', + 'COMPOSITE (multi-component briefs — settings panel with N rows, team list with N members, audit feed with N entries, onboarding screen with N steps): emit ONE `` per component under the same parent. Tags appear in render order (top-to-bottom for vertical, left-to-right for horizontal).', + ' {"name": "add_section_header_v0", "arguments": {"title": "Notifications"}}', + ' {"name": "add_setting_row_v0", "arguments": {...}}', + ' {"name": "add_setting_row_v0", "arguments": {...}}', + '', + 'FALLBACK: when no element tool fits a given component shape, emit a single batch_design call.', ' {"name": "batch_design", "arguments": {"operations": ""}}', 'The `operations` value is a single string containing the batch_design DSL.', '', - 'Do not combine multiple tags. Do not add prose before or after.', + 'Do not add prose before, after, or between tags.', '', ].join('\n'); diff --git a/scripts/ab-corpus/measure-prompt-sizes.ts b/scripts/ab-corpus/measure-prompt-sizes.ts index b91d9df91..62c4a3ebb 100644 --- a/scripts/ab-corpus/measure-prompt-sizes.ts +++ b/scripts/ab-corpus/measure-prompt-sizes.ts @@ -13,7 +13,11 @@ import { buildSystemPrompt } from './build-prompt'; -const cases: Array<{ label: string; variant: 'B' | 'T'; difficulty?: 'obvious' | 'optional' | 'composite' }> = [ +const cases: Array<{ + label: string; + variant: 'B' | 'T'; + difficulty?: 'obvious' | 'optional' | 'composite'; +}> = [ { label: 'B (any difficulty) ', variant: 'B' }, { label: 'T + difficulty="obvious" ', variant: 'T', difficulty: 'obvious' }, { label: 'T + difficulty="optional" ', variant: 'T', difficulty: 'optional' }, @@ -30,6 +34,10 @@ process.stdout.write(`---------------------+---------+------------------\n`); for (const c of cases) { const built = buildSystemPrompt(c.variant, c.difficulty ? { difficulty: c.difficulty } : {}); const len = built.system.length; - process.stdout.write(`${c.label} | ${String(len).padStart(7)} | ${String(Math.round(len / 4)).padStart(6)}\n`); + process.stdout.write( + `${c.label} | ${String(len).padStart(7)} | ${String(Math.round(len / 4)).padStart(6)}\n`, + ); } -process.stdout.write(`\nObvious-vs-composite savings: ${savings} chars (~${Math.round(savings / 4)} tokens)\n`); +process.stdout.write( + `\nObvious-vs-composite savings: ${savings} chars (~${Math.round(savings / 4)} tokens)\n`, +);