fix(ab-corpus): allow multi-tool output in T system prompt
Codex stop-time review caught a contradiction: elements.md taught
"emit one <op_tool> per component" while T_TOOL_CALL_INSTRUCTIONS
explicitly forbade it ("Respond with one tag, nothing else" + "Do
not combine multiple tags"). Live models were probably reading the
later, more authoritative trailing instructions and dropping back to
batch_design — which matches ab-v3's 0/25 composite-T multi-tool
runs even though the cookbook had recipes.
Rewrites the T output-format block to:
- declare "one or more <op_tool> tags" up front
- add an explicit COMPOSITE clause showing chained tags for
multi-component briefs (settings panel / team list / audit feed /
onboarding) with a 3-tag worked example
- keep the FALLBACK clause for batch_design when no element tool fits
- keep the "no prose between tags" rule
Tests now assert the multi-tool marker is present in every T variant
AND the forbidding phrases ("one tag, nothing else", "Do not combine
multiple tags") are GONE — the regression guard Codex effectively
asked for. Also pulls in an oxfmt auto-format on the measurement
helper that landed unformatted in a3c5bf2d.
This commit is contained in:
parent
2f31b62ca9
commit
b7e6097da5
|
|
@ -8,9 +8,17 @@ import { buildSystemPrompt } from '../build-prompt';
|
|||
// large enough that structural assertions catch any regression where
|
||||
// the diet silently no-ops.
|
||||
|
||||
const T_PRIMARY_MARKER = 'PRIMARY: when your intent matches an add_*_v0 element tool above';
|
||||
const T_MULTI_TOOL_MARKER = 'Respond with one or more `<op_tool>` tags';
|
||||
const T_COMPOSITE_MARKER = 'COMPOSITE (multi-component briefs';
|
||||
const B_BATCH_MARKER =
|
||||
'<op_tool>{"name": "batch_design", "arguments": {"operations": "<DSL_STRING>"}}</op_tool>';
|
||||
// Earlier versions of the T instructions explicitly forbade multi-tool
|
||||
// output ("Respond with one tag" / "Do not combine multiple tags");
|
||||
// that wording undercut the multi-tool teaching baked into elements.md.
|
||||
// Codex stop-time review caught the contradiction — guard against
|
||||
// regression by asserting the forbidding text is GONE.
|
||||
const FORBIDDEN_SINGLE_TAG_PHRASE = 'one `<op_tool>` tag, nothing else';
|
||||
const FORBIDDEN_NO_COMBINE_PHRASE = 'Do not combine multiple tags';
|
||||
|
||||
describe('buildSystemPrompt', () => {
|
||||
it('T + composite returns the largest prompt (cookbook present)', () => {
|
||||
|
|
@ -56,16 +64,25 @@ describe('buildSystemPrompt', () => {
|
|||
expect(distinct.size).toBe(1);
|
||||
});
|
||||
|
||||
it('every T variant carries the PRIMARY/FALLBACK output-format marker', () => {
|
||||
it('every T variant carries the multi-tool output-format marker', () => {
|
||||
for (const difficulty of ['obvious', 'optional', 'composite'] as const) {
|
||||
const built = buildSystemPrompt('T', { difficulty });
|
||||
expect(built.system).toContain(T_PRIMARY_MARKER);
|
||||
expect(built.system).toContain(T_MULTI_TOOL_MARKER);
|
||||
expect(built.system).toContain(T_COMPOSITE_MARKER);
|
||||
}
|
||||
});
|
||||
|
||||
it('B variant carries the batch_design marker but not the PRIMARY split', () => {
|
||||
it('T variant does NOT forbid multi-tool output (regression guard)', () => {
|
||||
for (const difficulty of ['obvious', 'optional', 'composite'] as const) {
|
||||
const built = buildSystemPrompt('T', { difficulty });
|
||||
expect(built.system).not.toContain(FORBIDDEN_SINGLE_TAG_PHRASE);
|
||||
expect(built.system).not.toContain(FORBIDDEN_NO_COMBINE_PHRASE);
|
||||
}
|
||||
});
|
||||
|
||||
it('B variant carries the batch_design marker but not the multi-tool split', () => {
|
||||
const built = buildSystemPrompt('B');
|
||||
expect(built.system).toContain(B_BATCH_MARKER);
|
||||
expect(built.system).not.toContain(T_PRIMARY_MARKER);
|
||||
expect(built.system).not.toContain(T_COMPOSITE_MARKER);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -43,18 +43,23 @@ const B_TOOL_CALL_INSTRUCTIONS = [
|
|||
|
||||
const T_TOOL_CALL_INSTRUCTIONS = [
|
||||
'',
|
||||
'OUTPUT FORMAT — EMIT AS TOOL CALL:',
|
||||
'OUTPUT FORMAT — EMIT AS TOOL CALL(S):',
|
||||
'',
|
||||
'Respond with one `<op_tool>` tag, nothing else. Choose based on intent:',
|
||||
'Respond with one or more `<op_tool>` tags, nothing else. The harness reads every `<op_tool>` tag in your output, so chain as many as the brief implies. Choose tools based on intent:',
|
||||
'',
|
||||
'PRIMARY: when your intent matches an add_*_v0 element tool above, emit:',
|
||||
'PRIMARY (single component): emit one tag for the add_*_v0 element tool that fits.',
|
||||
' <op_tool>{"name": "add_X_v0", "arguments": {...}}</op_tool>',
|
||||
'',
|
||||
'FALLBACK: when no element tool fits (heterogeneous layout, composite section, post-hoc styling), emit:',
|
||||
'COMPOSITE (multi-component briefs — settings panel with N rows, team list with N members, audit feed with N entries, onboarding screen with N steps): emit ONE `<op_tool>` per component under the same parent. Tags appear in render order (top-to-bottom for vertical, left-to-right for horizontal).',
|
||||
' <op_tool>{"name": "add_section_header_v0", "arguments": {"title": "Notifications"}}</op_tool>',
|
||||
' <op_tool>{"name": "add_setting_row_v0", "arguments": {...}}</op_tool>',
|
||||
' <op_tool>{"name": "add_setting_row_v0", "arguments": {...}}</op_tool>',
|
||||
'',
|
||||
'FALLBACK: when no element tool fits a given component shape, emit a single batch_design call.',
|
||||
' <op_tool>{"name": "batch_design", "arguments": {"operations": "<DSL_STRING>"}}</op_tool>',
|
||||
'The `operations` value is a single string containing the batch_design DSL.',
|
||||
'',
|
||||
'Do not combine multiple tags. Do not add prose before or after.',
|
||||
'Do not add prose before, after, or between tags.',
|
||||
'',
|
||||
].join('\n');
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,11 @@
|
|||
|
||||
import { buildSystemPrompt } from './build-prompt';
|
||||
|
||||
const cases: Array<{ label: string; variant: 'B' | 'T'; difficulty?: 'obvious' | 'optional' | 'composite' }> = [
|
||||
const cases: Array<{
|
||||
label: string;
|
||||
variant: 'B' | 'T';
|
||||
difficulty?: 'obvious' | 'optional' | 'composite';
|
||||
}> = [
|
||||
{ label: 'B (any difficulty) ', variant: 'B' },
|
||||
{ label: 'T + difficulty="obvious" ', variant: 'T', difficulty: 'obvious' },
|
||||
{ label: 'T + difficulty="optional" ', variant: 'T', difficulty: 'optional' },
|
||||
|
|
@ -30,6 +34,10 @@ process.stdout.write(`---------------------+---------+------------------\n`);
|
|||
for (const c of cases) {
|
||||
const built = buildSystemPrompt(c.variant, c.difficulty ? { difficulty: c.difficulty } : {});
|
||||
const len = built.system.length;
|
||||
process.stdout.write(`${c.label} | ${String(len).padStart(7)} | ${String(Math.round(len / 4)).padStart(6)}\n`);
|
||||
process.stdout.write(
|
||||
`${c.label} | ${String(len).padStart(7)} | ${String(Math.round(len / 4)).padStart(6)}\n`,
|
||||
);
|
||||
}
|
||||
process.stdout.write(`\nObvious-vs-composite savings: ${savings} chars (~${Math.round(savings / 4)} tokens)\n`);
|
||||
process.stdout.write(
|
||||
`\nObvious-vs-composite savings: ${savings} chars (~${Math.round(savings / 4)} tokens)\n`,
|
||||
);
|
||||
|
|
|
|||
Loading…
Reference in a new issue