feat(ab-corpus): gate elements-cookbook on T arm by difficulty

ab-v3 left T prompts at ~22k tokens (vs ~6-7k for B). The 18kb
elements-cookbook teaches per-tool arg shapes, which is what
composite multi-tool chains genuinely need; single-tool obvious
prompts can route correctly from just the decision tree + PREFER
list alone.

buildSystemPrompt now takes opts.difficulty. T+obvious strips the
cookbook (saves ~18kb on the 47/52 ab-v3 obvious prompts);
T+composite, T+optional, and undefined keep both halves. B variant
unchanged — still strips both, so the A/B comparison stays clean.

Verified by 8 new build-prompt.test.ts cases including a 10kb floor
on the obvious-vs-composite delta. ab-v4 will measure whether the
diet hurts arg compliance on weak models; per-domain split is the
Phase 2 fallback if obvious-T garbage rate creeps up.
This commit is contained in:
Fini 2026-04-29 09:49:42 +08:00
parent 7d419cffd1
commit f385f433e1
3 changed files with 112 additions and 2 deletions

View file

@ -0,0 +1,71 @@
import { describe, expect, it } from 'vitest';
import { buildSystemPrompt } from '../build-prompt';
// These tests work entirely from the prompt builder's observable output —
// length deltas + marker strings — so they don't need to import the
// internal pen-ai-skills registry (which doesn't resolve cleanly under
// vitest from a path outside apps/web). The relative cookbook size is
// large enough that structural assertions catch any regression where
// the diet silently no-ops.
const T_PRIMARY_MARKER = 'PRIMARY: when your intent matches an add_*_v0 element tool above';
const B_BATCH_MARKER =
'<op_tool>{"name": "batch_design", "arguments": {"operations": "<DSL_STRING>"}}</op_tool>';
describe('buildSystemPrompt', () => {
it('T + composite returns the largest prompt (cookbook present)', () => {
const composite = buildSystemPrompt('T', { difficulty: 'composite' });
const obvious = buildSystemPrompt('T', { difficulty: 'obvious' });
expect(composite.system.length).toBeGreaterThan(obvious.system.length);
});
it('T + undefined difficulty matches the composite size (safe default)', () => {
const undef = buildSystemPrompt('T');
const composite = buildSystemPrompt('T', { difficulty: 'composite' });
expect(undef.system.length).toBe(composite.system.length);
});
it("T + difficulty='optional' matches the composite size", () => {
const optional = buildSystemPrompt('T', { difficulty: 'optional' });
const composite = buildSystemPrompt('T', { difficulty: 'composite' });
expect(optional.system.length).toBe(composite.system.length);
});
it("T + difficulty='obvious' shaves at least 10kb off the prompt", () => {
const obvious = buildSystemPrompt('T', { difficulty: 'obvious' }).system.length;
const composite = buildSystemPrompt('T', { difficulty: 'composite' }).system.length;
// Cookbook is ~18kb; expect at least 10kb savings to guard against
// accidental regressions where the strip silently no-ops.
expect(composite - obvious).toBeGreaterThan(10_000);
});
it('B prompt is smaller than every T variant (both skills stripped)', () => {
const b = buildSystemPrompt('B').system.length;
const tComposite = buildSystemPrompt('T', { difficulty: 'composite' }).system.length;
const tObvious = buildSystemPrompt('T', { difficulty: 'obvious' }).system.length;
expect(b).toBeLessThan(tObvious);
expect(b).toBeLessThan(tComposite);
});
it('B is the same regardless of difficulty (variant comparison stays clean)', () => {
const sizes = (['obvious', 'optional', 'composite', undefined] as const).map((d) => {
const built = buildSystemPrompt('B', d ? { difficulty: d } : {});
return built.system.length;
});
const distinct = new Set(sizes);
expect(distinct.size).toBe(1);
});
it('every T variant carries the PRIMARY/FALLBACK output-format marker', () => {
for (const difficulty of ['obvious', 'optional', 'composite'] as const) {
const built = buildSystemPrompt('T', { difficulty });
expect(built.system).toContain(T_PRIMARY_MARKER);
}
});
it('B variant carries the batch_design marker but not the PRIMARY split', () => {
const built = buildSystemPrompt('B');
expect(built.system).toContain(B_BATCH_MARKER);
expect(built.system).not.toContain(T_PRIMARY_MARKER);
});
});

View file

@ -63,13 +63,52 @@ export interface BuiltPrompt {
variant: 'B' | 'T';
}
export function buildSystemPrompt(variant: 'B' | 'T'): BuiltPrompt {
export interface BuildPromptOpts {
/**
* Prompt difficulty signal. When variant=T, controls whether the
* elements-cookbook (~18kb of arg-shape examples) is included.
*
* - 'composite', 'optional', or undefined keep cookbook
* (multi-tool briefs need the chained recipes; safe default for
* free-form callers).
* - 'obvious' strip cookbook. Decision tree + PREFER list still
* teach tool selection; the per-tool minimal-usage examples are
* the cost we trade for ~80% smaller T prompts on single-tool
* prompts. Validated by ab-v4 sweep (Phase 2 of token-diet plan).
*
* Variant=B is unaffected baseline always strips both skills so
* the A/B comparison still isolates "tools vs no-tools".
*/
difficulty?: 'obvious' | 'optional' | 'composite';
}
export function buildSystemPrompt(variant: 'B' | 'T', opts: BuildPromptOpts = {}): BuiltPrompt {
// buildDesignPrompt() already concatenates every section the harness
// needs — schema, style, examples, DESIGN_TYPE_DETECTION, roles,
// layout, text rules, guidelines, variables, auto-replace,
// post-processing, AND elements (appended last session).
const full = buildDesignPrompt();
if (variant === 'T') {
if (opts.difficulty === 'obvious') {
// Save ~18kb by stripping the cookbook on single-tool prompts.
// Models still see the decision tree + PREFER list (which is
// enough to route to the right tool); they just lose the
// copy-paste arg-shape examples. Risk: small dip in arg
// compliance on weaker models — measured in ab-v4.
const cookbookSkill = getSkillByName('elements-cookbook');
if (!cookbookSkill) {
throw new Error(
'elements-cookbook skill not found in registry — cannot apply obvious-difficulty diet without it',
);
}
const stripped = full.replace(cookbookSkill.content, '');
if (stripped === full) {
throw new Error(
'elements-cookbook content was not present in the full prompt — buildDesignPrompt() may have been refactored; update build-prompt.ts to match',
);
}
return { system: stripped.trim() + '\n\n' + T_TOOL_CALL_INSTRUCTIONS, variant };
}
return { system: full + '\n\n' + T_TOOL_CALL_INSTRUCTIONS, variant };
}
// Baseline: strip BOTH elements skills (decision-tree + cookbook).

View file

@ -53,7 +53,7 @@ import type { ModelCall } from './stub-model';
* that's the path we're actively promoting.
*/
export async function realModelCall(call: ModelCall): Promise<ChatCallResult> {
const built = buildSystemPrompt(call.variant);
const built = buildSystemPrompt(call.variant, { difficulty: call.prompt.difficulty });
const user = call.prompt.prompt;
const model = call.model;