feat(ab-corpus): gate elements-cookbook on T arm by difficulty
ab-v3 left T prompts at ~22k tokens (vs ~6-7k for B). The 18kb elements-cookbook teaches per-tool arg shapes, which is what composite multi-tool chains genuinely need; single-tool obvious prompts can route correctly from just the decision tree + PREFER list alone. buildSystemPrompt now takes opts.difficulty. T+obvious strips the cookbook (saves ~18kb on the 47/52 ab-v3 obvious prompts); T+composite, T+optional, and undefined keep both halves. B variant unchanged — still strips both, so the A/B comparison stays clean. Verified by 8 new build-prompt.test.ts cases including a 10kb floor on the obvious-vs-composite delta. ab-v4 will measure whether the diet hurts arg compliance on weak models; per-domain split is the Phase 2 fallback if obvious-T garbage rate creeps up.
This commit is contained in:
parent
7d419cffd1
commit
f385f433e1
71
scripts/ab-corpus/__tests__/build-prompt.test.ts
Normal file
71
scripts/ab-corpus/__tests__/build-prompt.test.ts
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import { buildSystemPrompt } from '../build-prompt';
|
||||
|
||||
// These tests work entirely from the prompt builder's observable output —
|
||||
// length deltas + marker strings — so they don't need to import the
|
||||
// internal pen-ai-skills registry (which doesn't resolve cleanly under
|
||||
// vitest from a path outside apps/web). The relative cookbook size is
|
||||
// large enough that structural assertions catch any regression where
|
||||
// the diet silently no-ops.
|
||||
|
||||
const T_PRIMARY_MARKER = 'PRIMARY: when your intent matches an add_*_v0 element tool above';
|
||||
const B_BATCH_MARKER =
|
||||
'<op_tool>{"name": "batch_design", "arguments": {"operations": "<DSL_STRING>"}}</op_tool>';
|
||||
|
||||
describe('buildSystemPrompt', () => {
|
||||
it('T + composite returns the largest prompt (cookbook present)', () => {
|
||||
const composite = buildSystemPrompt('T', { difficulty: 'composite' });
|
||||
const obvious = buildSystemPrompt('T', { difficulty: 'obvious' });
|
||||
expect(composite.system.length).toBeGreaterThan(obvious.system.length);
|
||||
});
|
||||
|
||||
it('T + undefined difficulty matches the composite size (safe default)', () => {
|
||||
const undef = buildSystemPrompt('T');
|
||||
const composite = buildSystemPrompt('T', { difficulty: 'composite' });
|
||||
expect(undef.system.length).toBe(composite.system.length);
|
||||
});
|
||||
|
||||
it("T + difficulty='optional' matches the composite size", () => {
|
||||
const optional = buildSystemPrompt('T', { difficulty: 'optional' });
|
||||
const composite = buildSystemPrompt('T', { difficulty: 'composite' });
|
||||
expect(optional.system.length).toBe(composite.system.length);
|
||||
});
|
||||
|
||||
it("T + difficulty='obvious' shaves at least 10kb off the prompt", () => {
|
||||
const obvious = buildSystemPrompt('T', { difficulty: 'obvious' }).system.length;
|
||||
const composite = buildSystemPrompt('T', { difficulty: 'composite' }).system.length;
|
||||
// Cookbook is ~18kb; expect at least 10kb savings to guard against
|
||||
// accidental regressions where the strip silently no-ops.
|
||||
expect(composite - obvious).toBeGreaterThan(10_000);
|
||||
});
|
||||
|
||||
it('B prompt is smaller than every T variant (both skills stripped)', () => {
|
||||
const b = buildSystemPrompt('B').system.length;
|
||||
const tComposite = buildSystemPrompt('T', { difficulty: 'composite' }).system.length;
|
||||
const tObvious = buildSystemPrompt('T', { difficulty: 'obvious' }).system.length;
|
||||
expect(b).toBeLessThan(tObvious);
|
||||
expect(b).toBeLessThan(tComposite);
|
||||
});
|
||||
|
||||
it('B is the same regardless of difficulty (variant comparison stays clean)', () => {
|
||||
const sizes = (['obvious', 'optional', 'composite', undefined] as const).map((d) => {
|
||||
const built = buildSystemPrompt('B', d ? { difficulty: d } : {});
|
||||
return built.system.length;
|
||||
});
|
||||
const distinct = new Set(sizes);
|
||||
expect(distinct.size).toBe(1);
|
||||
});
|
||||
|
||||
it('every T variant carries the PRIMARY/FALLBACK output-format marker', () => {
|
||||
for (const difficulty of ['obvious', 'optional', 'composite'] as const) {
|
||||
const built = buildSystemPrompt('T', { difficulty });
|
||||
expect(built.system).toContain(T_PRIMARY_MARKER);
|
||||
}
|
||||
});
|
||||
|
||||
it('B variant carries the batch_design marker but not the PRIMARY split', () => {
|
||||
const built = buildSystemPrompt('B');
|
||||
expect(built.system).toContain(B_BATCH_MARKER);
|
||||
expect(built.system).not.toContain(T_PRIMARY_MARKER);
|
||||
});
|
||||
});
|
||||
|
|
@ -63,13 +63,52 @@ export interface BuiltPrompt {
|
|||
variant: 'B' | 'T';
|
||||
}
|
||||
|
||||
export function buildSystemPrompt(variant: 'B' | 'T'): BuiltPrompt {
|
||||
export interface BuildPromptOpts {
|
||||
/**
|
||||
* Prompt difficulty signal. When variant=T, controls whether the
|
||||
* elements-cookbook (~18kb of arg-shape examples) is included.
|
||||
*
|
||||
* - 'composite', 'optional', or undefined → keep cookbook
|
||||
* (multi-tool briefs need the chained recipes; safe default for
|
||||
* free-form callers).
|
||||
* - 'obvious' → strip cookbook. Decision tree + PREFER list still
|
||||
* teach tool selection; the per-tool minimal-usage examples are
|
||||
* the cost we trade for ~80% smaller T prompts on single-tool
|
||||
* prompts. Validated by ab-v4 sweep (Phase 2 of token-diet plan).
|
||||
*
|
||||
* Variant=B is unaffected — baseline always strips both skills so
|
||||
* the A/B comparison still isolates "tools vs no-tools".
|
||||
*/
|
||||
difficulty?: 'obvious' | 'optional' | 'composite';
|
||||
}
|
||||
|
||||
export function buildSystemPrompt(variant: 'B' | 'T', opts: BuildPromptOpts = {}): BuiltPrompt {
|
||||
// buildDesignPrompt() already concatenates every section the harness
|
||||
// needs — schema, style, examples, DESIGN_TYPE_DETECTION, roles,
|
||||
// layout, text rules, guidelines, variables, auto-replace,
|
||||
// post-processing, AND elements (appended last session).
|
||||
const full = buildDesignPrompt();
|
||||
if (variant === 'T') {
|
||||
if (opts.difficulty === 'obvious') {
|
||||
// Save ~18kb by stripping the cookbook on single-tool prompts.
|
||||
// Models still see the decision tree + PREFER list (which is
|
||||
// enough to route to the right tool); they just lose the
|
||||
// copy-paste arg-shape examples. Risk: small dip in arg
|
||||
// compliance on weaker models — measured in ab-v4.
|
||||
const cookbookSkill = getSkillByName('elements-cookbook');
|
||||
if (!cookbookSkill) {
|
||||
throw new Error(
|
||||
'elements-cookbook skill not found in registry — cannot apply obvious-difficulty diet without it',
|
||||
);
|
||||
}
|
||||
const stripped = full.replace(cookbookSkill.content, '');
|
||||
if (stripped === full) {
|
||||
throw new Error(
|
||||
'elements-cookbook content was not present in the full prompt — buildDesignPrompt() may have been refactored; update build-prompt.ts to match',
|
||||
);
|
||||
}
|
||||
return { system: stripped.trim() + '\n\n' + T_TOOL_CALL_INSTRUCTIONS, variant };
|
||||
}
|
||||
return { system: full + '\n\n' + T_TOOL_CALL_INSTRUCTIONS, variant };
|
||||
}
|
||||
// Baseline: strip BOTH elements skills (decision-tree + cookbook).
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ import type { ModelCall } from './stub-model';
|
|||
* that's the path we're actively promoting.
|
||||
*/
|
||||
export async function realModelCall(call: ModelCall): Promise<ChatCallResult> {
|
||||
const built = buildSystemPrompt(call.variant);
|
||||
const built = buildSystemPrompt(call.variant, { difficulty: call.prompt.difficulty });
|
||||
const user = call.prompt.prompt;
|
||||
const model = call.model;
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue