From f385f433e1213b10dd54bb0d20d42cdffbd5c818 Mon Sep 17 00:00:00 2001 From: Fini Date: Wed, 29 Apr 2026 09:49:42 +0800 Subject: [PATCH] feat(ab-corpus): gate elements-cookbook on T arm by difficulty MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ab-v3 left T prompts at ~22k tokens (vs ~6-7k for B). The 18kb elements-cookbook teaches per-tool arg shapes, which is what composite multi-tool chains genuinely need; single-tool obvious prompts can route correctly from just the decision tree + PREFER list alone. buildSystemPrompt now takes opts.difficulty. T+obvious strips the cookbook (saves ~18kb on the 47/52 ab-v3 obvious prompts); T+composite, T+optional, and undefined keep both halves. B variant unchanged — still strips both, so the A/B comparison stays clean. Verified by 8 new build-prompt.test.ts cases including a 10kb floor on the obvious-vs-composite delta. ab-v4 will measure whether the diet hurts arg compliance on weak models; per-domain split is the Phase 2 fallback if obvious-T garbage rate creeps up. --- .../ab-corpus/__tests__/build-prompt.test.ts | 71 +++++++++++++++++++ scripts/ab-corpus/build-prompt.ts | 41 ++++++++++- scripts/ab-corpus/real-model.ts | 2 +- 3 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 scripts/ab-corpus/__tests__/build-prompt.test.ts diff --git a/scripts/ab-corpus/__tests__/build-prompt.test.ts b/scripts/ab-corpus/__tests__/build-prompt.test.ts new file mode 100644 index 000000000..32b774f48 --- /dev/null +++ b/scripts/ab-corpus/__tests__/build-prompt.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from 'vitest'; +import { buildSystemPrompt } from '../build-prompt'; + +// These tests work entirely from the prompt builder's observable output — +// length deltas + marker strings — so they don't need to import the +// internal pen-ai-skills registry (which doesn't resolve cleanly under +// vitest from a path outside apps/web). The relative cookbook size is +// large enough that structural assertions catch any regression where +// the diet silently no-ops. + +const T_PRIMARY_MARKER = 'PRIMARY: when your intent matches an add_*_v0 element tool above'; +const B_BATCH_MARKER = + '{"name": "batch_design", "arguments": {"operations": ""}}'; + +describe('buildSystemPrompt', () => { + it('T + composite returns the largest prompt (cookbook present)', () => { + const composite = buildSystemPrompt('T', { difficulty: 'composite' }); + const obvious = buildSystemPrompt('T', { difficulty: 'obvious' }); + expect(composite.system.length).toBeGreaterThan(obvious.system.length); + }); + + it('T + undefined difficulty matches the composite size (safe default)', () => { + const undef = buildSystemPrompt('T'); + const composite = buildSystemPrompt('T', { difficulty: 'composite' }); + expect(undef.system.length).toBe(composite.system.length); + }); + + it("T + difficulty='optional' matches the composite size", () => { + const optional = buildSystemPrompt('T', { difficulty: 'optional' }); + const composite = buildSystemPrompt('T', { difficulty: 'composite' }); + expect(optional.system.length).toBe(composite.system.length); + }); + + it("T + difficulty='obvious' shaves at least 10kb off the prompt", () => { + const obvious = buildSystemPrompt('T', { difficulty: 'obvious' }).system.length; + const composite = buildSystemPrompt('T', { difficulty: 'composite' }).system.length; + // Cookbook is ~18kb; expect at least 10kb savings to guard against + // accidental regressions where the strip silently no-ops. + expect(composite - obvious).toBeGreaterThan(10_000); + }); + + it('B prompt is smaller than every T variant (both skills stripped)', () => { + const b = buildSystemPrompt('B').system.length; + const tComposite = buildSystemPrompt('T', { difficulty: 'composite' }).system.length; + const tObvious = buildSystemPrompt('T', { difficulty: 'obvious' }).system.length; + expect(b).toBeLessThan(tObvious); + expect(b).toBeLessThan(tComposite); + }); + + it('B is the same regardless of difficulty (variant comparison stays clean)', () => { + const sizes = (['obvious', 'optional', 'composite', undefined] as const).map((d) => { + const built = buildSystemPrompt('B', d ? { difficulty: d } : {}); + return built.system.length; + }); + const distinct = new Set(sizes); + expect(distinct.size).toBe(1); + }); + + it('every T variant carries the PRIMARY/FALLBACK output-format marker', () => { + for (const difficulty of ['obvious', 'optional', 'composite'] as const) { + const built = buildSystemPrompt('T', { difficulty }); + expect(built.system).toContain(T_PRIMARY_MARKER); + } + }); + + it('B variant carries the batch_design marker but not the PRIMARY split', () => { + const built = buildSystemPrompt('B'); + expect(built.system).toContain(B_BATCH_MARKER); + expect(built.system).not.toContain(T_PRIMARY_MARKER); + }); +}); diff --git a/scripts/ab-corpus/build-prompt.ts b/scripts/ab-corpus/build-prompt.ts index 9198b23de..992a93115 100644 --- a/scripts/ab-corpus/build-prompt.ts +++ b/scripts/ab-corpus/build-prompt.ts @@ -63,13 +63,52 @@ export interface BuiltPrompt { variant: 'B' | 'T'; } -export function buildSystemPrompt(variant: 'B' | 'T'): BuiltPrompt { +export interface BuildPromptOpts { + /** + * Prompt difficulty signal. When variant=T, controls whether the + * elements-cookbook (~18kb of arg-shape examples) is included. + * + * - 'composite', 'optional', or undefined → keep cookbook + * (multi-tool briefs need the chained recipes; safe default for + * free-form callers). + * - 'obvious' → strip cookbook. Decision tree + PREFER list still + * teach tool selection; the per-tool minimal-usage examples are + * the cost we trade for ~80% smaller T prompts on single-tool + * prompts. Validated by ab-v4 sweep (Phase 2 of token-diet plan). + * + * Variant=B is unaffected — baseline always strips both skills so + * the A/B comparison still isolates "tools vs no-tools". + */ + difficulty?: 'obvious' | 'optional' | 'composite'; +} + +export function buildSystemPrompt(variant: 'B' | 'T', opts: BuildPromptOpts = {}): BuiltPrompt { // buildDesignPrompt() already concatenates every section the harness // needs — schema, style, examples, DESIGN_TYPE_DETECTION, roles, // layout, text rules, guidelines, variables, auto-replace, // post-processing, AND elements (appended last session). const full = buildDesignPrompt(); if (variant === 'T') { + if (opts.difficulty === 'obvious') { + // Save ~18kb by stripping the cookbook on single-tool prompts. + // Models still see the decision tree + PREFER list (which is + // enough to route to the right tool); they just lose the + // copy-paste arg-shape examples. Risk: small dip in arg + // compliance on weaker models — measured in ab-v4. + const cookbookSkill = getSkillByName('elements-cookbook'); + if (!cookbookSkill) { + throw new Error( + 'elements-cookbook skill not found in registry — cannot apply obvious-difficulty diet without it', + ); + } + const stripped = full.replace(cookbookSkill.content, ''); + if (stripped === full) { + throw new Error( + 'elements-cookbook content was not present in the full prompt — buildDesignPrompt() may have been refactored; update build-prompt.ts to match', + ); + } + return { system: stripped.trim() + '\n\n' + T_TOOL_CALL_INSTRUCTIONS, variant }; + } return { system: full + '\n\n' + T_TOOL_CALL_INSTRUCTIONS, variant }; } // Baseline: strip BOTH elements skills (decision-tree + cookbook). diff --git a/scripts/ab-corpus/real-model.ts b/scripts/ab-corpus/real-model.ts index 63145a0aa..c4b2e24fb 100644 --- a/scripts/ab-corpus/real-model.ts +++ b/scripts/ab-corpus/real-model.ts @@ -53,7 +53,7 @@ import type { ModelCall } from './stub-model'; * that's the path we're actively promoting. */ export async function realModelCall(call: ModelCall): Promise { - const built = buildSystemPrompt(call.variant); + const built = buildSystemPrompt(call.variant, { difficulty: call.prompt.difficulty }); const user = call.prompt.prompt; const model = call.model;