/** * Build the system prompt for each A/B variant. * * Baseline (B): current OpenPencil design prompt MINUS both element- * tool skills (decision tree + cookbook). The model sees schema / * layout / text rules + examples and can only emit `batch_design` * DSL. * * Treatment (T): same baseline PLUS both element-tool skills PLUS the * `` output-format instruction. The model can either emit * an element-tool call (preferred, schema-narrow) or fall back to * `batch_design` DSL. * * Implementation: `buildDesignPrompt()` includes elements + elements- * cookbook by default. We derive B by string-subtracting each skill * body; T is base + trailing instructions. The cookbook lives in a * separate file so neither piece exceeds the 800-line per-file ceiling * — text-only LLMs (no MCP `tools/list`) need both halves, the * decision tree to pick the tool and the cookbook to know its arg * shape. String subtraction is brittle if elements.md / elements- * cookbook.md move, but it's O(minutes) to fix if it does — much less * churn than refactoring the prompt builder just for this eval. */ import { getSkillByName } from '@zseven-w/pen-ai-skills'; import { buildDesignPrompt } from '@zseven-w/pen-mcp'; // Both variants teach the SAME `` wrapper format so the only // thing that differs between B and T is the tool set — this isolates // "does having element tools help?" from "does teaching a tool-call // format help?". Without this, weak models in B would invent their own // format (observed with MiniMax M2.7 emitting `[TOOL_CALL]` pseudo-JS) // and the variant comparison becomes noise. const B_TOOL_CALL_INSTRUCTIONS = [ '', 'OUTPUT FORMAT — EMIT AS TOOL CALL:', '', 'Respond with exactly one line in this form, nothing else:', ' {"name": "batch_design", "arguments": {"operations": ""}}', 'The `operations` value is a single string containing the batch_design DSL (one operation per line — use \\n to separate). Do not add prose before or after the tag.', '', ].join('\n'); const T_TOOL_CALL_INSTRUCTIONS = [ '', 'OUTPUT FORMAT — EMIT AS TOOL CALL(S):', '', 'Respond with one or more `` tags, nothing else. The harness reads every `` tag in your output, so chain as many as the brief implies. Pick ONE strategy for the whole response — do NOT mix element tools with batch_design in the same output:', '', 'STRATEGY A — element tools (preferred): when every component in the brief fits an `add_*_v0` element tool, emit one tag per component, in render order (top-to-bottom for vertical layouts, left-to-right for horizontal). Single-component briefs produce exactly one tag; multi-component briefs (settings panel with N rows, team list with N members, audit feed with N entries, onboarding screen with N step cards) produce N+1 or more.', ' {"name": "add_section_header_v0", "arguments": {"title": "Notifications"}}', ' {"name": "add_setting_row_v0", "arguments": {...}}', ' {"name": "add_setting_row_v0", "arguments": {...}}', '', 'STRATEGY B — batch_design fallback: when the brief includes any component shape that NO element tool covers (heterogeneous custom layout, post-hoc styling, bespoke scaffolding), emit a SINGLE batch_design call covering the WHOLE response. Multiple `` tags carrying batch_design or any mix of batch_design + element tools is not supported — the harness drops the batch_design half and only the element calls run.', ' {"name": "batch_design", "arguments": {"operations": ""}}', 'The `operations` value is a single string containing the batch_design DSL covering every component in the brief.', '', 'Do not add prose before, after, or between tags. Do not mix Strategy A and Strategy B in the same output — choose element tools for every component or batch_design for every component.', '', ].join('\n'); export interface BuiltPrompt { system: string; variant: 'B' | 'T'; } export interface BuildPromptOpts { /** * Prompt difficulty signal. When variant=T, controls whether the * elements-cookbook (~18kb of arg-shape examples) is included. * * - 'composite', 'optional', or undefined → keep cookbook * (multi-tool briefs need the chained recipes; safe default for * free-form callers). * - 'obvious' → strip cookbook. Decision tree + PREFER list still * teach tool selection; the per-tool minimal-usage examples are * the cost we trade for ~80% smaller T prompts on single-tool * prompts. Validated by ab-v4 sweep (Phase 2 of token-diet plan). * * Variant=B is unaffected — baseline always strips both skills so * the A/B comparison still isolates "tools vs no-tools". */ difficulty?: 'obvious' | 'optional' | 'composite'; /** * Prompt category signal. When variant=T and the category is one of * the supported domains, strip `...` * blocks in elements.md whose tag doesn't match. Untagged content is * "general" and stays in every variant. * * Saves ~3-5kb per non-matching block. The per-domain annotations * are partial today (only the most obviously domain-specific tools * — bottom_nav / data_table_row / pricing_card etc.); ambiguous ones * stay untagged so the harness fails closed (they always load). * * undefined → all domain blocks load (safe default for free-form * callers and ab-v3-style runs that don't classify by category). */ category?: 'mobile' | 'dashboard' | 'landing'; } /** * Strips `...` blocks whose tag * doesn't match `keep`. Untagged content is preserved verbatim. * * The inline comment syntax keeps elements.md a single file (no * multi-file splits to maintain per domain) and keeps the diff * surgical — adding a tag is two HTML comments, no structural move. * * Block syntax (each marker on its own line): * * * ... mobile-only content ... * * * Multiple comma-separated tags are accepted (e.g. `mobile,dashboard`) * and a block matches if `keep` appears in the list. This is the * common case for tools like calendar_grid that legitimately span * domains. * * Whitespace contract: the regex consumes ONLY the single newlines * adjoining each marker, not surrounding blank lines. That preserves * the blank-line separator between adjacent kept blocks — without * this, the closing ` ``` ` of one recipe would butt up against the * `### ` heading of the next, breaking markdown structure (caught * by Codex stop-time review). */ function stripNonMatchingDomains(text: string, keep: string): string { return text.replace( /\n([\s\S]*?)\n/g, (_, tags: string, body: string) => { const allowed = tags.split(',').map((t) => t.trim()); return allowed.includes(keep) ? body : ''; }, ); } export function buildSystemPrompt(variant: 'B' | 'T', opts: BuildPromptOpts = {}): BuiltPrompt { // buildDesignPrompt() already concatenates every section the harness // needs — schema, style, examples, DESIGN_TYPE_DETECTION, roles, // layout, text rules, guidelines, variables, auto-replace, // post-processing, AND elements (appended last session). const full = buildDesignPrompt(); if (variant === 'T') { let payload = full; if (opts.difficulty === 'obvious') { // Save ~18kb by stripping the cookbook on single-tool prompts. // Models still see the decision tree + PREFER list (which is // enough to route to the right tool); they just lose the // copy-paste arg-shape examples. Risk: small dip in arg // compliance on weaker models — measured in ab-v4. const cookbookSkill = getSkillByName('elements-cookbook'); if (!cookbookSkill) { throw new Error( 'elements-cookbook skill not found in registry — cannot apply obvious-difficulty diet without it', ); } payload = payload.replace(cookbookSkill.content, ''); if (payload === full) { throw new Error( 'elements-cookbook content was not present in the full prompt — buildDesignPrompt() may have been refactored; update build-prompt.ts to match', ); } } if (opts.category) { payload = stripNonMatchingDomains(payload, opts.category); } return { system: payload.trim() + '\n\n' + T_TOOL_CALL_INSTRUCTIONS, variant }; } // Baseline: strip BOTH elements skills (decision-tree + cookbook). // The cookbook split landed when elements.md crossed the 800-line // ceiling — both halves carry element-tool guidance that B must not // see, otherwise the baseline leaks tool names + arg shapes and the // A/B delta no longer measures "tools vs no-tools". Exact-match // removal of each skill body keeps the rest of the prompt byte- // identical so any delta isn't explained by accidental shape drift. const elementsSkill = getSkillByName('elements'); const cookbookSkill = getSkillByName('elements-cookbook'); if (!elementsSkill) { throw new Error( 'elements skill not found in registry — cannot build baseline prompt without it (would leak elements content into B if left in place)', ); } if (!cookbookSkill) { throw new Error( 'elements-cookbook skill not found in registry — cannot build baseline prompt without stripping it (would leak the arg-shape examples into B)', ); } let stripped = full.replace(elementsSkill.content, ''); if (stripped === full) { throw new Error( 'elements content was not present in the full prompt — buildDesignPrompt() may have been refactored; update build-prompt.ts to match', ); } const beforeCookbook = stripped; stripped = stripped.replace(cookbookSkill.content, ''); if (stripped === beforeCookbook) { throw new Error( 'elements-cookbook content was not present in the full prompt — buildDesignPrompt() may have been refactored; update build-prompt.ts to match', ); } return { system: stripped.trim() + '\n\n' + B_TOOL_CALL_INSTRUCTIONS, variant }; }