diff --git a/packages/pen-ai-skills/skills/phases/generation/elements.md b/packages/pen-ai-skills/skills/phases/generation/elements.md index 4dc810d87..e86924ba4 100644 --- a/packages/pen-ai-skills/skills/phases/generation/elements.md +++ b/packages/pen-ai-skills/skills/phases/generation/elements.md @@ -451,6 +451,8 @@ add_setting_row_v0({ parent_id: "", title: "Push notifications", leading_i add_setting_row_v0({ parent_id: "", title: "Email digest", leading_icon: "mail", trailing: { kind: "switch", on: false } }) ``` + + ### Team / members list (avatars + pending invitations) ``` @@ -466,6 +468,10 @@ add_section_header_v0({ parent_id: "", title: "Pending invitations" }) add_invite_row_v0({ parent_id: "", email: "leon@acme.com", role: "Editor", status: "pending", action_label: "Resend" }) ``` + + + + ### Audit / activity feed (N log entries under a header) ``` @@ -479,6 +485,10 @@ add_activity_log_v0({ parent_id: "", actor: "Raj Patel", action: "updated add_activity_log_v0({ parent_id: "", actor: "Sarah Lee", action: "deleted archive-2024.zip", timestamp: "yesterday", icon: "trash", tone: "danger" }) ``` + + + + ### Faceted search filter sidebar (N filter groups stacked) ``` @@ -486,6 +496,10 @@ add_filter_group_v0({ parent_id: "", title: "Category", options: [{ lab add_filter_group_v0({ parent_id: "", title: "Brand", options: [{ label: "Nike", count: 32 }, { label: "Adidas", count: 28 }, { label: "Patagonia", count: 15, selected: true }, { label: "Arc'teryx", count: 9 }] }) ``` + + + + ### Onboarding "How it works" (N step cards) ``` @@ -497,6 +511,10 @@ add_step_card_v0({ parent_id: "", number: 3, title: "Set your goals", desc add_step_card_v0({ parent_id: "", number: 4, title: "You're all set", description: "You're ready to use the app. Tap below to continue.", completed: true }) ``` + + + + ### Pricing section (3 tiers, middle one featured) ``` @@ -505,6 +523,10 @@ add_pricing_card_v0({ parent_id: "", tier: "Pro", price: "29", period: "/m add_pricing_card_v0({ parent_id: "", tier: "Enterprise", price: "Custom", features: ["Dedicated support", "SSO", "SLA"], cta: "Contact sales" }) ``` + + + + ### Dashboard KPI strip (4 stat cards) ``` @@ -514,6 +536,8 @@ add_stat_card_v0({ parent_id: "", label: "Churn", value: "3.2%", icon: "us add_stat_card_v0({ parent_id: "", label: "Sessions", value: "5,471", icon: "activity" }) ``` + + ### OTP / 2FA verification screen ``` @@ -524,6 +548,8 @@ add_text_button_v0({ parent_id: "", label: "Verify" }) add_link_v0({ parent_id: "", label: "Resend code" }) ``` + + ### Support chat thread ``` @@ -533,6 +559,8 @@ add_chat_bubble_v0({ parent_id: "", message: "Sorry to hear! Let me chec add_attachment_row_v0({ parent_id: "", filename: "receipt.pdf", size: "240 KB", icon: "file-text" }) ``` + + ### Empty inbox / first-run onboarding ``` diff --git a/scripts/ab-corpus/__tests__/build-prompt.test.ts b/scripts/ab-corpus/__tests__/build-prompt.test.ts index 74f7aa484..f6171648f 100644 --- a/scripts/ab-corpus/__tests__/build-prompt.test.ts +++ b/scripts/ab-corpus/__tests__/build-prompt.test.ts @@ -103,4 +103,77 @@ describe('buildSystemPrompt', () => { expect(built.system).not.toContain(T_STRATEGY_A_MARKER); expect(built.system).not.toContain(T_STRATEGY_B_MARKER); }); + + it('T + category=mobile drops dashboard/landing recipes from cookbook', () => { + const mobile = buildSystemPrompt('T', { difficulty: 'composite', category: 'mobile' }); + // Dashboard-tagged recipe titles should be gone (Team / members list, + // Audit / activity feed, KPI strip, Faceted search filter sidebar). + expect(mobile.system).not.toContain('### Team / members list'); + expect(mobile.system).not.toContain('### Audit / activity feed'); + expect(mobile.system).not.toContain('### Dashboard KPI strip'); + expect(mobile.system).not.toContain('### Faceted search filter sidebar'); + // Landing-tagged recipe (Pricing) should be gone. + expect(mobile.system).not.toContain('### Pricing section'); + // Mobile-tagged recipes should remain. + expect(mobile.system).toContain('### Onboarding "How it works"'); + expect(mobile.system).toContain('### Support chat thread'); + }); + + it('T + category=dashboard drops mobile/landing recipes from cookbook', () => { + const dashboard = buildSystemPrompt('T', { difficulty: 'composite', category: 'dashboard' }); + // Mobile-tagged recipes should be gone. + expect(dashboard.system).not.toContain('### Onboarding "How it works"'); + expect(dashboard.system).not.toContain('### Support chat thread'); + // Landing-tagged recipe should be gone. + expect(dashboard.system).not.toContain('### Pricing section'); + // Dashboard-tagged recipes should remain. + expect(dashboard.system).toContain('### Team / members list'); + expect(dashboard.system).toContain('### Audit / activity feed'); + expect(dashboard.system).toContain('### Dashboard KPI strip'); + expect(dashboard.system).toContain('### Faceted search filter sidebar'); + }); + + it('T + category=landing drops mobile/dashboard recipes from cookbook', () => { + const landing = buildSystemPrompt('T', { difficulty: 'composite', category: 'landing' }); + // Mobile-tagged recipes should be gone. + expect(landing.system).not.toContain('### Onboarding "How it works"'); + expect(landing.system).not.toContain('### Support chat thread'); + // Dashboard-tagged recipes should be gone. + expect(landing.system).not.toContain('### Team / members list'); + expect(landing.system).not.toContain('### Dashboard KPI strip'); + // Landing-tagged recipe should remain. + expect(landing.system).toContain('### Pricing section'); + }); + + it('T + category preserves untagged content (Empty inbox / Login etc. stay general)', () => { + // Untagged recipes should be visible regardless of category. + for (const category of ['mobile', 'dashboard', 'landing'] as const) { + const built = buildSystemPrompt('T', { difficulty: 'composite', category }); + expect(built.system).toContain('### Empty inbox / first-run onboarding'); + // Login + Signup are untagged today (cross-domain ambiguity); they + // load for every category until we either tag with a comma list + // or split them into single-domain variants. + expect(built.system).toContain('### Login screen'); + expect(built.system).toContain('### Signup form'); + } + }); + + it('T + category undefined = no filtering (every tagged recipe loads)', () => { + const full = buildSystemPrompt('T', { difficulty: 'composite' }); + expect(full.system).toContain('### Team / members list'); + expect(full.system).toContain('### Onboarding "How it works"'); + expect(full.system).toContain('### Pricing section'); + expect(full.system).toContain('### Empty inbox / first-run onboarding'); + }); + + it('T + category produces a smaller prompt than uncategorized (filter actually fires)', () => { + const full = buildSystemPrompt('T', { difficulty: 'composite' }).system.length; + for (const category of ['mobile', 'dashboard', 'landing'] as const) { + const filtered = buildSystemPrompt('T', { difficulty: 'composite', category }).system.length; + expect(filtered).toBeLessThan(full); + // Each filter should drop at least 500 chars (loose floor — guards + // against the regex silently no-oping, not a tight savings target). + expect(full - filtered).toBeGreaterThan(500); + } + }); }); diff --git a/scripts/ab-corpus/build-prompt.ts b/scripts/ab-corpus/build-prompt.ts index cf48dfacd..2a8a037f8 100644 --- a/scripts/ab-corpus/build-prompt.ts +++ b/scripts/ab-corpus/build-prompt.ts @@ -82,6 +82,51 @@ export interface BuildPromptOpts { * the A/B comparison still isolates "tools vs no-tools". */ difficulty?: 'obvious' | 'optional' | 'composite'; + + /** + * Prompt category signal. When variant=T and the category is one of + * the supported domains, strip `...` + * blocks in elements.md whose tag doesn't match. Untagged content is + * "general" and stays in every variant. + * + * Saves ~3-5kb per non-matching block. The per-domain annotations + * are partial today (only the most obviously domain-specific tools + * — bottom_nav / data_table_row / pricing_card etc.); ambiguous ones + * stay untagged so the harness fails closed (they always load). + * + * undefined → all domain blocks load (safe default for free-form + * callers and ab-v3-style runs that don't classify by category). + */ + category?: 'mobile' | 'dashboard' | 'landing'; +} + +/** + * Strips `...` blocks whose tag + * doesn't match `keep`. Untagged content is preserved verbatim. + * + * The inline comment syntax keeps elements.md a single file (no + * multi-file splits to maintain per domain) and keeps the diff + * surgical — adding a tag is two HTML comments, no structural move. + * + * Block syntax: + * + * + * ... mobile-only content ... + * + * + * Multiple comma-separated tags are accepted (e.g. `mobile,dashboard`) + * and a block matches if `keep` appears in the list. This is the + * common case for tools like calendar_grid that legitimately span + * domains. + */ +function stripNonMatchingDomains(text: string, keep: string): string { + return text.replace( + /\s*([\s\S]*?)\s*\s*/g, + (_, tags: string, body: string) => { + const allowed = tags.split(',').map((t) => t.trim()); + return allowed.includes(keep) ? body : ''; + }, + ); } export function buildSystemPrompt(variant: 'B' | 'T', opts: BuildPromptOpts = {}): BuiltPrompt { @@ -91,6 +136,7 @@ export function buildSystemPrompt(variant: 'B' | 'T', opts: BuildPromptOpts = {} // post-processing, AND elements (appended last session). const full = buildDesignPrompt(); if (variant === 'T') { + let payload = full; if (opts.difficulty === 'obvious') { // Save ~18kb by stripping the cookbook on single-tool prompts. // Models still see the decision tree + PREFER list (which is @@ -103,15 +149,17 @@ export function buildSystemPrompt(variant: 'B' | 'T', opts: BuildPromptOpts = {} 'elements-cookbook skill not found in registry — cannot apply obvious-difficulty diet without it', ); } - const stripped = full.replace(cookbookSkill.content, ''); - if (stripped === full) { + payload = payload.replace(cookbookSkill.content, ''); + if (payload === full) { throw new Error( 'elements-cookbook content was not present in the full prompt — buildDesignPrompt() may have been refactored; update build-prompt.ts to match', ); } - return { system: stripped.trim() + '\n\n' + T_TOOL_CALL_INSTRUCTIONS, variant }; } - return { system: full + '\n\n' + T_TOOL_CALL_INSTRUCTIONS, variant }; + if (opts.category) { + payload = stripNonMatchingDomains(payload, opts.category); + } + return { system: payload.trim() + '\n\n' + T_TOOL_CALL_INSTRUCTIONS, variant }; } // Baseline: strip BOTH elements skills (decision-tree + cookbook). // The cookbook split landed when elements.md crossed the 800-line diff --git a/scripts/ab-corpus/measure-prompt-sizes.ts b/scripts/ab-corpus/measure-prompt-sizes.ts index 62c4a3ebb..ed425e86b 100644 --- a/scripts/ab-corpus/measure-prompt-sizes.ts +++ b/scripts/ab-corpus/measure-prompt-sizes.ts @@ -1,43 +1,92 @@ #!/usr/bin/env bun /** * Quick inline diagnostic — prints character + 4:1-token-estimate sizes - * for every (variant, difficulty) combination of `buildSystemPrompt`. - * Run when tuning the elements-cookbook diet to confirm the gate is - * actually shaving the bytes the test floor predicts: + * for every (variant, difficulty, category) combination of + * `buildSystemPrompt`. Run when tuning the elements diet to confirm + * the gates are actually shaving the bytes the test floor predicts: * * bun scripts/ab-corpus/measure-prompt-sizes.ts * * Not wired into the harness — purely for human inspection. The - * authoritative regression guard is build-prompt.test.ts (>10kb floor). + * authoritative regression guards are build-prompt.test.ts. */ import { buildSystemPrompt } from './build-prompt'; +type Difficulty = 'obvious' | 'optional' | 'composite'; +type Category = 'mobile' | 'dashboard' | 'landing'; + const cases: Array<{ label: string; variant: 'B' | 'T'; - difficulty?: 'obvious' | 'optional' | 'composite'; + difficulty?: Difficulty; + category?: Category; }> = [ - { label: 'B (any difficulty) ', variant: 'B' }, - { label: 'T + difficulty="obvious" ', variant: 'T', difficulty: 'obvious' }, - { label: 'T + difficulty="optional" ', variant: 'T', difficulty: 'optional' }, - { label: 'T + difficulty="composite" ', variant: 'T', difficulty: 'composite' }, - { label: 'T + (no difficulty, safe default) ', variant: 'T' }, + { label: 'B (any) ', variant: 'B' }, + { label: 'T + difficulty=obvious ', variant: 'T', difficulty: 'obvious' }, + { label: 'T + difficulty=composite ', variant: 'T', difficulty: 'composite' }, + { + label: 'T + difficulty=obvious + category=mobile ', + variant: 'T', + difficulty: 'obvious', + category: 'mobile', + }, + { + label: 'T + difficulty=obvious + category=dash ', + variant: 'T', + difficulty: 'obvious', + category: 'dashboard', + }, + { + label: 'T + difficulty=obvious + category=landing', + variant: 'T', + difficulty: 'obvious', + category: 'landing', + }, + { + label: 'T + difficulty=composite + category=mobile', + variant: 'T', + difficulty: 'composite', + category: 'mobile', + }, + { + label: 'T + difficulty=composite + category=dash ', + variant: 'T', + difficulty: 'composite', + category: 'dashboard', + }, + { + label: 'T + difficulty=composite + category=land ', + variant: 'T', + difficulty: 'composite', + category: 'landing', + }, ]; -const tObvious = buildSystemPrompt('T', { difficulty: 'obvious' }).system.length; -const tComposite = buildSystemPrompt('T', { difficulty: 'composite' }).system.length; -const savings = tComposite - tObvious; - -process.stdout.write(`Variant / Difficulty | Chars | ~Tokens (chars/4)\n`); -process.stdout.write(`---------------------+---------+------------------\n`); +process.stdout.write(`Variant / Difficulty / Category | Chars | ~Tokens\n`); +process.stdout.write(`-----------------------------------------+---------+---------\n`); for (const c of cases) { - const built = buildSystemPrompt(c.variant, c.difficulty ? { difficulty: c.difficulty } : {}); + const opts: { difficulty?: Difficulty; category?: Category } = {}; + if (c.difficulty) opts.difficulty = c.difficulty; + if (c.category) opts.category = c.category; + const built = buildSystemPrompt(c.variant, opts); const len = built.system.length; process.stdout.write( `${c.label} | ${String(len).padStart(7)} | ${String(Math.round(len / 4)).padStart(6)}\n`, ); } + +const tCompositeFull = buildSystemPrompt('T', { difficulty: 'composite' }).system.length; +const tCompositeMobile = buildSystemPrompt('T', { difficulty: 'composite', category: 'mobile' }) + .system.length; +const tCompositeDash = buildSystemPrompt('T', { difficulty: 'composite', category: 'dashboard' }) + .system.length; +const tCompositeLanding = buildSystemPrompt('T', { difficulty: 'composite', category: 'landing' }) + .system.length; + process.stdout.write( - `\nObvious-vs-composite savings: ${savings} chars (~${Math.round(savings / 4)} tokens)\n`, + `\nCategory savings on composite (the cookbook-loaded path):\n` + + ` mobile: ${tCompositeFull - tCompositeMobile} chars (~${Math.round((tCompositeFull - tCompositeMobile) / 4)} tokens)\n` + + ` dashboard: ${tCompositeFull - tCompositeDash} chars (~${Math.round((tCompositeFull - tCompositeDash) / 4)} tokens)\n` + + ` landing: ${tCompositeFull - tCompositeLanding} chars (~${Math.round((tCompositeFull - tCompositeLanding) / 4)} tokens)\n`, ); diff --git a/scripts/ab-corpus/real-model.ts b/scripts/ab-corpus/real-model.ts index c4b2e24fb..f16037239 100644 --- a/scripts/ab-corpus/real-model.ts +++ b/scripts/ab-corpus/real-model.ts @@ -53,7 +53,10 @@ import type { ModelCall } from './stub-model'; * that's the path we're actively promoting. */ export async function realModelCall(call: ModelCall): Promise { - const built = buildSystemPrompt(call.variant, { difficulty: call.prompt.difficulty }); + const built = buildSystemPrompt(call.variant, { + difficulty: call.prompt.difficulty, + category: call.prompt.category, + }); const user = call.prompt.prompt; const model = call.model;