feat(ab-corpus): per-domain cookbook filter (Phase 2 of token diet)
ab-v3 / ab-v4 showed Phase 1A (cookbook strip on obvious difficulty)
shaved ~4.4k tokens off T-obvious. Phase 2 adds a per-category gate
that strips cookbook recipes whose domain doesn't match the prompt's
category — mobile briefs don't see dashboard recipes, dashboard
briefs don't see mobile / landing recipes, etc.
Mechanism: HTML comment block markers in elements.md
(`<!-- @domain:dashboard --> ... <!-- /@domain -->`) plus a
stripNonMatchingDomains() pass in buildSystemPrompt that drops blocks
whose tag list doesn't include the active category. Untagged content
is "general" and stays in every variant — the safe default.
Tagged 7 single-domain cookbook recipes:
- dashboard: Team / members list, Audit / activity feed, Faceted
search filter sidebar, Dashboard KPI strip
- landing: Pricing section
- mobile: Onboarding "How it works", Support chat thread
Cross-domain recipes (Login, Signup, Settings page, OTP, Empty
inbox) stay untagged so they load for every category. Decision tree
+ PREFER list also untagged today; the per-tool annotations there
would be a much larger judgment pass for marginal additional savings.
Token measurements (chars / 4 estimate):
full mobile dashboard landing
- T + composite 19.0k 17.9k 18.4k 17.7k
(-1.1k) (-0.6k) (-1.3k)
- T + obvious 14.6k 13.5k 14.0k 13.3k
(-1.1k) (-0.6k) (-1.3k)
Modest absolute savings — Phase 2 only filters cookbook RECIPES (in
elements.md), and most cookbook content is in elements-cookbook.md
which Phase 1A already strips on obvious. To hit the 6-8k T target
we still need decision-tree compression or PREFER-list trim, but
both are lossier than this gate. Phase 3 candidates noted in the
ab-v4 results doc.
real-model.ts plumbs call.prompt.category through to buildSystemPrompt.
3766 vitest pass (+6 category filter tests including a 500-char
floor regression guard that the filter actually shaves bytes).
This commit is contained in:
Fini
parent
bacafae52a
commit
c835976479
|
|
@ -451,6 +451,8 @@ add_setting_row_v0({ parent_id: "<page>", title: "Push notifications", leading_i
|
|||
add_setting_row_v0({ parent_id: "<page>", title: "Email digest", leading_icon: "mail", trailing: { kind: "switch", on: false } })
|
||||
```
|
||||
|
||||
<!-- @domain:dashboard -->
|
||||
|
||||
### Team / members list (avatars + pending invitations)
|
||||
|
||||
```
|
||||
|
|
@ -466,6 +468,10 @@ add_section_header_v0({ parent_id: "<page>", title: "Pending invitations" })
|
|||
add_invite_row_v0({ parent_id: "<page>", email: "leon@acme.com", role: "Editor", status: "pending", action_label: "Resend" })
|
||||
```
|
||||
|
||||
<!-- /@domain -->
|
||||
|
||||
<!-- @domain:dashboard -->
|
||||
|
||||
### Audit / activity feed (N log entries under a header)
|
||||
|
||||
```
|
||||
|
|
@ -479,6 +485,10 @@ add_activity_log_v0({ parent_id: "<panel>", actor: "Raj Patel", action: "updated
|
|||
add_activity_log_v0({ parent_id: "<panel>", actor: "Sarah Lee", action: "deleted archive-2024.zip", timestamp: "yesterday", icon: "trash", tone: "danger" })
|
||||
```
|
||||
|
||||
<!-- /@domain -->
|
||||
|
||||
<!-- @domain:dashboard -->
|
||||
|
||||
### Faceted search filter sidebar (N filter groups stacked)
|
||||
|
||||
```
|
||||
|
|
@ -486,6 +496,10 @@ add_filter_group_v0({ parent_id: "<sidebar>", title: "Category", options: [{ lab
|
|||
add_filter_group_v0({ parent_id: "<sidebar>", title: "Brand", options: [{ label: "Nike", count: 32 }, { label: "Adidas", count: 28 }, { label: "Patagonia", count: 15, selected: true }, { label: "Arc'teryx", count: 9 }] })
|
||||
```
|
||||
|
||||
<!-- /@domain -->
|
||||
|
||||
<!-- @domain:mobile -->
|
||||
|
||||
### Onboarding "How it works" (N step cards)
|
||||
|
||||
```
|
||||
|
|
@ -497,6 +511,10 @@ add_step_card_v0({ parent_id: "<page>", number: 3, title: "Set your goals", desc
|
|||
add_step_card_v0({ parent_id: "<page>", number: 4, title: "You're all set", description: "You're ready to use the app. Tap below to continue.", completed: true })
|
||||
```
|
||||
|
||||
<!-- /@domain -->
|
||||
|
||||
<!-- @domain:landing -->
|
||||
|
||||
### Pricing section (3 tiers, middle one featured)
|
||||
|
||||
```
|
||||
|
|
@ -505,6 +523,10 @@ add_pricing_card_v0({ parent_id: "<page>", tier: "Pro", price: "29", period: "/m
|
|||
add_pricing_card_v0({ parent_id: "<page>", tier: "Enterprise", price: "Custom", features: ["Dedicated support", "SSO", "SLA"], cta: "Contact sales" })
|
||||
```
|
||||
|
||||
<!-- /@domain -->
|
||||
|
||||
<!-- @domain:dashboard -->
|
||||
|
||||
### Dashboard KPI strip (4 stat cards)
|
||||
|
||||
```
|
||||
|
|
@ -514,6 +536,8 @@ add_stat_card_v0({ parent_id: "<page>", label: "Churn", value: "3.2%", icon: "us
|
|||
add_stat_card_v0({ parent_id: "<page>", label: "Sessions", value: "5,471", icon: "activity" })
|
||||
```
|
||||
|
||||
<!-- /@domain -->
|
||||
|
||||
### OTP / 2FA verification screen
|
||||
|
||||
```
|
||||
|
|
@ -524,6 +548,8 @@ add_text_button_v0({ parent_id: "<page>", label: "Verify" })
|
|||
add_link_v0({ parent_id: "<page>", label: "Resend code" })
|
||||
```
|
||||
|
||||
<!-- @domain:mobile -->
|
||||
|
||||
### Support chat thread
|
||||
|
||||
```
|
||||
|
|
@ -533,6 +559,8 @@ add_chat_bubble_v0({ parent_id: "<thread>", message: "Sorry to hear! Let me chec
|
|||
add_attachment_row_v0({ parent_id: "<thread>", filename: "receipt.pdf", size: "240 KB", icon: "file-text" })
|
||||
```
|
||||
|
||||
<!-- /@domain -->
|
||||
|
||||
### Empty inbox / first-run onboarding
|
||||
|
||||
```
|
||||
|
|
|
|||
|
|
@ -103,4 +103,77 @@ describe('buildSystemPrompt', () => {
|
|||
expect(built.system).not.toContain(T_STRATEGY_A_MARKER);
|
||||
expect(built.system).not.toContain(T_STRATEGY_B_MARKER);
|
||||
});
|
||||
|
||||
it('T + category=mobile drops dashboard/landing recipes from cookbook', () => {
|
||||
const mobile = buildSystemPrompt('T', { difficulty: 'composite', category: 'mobile' });
|
||||
// Dashboard-tagged recipe titles should be gone (Team / members list,
|
||||
// Audit / activity feed, KPI strip, Faceted search filter sidebar).
|
||||
expect(mobile.system).not.toContain('### Team / members list');
|
||||
expect(mobile.system).not.toContain('### Audit / activity feed');
|
||||
expect(mobile.system).not.toContain('### Dashboard KPI strip');
|
||||
expect(mobile.system).not.toContain('### Faceted search filter sidebar');
|
||||
// Landing-tagged recipe (Pricing) should be gone.
|
||||
expect(mobile.system).not.toContain('### Pricing section');
|
||||
// Mobile-tagged recipes should remain.
|
||||
expect(mobile.system).toContain('### Onboarding "How it works"');
|
||||
expect(mobile.system).toContain('### Support chat thread');
|
||||
});
|
||||
|
||||
it('T + category=dashboard drops mobile/landing recipes from cookbook', () => {
|
||||
const dashboard = buildSystemPrompt('T', { difficulty: 'composite', category: 'dashboard' });
|
||||
// Mobile-tagged recipes should be gone.
|
||||
expect(dashboard.system).not.toContain('### Onboarding "How it works"');
|
||||
expect(dashboard.system).not.toContain('### Support chat thread');
|
||||
// Landing-tagged recipe should be gone.
|
||||
expect(dashboard.system).not.toContain('### Pricing section');
|
||||
// Dashboard-tagged recipes should remain.
|
||||
expect(dashboard.system).toContain('### Team / members list');
|
||||
expect(dashboard.system).toContain('### Audit / activity feed');
|
||||
expect(dashboard.system).toContain('### Dashboard KPI strip');
|
||||
expect(dashboard.system).toContain('### Faceted search filter sidebar');
|
||||
});
|
||||
|
||||
it('T + category=landing drops mobile/dashboard recipes from cookbook', () => {
|
||||
const landing = buildSystemPrompt('T', { difficulty: 'composite', category: 'landing' });
|
||||
// Mobile-tagged recipes should be gone.
|
||||
expect(landing.system).not.toContain('### Onboarding "How it works"');
|
||||
expect(landing.system).not.toContain('### Support chat thread');
|
||||
// Dashboard-tagged recipes should be gone.
|
||||
expect(landing.system).not.toContain('### Team / members list');
|
||||
expect(landing.system).not.toContain('### Dashboard KPI strip');
|
||||
// Landing-tagged recipe should remain.
|
||||
expect(landing.system).toContain('### Pricing section');
|
||||
});
|
||||
|
||||
it('T + category preserves untagged content (Empty inbox / Login etc. stay general)', () => {
|
||||
// Untagged recipes should be visible regardless of category.
|
||||
for (const category of ['mobile', 'dashboard', 'landing'] as const) {
|
||||
const built = buildSystemPrompt('T', { difficulty: 'composite', category });
|
||||
expect(built.system).toContain('### Empty inbox / first-run onboarding');
|
||||
// Login + Signup are untagged today (cross-domain ambiguity); they
|
||||
// load for every category until we either tag with a comma list
|
||||
// or split them into single-domain variants.
|
||||
expect(built.system).toContain('### Login screen');
|
||||
expect(built.system).toContain('### Signup form');
|
||||
}
|
||||
});
|
||||
|
||||
it('T + category undefined = no filtering (every tagged recipe loads)', () => {
|
||||
const full = buildSystemPrompt('T', { difficulty: 'composite' });
|
||||
expect(full.system).toContain('### Team / members list');
|
||||
expect(full.system).toContain('### Onboarding "How it works"');
|
||||
expect(full.system).toContain('### Pricing section');
|
||||
expect(full.system).toContain('### Empty inbox / first-run onboarding');
|
||||
});
|
||||
|
||||
it('T + category produces a smaller prompt than uncategorized (filter actually fires)', () => {
|
||||
const full = buildSystemPrompt('T', { difficulty: 'composite' }).system.length;
|
||||
for (const category of ['mobile', 'dashboard', 'landing'] as const) {
|
||||
const filtered = buildSystemPrompt('T', { difficulty: 'composite', category }).system.length;
|
||||
expect(filtered).toBeLessThan(full);
|
||||
// Each filter should drop at least 500 chars (loose floor — guards
|
||||
// against the regex silently no-oping, not a tight savings target).
|
||||
expect(full - filtered).toBeGreaterThan(500);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -82,6 +82,51 @@ export interface BuildPromptOpts {
|
|||
* the A/B comparison still isolates "tools vs no-tools".
|
||||
*/
|
||||
difficulty?: 'obvious' | 'optional' | 'composite';
|
||||
|
||||
/**
|
||||
* Prompt category signal. When variant=T and the category is one of
|
||||
* the supported domains, strip `<!-- @domain:X -->...<!-- /@domain -->`
|
||||
* blocks in elements.md whose tag doesn't match. Untagged content is
|
||||
* "general" and stays in every variant.
|
||||
*
|
||||
* Saves ~3-5kb per non-matching block. The per-domain annotations
|
||||
* are partial today (only the most obviously domain-specific tools
|
||||
* — bottom_nav / data_table_row / pricing_card etc.); ambiguous ones
|
||||
* stay untagged so the harness fails closed (they always load).
|
||||
*
|
||||
* undefined → all domain blocks load (safe default for free-form
|
||||
* callers and ab-v3-style runs that don't classify by category).
|
||||
*/
|
||||
category?: 'mobile' | 'dashboard' | 'landing';
|
||||
}
|
||||
|
||||
/**
|
||||
* Strips `<!-- @domain:X -->...<!-- /@domain -->` blocks whose tag
|
||||
* doesn't match `keep`. Untagged content is preserved verbatim.
|
||||
*
|
||||
* The inline comment syntax keeps elements.md a single file (no
|
||||
* multi-file splits to maintain per domain) and keeps the diff
|
||||
* surgical — adding a tag is two HTML comments, no structural move.
|
||||
*
|
||||
* Block syntax:
|
||||
*
|
||||
* <!-- @domain:mobile -->
|
||||
* ... mobile-only content ...
|
||||
* <!-- /@domain -->
|
||||
*
|
||||
* Multiple comma-separated tags are accepted (e.g. `mobile,dashboard`)
|
||||
* and a block matches if `keep` appears in the list. This is the
|
||||
* common case for tools like calendar_grid that legitimately span
|
||||
* domains.
|
||||
*/
|
||||
function stripNonMatchingDomains(text: string, keep: string): string {
|
||||
return text.replace(
|
||||
/<!-- @domain:([a-z,\s]+) -->\s*([\s\S]*?)\s*<!-- \/@domain -->\s*/g,
|
||||
(_, tags: string, body: string) => {
|
||||
const allowed = tags.split(',').map((t) => t.trim());
|
||||
return allowed.includes(keep) ? body : '';
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
export function buildSystemPrompt(variant: 'B' | 'T', opts: BuildPromptOpts = {}): BuiltPrompt {
|
||||
|
|
@ -91,6 +136,7 @@ export function buildSystemPrompt(variant: 'B' | 'T', opts: BuildPromptOpts = {}
|
|||
// post-processing, AND elements (appended last session).
|
||||
const full = buildDesignPrompt();
|
||||
if (variant === 'T') {
|
||||
let payload = full;
|
||||
if (opts.difficulty === 'obvious') {
|
||||
// Save ~18kb by stripping the cookbook on single-tool prompts.
|
||||
// Models still see the decision tree + PREFER list (which is
|
||||
|
|
@ -103,15 +149,17 @@ export function buildSystemPrompt(variant: 'B' | 'T', opts: BuildPromptOpts = {}
|
|||
'elements-cookbook skill not found in registry — cannot apply obvious-difficulty diet without it',
|
||||
);
|
||||
}
|
||||
const stripped = full.replace(cookbookSkill.content, '');
|
||||
if (stripped === full) {
|
||||
payload = payload.replace(cookbookSkill.content, '');
|
||||
if (payload === full) {
|
||||
throw new Error(
|
||||
'elements-cookbook content was not present in the full prompt — buildDesignPrompt() may have been refactored; update build-prompt.ts to match',
|
||||
);
|
||||
}
|
||||
return { system: stripped.trim() + '\n\n' + T_TOOL_CALL_INSTRUCTIONS, variant };
|
||||
}
|
||||
return { system: full + '\n\n' + T_TOOL_CALL_INSTRUCTIONS, variant };
|
||||
if (opts.category) {
|
||||
payload = stripNonMatchingDomains(payload, opts.category);
|
||||
}
|
||||
return { system: payload.trim() + '\n\n' + T_TOOL_CALL_INSTRUCTIONS, variant };
|
||||
}
|
||||
// Baseline: strip BOTH elements skills (decision-tree + cookbook).
|
||||
// The cookbook split landed when elements.md crossed the 800-line
|
||||
|
|
|
|||
|
|
@ -1,43 +1,92 @@
|
|||
#!/usr/bin/env bun
|
||||
/**
|
||||
* Quick inline diagnostic — prints character + 4:1-token-estimate sizes
|
||||
* for every (variant, difficulty) combination of `buildSystemPrompt`.
|
||||
* Run when tuning the elements-cookbook diet to confirm the gate is
|
||||
* actually shaving the bytes the test floor predicts:
|
||||
* for every (variant, difficulty, category) combination of
|
||||
* `buildSystemPrompt`. Run when tuning the elements diet to confirm
|
||||
* the gates are actually shaving the bytes the test floor predicts:
|
||||
*
|
||||
* bun scripts/ab-corpus/measure-prompt-sizes.ts
|
||||
*
|
||||
* Not wired into the harness — purely for human inspection. The
|
||||
* authoritative regression guard is build-prompt.test.ts (>10kb floor).
|
||||
* authoritative regression guards are build-prompt.test.ts.
|
||||
*/
|
||||
|
||||
import { buildSystemPrompt } from './build-prompt';
|
||||
|
||||
type Difficulty = 'obvious' | 'optional' | 'composite';
|
||||
type Category = 'mobile' | 'dashboard' | 'landing';
|
||||
|
||||
const cases: Array<{
|
||||
label: string;
|
||||
variant: 'B' | 'T';
|
||||
difficulty?: 'obvious' | 'optional' | 'composite';
|
||||
difficulty?: Difficulty;
|
||||
category?: Category;
|
||||
}> = [
|
||||
{ label: 'B (any difficulty) ', variant: 'B' },
|
||||
{ label: 'T + difficulty="obvious" ', variant: 'T', difficulty: 'obvious' },
|
||||
{ label: 'T + difficulty="optional" ', variant: 'T', difficulty: 'optional' },
|
||||
{ label: 'T + difficulty="composite" ', variant: 'T', difficulty: 'composite' },
|
||||
{ label: 'T + (no difficulty, safe default) ', variant: 'T' },
|
||||
{ label: 'B (any) ', variant: 'B' },
|
||||
{ label: 'T + difficulty=obvious ', variant: 'T', difficulty: 'obvious' },
|
||||
{ label: 'T + difficulty=composite ', variant: 'T', difficulty: 'composite' },
|
||||
{
|
||||
label: 'T + difficulty=obvious + category=mobile ',
|
||||
variant: 'T',
|
||||
difficulty: 'obvious',
|
||||
category: 'mobile',
|
||||
},
|
||||
{
|
||||
label: 'T + difficulty=obvious + category=dash ',
|
||||
variant: 'T',
|
||||
difficulty: 'obvious',
|
||||
category: 'dashboard',
|
||||
},
|
||||
{
|
||||
label: 'T + difficulty=obvious + category=landing',
|
||||
variant: 'T',
|
||||
difficulty: 'obvious',
|
||||
category: 'landing',
|
||||
},
|
||||
{
|
||||
label: 'T + difficulty=composite + category=mobile',
|
||||
variant: 'T',
|
||||
difficulty: 'composite',
|
||||
category: 'mobile',
|
||||
},
|
||||
{
|
||||
label: 'T + difficulty=composite + category=dash ',
|
||||
variant: 'T',
|
||||
difficulty: 'composite',
|
||||
category: 'dashboard',
|
||||
},
|
||||
{
|
||||
label: 'T + difficulty=composite + category=land ',
|
||||
variant: 'T',
|
||||
difficulty: 'composite',
|
||||
category: 'landing',
|
||||
},
|
||||
];
|
||||
|
||||
const tObvious = buildSystemPrompt('T', { difficulty: 'obvious' }).system.length;
|
||||
const tComposite = buildSystemPrompt('T', { difficulty: 'composite' }).system.length;
|
||||
const savings = tComposite - tObvious;
|
||||
|
||||
process.stdout.write(`Variant / Difficulty | Chars | ~Tokens (chars/4)\n`);
|
||||
process.stdout.write(`---------------------+---------+------------------\n`);
|
||||
process.stdout.write(`Variant / Difficulty / Category | Chars | ~Tokens\n`);
|
||||
process.stdout.write(`-----------------------------------------+---------+---------\n`);
|
||||
for (const c of cases) {
|
||||
const built = buildSystemPrompt(c.variant, c.difficulty ? { difficulty: c.difficulty } : {});
|
||||
const opts: { difficulty?: Difficulty; category?: Category } = {};
|
||||
if (c.difficulty) opts.difficulty = c.difficulty;
|
||||
if (c.category) opts.category = c.category;
|
||||
const built = buildSystemPrompt(c.variant, opts);
|
||||
const len = built.system.length;
|
||||
process.stdout.write(
|
||||
`${c.label} | ${String(len).padStart(7)} | ${String(Math.round(len / 4)).padStart(6)}\n`,
|
||||
);
|
||||
}
|
||||
|
||||
const tCompositeFull = buildSystemPrompt('T', { difficulty: 'composite' }).system.length;
|
||||
const tCompositeMobile = buildSystemPrompt('T', { difficulty: 'composite', category: 'mobile' })
|
||||
.system.length;
|
||||
const tCompositeDash = buildSystemPrompt('T', { difficulty: 'composite', category: 'dashboard' })
|
||||
.system.length;
|
||||
const tCompositeLanding = buildSystemPrompt('T', { difficulty: 'composite', category: 'landing' })
|
||||
.system.length;
|
||||
|
||||
process.stdout.write(
|
||||
`\nObvious-vs-composite savings: ${savings} chars (~${Math.round(savings / 4)} tokens)\n`,
|
||||
`\nCategory savings on composite (the cookbook-loaded path):\n` +
|
||||
` mobile: ${tCompositeFull - tCompositeMobile} chars (~${Math.round((tCompositeFull - tCompositeMobile) / 4)} tokens)\n` +
|
||||
` dashboard: ${tCompositeFull - tCompositeDash} chars (~${Math.round((tCompositeFull - tCompositeDash) / 4)} tokens)\n` +
|
||||
` landing: ${tCompositeFull - tCompositeLanding} chars (~${Math.round((tCompositeFull - tCompositeLanding) / 4)} tokens)\n`,
|
||||
);
|
||||
|
|
|
|||
|
|
@ -53,7 +53,10 @@ import type { ModelCall } from './stub-model';
|
|||
* that's the path we're actively promoting.
|
||||
*/
|
||||
export async function realModelCall(call: ModelCall): Promise<ChatCallResult> {
|
||||
const built = buildSystemPrompt(call.variant, { difficulty: call.prompt.difficulty });
|
||||
const built = buildSystemPrompt(call.variant, {
|
||||
difficulty: call.prompt.difficulty,
|
||||
category: call.prompt.category,
|
||||
});
|
||||
const user = call.prompt.prompt;
|
||||
const model = call.model;
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue