123 lines
5 KiB
TypeScript
123 lines
5 KiB
TypeScript
/**
|
|
* Offline model stub for --dry-run mode. Returns hardcoded outputs that
|
|
* cover all four kinds the scorer can see (tool_call, batch_design,
|
|
* garbage, plus a "mostly right" batch_design that passes shape checks
|
|
* too). Used to verify the full pipeline without burning API credits.
|
|
*
|
|
* Keyed by (promptId, variant) so individual prompts can be told to
|
|
* return specific shapes. Fallback response when a prompt isn't
|
|
* explicitly fixtured: treatment variant returns a generic
|
|
* batch_design (simulates "weak model didn't route to element tool"),
|
|
* baseline returns a minimal valid design.
|
|
*
|
|
* When the real model adapter lands, swap this out with a thin
|
|
* wrapper around apps/web/src/services/ai/ai-service.ts::streamChat
|
|
* (or a provider-native fetch). The return-shape contract — a single
|
|
* raw string the scorer parses — stays the same.
|
|
*/
|
|
|
|
import type { CorpusPrompt } from '@zseven-w/pen-ai-skills';
|
|
|
|
export interface ModelCall {
|
|
model: string;
|
|
prompt: CorpusPrompt;
|
|
variant: 'B' | 'T';
|
|
/** Full resolved system+user prompt text sent to the model. Ignored
|
|
* by the stub, kept so the real adapter has the same signature. */
|
|
systemPrompt: string;
|
|
userPrompt: string;
|
|
}
|
|
|
|
export async function stubModelCall(call: ModelCall): Promise<string> {
|
|
const key = `${call.prompt.id}|${call.variant}`;
|
|
const fixture = FIXTURES[key];
|
|
if (fixture) return fixture;
|
|
// Fallback: treatment-arm on unfixtured prompts simulates a weak
|
|
// model that knew about the tools but picked batch_design anyway
|
|
// (interesting M5 signal); baseline always emits batch_design DSL.
|
|
return DEFAULT_BATCH_DESIGN;
|
|
}
|
|
|
|
// batch_design's DSL parser expects one operation per line. Keep this
|
|
// on a single physical line or it fails at apply time — the point of
|
|
// a fallback stub is to produce a *legal* design so M1 passes.
|
|
const DEFAULT_BATCH_DESIGN =
|
|
'root=I(null, {"type":"frame","name":"Root","role":"section","width":375,"height":812,"layout":"vertical","children":[]})';
|
|
|
|
/**
|
|
* A few hand-crafted fixtures so the dry-run report has variety in it
|
|
* (some M1 passes, some fails, some tool_calls, some batch_design). The
|
|
* shapes here aren't meant to be realistic model outputs — they're just
|
|
* enough to exercise the pipeline's branches.
|
|
*/
|
|
const FIXTURES: Record<string, string> = {
|
|
// mobile-filter-chips (obvious) in treatment: correct tool_call routing
|
|
'mobile-filter-chips|T': [
|
|
'<op_tool>',
|
|
'{"name": "add_nav_chip_row_v0", "arguments": {',
|
|
' "items": [',
|
|
' {"label": "All", "active": true},',
|
|
' {"label": "Videos", "icon": "video"},',
|
|
' {"label": "Photos", "icon": "image"},',
|
|
' {"label": "Music", "icon": "music"},',
|
|
' {"label": "Articles", "icon": "file-text"},',
|
|
' {"label": "Podcasts", "icon": "headphones"},',
|
|
' {"label": "Events", "icon": "calendar"},',
|
|
' {"label": "Books", "icon": "book"}',
|
|
' ]',
|
|
'}}',
|
|
'</op_tool>',
|
|
].join('\n'),
|
|
// mobile-empty-inbox (obvious) in treatment: correct tool routing
|
|
'mobile-empty-inbox|T': [
|
|
'<op_tool>',
|
|
'{"name": "add_empty_state_v0", "arguments": {',
|
|
' "title": "No messages yet",',
|
|
' "subtitle": "New conversations will appear here",',
|
|
' "icon": "inbox",',
|
|
' "cta_label": "Start a conversation"',
|
|
'}}',
|
|
'</op_tool>',
|
|
].join('\n'),
|
|
// dashboard-alert-banner (obvious) in treatment: correct routing
|
|
'dashboard-alert-banner|T': [
|
|
'<op_tool>',
|
|
'{"name": "add_alert_v0", "arguments": {',
|
|
' "message": "Your subscription expires in 7 days.",',
|
|
' "icon": "triangle-alert",',
|
|
' "dismissible": true',
|
|
'}}',
|
|
'</op_tool>',
|
|
].join('\n'),
|
|
// landing-testimonial (obvious) in treatment: correct routing
|
|
'landing-testimonial|T': [
|
|
'<op_tool>',
|
|
'{"name": "add_quote_block_v0", "arguments": {',
|
|
' "quote": "OpenPencil replaced two tools in our stack.",',
|
|
' "author": "Alex Chen"',
|
|
'}}',
|
|
'</op_tool>',
|
|
].join('\n'),
|
|
// landing-product-ratings (obvious) in treatment: WRONG tool routing.
|
|
// Prompt asks for star rating (expected_tool_if_any=add_rating_stars_v0)
|
|
// but stub simulates a weak model routing to add_badge_v0 — the
|
|
// badge IS a schema-valid tool call, just wrong intent. Exercises
|
|
// the wrong-tool routing classification.
|
|
'landing-product-ratings|T': [
|
|
'<op_tool>',
|
|
'{"name": "add_badge_v0", "arguments": {',
|
|
' "label": "4.8"',
|
|
'}}',
|
|
'</op_tool>',
|
|
].join('\n'),
|
|
// Baseline garbage response (simulates a weak model producing prose)
|
|
'mobile-tabs-demo|B':
|
|
'I can help you design that! Let me create a tabs demo for you with three underline tabs.',
|
|
// Treatment garbage on an OBVIOUS prompt — exercises the Garbage
|
|
// column in the routing breakdown. Codex stop-hook 2026-04-20 fix:
|
|
// if garbage isn't counted in the M5 denominator, right-tool rates
|
|
// look rosy even when most outputs fail to parse.
|
|
'dashboard-wizard-stepper|T':
|
|
'Sure! Here is a stepper wizard header: <thinking about design> ... actually let me just describe it.',
|
|
};
|