diff --git a/crates/op-ai-skills/skills/phases/planning/style-guide-selector.md b/crates/op-ai-skills/skills/phases/planning/style-guide-selector.md index b42b65bc2..fc68381d6 100644 --- a/crates/op-ai-skills/skills/phases/planning/style-guide-selector.md +++ b/crates/op-ai-skills/skills/phases/planning/style-guide-selector.md @@ -4,7 +4,7 @@ description: Select a pre-built visual style guide based on user request phase: [planning] trigger: null priority: 3 -budget: 500 +budget: 1500 category: base --- diff --git a/crates/op-ai-skills/src/resolve.rs b/crates/op-ai-skills/src/resolve.rs index 9a2cb883e..187a52ef5 100644 --- a/crates/op-ai-skills/src/resolve.rs +++ b/crates/op-ai-skills/src/resolve.rs @@ -273,7 +273,7 @@ mod tests { }; let ctx = resolve_skills(Phase::Planning, "plan a landing page", &opts); assert!(ctx.memory.document_context.is_some()); - assert_eq!(ctx.budget_max, 4000); + assert_eq!(ctx.budget_max, 6000); } #[test] diff --git a/crates/op-ai-skills/src/types.rs b/crates/op-ai-skills/src/types.rs index 971ef6e36..1ac724da9 100644 --- a/crates/op-ai-skills/src/types.rs +++ b/crates/op-ai-skills/src/types.rs @@ -57,9 +57,18 @@ impl Phase { /// headroom the codebase had already decided was correct. Tier-scaled /// callers (Basic/Standard mobile/desktop) are unaffected: they pass an /// explicit `budget_override` and never fall through to this default. + /// + /// Planning moved 4000 → 6000 for a related reason (2026-07-28). Its + /// three `Base` skills are budget-EXEMPT but still counted against the + /// total, and they need ~4500 tokens on their own once + /// `style-guide-selector` carries the injected style-guide catalog. At + /// 4000 the phase was already over budget before Step 3 ran, so + /// `landing-page-predesign` — the phase's only Domain skill — could never + /// be included on ANY prompt, matched or not. The ceiling now covers the + /// base set plus that skill with headroom. pub fn default_budget(self) -> u32 { match self { - Phase::Planning => 4000, + Phase::Planning => 6000, Phase::Generation => 12000, Phase::Validation => 3000, Phase::Maintenance => 5000, @@ -69,7 +78,7 @@ impl Phase { /// Per-phase default token budgets — the TS `DEFAULT_BUDGETS` record. pub const DEFAULT_BUDGETS: [(Phase, u32); 4] = [ - (Phase::Planning, 4000), + (Phase::Planning, 6000), (Phase::Generation, 12000), (Phase::Validation, 3000), (Phase::Maintenance, 5000), @@ -322,7 +331,7 @@ mod tests { #[test] fn default_budget_table() { - assert_eq!(Phase::Planning.default_budget(), 4000); + assert_eq!(Phase::Planning.default_budget(), 6000); assert_eq!(Phase::Generation.default_budget(), 12000); assert_eq!(Phase::Validation.default_budget(), 3000); assert_eq!(Phase::Maintenance.default_budget(), 5000); diff --git a/crates/op-orchestrator/src/prompt_skill_budget_tests.rs b/crates/op-orchestrator/src/prompt_skill_budget_tests.rs new file mode 100644 index 000000000..f0078c6a8 --- /dev/null +++ b/crates/op-orchestrator/src/prompt_skill_budget_tests.rs @@ -0,0 +1,183 @@ +//! Budget guards for planning skills whose content is AUGMENTED at runtime. +//! +//! `op-ai-skills` has `no_skill_silently_exceeds_its_own_budget`, but that +//! test measures the file on disk — and `style-guide-selector.md` carries a +//! `{{availableStyleGuides}}` placeholder that the orchestrator substitutes +//! with the whole style-guide catalog before the trimmer ever sees it. The +//! static corpus was 210 tokens against a 500 budget; the injected catalog +//! took the real prompt to 819 tokens, and Step 1 of `trim_by_budget_pinned` +//! silently chopped 1287 chars off the tail — the selection rules that tell +//! the model what to DO with the catalog — on every planning call +//! (2026-07-28 production log). +//! +//! The corpus test cannot catch this: `op-ai-skills` is below the +//! orchestrator in the dependency graph and cannot see the catalog. So the +//! guard has to live here, on the side that owns the augmentation. + +use std::collections::HashMap; + +use jian_ops_schema::{DesignMdColor, DesignMdSpec, DesignMdTypography}; +use op_ai_skills::budget::estimate_tokens; +use op_ai_skills::resolver::inject_dynamic_content; + +use super::*; + +/// Every placeholder the orchestrator substitutes into a PLANNING skill. +/// Adding a runtime-augmented planning key without adding it here is what +/// this guard exists to make impossible — a new key with no worst-case entry +/// leaves the same blind spot `{{availableStyleGuides}}` sat in. +const AUGMENTED_PLANNING_KEYS: [&str; 1] = ["availableStyleGuides"]; + +/// A design.md spec filled to the size limits `build_design_md_style_policy` +/// enforces (200/300/400-char truncations, 10 palette rows, 6 surface rows), +/// so the design.md branch is measured at ITS worst case too — that branch +/// interpolates USER content, and a fixed budget has to cover it. +fn saturated_design_md() -> DesignMdSpec { + let color = |i: usize| DesignMdColor { + name: format!("Palette Color Number {i}"), + hex: format!("#0000{i:02}"), + role: format!("surface role {i} — card, panel and sidebar backgrounds"), + }; + DesignMdSpec { + raw: String::new(), + project_name: Some("A Rather Long Project Name For Measurement".into()), + visual_theme: Some("v".repeat(400)), + color_palette: Some((0..20).map(color).collect()), + typography: Some(DesignMdTypography { + font_family: Some("f".repeat(120)), + headings: Some("h".repeat(120)), + body: Some("b".repeat(120)), + scale: Some("s".repeat(400)), + }), + component_styles: Some("c".repeat(600)), + layout_principles: Some("l".repeat(800)), + generation_notes: Some("n".repeat(800)), + } +} + +/// The worst-case value of `{{availableStyleGuides}}` over every branch that +/// can produce one: both catalog planning modes crossed with every model tier +/// (the tier sets the snippet count), plus the design.md branch. +fn worst_case_style_guide_context() -> (String, String) { + // One model id per tier — `snippet_limit` is the only tier-sensitive + // input, and it is what makes the catalog branch grow. + let models = ["claude-opus", "glm-4", "minimax-m3", ""]; + let prompts = [ + "a fintech dashboard", + "a dark minimalist mobile music app landing page", + "xyz123", + ]; + let design_md = saturated_design_md(); + let mut worst = (String::new(), String::new()); + for mode in [PlanningMode::Rich, PlanningMode::Minimal] { + for model in models { + for prompt in prompts { + for spec in [None, Some(&design_md)] { + let ctx = build_planning_style_guide_context(prompt, Some(model), mode, spec); + if ctx.available_style_guides.chars().count() > worst.0.chars().count() { + let label = format!( + "mode={mode:?} model={model:?} design_md={} prompt={prompt:?}", + spec.is_some() + ); + worst = (ctx.available_style_guides, label); + } + } + } + } + } + worst +} + +#[test] +fn runtime_augmented_planning_skills_fit_their_own_budget() { + let (context, label) = worst_case_style_guide_context(); + let dynamic = HashMap::from([("availableStyleGuides".to_string(), context.clone())]); + + let mut offenders = Vec::new(); + for skill in op_ai_skills::get_skills_by_phase(op_ai_skills::Phase::Planning) { + let augmented = inject_dynamic_content(&skill.content, &dynamic); + if augmented == skill.content { + continue; // no placeholder — the corpus test already covers it + } + let actual = estimate_tokens(&augmented); + if actual > skill.meta.budget { + offenders.push(format!( + "{} (budget={}, augmented={actual}, over by {}) — worst case {label}", + skill.meta.name, + skill.meta.budget, + actual - skill.meta.budget + )); + } + } + assert!( + offenders.is_empty(), + "planning skills are truncated AFTER runtime augmentation, silently dropping \ + their tail from every planning prompt: {offenders:?}" + ); +} + +#[test] +fn every_augmented_placeholder_has_a_worst_case_in_this_guard() { + // The guard is only as good as its key list. Any `{{placeholder}}` a + // planning skill declares must be one this file measures — otherwise a + // new augmented key reopens exactly the hole this file closes. + for skill in op_ai_skills::get_skills_by_phase(op_ai_skills::Phase::Planning) { + for (index, _) in skill.content.match_indices("{{") { + let rest = &skill.content[index + 2..]; + let Some(end) = rest.find("}}") else { continue }; + let key = &rest[..end]; + assert!( + AUGMENTED_PLANNING_KEYS.contains(&key), + "planning skill {:?} interpolates {{{{{key}}}}}, which this budget guard \ + does not measure — add its worst case to AUGMENTED_PLANNING_KEYS", + skill.meta.name + ); + } + } +} + +#[test] +fn no_planning_skill_is_dropped_or_truncated_by_the_phase_budget() { + // End-to-end through the real resolver: the per-skill cap AND the phase + // total. The landing-page prompt is deliberate — `landing-page-predesign` + // is the phase's only Domain skill, so it is the one that gets squeezed + // out when the base skills eat the whole total. + let (context, label) = worst_case_style_guide_context(); + let opts = op_ai_skills::ResolveOptions { + dynamic_content: HashMap::from([("availableStyleGuides".to_string(), context)]), + ..Default::default() + }; + let prompt = "a marketing landing page for a fintech product"; + let ctx = op_ai_skills::resolve_skills(op_ai_skills::Phase::Planning, prompt, &opts); + + let truncated: Vec<&str> = ctx + .report + .included + .iter() + .filter(|entry| entry.truncated) + .map(|entry| entry.name.as_str()) + .collect(); + assert!( + truncated.is_empty(), + "truncated planning skills {truncated:?} (worst case {label}); \ + used {}/{} tokens", + ctx.report.budget_used, + ctx.report.budget_max + ); + + let starved: Vec<&str> = ctx + .report + .dropped + .iter() + .filter(|entry| entry.reason == op_ai_skills::DropReason::BudgetExhausted) + .map(|entry| entry.name.as_str()) + .collect(); + assert!( + starved.is_empty(), + "planning skills dropped for budget {starved:?}; used {}/{} tokens — the phase \ + total must cover the base skills (which are budget-exempt and therefore always \ + counted) plus every intent-matched Domain skill", + ctx.report.budget_used, + ctx.report.budget_max + ); +} diff --git a/crates/op-orchestrator/src/prompt_tests.rs b/crates/op-orchestrator/src/prompt_tests.rs index 7c2e5a187..d456e755a 100644 --- a/crates/op-orchestrator/src/prompt_tests.rs +++ b/crates/op-orchestrator/src/prompt_tests.rs @@ -82,6 +82,8 @@ fn subtask() -> crate::plan::Subtask { mod components_tests; #[path = "prompt_planning_tests.rs"] mod planning_tests; +#[path = "prompt_skill_budget_tests.rs"] +mod skill_budget_tests; #[path = "prompt_subagent_content_tests.rs"] mod subagent_content_tests; #[path = "prompt_timeout_feedback_tests.rs"]