fix(ai): budget runtime-augmented planning skills and raise the phase ceiling
The per-skill budget guard measures skill files as they sit on disk, but style-guide-selector grows at resolve time when the available-style-guide catalog replaces its placeholder — 819 tokens against a 500 budget, so the catalog tail was silently cut from every planning prompt. Worse, the three base planning skills alone already exceeded the 4000-token phase ceiling, which made the only domain skill of the phase undeliverable on any prompt, with no warning anywhere. The skill budget now covers its worst-case injection, the planning ceiling rises to 6000, and three new guards keep the hole shut: augmented skills must fit their own budget, every augmentation placeholder must declare a worst case, and the end-to- end resolver must deliver the full phase without drops or truncation. Claude-Session: https://claude.ai/code/session_01FqKQqNj8exYwopGDpYUU7x
This commit is contained in:
parent
2ca873346c
commit
b74c94d2da
|
|
@ -4,7 +4,7 @@ description: Select a pre-built visual style guide based on user request
|
|||
phase: [planning]
|
||||
trigger: null
|
||||
priority: 3
|
||||
budget: 500
|
||||
budget: 1500
|
||||
category: base
|
||||
---
|
||||
|
||||
|
|
|
|||
|
|
@ -273,7 +273,7 @@ mod tests {
|
|||
};
|
||||
let ctx = resolve_skills(Phase::Planning, "plan a landing page", &opts);
|
||||
assert!(ctx.memory.document_context.is_some());
|
||||
assert_eq!(ctx.budget_max, 4000);
|
||||
assert_eq!(ctx.budget_max, 6000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -57,9 +57,18 @@ impl Phase {
|
|||
/// headroom the codebase had already decided was correct. Tier-scaled
|
||||
/// callers (Basic/Standard mobile/desktop) are unaffected: they pass an
|
||||
/// explicit `budget_override` and never fall through to this default.
|
||||
///
|
||||
/// Planning moved 4000 → 6000 for a related reason (2026-07-28). Its
|
||||
/// three `Base` skills are budget-EXEMPT but still counted against the
|
||||
/// total, and they need ~4500 tokens on their own once
|
||||
/// `style-guide-selector` carries the injected style-guide catalog. At
|
||||
/// 4000 the phase was already over budget before Step 3 ran, so
|
||||
/// `landing-page-predesign` — the phase's only Domain skill — could never
|
||||
/// be included on ANY prompt, matched or not. The ceiling now covers the
|
||||
/// base set plus that skill with headroom.
|
||||
pub fn default_budget(self) -> u32 {
|
||||
match self {
|
||||
Phase::Planning => 4000,
|
||||
Phase::Planning => 6000,
|
||||
Phase::Generation => 12000,
|
||||
Phase::Validation => 3000,
|
||||
Phase::Maintenance => 5000,
|
||||
|
|
@ -69,7 +78,7 @@ impl Phase {
|
|||
|
||||
/// Per-phase default token budgets — the TS `DEFAULT_BUDGETS` record.
|
||||
pub const DEFAULT_BUDGETS: [(Phase, u32); 4] = [
|
||||
(Phase::Planning, 4000),
|
||||
(Phase::Planning, 6000),
|
||||
(Phase::Generation, 12000),
|
||||
(Phase::Validation, 3000),
|
||||
(Phase::Maintenance, 5000),
|
||||
|
|
@ -322,7 +331,7 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn default_budget_table() {
|
||||
assert_eq!(Phase::Planning.default_budget(), 4000);
|
||||
assert_eq!(Phase::Planning.default_budget(), 6000);
|
||||
assert_eq!(Phase::Generation.default_budget(), 12000);
|
||||
assert_eq!(Phase::Validation.default_budget(), 3000);
|
||||
assert_eq!(Phase::Maintenance.default_budget(), 5000);
|
||||
|
|
|
|||
183
crates/op-orchestrator/src/prompt_skill_budget_tests.rs
Normal file
183
crates/op-orchestrator/src/prompt_skill_budget_tests.rs
Normal file
|
|
@ -0,0 +1,183 @@
|
|||
//! Budget guards for planning skills whose content is AUGMENTED at runtime.
|
||||
//!
|
||||
//! `op-ai-skills` has `no_skill_silently_exceeds_its_own_budget`, but that
|
||||
//! test measures the file on disk — and `style-guide-selector.md` carries a
|
||||
//! `{{availableStyleGuides}}` placeholder that the orchestrator substitutes
|
||||
//! with the whole style-guide catalog before the trimmer ever sees it. The
|
||||
//! static corpus was 210 tokens against a 500 budget; the injected catalog
|
||||
//! took the real prompt to 819 tokens, and Step 1 of `trim_by_budget_pinned`
|
||||
//! silently chopped 1287 chars off the tail — the selection rules that tell
|
||||
//! the model what to DO with the catalog — on every planning call
|
||||
//! (2026-07-28 production log).
|
||||
//!
|
||||
//! The corpus test cannot catch this: `op-ai-skills` is below the
|
||||
//! orchestrator in the dependency graph and cannot see the catalog. So the
|
||||
//! guard has to live here, on the side that owns the augmentation.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use jian_ops_schema::{DesignMdColor, DesignMdSpec, DesignMdTypography};
|
||||
use op_ai_skills::budget::estimate_tokens;
|
||||
use op_ai_skills::resolver::inject_dynamic_content;
|
||||
|
||||
use super::*;
|
||||
|
||||
/// Every placeholder the orchestrator substitutes into a PLANNING skill.
|
||||
/// Adding a runtime-augmented planning key without adding it here is what
|
||||
/// this guard exists to make impossible — a new key with no worst-case entry
|
||||
/// leaves the same blind spot `{{availableStyleGuides}}` sat in.
|
||||
const AUGMENTED_PLANNING_KEYS: [&str; 1] = ["availableStyleGuides"];
|
||||
|
||||
/// A design.md spec filled to the size limits `build_design_md_style_policy`
|
||||
/// enforces (200/300/400-char truncations, 10 palette rows, 6 surface rows),
|
||||
/// so the design.md branch is measured at ITS worst case too — that branch
|
||||
/// interpolates USER content, and a fixed budget has to cover it.
|
||||
fn saturated_design_md() -> DesignMdSpec {
|
||||
let color = |i: usize| DesignMdColor {
|
||||
name: format!("Palette Color Number {i}"),
|
||||
hex: format!("#0000{i:02}"),
|
||||
role: format!("surface role {i} — card, panel and sidebar backgrounds"),
|
||||
};
|
||||
DesignMdSpec {
|
||||
raw: String::new(),
|
||||
project_name: Some("A Rather Long Project Name For Measurement".into()),
|
||||
visual_theme: Some("v".repeat(400)),
|
||||
color_palette: Some((0..20).map(color).collect()),
|
||||
typography: Some(DesignMdTypography {
|
||||
font_family: Some("f".repeat(120)),
|
||||
headings: Some("h".repeat(120)),
|
||||
body: Some("b".repeat(120)),
|
||||
scale: Some("s".repeat(400)),
|
||||
}),
|
||||
component_styles: Some("c".repeat(600)),
|
||||
layout_principles: Some("l".repeat(800)),
|
||||
generation_notes: Some("n".repeat(800)),
|
||||
}
|
||||
}
|
||||
|
||||
/// The worst-case value of `{{availableStyleGuides}}` over every branch that
|
||||
/// can produce one: both catalog planning modes crossed with every model tier
|
||||
/// (the tier sets the snippet count), plus the design.md branch.
|
||||
fn worst_case_style_guide_context() -> (String, String) {
|
||||
// One model id per tier — `snippet_limit` is the only tier-sensitive
|
||||
// input, and it is what makes the catalog branch grow.
|
||||
let models = ["claude-opus", "glm-4", "minimax-m3", ""];
|
||||
let prompts = [
|
||||
"a fintech dashboard",
|
||||
"a dark minimalist mobile music app landing page",
|
||||
"xyz123",
|
||||
];
|
||||
let design_md = saturated_design_md();
|
||||
let mut worst = (String::new(), String::new());
|
||||
for mode in [PlanningMode::Rich, PlanningMode::Minimal] {
|
||||
for model in models {
|
||||
for prompt in prompts {
|
||||
for spec in [None, Some(&design_md)] {
|
||||
let ctx = build_planning_style_guide_context(prompt, Some(model), mode, spec);
|
||||
if ctx.available_style_guides.chars().count() > worst.0.chars().count() {
|
||||
let label = format!(
|
||||
"mode={mode:?} model={model:?} design_md={} prompt={prompt:?}",
|
||||
spec.is_some()
|
||||
);
|
||||
worst = (ctx.available_style_guides, label);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
worst
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_augmented_planning_skills_fit_their_own_budget() {
|
||||
let (context, label) = worst_case_style_guide_context();
|
||||
let dynamic = HashMap::from([("availableStyleGuides".to_string(), context.clone())]);
|
||||
|
||||
let mut offenders = Vec::new();
|
||||
for skill in op_ai_skills::get_skills_by_phase(op_ai_skills::Phase::Planning) {
|
||||
let augmented = inject_dynamic_content(&skill.content, &dynamic);
|
||||
if augmented == skill.content {
|
||||
continue; // no placeholder — the corpus test already covers it
|
||||
}
|
||||
let actual = estimate_tokens(&augmented);
|
||||
if actual > skill.meta.budget {
|
||||
offenders.push(format!(
|
||||
"{} (budget={}, augmented={actual}, over by {}) — worst case {label}",
|
||||
skill.meta.name,
|
||||
skill.meta.budget,
|
||||
actual - skill.meta.budget
|
||||
));
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
offenders.is_empty(),
|
||||
"planning skills are truncated AFTER runtime augmentation, silently dropping \
|
||||
their tail from every planning prompt: {offenders:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_augmented_placeholder_has_a_worst_case_in_this_guard() {
|
||||
// The guard is only as good as its key list. Any `{{placeholder}}` a
|
||||
// planning skill declares must be one this file measures — otherwise a
|
||||
// new augmented key reopens exactly the hole this file closes.
|
||||
for skill in op_ai_skills::get_skills_by_phase(op_ai_skills::Phase::Planning) {
|
||||
for (index, _) in skill.content.match_indices("{{") {
|
||||
let rest = &skill.content[index + 2..];
|
||||
let Some(end) = rest.find("}}") else { continue };
|
||||
let key = &rest[..end];
|
||||
assert!(
|
||||
AUGMENTED_PLANNING_KEYS.contains(&key),
|
||||
"planning skill {:?} interpolates {{{{{key}}}}}, which this budget guard \
|
||||
does not measure — add its worst case to AUGMENTED_PLANNING_KEYS",
|
||||
skill.meta.name
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_planning_skill_is_dropped_or_truncated_by_the_phase_budget() {
|
||||
// End-to-end through the real resolver: the per-skill cap AND the phase
|
||||
// total. The landing-page prompt is deliberate — `landing-page-predesign`
|
||||
// is the phase's only Domain skill, so it is the one that gets squeezed
|
||||
// out when the base skills eat the whole total.
|
||||
let (context, label) = worst_case_style_guide_context();
|
||||
let opts = op_ai_skills::ResolveOptions {
|
||||
dynamic_content: HashMap::from([("availableStyleGuides".to_string(), context)]),
|
||||
..Default::default()
|
||||
};
|
||||
let prompt = "a marketing landing page for a fintech product";
|
||||
let ctx = op_ai_skills::resolve_skills(op_ai_skills::Phase::Planning, prompt, &opts);
|
||||
|
||||
let truncated: Vec<&str> = ctx
|
||||
.report
|
||||
.included
|
||||
.iter()
|
||||
.filter(|entry| entry.truncated)
|
||||
.map(|entry| entry.name.as_str())
|
||||
.collect();
|
||||
assert!(
|
||||
truncated.is_empty(),
|
||||
"truncated planning skills {truncated:?} (worst case {label}); \
|
||||
used {}/{} tokens",
|
||||
ctx.report.budget_used,
|
||||
ctx.report.budget_max
|
||||
);
|
||||
|
||||
let starved: Vec<&str> = ctx
|
||||
.report
|
||||
.dropped
|
||||
.iter()
|
||||
.filter(|entry| entry.reason == op_ai_skills::DropReason::BudgetExhausted)
|
||||
.map(|entry| entry.name.as_str())
|
||||
.collect();
|
||||
assert!(
|
||||
starved.is_empty(),
|
||||
"planning skills dropped for budget {starved:?}; used {}/{} tokens — the phase \
|
||||
total must cover the base skills (which are budget-exempt and therefore always \
|
||||
counted) plus every intent-matched Domain skill",
|
||||
ctx.report.budget_used,
|
||||
ctx.report.budget_max
|
||||
);
|
||||
}
|
||||
|
|
@ -82,6 +82,8 @@ fn subtask() -> crate::plan::Subtask {
|
|||
mod components_tests;
|
||||
#[path = "prompt_planning_tests.rs"]
|
||||
mod planning_tests;
|
||||
#[path = "prompt_skill_budget_tests.rs"]
|
||||
mod skill_budget_tests;
|
||||
#[path = "prompt_subagent_content_tests.rs"]
|
||||
mod subagent_content_tests;
|
||||
#[path = "prompt_timeout_feedback_tests.rs"]
|
||||
|
|
|
|||
Loading…
Reference in a new issue