|
| 1 | +#!/usr/bin/env python3 |
| 2 | +"""Golden-prompt runner. Executes tests/golden-prompts.md against a live model. |
| 3 | +
|
| 4 | +Runs each prompt through `claude -p` with this repo loaded via --plugin-dir, |
| 5 | +then asserts: |
| 6 | + - activation rows (the per-bucket tables): the named skill activates, and in |
| 7 | + --full mode the output uses the contracted verdict vocabulary; |
| 8 | + - routing rows ("must NOT misfire" table): the intended skill activates and |
| 9 | + the adjacent skill does not; the dormancy row activates nothing. |
| 10 | +
|
| 11 | +This costs API tokens and takes minutes — it is NOT run by CI on push. |
| 12 | +Run it before cutting a release, or after editing any frontmatter description |
| 13 | +(routing) or role structure (contracts). |
| 14 | +
|
| 15 | +Usage: |
| 16 | + python3 scripts/run_golden_prompts.py --list # parse and show cases, no API calls |
| 17 | + python3 scripts/run_golden_prompts.py --routing # routing table only (cheap: capped turns) |
| 18 | + python3 scripts/run_golden_prompts.py --full # everything, uncapped, checks verdict vocab |
| 19 | + python3 scripts/run_golden_prompts.py --only people-tool-selection,general-roi-gate |
| 20 | + python3 scripts/run_golden_prompts.py --model sonnet # cross-model run |
| 21 | +""" |
| 22 | + |
| 23 | +from __future__ import annotations |
| 24 | + |
| 25 | +import argparse |
| 26 | +import json |
| 27 | +import os |
| 28 | +import re |
| 29 | +import subprocess |
| 30 | +import sys |
| 31 | +import tempfile |
| 32 | +import time |
| 33 | + |
| 34 | +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
| 35 | +GOLDEN = os.path.join(ROOT, "tests", "golden-prompts.md") |
| 36 | + |
| 37 | +# Read-only tools only: skills may consult references/, but must not mutate. |
| 38 | +ALLOWED_TOOLS = "Skill Read Glob Grep" |
| 39 | + |
| 40 | + |
| 41 | +def parse_cases(): |
| 42 | + """Return (activation_cases, routing_cases) from golden-prompts.md.""" |
| 43 | + activation, routing = [], [] |
| 44 | + in_routing = False |
| 45 | + for line in open(GOLDEN): |
| 46 | + if line.startswith("## Routing prompts"): |
| 47 | + in_routing = True |
| 48 | + continue |
| 49 | + if not line.startswith("|") or line.startswith("|-") or line.startswith("|---"): |
| 50 | + continue |
| 51 | + cells = [c.strip() for c in line.strip().strip("|").split("|")] |
| 52 | + if in_routing: |
| 53 | + if len(cells) < 3 or cells[0] in ("Prompt", "---"): |
| 54 | + continue |
| 55 | + prompt = cells[0].strip('"') |
| 56 | + must = m.group(1) if (m := re.search(r"`([a-z0-9-]+)`", cells[1])) else None |
| 57 | + must_not = m.group(1) if (m := re.search(r"`([a-z0-9-]+)`", cells[2])) else "ANY" |
| 58 | + routing.append({"prompt": prompt, "must": must, "must_not": must_not}) |
| 59 | + else: |
| 60 | + m = re.match(r"`([a-z0-9-]+)`", cells[0]) |
| 61 | + if not m or len(cells) < 3: |
| 62 | + continue |
| 63 | + prompt = cells[1].strip('"') |
| 64 | + vocab_groups = [ |
| 65 | + [opt.strip() for opt in grp.split(" / ")] |
| 66 | + for grp in re.findall(r"\*\*(.+?)\*\*", cells[2]) |
| 67 | + ] |
| 68 | + activation.append({"skill": m.group(1), "prompt": prompt, "vocab": vocab_groups}) |
| 69 | + return activation, routing |
| 70 | + |
| 71 | + |
| 72 | +def run_claude(prompt: str, model: str | None, max_turns: int | None, timeout: int): |
| 73 | + """Run one headless prompt. Returns (activated_skills, result_text, error).""" |
| 74 | + cmd = [ |
| 75 | + "claude", "-p", prompt, |
| 76 | + "--plugin-dir", ROOT, |
| 77 | + "--output-format", "stream-json", "--verbose", |
| 78 | + "--allowedTools", ALLOWED_TOOLS, |
| 79 | + ] |
| 80 | + if model: |
| 81 | + cmd += ["--model", model] |
| 82 | + if max_turns: |
| 83 | + cmd += ["--max-turns", str(max_turns)] |
| 84 | + # Neutral cwd: don't let this repo's CLAUDE.md leak into the routing test. |
| 85 | + with tempfile.TemporaryDirectory() as neutral: |
| 86 | + try: |
| 87 | + proc = subprocess.run( |
| 88 | + cmd, capture_output=True, text=True, timeout=timeout, cwd=neutral |
| 89 | + ) |
| 90 | + except subprocess.TimeoutExpired: |
| 91 | + return set(), "", "timeout" |
| 92 | + if proc.returncode != 0 and not proc.stdout: |
| 93 | + return set(), "", (proc.stderr or "claude exited non-zero").strip()[:300] |
| 94 | + |
| 95 | + activated, result_text, result_error = set(), "", None |
| 96 | + for raw in proc.stdout.splitlines(): |
| 97 | + try: |
| 98 | + event = json.loads(raw) |
| 99 | + except json.JSONDecodeError: |
| 100 | + continue |
| 101 | + if event.get("type") == "assistant": |
| 102 | + for block in event.get("message", {}).get("content", []): |
| 103 | + if block.get("type") == "tool_use" and block.get("name") == "Skill": |
| 104 | + activated.add(block.get("input", {}).get("skill", "").split(":")[-1]) |
| 105 | + elif event.get("type") == "result": |
| 106 | + result_text = event.get("result") or "" |
| 107 | + if event.get("is_error") or event.get("subtype") != "success": |
| 108 | + # A failed run must not masquerade as "skill stayed dormant". |
| 109 | + result_error = f"{event.get('subtype')}: {result_text[:200]}" |
| 110 | + return activated, result_text, result_error |
| 111 | + |
| 112 | + |
| 113 | +def vocab_hit(options: list[str], text: str) -> bool: |
| 114 | + """True if any option appears in text (hyphen/space interchangeable).""" |
| 115 | + for opt in options: |
| 116 | + pattern = re.escape(opt).replace(r"\-", "[-\\s]").replace(r"\ ", "[-\\s]") |
| 117 | + if re.search(rf"\b{pattern}\b", text): |
| 118 | + return True |
| 119 | + return False |
| 120 | + |
| 121 | + |
| 122 | +def main() -> int: |
| 123 | + ap = argparse.ArgumentParser(description=__doc__) |
| 124 | + ap.add_argument("--list", action="store_true", help="show parsed cases, no API calls") |
| 125 | + ap.add_argument("--routing", action="store_true", help="routing table only") |
| 126 | + ap.add_argument("--full", action="store_true", help="activation + verdict vocabulary, uncapped") |
| 127 | + ap.add_argument("--only", help="comma-separated skill slugs to test") |
| 128 | + ap.add_argument("--model", help="model alias to run against (default: your configured model)") |
| 129 | + ap.add_argument("--sleep", type=int, default=15, help="seconds between runs (rate-limit headroom)") |
| 130 | + args = ap.parse_args() |
| 131 | + |
| 132 | + activation, routing = parse_cases() |
| 133 | + if args.only: |
| 134 | + keep = set(args.only.split(",")) |
| 135 | + activation = [c for c in activation if c["skill"] in keep] |
| 136 | + routing = [c for c in routing if c["must"] in keep or c["must_not"] in keep] |
| 137 | + |
| 138 | + if args.list: |
| 139 | + for c in activation: |
| 140 | + print(f"activate {c['skill']:35s} vocab groups: {len(c['vocab'])}") |
| 141 | + for c in routing: |
| 142 | + print(f"route {str(c['must']):35s} not: {c['must_not']}") |
| 143 | + print(f"\n{len(activation)} activation + {len(routing)} routing cases") |
| 144 | + return 0 |
| 145 | + |
| 146 | + failures = [] |
| 147 | + run_activation = not args.routing |
| 148 | + |
| 149 | + if run_activation: |
| 150 | + for c in activation: |
| 151 | + max_turns = None if args.full else 3 |
| 152 | + timeout = 600 if args.full else 180 |
| 153 | + activated, text, error = run_claude(c["prompt"], args.model, max_turns, timeout) |
| 154 | + time.sleep(args.sleep) |
| 155 | + if error: |
| 156 | + failures.append(f"{c['skill']}: run failed ({error})") |
| 157 | + print(f"ERROR {c['skill']}: {error}") |
| 158 | + continue |
| 159 | + ok = c["skill"] in activated |
| 160 | + if not ok: |
| 161 | + failures.append(f"{c['skill']}: did not activate (activated: {sorted(activated) or 'none'})") |
| 162 | + if args.full and ok: |
| 163 | + for group in c["vocab"]: |
| 164 | + if not vocab_hit(group, text): |
| 165 | + ok = False |
| 166 | + failures.append(f"{c['skill']}: output missing vocabulary {group}") |
| 167 | + print(f"{'PASS' if ok else 'FAIL'} activate {c['skill']}") |
| 168 | + |
| 169 | + for c in routing: |
| 170 | + activated, _, error = run_claude(c["prompt"], args.model, 3, 180) |
| 171 | + time.sleep(args.sleep) |
| 172 | + if error: |
| 173 | + failures.append(f"routing '{c['prompt'][:40]}…': run failed ({error})") |
| 174 | + print(f"ERROR routing '{c['prompt'][:40]}…': {error}") |
| 175 | + continue |
| 176 | + ok = True |
| 177 | + if c["must"] and c["must"] not in activated: |
| 178 | + ok = False |
| 179 | + failures.append(f"routing '{c['prompt'][:40]}…': {c['must']} did not activate ({sorted(activated) or 'none'})") |
| 180 | + if c["must_not"] == "ANY": |
| 181 | + if activated: |
| 182 | + ok = False |
| 183 | + failures.append(f"dormancy '{c['prompt'][:40]}…': activated {sorted(activated)}") |
| 184 | + elif c["must_not"] in activated: |
| 185 | + ok = False |
| 186 | + failures.append(f"routing '{c['prompt'][:40]}…': misfired {c['must_not']}") |
| 187 | + print(f"{'PASS' if ok else 'FAIL'} route {c['must'] or '(dormancy)'}") |
| 188 | + |
| 189 | + print() |
| 190 | + if failures: |
| 191 | + print(f"FAIL — {len(failures)} issue(s):") |
| 192 | + for f in failures: |
| 193 | + print(f" - {f}") |
| 194 | + return 1 |
| 195 | + print("OK — all golden prompts passed") |
| 196 | + return 0 |
| 197 | + |
| 198 | + |
| 199 | +if __name__ == "__main__": |
| 200 | + sys.exit(main()) |
0 commit comments