From 5f6db0bbabfffa523ea334b9ebf1f2ec3e0500cb Mon Sep 17 00:00:00 2001 From: Shiori Amamiya Date: Wed, 30 Sep 2026 14:36:37 -0400 Subject: [PATCH 1/2] eval tutorial: malformed gate input raises instead of producing a verdict decide() passed all three conditions on empty result lists, and zip() truncation let a candidate scored on 5 of 20 cases pass the regression and floor checks. Found by Astra; reproduced independently by Shiori and Tama. The new test fails against the previous run_eval.py. Demo source: evals-tutorial 9eaecf6 (byte-identical). Co-Authored-By: Claude Opus 5.5 --- evals/write-an-eval-that-can-say-no/run_eval.py | 14 +++++++++++--- evals/write-an-eval-that-can-say-no/test_gate.py | 9 +++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/evals/write-an-eval-that-can-say-no/run_eval.py b/evals/write-an-eval-that-can-say-no/run_eval.py index a1a146c..5d1a560 100644 --- a/evals/write-an-eval-that-can-say-no/run_eval.py +++ b/evals/write-an-eval-that-can-say-no/run_eval.py @@ -24,6 +24,10 @@ def need(share, total): def decide(gate, inc_held, new_held, inc_cases, new_cases): """The whole gate as one pure function over per-row right/wrong lists.""" + # An empty or mismatched input must be an error, never a verdict: zip() would + # silently truncate, and zero rows would pass every condition. + if not (len(inc_held) == len(new_held) > 0 and len(inc_cases) == len(new_cases) > 0): + raise ValueError("the gate needs non-empty, equal-length results for both models") n_held, n_cases = len(new_held), len(new_cases) gain = sum(new_held) - sum(inc_held) regressions = [i for i, (a, b) in enumerate(zip(inc_cases, new_cases)) if a and not b] @@ -51,11 +55,15 @@ def main(): cases = [json.loads(l) for l in Path("evals/cases.jsonl").read_text().splitlines()] held = [json.loads(l) for l in Path("evals/heldout.jsonl").read_text().splitlines()] # the frozen bytes + def right(m, rows): + preds = m.predict([r["text"] for r in rows]) + if len(preds) != len(rows): + raise ValueError(f"got {len(preds)} predictions for {len(rows)} rows") + return [int(p) == r["label"] for p, r in zip(preds, rows)] + def score(name): m = joblib.load(f"{name}.joblib") - held_ok = [int(p) == r["label"] for p, r in zip(m.predict([r["text"] for r in held]), held)] - case_ok = [int(p) == c["label"] for p, c in zip(m.predict([c["text"] for c in cases]), cases)] - return held_ok, case_ok + return right(m, held), right(m, cases) inc_held, inc_cases = score("incumbent") name = sys.argv[1] if len(sys.argv) > 1 else "candidate" diff --git a/evals/write-an-eval-that-can-say-no/test_gate.py b/evals/write-an-eval-that-can-say-no/test_gate.py index f4dcc80..c2712f9 100644 --- a/evals/write-an-eval-that-can-say-no/test_gate.py +++ b/evals/write-an-eval-that-can-say-no/test_gate.py @@ -34,6 +34,15 @@ def test_floor_fails_alone(self): checks, _ = decide(GATE, INC_HELD, [True] * 214 + [False] * 86, weak_inc, [True] * 13 + [False] * 7) self.assertEqual(list(checks.values()), [True, True, False]) + def test_malformed_input_is_an_error_not_a_verdict(self): + for inc_held, new_held, inc_cases, new_cases in [ + ([], [], [], []), # nothing scored + (INC_HELD, INC_HELD, INC_CASES, [True] * 5), # candidate scored on 5 of 20 cases + (INC_HELD, INC_HELD[:299], INC_CASES, INC_CASES), # one held-out row missing + ]: + with self.assertRaises(ValueError): + decide(GATE, inc_held, new_held, inc_cases, new_cases) + if __name__ == "__main__": unittest.main() From f400baba3b7527134fa444862da12b3d6c4b81af Mon Sep 17 00:00:00 2001 From: Shiori Amamiya Date: Wed, 30 Sep 2026 14:36:48 -0400 Subject: [PATCH 2/2] eval tutorial README: list the malformed-input test Co-Authored-By: Claude Opus 5.5 --- evals/write-an-eval-that-can-say-no/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evals/write-an-eval-that-can-say-no/README.md b/evals/write-an-eval-that-can-say-no/README.md index 478869b..79e91e2 100644 --- a/evals/write-an-eval-that-can-say-no/README.md +++ b/evals/write-an-eval-that-can-say-no/README.md @@ -39,7 +39,7 @@ For the optional uncertainty calculation, run `uv run python bootstrap.py`. It r - `freeze.py` — downloads a pinned SST-2 revision and regenerates the held-out rows. - `train.py` — trains the incumbent, candidate, and shuffled-label control. - `run_eval.py` — scores them and applies the gate. -- `test_gate.py` — a passing fixture, the exact threshold boundary, and independent failures for each condition. +- `test_gate.py` — a passing fixture, the exact threshold boundary, independent failures for each condition, and malformed input that must raise an error rather than produce a verdict. - `bootstrap.py` — optional paired bootstrap over the held-out rows. The visible cases are meant for iteration. In a real release process, keep an independent promotion set away from the jobs and people tuning the model, and combine the offline gate with review and live checks. A pass here only means the declared offline conditions were met.