eval tutorial: malformed gate input raises instead of producing a verdict

evals/write-an-eval-that-can-say-no/README.md

@@ -39,7 +39,7 @@ For the optional uncertainty calculation, run `uv run python bootstrap.py`. It r
 - `freeze.py` — downloads a pinned SST-2 revision and regenerates the held-out rows.
 - `train.py` — trains the incumbent, candidate, and shuffled-label control.
 - `run_eval.py` — scores them and applies the gate.
-- `test_gate.py` — a passing fixture, the exact threshold boundary, and independent failures for each condition.
+- `test_gate.py` — a passing fixture, the exact threshold boundary, independent failures for each condition, and malformed input that must raise an error rather than produce a verdict.
 - `bootstrap.py` — optional paired bootstrap over the held-out rows.
 
 The visible cases are meant for iteration. In a real release process, keep an independent promotion set away from the jobs and people tuning the model, and combine the offline gate with review and live checks. A pass here only means the declared offline conditions were met.

evals/write-an-eval-that-can-say-no/run_eval.py

@@ -24,6 +24,10 @@ def need(share, total):
 
 def decide(gate, inc_held, new_held, inc_cases, new_cases):
     """The whole gate as one pure function over per-row right/wrong lists."""
+    # An empty or mismatched input must be an error, never a verdict: zip() would
+    # silently truncate, and zero rows would pass every condition.
+    if not (len(inc_held) == len(new_held) > 0 and len(inc_cases) == len(new_cases) > 0):
+        raise ValueError("the gate needs non-empty, equal-length results for both models")
     n_held, n_cases = len(new_held), len(new_cases)
     gain = sum(new_held) - sum(inc_held)
     regressions = [i for i, (a, b) in enumerate(zip(inc_cases, new_cases)) if a and not b]
@@ -51,11 +55,15 @@ def main():
     cases = [json.loads(l) for l in Path("evals/cases.jsonl").read_text().splitlines()]
     held = [json.loads(l) for l in Path("evals/heldout.jsonl").read_text().splitlines()]  # the frozen bytes
 
+    def right(m, rows):
+        preds = m.predict([r["text"] for r in rows])
+        if len(preds) != len(rows):
+            raise ValueError(f"got {len(preds)} predictions for {len(rows)} rows")
+        return [int(p) == r["label"] for p, r in zip(preds, rows)]
+
     def score(name):
         m = joblib.load(f"{name}.joblib")
-        held_ok = [int(p) == r["label"] for p, r in zip(m.predict([r["text"] for r in held]), held)]
-        case_ok = [int(p) == c["label"] for p, c in zip(m.predict([c["text"] for c in cases]), cases)]
-        return held_ok, case_ok
+        return right(m, held), right(m, cases)
 
     inc_held, inc_cases = score("incumbent")
     name = sys.argv[1] if len(sys.argv) > 1 else "candidate"

evals/write-an-eval-that-can-say-no/test_gate.py

@@ -34,6 +34,15 @@ def test_floor_fails_alone(self):
         checks, _ = decide(GATE, INC_HELD, [True] * 214 + [False] * 86, weak_inc, [True] * 13 + [False] * 7)
         self.assertEqual(list(checks.values()), [True, True, False])
 
+    def test_malformed_input_is_an_error_not_a_verdict(self):
+        for inc_held, new_held, inc_cases, new_cases in [
+            ([], [], [], []),                                  # nothing scored
+            (INC_HELD, INC_HELD, INC_CASES, [True] * 5),       # candidate scored on 5 of 20 cases
+            (INC_HELD, INC_HELD[:299], INC_CASES, INC_CASES),  # one held-out row missing
+        ]:
+            with self.assertRaises(ValueError):
+                decide(GATE, inc_held, new_held, inc_cases, new_cases)
+
 
 if __name__ == "__main__":
     unittest.main()