eval tutorial: framed frozen hash, held-out rows hash, honest gate header

evals/write-an-eval-that-can-say-no/README.md

@@ -18,7 +18,7 @@ uv run python train.py
 uv run python -m unittest test_gate
 ```
 
-The freeze step regenerates `evals/heldout.jsonl` from the dataset revision pinned in `freeze.py`. That file is intentionally untracked because the Hub lists the dataset license as unknown. The committed hash is the expected value; the `git diff` command above checks that the regenerated rows, authored cases, and gate policy still produce it.
+The freeze step regenerates `evals/heldout.jsonl` from the dataset revision pinned in `freeze.py`. That file is intentionally untracked because the Hub lists the dataset license as unknown. The committed hash is the expected value; the `git diff` command above checks that the regenerated rows, authored cases, and gate policy still produce it. `freeze.py` also prints a hash of the held-out rows alone: `a3a3519a046a328d` means your data matches ours, independent of how the other files were saved.
 
 Now run the candidate and the known-bad control. **Exit 1 is expected for both**: it means the gate rejected the model.
 

evals/write-an-eval-that-can-say-no/evals/FROZEN.sha256

@@ -1 +1 @@
-83d5f62880ac8ac3c11208e672090ae0ebc2f85a0b218dd0f977e0deaa9984bd
+25c4473f259533b2febf0a1ce053cfc88a6d94c4eba577591b99d8f39f717b85

evals/write-an-eval-that-can-say-no/evals/gate.toml

@@ -4,5 +4,5 @@
 [gate]
 # The candidate replaces the incumbent only if all of these hold:
 min_heldout_gain = 0.02      # beats the incumbent on the held-out sample by >= 2 percentage points
-max_case_regressions = 0     # fixes nothing by breaking an authored case the incumbent got right
+max_case_regressions = 0     # breaks no authored case the incumbent got right
 min_case_accuracy = 0.70     # and clears an absolute floor on the authored cases

evals/write-an-eval-that-can-say-no/freeze.py

@@ -5,19 +5,35 @@
 """
 import hashlib, json, random
 from pathlib import Path
-from datasets import load_dataset
 
 REVISION = "8d51e7e4887a4caaa95b3fbebbf53c0490b58bbb"  # exact stanfordnlp/sst2 commit
 SEED, N_HELDOUT = 1234, 300
+FROZEN = ("evals/cases.jsonl", "evals/heldout.jsonl", "evals/gate.toml")
 
-val = load_dataset("stanfordnlp/sst2", split="validation", revision=REVISION)
-idx = sorted(random.Random(SEED).sample(range(len(val)), N_HELDOUT))
-with open("evals/heldout.jsonl", "w") as f:           # the rows themselves, not pointers
-    for row in val.select(idx):
-        f.write(json.dumps({"text": row["sentence"], "label": row["label"]}) + "\n")
-
-h = hashlib.sha256()
-for name in ("evals/cases.jsonl", "evals/heldout.jsonl", "evals/gate.toml"):
-    h.update(Path(name).read_bytes())
-Path("evals/FROZEN.sha256").write_text(h.hexdigest() + "\n")
-print("frozen", h.hexdigest()[:16], f"({N_HELDOUT} held-out rows, 20 authored cases)")
+
+def frozen_digest(names=FROZEN):
+    """SHA-256 over each file's name, byte length and bytes. Framing each file
+    means bytes cannot move from one file into the next without changing it."""
+    h = hashlib.sha256()
+    for name in names:
+        data = Path(name).read_bytes()
+        h.update(name.encode() + b"\0" + len(data).to_bytes(8, "big") + data)
+    return h.hexdigest()
+
+
+def main():
+    from datasets import load_dataset
+    val = load_dataset("stanfordnlp/sst2", split="validation", revision=REVISION)
+    idx = sorted(random.Random(SEED).sample(range(len(val)), N_HELDOUT))
+    rows = [json.dumps({"text": r["sentence"], "label": r["label"]}) + "\n" for r in val.select(idx)]
+    Path("evals/heldout.jsonl").write_bytes("".join(rows).encode("utf-8"))  # the rows themselves, LF on every OS
+
+    digest = frozen_digest()
+    Path("evals/FROZEN.sha256").write_text(digest + "\n")
+    rows_hash = hashlib.sha256(Path("evals/heldout.jsonl").read_bytes()).hexdigest()
+    print(f"held-out rows {rows_hash[:16]} ({N_HELDOUT} of {len(val)} validation rows)")
+    print(f"frozen        {digest[:16]} (cases + held-out rows + gate)")
+
+
+if __name__ == "__main__":
+    main()

evals/write-an-eval-that-can-say-no/run_eval.py

@@ -6,12 +6,10 @@
 2 = eval inputs were altered, 3 = the eval could not run (e.g. a model file
 is missing). A crash must never look like a verdict.
 """
-import hashlib, json, math, sys, tomllib
+import json, math, sys, tomllib
 from fractions import Fraction
 from pathlib import Path
 
-FROZEN = ("evals/cases.jsonl", "evals/heldout.jsonl", "evals/gate.toml")
-
 
 def need(share, total):
     """Smallest whole count that meets `share` of `total`, in exact arithmetic.
@@ -44,10 +42,9 @@ def decide(gate, inc_held, new_held, inc_cases, new_cases):
 def main():
     # Everything that can fail lives inside main(), so any crash becomes exit 3.
     import joblib  # third-party: a missing install is a crash, not a verdict
-    h = hashlib.sha256()
-    for name in FROZEN:
-        h.update(Path(name).read_bytes())
-    if h.hexdigest() != Path("evals/FROZEN.sha256").read_text().strip():
+    from freeze import frozen_digest  # step 3's file; missing, it is a crash (3), not a verdict
+    digest = frozen_digest()
+    if digest != Path("evals/FROZEN.sha256").read_text().strip():
         print("REFUSED: eval inputs changed after freezing. Re-freeze deliberately, and say why.", file=sys.stderr)
         return 2
 
@@ -69,17 +66,18 @@ def score(name):
     name = sys.argv[1] if len(sys.argv) > 1 else "candidate"
     new_held, new_cases = score(name)
 
-    print(f"{'':12}{'held-out (300)':>16}{'authored (20)':>16}")
-    print(f"{'incumbent':12}{sum(inc_held) / len(held):>16.3f}{sum(inc_cases):>13}/20")
-    print(f"{name:12}{sum(new_held) / len(held):>16.3f}{sum(new_cases):>13}/20")
+    n_held, n_cases = len(held), len(cases)
+    print(f"{'':12}{f'held-out ({n_held})':>16}{f'authored ({n_cases})':>16}")
+    print(f"{'incumbent':12}{sum(inc_held) / n_held:>16.3f}{sum(inc_cases):>13}/{n_cases}")
+    print(f"{name:12}{sum(new_held) / n_held:>16.3f}{sum(new_cases):>13}/{n_cases}")
 
     print(f"\nMisses for {name} on authored cases (read these, don't just count them):")
     for c, ok in zip(cases, new_cases):
         if not ok:
             print(f"  {c['id']:7} [{c['why']}] {c['text']!r} (expected {'pos' if c['label'] else 'neg'})")
 
     checks, regressions = decide(gate, inc_held, new_held, inc_cases, new_cases)
-    print("\nGate (declared before any model was trained):")
+    print(f"\nGate (frozen inputs {digest[:16]}):")
     for label, passed in checks.items():
         extra = f" {[cases[i]['id'] for i in regressions]}" if "regressions" in label and regressions else ""
         print(f"  {'PASS' if passed else 'FAIL'}  {label}{extra}")