diff --git a/evals/write-an-eval-that-can-say-no/README.md b/evals/write-an-eval-that-can-say-no/README.md index 79e91e2..997755c 100644 --- a/evals/write-an-eval-that-can-say-no/README.md +++ b/evals/write-an-eval-that-can-say-no/README.md @@ -18,7 +18,7 @@ uv run python train.py uv run python -m unittest test_gate ``` -The freeze step regenerates `evals/heldout.jsonl` from the dataset revision pinned in `freeze.py`. That file is intentionally untracked because the Hub lists the dataset license as unknown. The committed hash is the expected value; the `git diff` command above checks that the regenerated rows, authored cases, and gate policy still produce it. +The freeze step regenerates `evals/heldout.jsonl` from the dataset revision pinned in `freeze.py`. That file is intentionally untracked because the Hub lists the dataset license as unknown. The committed hash is the expected value; the `git diff` command above checks that the regenerated rows, authored cases, and gate policy still produce it. `freeze.py` also prints a hash of the held-out rows alone: `a3a3519a046a328d` means your data matches ours, independent of how the other files were saved. Now run the candidate and the known-bad control. **Exit 1 is expected for both**: it means the gate rejected the model. diff --git a/evals/write-an-eval-that-can-say-no/evals/FROZEN.sha256 b/evals/write-an-eval-that-can-say-no/evals/FROZEN.sha256 index feaf7ae..a2ca2ae 100644 --- a/evals/write-an-eval-that-can-say-no/evals/FROZEN.sha256 +++ b/evals/write-an-eval-that-can-say-no/evals/FROZEN.sha256 @@ -1 +1 @@ -83d5f62880ac8ac3c11208e672090ae0ebc2f85a0b218dd0f977e0deaa9984bd +25c4473f259533b2febf0a1ce053cfc88a6d94c4eba577591b99d8f39f717b85 diff --git a/evals/write-an-eval-that-can-say-no/evals/gate.toml b/evals/write-an-eval-that-can-say-no/evals/gate.toml index 419f241..064ca9e 100644 --- a/evals/write-an-eval-that-can-say-no/evals/gate.toml +++ b/evals/write-an-eval-that-can-say-no/evals/gate.toml @@ -4,5 +4,5 @@ [gate] # The candidate replaces the incumbent only if all of these hold: min_heldout_gain = 0.02 # beats the incumbent on the held-out sample by >= 2 percentage points -max_case_regressions = 0 # fixes nothing by breaking an authored case the incumbent got right +max_case_regressions = 0 # breaks no authored case the incumbent got right min_case_accuracy = 0.70 # and clears an absolute floor on the authored cases diff --git a/evals/write-an-eval-that-can-say-no/freeze.py b/evals/write-an-eval-that-can-say-no/freeze.py index a64c5ad..3c24453 100644 --- a/evals/write-an-eval-that-can-say-no/freeze.py +++ b/evals/write-an-eval-that-can-say-no/freeze.py @@ -5,19 +5,35 @@ """ import hashlib, json, random from pathlib import Path -from datasets import load_dataset REVISION = "8d51e7e4887a4caaa95b3fbebbf53c0490b58bbb" # exact stanfordnlp/sst2 commit SEED, N_HELDOUT = 1234, 300 +FROZEN = ("evals/cases.jsonl", "evals/heldout.jsonl", "evals/gate.toml") -val = load_dataset("stanfordnlp/sst2", split="validation", revision=REVISION) -idx = sorted(random.Random(SEED).sample(range(len(val)), N_HELDOUT)) -with open("evals/heldout.jsonl", "w") as f: # the rows themselves, not pointers - for row in val.select(idx): - f.write(json.dumps({"text": row["sentence"], "label": row["label"]}) + "\n") - -h = hashlib.sha256() -for name in ("evals/cases.jsonl", "evals/heldout.jsonl", "evals/gate.toml"): - h.update(Path(name).read_bytes()) -Path("evals/FROZEN.sha256").write_text(h.hexdigest() + "\n") -print("frozen", h.hexdigest()[:16], f"({N_HELDOUT} held-out rows, 20 authored cases)") + +def frozen_digest(names=FROZEN): + """SHA-256 over each file's name, byte length and bytes. Framing each file + means bytes cannot move from one file into the next without changing it.""" + h = hashlib.sha256() + for name in names: + data = Path(name).read_bytes() + h.update(name.encode() + b"\0" + len(data).to_bytes(8, "big") + data) + return h.hexdigest() + + +def main(): + from datasets import load_dataset + val = load_dataset("stanfordnlp/sst2", split="validation", revision=REVISION) + idx = sorted(random.Random(SEED).sample(range(len(val)), N_HELDOUT)) + rows = [json.dumps({"text": r["sentence"], "label": r["label"]}) + "\n" for r in val.select(idx)] + Path("evals/heldout.jsonl").write_bytes("".join(rows).encode("utf-8")) # the rows themselves, LF on every OS + + digest = frozen_digest() + Path("evals/FROZEN.sha256").write_text(digest + "\n") + rows_hash = hashlib.sha256(Path("evals/heldout.jsonl").read_bytes()).hexdigest() + print(f"held-out rows {rows_hash[:16]} ({N_HELDOUT} of {len(val)} validation rows)") + print(f"frozen {digest[:16]} (cases + held-out rows + gate)") + + +if __name__ == "__main__": + main() diff --git a/evals/write-an-eval-that-can-say-no/run_eval.py b/evals/write-an-eval-that-can-say-no/run_eval.py index 5d1a560..abf567d 100644 --- a/evals/write-an-eval-that-can-say-no/run_eval.py +++ b/evals/write-an-eval-that-can-say-no/run_eval.py @@ -6,12 +6,10 @@ 2 = eval inputs were altered, 3 = the eval could not run (e.g. a model file is missing). A crash must never look like a verdict. """ -import hashlib, json, math, sys, tomllib +import json, math, sys, tomllib from fractions import Fraction from pathlib import Path -FROZEN = ("evals/cases.jsonl", "evals/heldout.jsonl", "evals/gate.toml") - def need(share, total): """Smallest whole count that meets `share` of `total`, in exact arithmetic. @@ -44,10 +42,9 @@ def decide(gate, inc_held, new_held, inc_cases, new_cases): def main(): # Everything that can fail lives inside main(), so any crash becomes exit 3. import joblib # third-party: a missing install is a crash, not a verdict - h = hashlib.sha256() - for name in FROZEN: - h.update(Path(name).read_bytes()) - if h.hexdigest() != Path("evals/FROZEN.sha256").read_text().strip(): + from freeze import frozen_digest # step 3's file; missing, it is a crash (3), not a verdict + digest = frozen_digest() + if digest != Path("evals/FROZEN.sha256").read_text().strip(): print("REFUSED: eval inputs changed after freezing. Re-freeze deliberately, and say why.", file=sys.stderr) return 2 @@ -69,9 +66,10 @@ def score(name): name = sys.argv[1] if len(sys.argv) > 1 else "candidate" new_held, new_cases = score(name) - print(f"{'':12}{'held-out (300)':>16}{'authored (20)':>16}") - print(f"{'incumbent':12}{sum(inc_held) / len(held):>16.3f}{sum(inc_cases):>13}/20") - print(f"{name:12}{sum(new_held) / len(held):>16.3f}{sum(new_cases):>13}/20") + n_held, n_cases = len(held), len(cases) + print(f"{'':12}{f'held-out ({n_held})':>16}{f'authored ({n_cases})':>16}") + print(f"{'incumbent':12}{sum(inc_held) / n_held:>16.3f}{sum(inc_cases):>13}/{n_cases}") + print(f"{name:12}{sum(new_held) / n_held:>16.3f}{sum(new_cases):>13}/{n_cases}") print(f"\nMisses for {name} on authored cases (read these, don't just count them):") for c, ok in zip(cases, new_cases): @@ -79,7 +77,7 @@ def score(name): print(f" {c['id']:7} [{c['why']}] {c['text']!r} (expected {'pos' if c['label'] else 'neg'})") checks, regressions = decide(gate, inc_held, new_held, inc_cases, new_cases) - print("\nGate (declared before any model was trained):") + print(f"\nGate (frozen inputs {digest[:16]}):") for label, passed in checks.items(): extra = f" {[cases[i]['id'] for i in regressions]}" if "regressions" in label and regressions else "" print(f" {'PASS' if passed else 'FAIL'} {label}{extra}")