diff --git a/evals/write-an-eval-that-can-say-no/README.md b/evals/write-an-eval-that-can-say-no/README.md index 997755c..0a1b94c 100644 --- a/evals/write-an-eval-that-can-say-no/README.md +++ b/evals/write-an-eval-that-can-say-no/README.md @@ -31,6 +31,8 @@ The scorer uses distinct exits: `0` eligible for the next release step, `1` reje For the optional uncertainty calculation, run `uv run python bootstrap.py`. It resamples the same 300 held-out rows for both models and reports the paired interval printed in the article. +To reproduce the article's 20-seed sensitivity check, run `uv run python sensitivity.py` after `freeze.py`. It retrains both models on training shuffle seeds 0–19 with the same recipe, applies the same gate to each pair, and prints one line for each seed plus the summary (20 / 16 / 4 / 2 / 13). It is a check run after the result, not part of the gate. + ## Files - `evals/gate.toml` — three illustrative release conditions, written before training. @@ -41,6 +43,7 @@ For the optional uncertainty calculation, run `uv run python bootstrap.py`. It r - `run_eval.py` — scores them and applies the gate. - `test_gate.py` — a passing fixture, the exact threshold boundary, independent failures for each condition, and malformed input that must raise an error rather than produce a verdict. - `bootstrap.py` — optional paired bootstrap over the held-out rows. +- `sensitivity.py` — optional check run after the result: retrains both models on training shuffle seeds 0–19 and applies the same gate, reproducing the article's 20-run counts. It is not part of the gate. The visible cases are meant for iteration. In a real release process, keep an independent promotion set away from the jobs and people tuning the model, and combine the offline gate with review and live checks. A pass here only means the declared offline conditions were met. diff --git a/evals/write-an-eval-that-can-say-no/sensitivity.py b/evals/write-an-eval-that-can-say-no/sensitivity.py new file mode 100644 index 0000000..d16bbb9 --- /dev/null +++ b/evals/write-an-eval-that-can-say-no/sensitivity.py @@ -0,0 +1,55 @@ +# sensitivity.py +"""Optional: how typical is the tutorial's result? Retrain the incumbent and +candidate with train.py's exact recipe on training shuffle seeds 0-19 and apply +the same gate to each pair. + +This is a check run after the result, not part of the gate. It measures how +much the verdict varies between training runs of this recipe, not how a model +behaves on real traffic. Takes under a minute. +""" +import json, tomllib +from collections import Counter +from pathlib import Path +from datasets import load_dataset +from sklearn.feature_extraction.text import TfidfVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.pipeline import make_pipeline +from run_eval import decide, need + +REVISION = "8d51e7e4887a4caaa95b3fbebbf53c0490b58bbb" # same pinned commit as train.py +gate = tomllib.loads(Path("evals/gate.toml").read_text())["gate"] +cases = [json.loads(l) for l in Path("evals/cases.jsonl").read_text().splitlines()] +held = [json.loads(l) for l in Path("evals/heldout.jsonl").read_text().splitlines()] +base = load_dataset("stanfordnlp/sst2", split="train", revision=REVISION) + +def fit(train, n, ngrams): # train.py's fit(), without the shuffled-label option + rows = train.select(range(n)) + model = make_pipeline(TfidfVectorizer(ngram_range=ngrams, min_df=2), + LogisticRegression(max_iter=1000)) + return model.fit(rows["sentence"], list(rows["label"])) + +def right(model, rows): + return [int(p) == r["label"] for p, r in zip(model.predict([r["text"] for r in rows]), rows)] + +seeds = range(20) +tally, regressed = Counter(), Counter() +for seed in seeds: + train = base.shuffle(seed=seed) + inc, cand = fit(train, 2_000, (1, 1)), fit(train, 20_000, (1, 2)) + inc_held, inc_cases, new_held, new_cases = right(inc, held), right(inc, cases), right(cand, held), right(cand, cases) + checks, regressions = decide(gate, inc_held, new_held, inc_cases, new_cases) + gain_ok, _, _ = checks.values() + tally["gain cleared"] += gain_ok + tally["at least one regression"] += bool(regressions) + tally["whole gate passed"] += all(checks.values()) + tally["both 14/20"] += sum(inc_cases) == 14 and sum(new_cases) == 14 + tally["incumbent under the floor"] += sum(inc_cases) < need(gate["min_case_accuracy"], len(cases)) + regressed.update(cases[i]["id"] for i in regressions) + print(f"seed {seed:2}: incumbent {sum(inc_held)}/{len(held)} {sum(inc_cases)}/{len(cases)}, " + f"candidate {sum(new_held)}/{len(held)} {sum(new_cases)}/{len(cases)}, " + f"regressed {[cases[i]['id'] for i in regressions]}, gate {'PASS' if all(checks.values()) else 'fail'}") + +print(f"\nOut of {len(seeds)} training shuffles:") +for label, n in tally.items(): + print(f" {label:26} {n}") +print(" regressions by case ", ", ".join(f"{c} {n}" for c, n in regressed.most_common()))