diff --git a/docs/known-issues.md b/docs/known-issues.md index b7efac2..02065c7 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -21,13 +21,12 @@ approved dataset export. A failed preparation is not evidence that the release file's hash is wrong. The extraction retains the original check; it does not silently rewrite approved context to get past it. -## Evidence validation does not distinguish added and deleted lines +## Scoring evidence is checked against the post-change text -The original judge validator checks whether quoted evidence occurs in the -patch text. A quote from a deleted line can satisfy that check. The original -Terra prompt, schema, evidence validator, and numeric scorer are preserved, -including this limitation. Numeric parity does not establish that every -criterion judgment is correct. +Evidence quotes must occur in added lines or in the post-change projection of +the diff. Unchanged context remains eligible evidence, while text found only +in deleted lines cannot support a judgment. This exact-match validation does +not establish that every criterion judgment is correct. ## Historical artifacts have different coverage and versions diff --git a/dogbench/judge.py b/dogbench/judge.py index 12818e8..de3c689 100644 --- a/dogbench/judge.py +++ b/dogbench/judge.py @@ -197,10 +197,9 @@ def run_codex(prompt: Path, schema: Path, output: Path, log: Path, model: str) - def validate_score(payload: dict[str, Any], blocks: dict[str, dict[str, Any]], patch: str) -> None: - # Scorers quote authored text without unified-diff `+` markers. Require a - # contiguous match against either the raw patch or its exact added-line - # projection; fuzzy or semantic-only matches remain invalid. - raw_view = " ".join(patch.split()) + # Scorers quote authored text without unified-diff markers. Match only the + # added lines or the post-change projection, excluding deleted content; + # fuzzy or semantic-only matches remain invalid. added_view = " ".join("\n".join( line[1:] for line in patch.splitlines() if line.startswith("+") and not line.startswith("+++") @@ -263,8 +262,7 @@ def validate_score(payload: dict[str, Any], blocks: dict[str, dict[str, Any]], p "contradicting_patch_quote"): quote = row.get(field) normalized = " ".join(quote.split()) if quote else "" - if (quote and normalized not in raw_view and normalized not in added_view - and normalized not in rendered_added_view + if (quote and normalized not in added_view and normalized not in rendered_added_view and normalized not in authored_view): raise ValueError(f"{cid}: {field} is not an exact patch quote") diff --git a/tests/test_judge.py b/tests/test_judge.py index c484255..195e26d 100644 --- a/tests/test_judge.py +++ b/tests/test_judge.py @@ -77,6 +77,27 @@ def test_evidence_rejects_invented_quote_and_missing_criterion(self): with self.assertRaises(ValueError): validate_score(payload, blocks, PATCH) + def test_evidence_must_come_from_added_or_post_change_lines(self): + patch = """diff --git a/docs.md b/docs.md +--- a/docs.md ++++ b/docs.md +@@ -1,2 +1,2 @@ +-Removed evidence sentence. ++Added evidence sentence. + context evidence sentence. +""" + blocks = criterion_blocks(RUBRIC) + + removed = {"criteria": [judgment( + supporting_patch_quote="Removed evidence sentence.")], "summary": ""} + with self.assertRaises(ValueError): + validate_score(removed, blocks, patch) + + for quote in ("context evidence sentence.", "Added evidence sentence."): + with self.subTest(quote=quote): + payload = {"criteria": [judgment(supporting_patch_quote=quote)], "summary": ""} + validate_score(payload, blocks, patch) + def test_saved_hash_bound_judgment_scores_without_model_call(self): payload = {"criteria": [judgment()], "summary": "Pass", "original_field": {"kept": True}} predictions = [