Reject scoring evidence found only in deleted lines

docs/known-issues.md

@@ -21,13 +21,12 @@ approved dataset export. A failed preparation is not evidence that the release
 file's hash is wrong. The extraction retains the original check; it does not
 silently rewrite approved context to get past it.
 
-## Evidence validation does not distinguish added and deleted lines
+## Scoring evidence is checked against the post-change text
 
-The original judge validator checks whether quoted evidence occurs in the
-patch text. A quote from a deleted line can satisfy that check. The original
-Terra prompt, schema, evidence validator, and numeric scorer are preserved,
-including this limitation. Numeric parity does not establish that every
-criterion judgment is correct.
+Evidence quotes must occur in added lines or in the post-change projection of
+the diff. Unchanged context remains eligible evidence, while text found only
+in deleted lines cannot support a judgment. This exact-match validation does
+not establish that every criterion judgment is correct.
 
 ## Historical artifacts have different coverage and versions
 

dogbench/judge.py

@@ -197,10 +197,9 @@ def run_codex(prompt: Path, schema: Path, output: Path, log: Path, model: str) -
 
 
 def validate_score(payload: dict[str, Any], blocks: dict[str, dict[str, Any]], patch: str) -> None:
-    # Scorers quote authored text without unified-diff `+` markers. Require a
-    # contiguous match against either the raw patch or its exact added-line
-    # projection; fuzzy or semantic-only matches remain invalid.
-    raw_view = " ".join(patch.split())
+    # Scorers quote authored text without unified-diff markers. Match only the
+    # added lines or the post-change projection, excluding deleted content;
+    # fuzzy or semantic-only matches remain invalid.
     added_view = " ".join("\n".join(
         line[1:] for line in patch.splitlines()
         if line.startswith("+") and not line.startswith("+++")
@@ -263,8 +262,7 @@ def validate_score(payload: dict[str, Any], blocks: dict[str, dict[str, Any]], p
                       "contradicting_patch_quote"):
             quote = row.get(field)
             normalized = " ".join(quote.split()) if quote else ""
-            if (quote and normalized not in raw_view and normalized not in added_view
-                    and normalized not in rendered_added_view
+            if (quote and normalized not in added_view and normalized not in rendered_added_view
                     and normalized not in authored_view):
                 raise ValueError(f"{cid}: {field} is not an exact patch quote")
 

tests/test_judge.py

@@ -77,6 +77,27 @@ def test_evidence_rejects_invented_quote_and_missing_criterion(self):
             with self.assertRaises(ValueError):
                 validate_score(payload, blocks, PATCH)
 
+    def test_evidence_must_come_from_added_or_post_change_lines(self):
+        patch = """diff --git a/docs.md b/docs.md
+--- a/docs.md
++++ b/docs.md
+@@ -1,2 +1,2 @@
+-Removed evidence sentence.
++Added evidence sentence.
+ context evidence sentence.
+"""
+        blocks = criterion_blocks(RUBRIC)
+
+        removed = {"criteria": [judgment(
+            supporting_patch_quote="Removed evidence sentence.")], "summary": ""}
+        with self.assertRaises(ValueError):
+            validate_score(removed, blocks, patch)
+
+        for quote in ("context evidence sentence.", "Added evidence sentence."):
+            with self.subTest(quote=quote):
+                payload = {"criteria": [judgment(supporting_patch_quote=quote)], "summary": ""}
+                validate_score(payload, blocks, patch)
+
     def test_saved_hash_bound_judgment_scores_without_model_call(self):
         payload = {"criteria": [judgment()], "summary": "Pass", "original_field": {"kept": True}}
         predictions = [