diff --git a/src/bioimageio/core/_resource_tests.py b/src/bioimageio/core/_resource_tests.py
index 51ae745d..f663cbf2 100644
--- a/src/bioimageio/core/_resource_tests.py
+++ b/src/bioimageio/core/_resource_tests.py
@@ -847,7 +847,7 @@ def evaluate_mismatched_elements(
 
         rtol_value = rtol * abs(expected_np)
         abs_diff = abs(actual_np - expected_np)
-        mismatched = abs_diff > atol + rtol_value
+        mismatched = ~np.isclose(actual_np, expected_np, rtol=rtol, atol=atol)
         mismatched_elements = mismatched.sum().item()
 
         mismatched_ppm = mismatched_elements / expected_np.size * 1e6
diff --git a/tests/test_resource_tests.py b/tests/test_resource_tests.py
index 892fbe77..edf14ce9 100644
--- a/tests/test_resource_tests.py
+++ b/tests/test_resource_tests.py
@@ -4,6 +4,7 @@
 import pytest
 
 from bioimageio.core import __version__
+from bioimageio.core.tensor import Tensor
 from bioimageio.spec import InvalidDescr, ValidationContext
 
 
@@ -80,3 +81,18 @@ def test_failed_reproducibility(unet2d_nuclei_broad_model: str, tmp_path: str):
     summary = test_model(model)
 
     assert summary.status == "valid-format"
+
+
+def test_reproducibility_rejects_nan_output():
+    from bioimageio.core._resource_tests import evaluate_mismatched_elements
+
+    actual = Tensor(np.full((2,), np.nan), dims=("x",))
+    expected = Tensor(np.ones((2,)), dims=("x",))
+
+    mismatched_ppm, msg, error_msg = evaluate_mismatched_elements(
+        actual, expected, rtol=1e-3, atol=1e-3, name="output"
+    )
+
+    assert error_msg is None
+    assert mismatched_ppm == 1_000_000
+    assert "2 of 2 elements disagree" in msg