From d84d4eb2729356e1c4f214d94473c33d25254db0 Mon Sep 17 00:00:00 2001 From: NuojCheng Date: Thu, 12 Nov 2026 20:59:44 +0000 Subject: [PATCH] Use the configured PRNG impl in standalone_checkpointer train.py sets jax_default_prng_impl to config.prng_impl (unsafe_rbg by default), and NNX checkpoints store RNG keys whose shape depends on it. standalone_checkpointer never set it, so restoring a train.py checkpoint failed with "Requested shape: (layers, 2) is not compatible with the stored shape: (layers, 4)". --- src/maxtext/utils/standalone_checkpointer.py | 2 ++ .../unit/standalone_checkpointer_nnx_test.py | 32 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/src/maxtext/utils/standalone_checkpointer.py b/src/maxtext/utils/standalone_checkpointer.py index dcb4d4bcee..45575ec510 100644 --- a/src/maxtext/utils/standalone_checkpointer.py +++ b/src/maxtext/utils/standalone_checkpointer.py @@ -206,6 +206,8 @@ def add_entropy_to_checkpoint(state): def main(argv: Sequence[str]) -> None: os.environ["TF_CPP_MIN_LOG_LEVEL"] = "0" config = pyconfig.initialize(argv) + # Match train.py: checkpoints store NNX RNG keys, whose shape depends on the PRNG impl. + jax.config.update("jax_default_prng_impl", config.prng_impl) train_utils.validate_train_config(config) print(f"Found {jax.device_count()} devices.") print(f"Found {jax.process_count()} processes.") diff --git a/tests/unit/standalone_checkpointer_nnx_test.py b/tests/unit/standalone_checkpointer_nnx_test.py index 31ae42f9e1..422895b2fe 100644 --- a/tests/unit/standalone_checkpointer_nnx_test.py +++ b/tests/unit/standalone_checkpointer_nnx_test.py @@ -14,6 +14,7 @@ """Unit tests for add_entropy_to_checkpoint on NNX state.""" +import os import unittest from unittest import mock @@ -23,6 +24,7 @@ from flax import nnx from maxtext.common import train_state_nnx +from maxtext.utils import standalone_checkpointer from maxtext.utils.standalone_checkpointer import add_entropy_to_checkpoint, checkpoint_loop @@ -266,5 +268,35 @@ def test_checkpoint_loop_saves_in_linen_layout(self): mock_to_linen.assert_called_once() +class MainTest(unittest.TestCase): + """Tests for main().""" + + def test_main_uses_configured_prng_impl(self): + """main() restores with config.prng_impl, which train.py also uses, rather than JAX's default impl.""" + seen = {} + + def _record_prng_impl(_config): + seen["prng_impl"] = jax.config.jax_default_prng_impl + + previous = jax.config.jax_default_prng_impl + jax.config.update("jax_default_prng_impl", "threefry2x32") + try: + with ( + mock.patch.object( + standalone_checkpointer.pyconfig, + "initialize", + return_value=mock.MagicMock(prng_impl="rbg", dataset_path=""), + ), + mock.patch.object(standalone_checkpointer.train_utils, "validate_train_config"), + mock.patch.object(standalone_checkpointer, "checkpoint_loop", side_effect=_record_prng_impl), + mock.patch.dict(os.environ), + ): + standalone_checkpointer.main([None]) + finally: + jax.config.update("jax_default_prng_impl", previous) + + self.assertEqual(seen["prng_impl"], "rbg") + + if __name__ == "__main__": unittest.main()