diff --git a/src/maxtext/trainers/pre_train/train_compile.py b/src/maxtext/trainers/pre_train/train_compile.py index d8e17f1ac1..97e4bc6d26 100644 --- a/src/maxtext/trainers/pre_train/train_compile.py +++ b/src/maxtext/trainers/pre_train/train_compile.py @@ -88,6 +88,7 @@ def get_topology_mesh(config): num_slices=config.compile_topology_num_slices, wrap=target_hardware.wrap, ).devices + jax.config.update("jax_use_shardy_partitioner", config.shardy) jax.config.update( "jax_remove_size_one_mesh_axis_from_type", config.remove_size_one_mesh_axis_from_type, @@ -303,6 +304,7 @@ def main(argv: Sequence[str]) -> None: # Parse and validate configuration config = pyconfig.initialize(argv) jax.config.update("jax_default_prng_impl", config.prng_impl) + jax.config.update("jax_use_shardy_partitioner", config.shardy) validate_config(config) # Create target mesh diff --git a/tests/conftest.py b/tests/conftest.py index 1f0d0a22c5..93b91600c0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -259,14 +259,17 @@ def restore_jax_remove_size_one_mesh_axis_from_type(): @contextlib.contextmanager def _restore_jax_prng_and_libtpu_env(): - """Saves and restores global JAX PRNG config and LIBTPU_INIT_ARGS across a scope.""" + """Saves and restores global JAX PRNG/Shardy config and LIBTPU_INIT_ARGS across a scope.""" previous_prng = getattr(jax.config, "jax_default_prng_impl", None) + previous_shardy = getattr(jax.config, "jax_use_shardy_partitioner", None) previous_libtpu = os.environ.get("LIBTPU_INIT_ARGS") try: yield finally: if previous_prng is not None and getattr(jax.config, "jax_default_prng_impl", None) != previous_prng: jax.config.update("jax_default_prng_impl", previous_prng) + if previous_shardy is not None and getattr(jax.config, "jax_use_shardy_partitioner", None) != previous_shardy: + jax.config.update("jax_use_shardy_partitioner", previous_shardy) if previous_libtpu is None: os.environ.pop("LIBTPU_INIT_ARGS", None) else: diff --git a/tests/unit/marker_propagation_test.py b/tests/unit/marker_propagation_test.py index da72acb66f..695f92c793 100644 --- a/tests/unit/marker_propagation_test.py +++ b/tests/unit/marker_propagation_test.py @@ -21,6 +21,7 @@ from absl.testing import parameterized import jax +from tests.conftest import _restore_jax_prng_and_libtpu_env from tests.conftest import pytest_collection_modifyitems @@ -111,6 +112,16 @@ def get_added_names(item): self.assertNotIn("cpu_only", get_added_names(item_gpu)) self.assertNotIn("cpu_only", get_added_names(item_cpu)) + def test_restore_jax_use_shardy_partitioner(self): + """Verifies _restore_jax_prng_and_libtpu_env restores jax_use_shardy_partitioner.""" + prev_shardy = getattr(jax.config, "jax_use_shardy_partitioner", True) + self.addCleanup(jax.config.update, "jax_use_shardy_partitioner", prev_shardy) + jax.config.update("jax_use_shardy_partitioner", True) + with _restore_jax_prng_and_libtpu_env(): + jax.config.update("jax_use_shardy_partitioner", False) + self.assertFalse(jax.config.jax_use_shardy_partitioner) + self.assertTrue(jax.config.jax_use_shardy_partitioner) + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/train_compile_test.py b/tests/unit/train_compile_test.py index 2d558538b4..a124e3f061 100644 --- a/tests/unit/train_compile_test.py +++ b/tests/unit/train_compile_test.py @@ -30,6 +30,8 @@ from tempfile import gettempdir, NamedTemporaryFile +from maxtext.configs import pyconfig +from maxtext.trainers.pre_train.train_compile import get_topology_mesh from maxtext.trainers.pre_train.train_compile import main as train_compile_main from maxtext.utils.globals import MAXTEXT_ASSETS_ROOT from tests.utils.test_helpers import get_test_config_path @@ -1819,3 +1821,20 @@ def test_envy(self, scan_layers): "override_model_config=True", ) ) + + def test_get_topology_mesh_updates_shardy_partitioner(self): + """Verifies get_topology_mesh updates jax_use_shardy_partitioner from config.shardy.""" + prev_shardy = getattr(jax.config, "jax_use_shardy_partitioner", True) + self.addCleanup(jax.config.update, "jax_use_shardy_partitioner", prev_shardy) + jax.config.update("jax_use_shardy_partitioner", False) + config = pyconfig.initialize( + ( + "", + get_test_config_path(), + "compile_topology=v6e-4", + "compile_topology_num_slices=1", + "shardy=True", + ) + ) + get_topology_mesh(config) + self.assertTrue(jax.config.jax_use_shardy_partitioner)