Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/maxtext/trainers/pre_train/train_compile.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ def get_topology_mesh(config):
num_slices=config.compile_topology_num_slices,
wrap=target_hardware.wrap,
).devices
jax.config.update("jax_use_shardy_partitioner", config.shardy)
jax.config.update(
"jax_remove_size_one_mesh_axis_from_type",
config.remove_size_one_mesh_axis_from_type,
Expand Down Expand Up @@ -303,6 +304,7 @@ def main(argv: Sequence[str]) -> None:
# Parse and validate configuration
config = pyconfig.initialize(argv)
jax.config.update("jax_default_prng_impl", config.prng_impl)
jax.config.update("jax_use_shardy_partitioner", config.shardy)
validate_config(config)

# Create target mesh
Expand Down
5 changes: 4 additions & 1 deletion tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -259,14 +259,17 @@ def restore_jax_remove_size_one_mesh_axis_from_type():

@contextlib.contextmanager
def _restore_jax_prng_and_libtpu_env():
"""Saves and restores global JAX PRNG config and LIBTPU_INIT_ARGS across a scope."""
"""Saves and restores global JAX PRNG/Shardy config and LIBTPU_INIT_ARGS across a scope."""
previous_prng = getattr(jax.config, "jax_default_prng_impl", None)
previous_shardy = getattr(jax.config, "jax_use_shardy_partitioner", None)
previous_libtpu = os.environ.get("LIBTPU_INIT_ARGS")
try:
yield
finally:
if previous_prng is not None and getattr(jax.config, "jax_default_prng_impl", None) != previous_prng:
jax.config.update("jax_default_prng_impl", previous_prng)
if previous_shardy is not None and getattr(jax.config, "jax_use_shardy_partitioner", None) != previous_shardy:
jax.config.update("jax_use_shardy_partitioner", previous_shardy)
if previous_libtpu is None:
os.environ.pop("LIBTPU_INIT_ARGS", None)
else:
Expand Down
11 changes: 11 additions & 0 deletions tests/unit/marker_propagation_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
from absl.testing import parameterized
import jax

from tests.conftest import _restore_jax_prng_and_libtpu_env
from tests.conftest import pytest_collection_modifyitems


Expand Down Expand Up @@ -111,6 +112,16 @@ def get_added_names(item):
self.assertNotIn("cpu_only", get_added_names(item_gpu))
self.assertNotIn("cpu_only", get_added_names(item_cpu))

def test_restore_jax_use_shardy_partitioner(self):
"""Verifies _restore_jax_prng_and_libtpu_env restores jax_use_shardy_partitioner."""
prev_shardy = getattr(jax.config, "jax_use_shardy_partitioner", True)
self.addCleanup(jax.config.update, "jax_use_shardy_partitioner", prev_shardy)
jax.config.update("jax_use_shardy_partitioner", True)
with _restore_jax_prng_and_libtpu_env():
jax.config.update("jax_use_shardy_partitioner", False)
self.assertFalse(jax.config.jax_use_shardy_partitioner)
self.assertTrue(jax.config.jax_use_shardy_partitioner)


if __name__ == "__main__":
unittest.main()
19 changes: 19 additions & 0 deletions tests/unit/train_compile_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@
from tempfile import gettempdir, NamedTemporaryFile


from maxtext.configs import pyconfig
from maxtext.trainers.pre_train.train_compile import get_topology_mesh
from maxtext.trainers.pre_train.train_compile import main as train_compile_main
from maxtext.utils.globals import MAXTEXT_ASSETS_ROOT
from tests.utils.test_helpers import get_test_config_path
Expand Down Expand Up @@ -1819,3 +1821,20 @@ def test_envy(self, scan_layers):
"override_model_config=True",
)
)

def test_get_topology_mesh_updates_shardy_partitioner(self):
"""Verifies get_topology_mesh updates jax_use_shardy_partitioner from config.shardy."""
prev_shardy = getattr(jax.config, "jax_use_shardy_partitioner", True)
self.addCleanup(jax.config.update, "jax_use_shardy_partitioner", prev_shardy)
jax.config.update("jax_use_shardy_partitioner", False)
config = pyconfig.initialize(
(
"",
get_test_config_path(),
"compile_topology=v6e-4",
"compile_topology_num_slices=1",
"shardy=True",
)
)
get_topology_mesh(config)
self.assertTrue(jax.config.jax_use_shardy_partitioner)
Loading