From 48c44ce430f442216f719e0f08cdf55ed498425f Mon Sep 17 00:00:00 2001 From: Ryan Liu Date: Tue, 27 Oct 2026 07:07:56 +0000 Subject: [PATCH] Set unsafe_rbg PRNG impl in generate_param_only_checkpoint # Description Root cause: #c41b2ae3a ("Refactor MaxText's checkpointing system to primarily use the Orbax v1 API") replaced the v0 full-state restore in `_load_linen_checkpoint_into_nnx` (`PyTreeRestore(item=linen_abstract, partial_restore=True)`) with `ocp.load(root, linen_abstract)`. That exposed a pre-existing mismatch in `generate_param_only_checkpoint`: it restores a full training state with `load_full_state_path`, but unlike `train.py` (and `decode.py`, `to_maxtext.py`, `forward_pass_logit_checker.py`, ...) it never set `jax_default_prng_impl`, so it builds its restore target with JAX's default `threefry2x32` keys while the checkpoint was written with `unsafe_rbg` keys. The `rngs.*.key` arrays that #fbb35123a persists under `nnx_aux` therefore disagree in shape -- `(num_layers, 2)` requested vs `(num_layers, 4)` stored. The v0 restore accepted that and the conversion kept passing; the v1 load rejects it and the conversion has failed ever since: ValueError: Requested shape: (18, 2) is not compatible with the stored shape: (18, 4) This breaks the finetune -> param-only -> decode flow in `tests/end_to_end/tpu/{gemma/2b,llama2/7b,gemma/7b,llama2/70b}` (b/576849994) at the first step that reads the fine-tuned checkpoint, so everything after it in those scripts is unreachable. Set the same PRNG impl as the other entry points before `pyconfig.initialize`. The mismatch is on the caller's side -- the checkpoint and the v1 loader are both right -- hence the fix here rather than in `checkpointing.py`. The integration test did not catch this because it runs `train_main` and `generate_param_only_ckpt_main` in one process, so the convert step inherited `unsafe_rbg` from training. It now resets the impl to JAX's default before the convert step, modelling the separate-process flow of the end-to-end scripts, and adds a `cpu_only` entry point using a local checkpoint directory so CPU CI exercises the flow and covers `generate_param_only_checkpoint.main`. Fixes b/576849994 # Tests - `pylint` 10.00/10, `pyink --check` clean on both files. - `tests/integration/generate_param_only_checkpoint_test.py::test_param_ckpt_generation_with_dot_product_cpu`: passed. - E2E on v5p-8 via the `maxtext_end_to_end` DAG with a test image built from this branch: passed for gemma-2b and gemma-7b. # Checklist Before submitting this PR, please make sure (put X in square brackets): - [X] I have performed a self-review of my code. For an optional AI review, add the `gemini-review` label. - [X] I have necessary comments in my code, particularly in hard-to-understand areas. - [X] I have run end-to-end tests and provided workload links above if applicable. - [X] I have made or will make corresponding changes to the doc if needed, including adding new documentation pages to the relevant Table of Contents (toctree directive) as explained in [our documentation](https://maxtext.readthedocs.io/en/latest/development.html#adding-new-documentation-files). --- .../utils/generate_param_only_checkpoint.py | 4 +++ .../generate_param_only_checkpoint_test.py | 34 +++++++++++++++++-- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/maxtext/utils/generate_param_only_checkpoint.py b/src/maxtext/utils/generate_param_only_checkpoint.py index aa979786ec..a2cd2807af 100644 --- a/src/maxtext/utils/generate_param_only_checkpoint.py +++ b/src/maxtext/utils/generate_param_only_checkpoint.py @@ -397,6 +397,10 @@ def generate_decode_checkpoint(config): def main(argv: Sequence[str]) -> None: print(argv) + # Match train.py (and the other entry points): the full-state checkpoint was written with + # unsafe_rbg keys, so the restore target must use the same PRNG impl or the nnx_aux RNG key + # shapes disagree ((layers, 2) threefry vs (layers, 4) rbg) and Orbax refuses to load. + jax.config.update("jax_default_prng_impl", "unsafe_rbg") config = pyconfig.initialize(argv) generate_decode_checkpoint(config) diff --git a/tests/integration/generate_param_only_checkpoint_test.py b/tests/integration/generate_param_only_checkpoint_test.py index ac0f949ff9..a9b63a993a 100644 --- a/tests/integration/generate_param_only_checkpoint_test.py +++ b/tests/integration/generate_param_only_checkpoint_test.py @@ -18,14 +18,13 @@ """ from datetime import datetime import os +import jax import pytest from maxtext.inference.decode import main as decode_main from maxtext.common.gcloud_stub import is_decoupled, jetstream config_lib, _, _, _, _ = jetstream() -if getattr(config_lib, "_IS_STUB", False): - pytest.skip("JetStream is not installed (stubbed)", allow_module_level=True) from maxtext.trainers.pre_train.train import main as train_main from maxtext.utils.globals import MAXTEXT_ASSETS_ROOT from maxtext.utils.generate_param_only_checkpoint import main as generate_param_only_ckpt_main @@ -52,9 +51,14 @@ def run_e2e_test_flow( state_path=None, train_attention_type=None, decode_config_extra=None, + base_output_directory=None, + run_decode=True, ): """Helper function to run training, generate parameter-only checkpoint, and decode.""" - base_output_directory = get_test_base_output_directory() + if run_decode and getattr(config_lib, "_IS_STUB", False): + pytest.skip("JetStream is not installed (stubbed)") + if base_output_directory is None: + base_output_directory = get_test_base_output_directory() dataset_path = get_test_dataset_path() run_date = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") train_attention = train_attention_type if train_attention_type is not None else attention_type @@ -65,6 +69,7 @@ def run_e2e_test_flow( f"base_output_directory={base_output_directory}", "async_checkpointing=False", "enable_checkpointing=True", + "enable_tensorboard=False", f"hardware={hardware}", "max_target_length=128", "per_device_batch_size=1", @@ -85,7 +90,9 @@ def run_e2e_test_flow( attention_type=train_attention, dataset_type="synthetic", dataset_path=dataset_path, + base_output_directory=base_output_directory, ) + + ["enable_tensorboard=False"] ) state_path = f"{base_output_directory}/runner_{run_date}/checkpoints/0/items" @@ -99,8 +106,15 @@ def run_e2e_test_flow( ] + pathways_command ) + # The end-to-end scripts run this step as its own process, which starts on JAX's default + # threefry impl rather than the unsafe_rbg that train_main left behind in this process. + # Reset it so the restore target is built the way it is in production (b/576849994). + jax.config.update("jax_default_prng_impl", "threefry2x32") generate_param_only_ckpt_main(generate_param_only_ckpt_config) + if not run_decode: + return + # Run inference on parameter-only checkpoint decode_config = ( shared_config @@ -221,3 +235,17 @@ def test_param_ckpt_generation_with_pre_generated_ckpt(capsys): captured = capsys.readouterr() expected_output = "Input `I love to`" assert expected_output in captured.out + + +@pytest.mark.integration_test +@pytest.mark.cpu_only +def test_param_ckpt_generation_with_dot_product_cpu(tmp_path): + """Tests the parameter-only checkpoint generation flow on CPU with dot product attention.""" + run_e2e_test_flow( + hardware="cpu", + attention_type="dot_product", + model_config=get_model_params(""), + base_output_directory=str(tmp_path), + run_decode=False, + ) + assert any(tmp_path.glob("generate_param_*/checkpoints/0/items"))