From 5d826b9ab11ca0441ca0d819e7b893b6138a5bfc Mon Sep 17 00:00:00 2001 From: Ryan Liu Date: Thu, 29 Oct 2026 06:36:35 +0000 Subject: [PATCH] Port llama_mistral_mixtral_orbax_to_hf to the NNX training state # Description Root cause: #b1ac62ab0 ("NNX: flip pure_nnx/enable_nnx/pure_nnx_decoder defaults to True") made `_read_train_checkpoint` -- which this standalone script borrows from `generate_param_only_checkpoint` -- return the NNX state by default: a flat `nnx.State` with the weights under `model` and the optimizer under `optimizer`, instead of a Linen `TrainState`. #bbb843f1d ("[NNX] Delete Linen (5/5)") then removed the Linen path altogether. The script was never ported and still indexes the old layout, `training_state.params["params"]`, so every conversion dies on the first weight it touches: AttributeError: No attribute 'params' in State In `tests/end_to_end/tpu/llama2/7b/test_llama2_7b.sh` this is the MaxText -> HF step after the first logit check; it stayed hidden behind the earlier failures in that script (the logit checker tokenizer, #, and the `generate_param_only_checkpoint` PRNG mismatch, #) and is only reachable once those are in. `tests/end_to_end/tpu/llama3.1/8b/{2,3}_test_llama3.1_8b.sh` run the same script. Reshape the loaded state once with `train_state_nnx.to_checkpoint_dict` -- the helper the checkpoint writer uses, so the result is exactly the on-disk Linen layout -- and hand `convert_state_to_hf` the `["params"]["params"]` subtree it was written against. The porting code is unchanged apart from the shorter indexing; `convert_gpt3_ckpt_from_paxml` already loads its state the same way. # Tests - `pylint` 10.00/10, `pyink --check` clean. - CPU (`JAX_PLATFORMS=cpu`, stable image): tiny llama2 (dense) and mixtral (MoE, 2 experts) models built through `load_model_params` -- a fresh init is the same `nnx.State` the restore path returns -- then `convert_state_to_hf` with matching `MODEL_PARAMS_DICT` entries: every HF tensor name and shape as expected, values equal to the MaxText arrays. The pre-fix script fails on the same models with the `AttributeError` above. - E2E on v5p-8 via the `maxtext_end_to_end` DAG with a test image built from this branch: pending (fill in workload link for llama2-7b). # Checklist Before submitting this PR, please make sure (put X in square brackets): - [X] I have performed a self-review of my code. For an optional AI review, add the `gemini-review` label. - [X] I have necessary comments in my code, particularly in hard-to-understand areas. - [X] I have run end-to-end tests and provided workload links above if applicable. - [X] I have made or will make corresponding changes to the doc if needed, including adding new documentation pages to the relevant Table of Contents (toctree directive) as explained in [our documentation](https://maxtext.readthedocs.io/en/latest/development.html#adding-new-documentation-files). --- .../llama_mistral_mixtral_orbax_to_hf.py | 66 +++++++++---------- 1 file changed, 31 insertions(+), 35 deletions(-) diff --git a/src/maxtext/checkpoint_conversion/standalone_scripts/llama_mistral_mixtral_orbax_to_hf.py b/src/maxtext/checkpoint_conversion/standalone_scripts/llama_mistral_mixtral_orbax_to_hf.py index 3e0f1f4574..781cb6c2fe 100644 --- a/src/maxtext/checkpoint_conversion/standalone_scripts/llama_mistral_mixtral_orbax_to_hf.py +++ b/src/maxtext/checkpoint_conversion/standalone_scripts/llama_mistral_mixtral_orbax_to_hf.py @@ -51,6 +51,7 @@ from maxtext.utils.generate_param_only_checkpoint import _read_train_checkpoint from maxtext.checkpoint_conversion.standalone_scripts import llama_or_mistral_ckpt from maxtext.common import checkpointing +from maxtext.common import train_state_nnx from maxtext.utils import max_logging from maxtext.utils import maxtext_utils from maxtext.utils.max_utils import unpermute_from_match_maxtext_rope @@ -83,9 +84,9 @@ def load_hf_model(model_size): return model -def load_model_state(config): +def load_model_params(config): """ - Loads the MaxText model's TrainState from the Orbax checkpoint + Loads the MaxText model's weights from the Orbax checkpoint, in the Linen checkpoint layout """ devices_array = maxtext_utils.create_device_mesh(config) mesh = Mesh(devices_array, config.mesh_axes) @@ -101,12 +102,15 @@ def load_model_state(config): # Read training state from config.load_paramaters_path max_logging.log(f"Read training checkpoint from: {config.load_full_state_path}") training_state, _ = _read_train_checkpoint(config, checkpoint_manager, mesh) - return training_state + # `_read_train_checkpoint` returns a flat nnx.State (weights under `model`, plus optimizer + # and RNG state). Reshape it to the on-disk Linen layout the porting below indexes, i.e. + # what used to be `TrainState.params["params"]`. + return train_state_nnx.to_checkpoint_dict(training_state)["params"]["params"] -def convert_state_to_hf(training_state, model_size): +def convert_state_to_hf(params, model_size): """ - Port the parameters from the Orbax training_state into the hf_model + Port the parameters from the MaxText weights (Linen checkpoint layout) into the hf_model """ if model_size not in llama_or_mistral_ckpt.MODEL_PARAMS_DICT: @@ -124,7 +128,7 @@ def convert_state_to_hf(training_state, model_size): # Port the embedding weights hf_model_params["model.embed_tokens.weight"] = torch.tensor( - np.asarray(training_state.params["params"]["token_embedder"]["embedding"]), dtype=torch.float16 + np.asarray(params["token_embedder"]["embedding"]), dtype=torch.float16 ) for layer_int in tqdm(range(base_num_decoder_layers), desc="Porting parameters layerwise"): @@ -135,9 +139,7 @@ def convert_state_to_hf(training_state, model_size): np.asarray( unpermute_from_match_maxtext_rope( reverse_scale( - training_state.params["params"]["decoder"]["layers"]["self_attention"]["query"]["kernel"][ - :, layer_int, :, : - ], + params["decoder"]["layers"]["self_attention"]["query"]["kernel"][:, layer_int, :, :], head_dim, ), model_size, @@ -151,9 +153,7 @@ def convert_state_to_hf(training_state, model_size): hf_model_params[f"model.layers.{layer_int}.self_attn.k_proj.weight"] = torch.tensor( np.asarray( unpermute_from_match_maxtext_rope( - training_state.params["params"]["decoder"]["layers"]["self_attention"]["key"]["kernel"][ - :, layer_int, :, : - ], + params["decoder"]["layers"]["self_attention"]["key"]["kernel"][:, layer_int, :, :], model_size, ) .reshape(base_num_query_heads * head_dim, base_num_kv_heads * head_dim) @@ -163,7 +163,7 @@ def convert_state_to_hf(training_state, model_size): ) hf_model_params[f"model.layers.{layer_int}.self_attn.v_proj.weight"] = torch.tensor( np.asarray( - training_state.params["params"]["decoder"]["layers"]["self_attention"]["value"]["kernel"][:, layer_int, :, :] + params["decoder"]["layers"]["self_attention"]["value"]["kernel"][:, layer_int, :, :] .reshape(base_num_query_heads * head_dim, base_num_kv_heads * head_dim) .T ), @@ -171,7 +171,7 @@ def convert_state_to_hf(training_state, model_size): ) hf_model_params[f"model.layers.{layer_int}.self_attn.o_proj.weight"] = torch.tensor( np.asarray( - training_state.params["params"]["decoder"]["layers"]["self_attention"]["out"]["kernel"][:, layer_int, :, :] + params["decoder"]["layers"]["self_attention"]["out"]["kernel"][:, layer_int, :, :] .reshape(base_num_query_heads * head_dim, base_num_query_heads * head_dim) .T ), @@ -181,64 +181,60 @@ def convert_state_to_hf(training_state, model_size): # MLP Layers if num_experts is None: hf_model_params[f"model.layers.{layer_int}.mlp.gate_proj.weight"] = torch.tensor( - np.asarray(training_state.params["params"]["decoder"]["layers"]["mlp"]["wi_0"]["kernel"][:, layer_int, :].T), + np.asarray(params["decoder"]["layers"]["mlp"]["wi_0"]["kernel"][:, layer_int, :].T), dtype=torch.float16, ) hf_model_params[f"model.layers.{layer_int}.mlp.up_proj.weight"] = torch.tensor( - np.asarray(training_state.params["params"]["decoder"]["layers"]["mlp"]["wi_1"]["kernel"][:, layer_int, :].T), + np.asarray(params["decoder"]["layers"]["mlp"]["wi_1"]["kernel"][:, layer_int, :].T), dtype=torch.float16, ) hf_model_params[f"model.layers.{layer_int}.mlp.down_proj.weight"] = torch.tensor( - np.asarray(training_state.params["params"]["decoder"]["layers"]["mlp"]["wo"]["kernel"][:, layer_int, :].T), + np.asarray(params["decoder"]["layers"]["mlp"]["wo"]["kernel"][:, layer_int, :].T), dtype=torch.float16, ) else: hf_model_params[f"model.layers.{layer_int}.block_sparse_moe.gate.weight"] = torch.tensor( - np.asarray( - training_state.params["params"]["decoder"]["layers"]["MoeBlock_0"]["gate"]["kernel"][:, layer_int, :].T - ), + np.asarray(params["decoder"]["layers"]["MoeBlock_0"]["gate"]["kernel"][:, layer_int, :].T), dtype=torch.float16, ) for k in range(num_experts): hf_model_params[f"model.layers.{layer_int}.block_sparse_moe.experts.{k}.w1.weight"] = torch.tensor( - np.asarray(training_state.params["params"]["decoder"]["layers"]["MoeBlock_0"]["wi_0"][k, layer_int, :, :].T), + np.asarray(params["decoder"]["layers"]["MoeBlock_0"]["wi_0"][k, layer_int, :, :].T), dtype=torch.float16, ) hf_model_params[f"model.layers.{layer_int}.block_sparse_moe.experts.{k}.w2.weight"] = torch.tensor( - np.asarray(training_state.params["params"]["decoder"]["layers"]["MoeBlock_0"]["wo"][k, layer_int, :, :].T), + np.asarray(params["decoder"]["layers"]["MoeBlock_0"]["wo"][k, layer_int, :, :].T), dtype=torch.float16, ) hf_model_params[f"model.layers.{layer_int}.block_sparse_moe.experts.{k}.w3.weight"] = torch.tensor( - np.asarray(training_state.params["params"]["decoder"]["layers"]["MoeBlock_0"]["wi_1"][k, layer_int, :, :].T), + np.asarray(params["decoder"]["layers"]["MoeBlock_0"]["wi_1"][k, layer_int, :, :].T), dtype=torch.float16, ) # Pre/post attention layer norm hf_model_params[f"model.layers.{layer_int}.input_layernorm.weight"] = torch.tensor( np.asarray( - training_state.params["params"]["decoder"]["layers"]["pre_self_attention_layer_norm"]["scale"][ - :, layer_int - ].reshape(base_num_query_heads * head_dim) + params["decoder"]["layers"]["pre_self_attention_layer_norm"]["scale"][:, layer_int].reshape( + base_num_query_heads * head_dim + ) ), dtype=torch.float16, ) hf_model_params[f"model.layers.{layer_int}.post_attention_layernorm.weight"] = torch.tensor( np.asarray( - training_state.params["params"]["decoder"]["layers"]["post_self_attention_layer_norm"]["scale"][ - :, layer_int - ].reshape(base_num_query_heads * head_dim) + params["decoder"]["layers"]["post_self_attention_layer_norm"]["scale"][:, layer_int].reshape( + base_num_query_heads * head_dim + ) ), dtype=torch.float16, ) # LM head and layernorm hf_model_params["lm_head.weight"] = torch.tensor( - np.asarray(training_state.params["params"]["decoder"]["logits_dense"]["kernel"].T), dtype=torch.float16 + np.asarray(params["decoder"]["logits_dense"]["kernel"].T), dtype=torch.float16 ) hf_model_params["model.norm.weight"] = torch.tensor( - np.asarray( - training_state.params["params"]["decoder"]["decoder_norm"]["scale"].reshape(base_num_query_heads * head_dim) - ), + np.asarray(params["decoder"]["decoder_norm"]["scale"].reshape(base_num_query_heads * head_dim)), dtype=torch.float16, ) @@ -250,8 +246,8 @@ def convert_orbax_hf(hf_model_path, config): Landing function to convert MaxText model's checkpoint to HuggingFace format """ hf_model = load_hf_model(config.model_name) - training_state = load_model_state(config) - new_hf_model_params = convert_state_to_hf(training_state, config.model_name) + params = load_model_params(config) + new_hf_model_params = convert_state_to_hf(params, config.model_name) print(f"Saving HuggingFace model to path = {hf_model_path}") hf_model.save_pretrained(hf_model_path, state_dict=new_hf_model_params)