From 34abe0acc87a6814fe77bb8b2326f7be76199786 Mon Sep 17 00:00:00 2001 From: EmmaLien Date: Tue, 27 Oct 2026 05:59:18 +0000 Subject: [PATCH] feat(testing): adapt the E2E pre-/post-training scripts to TPU v6e The E2E tests are moving from v5p to v6e. A v6e chip has 32 GiB of HBM (v5p: 95 GiB) and v6e slices are 2D, so several scripts no longer fit in memory or cannot build their meshes as written. Write the v6e settings into those scripts: - vllm_decode: weight_dtype=bfloat16 (vllm_decode loads fp32 weights by default) and parallelism that matches the v6e slice sizes. - Training: more expert parallelism for the MoE models, allow_split_physical_axes=True where EP/TP must split the 2D topology, bfloat16 weights where fp32 does not fit, and Gemma4 pre-training from the scanned checkpoint. - RL: larger rollout TP, trainer meshes sized for half of a v6e-64 job, opt_type=sgd for Gemma4 and Llama, and rl.reshard_chunk_size=1 for Gemma4. - Qwen3.5 RL on v6e-64: an EP8 x FSDP2 x TP2 trainer and EP8 x TP2 rollouts. TP-only rollouts pad the MoE intermediate and do not fit, and TP x EP stays at 16 because vLLM pads the 2 KV heads to TP x EP, which must not exceed the 16 query heads. Every changed script was run end to end on v6e. After this change the scripts are sized for v6e and are not expected to fit on v5p as is. --- .../tpu/gemma3/4b/test_gemma3_lora.sh | 6 +++-- .../end_to_end/tpu/gemma4/26b/test_gemma4.sh | 11 ++++++---- .../tpu/gemma4/26b/test_gemma4_rl.sh | 19 +++++++++++----- .../tpu/gemma4/26b/test_gemma4_sft.sh | 12 +++++++--- .../tpu/gpt_oss/20b/test_gpt_oss.sh | 3 ++- .../tpu/llama3.1/70b/test_llama3.1_70b.sh | 3 ++- .../tpu/llama3.1/70b/test_llama3.1_70b_rl.sh | 13 ++++++----- .../tpu/llama3.1/70b/test_llama3.1_70b_sft.sh | 11 ++++++---- .../tpu/qwen3.5/35b/test_qwen3.5_rl.sh | 7 +++++- .../end_to_end/tpu/qwen3/30b/test_qwen3_rl.sh | 22 ++++++++++++------- .../tpu/qwen3/30b/test_qwen3_sft.sh | 17 ++++++++------ 11 files changed, 83 insertions(+), 41 deletions(-) diff --git a/tests/end_to_end/tpu/gemma3/4b/test_gemma3_lora.sh b/tests/end_to_end/tpu/gemma3/4b/test_gemma3_lora.sh index 3266545050..1e03e735fc 100755 --- a/tests/end_to_end/tpu/gemma3/4b/test_gemma3_lora.sh +++ b/tests/end_to_end/tpu/gemma3/4b/test_gemma3_lora.sh @@ -35,7 +35,8 @@ python3 -m maxtext.inference.vllm_decode \ prompt="Suggest some famous landmarks in London." \ use_chat_template=True \ scan_layers=false \ - enable_single_controller=${use_pathways} + enable_single_controller=${use_pathways} \ + weight_dtype=bfloat16 # Step 2: Run LoRA on the converted checkpoint python3 -m maxtext.trainers.post_train.sft.train_sft \ @@ -67,4 +68,5 @@ python3 -m maxtext.inference.vllm_decode \ prompt="Suggest some famous landmarks in London." \ use_chat_template=True \ scan_layers=true \ - enable_single_controller=${use_pathways} + enable_single_controller=${use_pathways} \ + weight_dtype=bfloat16 diff --git a/tests/end_to_end/tpu/gemma4/26b/test_gemma4.sh b/tests/end_to_end/tpu/gemma4/26b/test_gemma4.sh index 6989a0edbb..dc97aee22a 100644 --- a/tests/end_to_end/tpu/gemma4/26b/test_gemma4.sh +++ b/tests/end_to_end/tpu/gemma4/26b/test_gemma4.sh @@ -30,6 +30,7 @@ USE_MULTIMODAL=false # Non-Googlers please remember to point `BASE_OUTPUT_DIRECTORY` to the GCS paths where you have the scanned and unscanned checkpoints stored BASE_OUTPUT_DIRECTORY=gs://runner-maxtext-logs/${MODEL_NAME} UNSCANNED_CKPT_PATH=${BASE_OUTPUT_DIRECTORY}/to_maxtext/unscanned/${run_id}/0/items +SCANNED_CKPT_PATH=${BASE_OUTPUT_DIRECTORY}/to_maxtext/scanned/${run_id}/0/items # Non-Googlers please remember to point `DATASET_PATH` to the GCS bucket where you have your training data DATASET_PATH=gs://maxtext-dataset @@ -61,11 +62,13 @@ python3 -m maxtext.trainers.pre_train.train \ base_output_directory=${BASE_OUTPUT_DIRECTORY}/train \ dataset_type=grain grain_file_type=tfrecord \ dataset_path=${DATASET_PATH} tokenizer_type="huggingface" \ - load_parameters_path=${UNSCANNED_CKPT_PATH} \ + load_parameters_path=${SCANNED_CKPT_PATH} \ per_device_batch_size=1 run_name=${run_id} \ max_target_length=8192 steps=2 async_checkpointing=false \ checkpoint_storage_use_zarr3=False checkpoint_storage_use_ocdbt=False \ - model_name=${MODEL_NAME} scan_layers=false use_multimodal=${USE_MULTIMODAL} + model_name=${MODEL_NAME} scan_layers=true use_multimodal=${USE_MULTIMODAL} \ + ici_expert_parallelism=8 \ + weight_dtype=bfloat16 # Step 3: Run inference on the checkpoint generated from the previous run if [ ${USE_MULTIMODAL} == true ]; then @@ -74,7 +77,7 @@ if [ ${USE_MULTIMODAL} == true ]; then load_parameters_path=${BASE_OUTPUT_DIRECTORY}/train/${run_id}/checkpoints/1/items \ per_device_batch_size=1 run_name=${run_id} \ max_prefill_predict_length=272 max_target_length=300 steps=1 async_checkpointing=false \ - scan_layers=false use_multimodal=true \ + scan_layers=true use_multimodal=true \ checkpoint_storage_use_zarr3=False checkpoint_storage_use_ocdbt=False \ prompt=\'Describe\ image\ \\' image_path=\'tests/assets/test_image.jpg\' attention=\'dot_product\' else @@ -84,5 +87,5 @@ else per_device_batch_size=1 run_name=${run_id} \ max_prefill_predict_length=8 max_target_length=16 steps=1 async_checkpointing=false \ checkpoint_storage_use_zarr3=False checkpoint_storage_use_ocdbt=False \ - scan_layers=false prompt='I love to' attention=\'dot_product\' + scan_layers=true prompt='I love to' attention=\'dot_product\' fi \ No newline at end of file diff --git a/tests/end_to_end/tpu/gemma4/26b/test_gemma4_rl.sh b/tests/end_to_end/tpu/gemma4/26b/test_gemma4_rl.sh index 86eb868e1b..f391228a52 100644 --- a/tests/end_to_end/tpu/gemma4/26b/test_gemma4_rl.sh +++ b/tests/end_to_end/tpu/gemma4/26b/test_gemma4_rl.sh @@ -36,7 +36,9 @@ python3 -m maxtext.inference.vllm_decode \ scan_layers=false \ enable_single_controller=${use_pathways} \ prefuse_moe_weights=True \ - ici_tensor_parallelism=4 + ici_tensor_parallelism=8 \ + ici_data_parallelism=8 \ + weight_dtype=bfloat16 # Step 2: Run RL on the converted checkpoint python3 -m maxtext.trainers.post_train.rl.train_rl \ @@ -48,7 +50,7 @@ python3 -m maxtext.trainers.post_train.rl.train_rl \ num_batches=2 \ batch_size=4 \ train_micro_batch_size=4 \ - ici_expert_parallelism=4 \ + ici_expert_parallelism=8 \ ici_fsdp_parallelism=2 \ ici_data_parallelism=1 \ num_test_batches=2 \ @@ -56,11 +58,16 @@ python3 -m maxtext.trainers.post_train.rl.train_rl \ enable_single_controller=${use_pathways} \ checkpoint_storage_use_zarr3=False \ checkpoint_storage_use_ocdbt=False \ - rollout_tensor_parallelism=2 \ + rollout_tensor_parallelism=4 \ vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \ vllm_additional_config='{"maxtext_config": {"model_name": "gemma4-26b", "log_config": "false", "prefuse_moe_weights": "true"}}' \ remat_policy=full \ - hbm_utilization_vllm=0.55 + hbm_utilization_vllm=0.55 \ + ici_tensor_parallelism=2 \ + allow_split_physical_axes=True \ + opt_type=sgd \ + rl.num_generations=4 \ + rl.reshard_chunk_size=1 # Step 3: Run inference on the checkpoint generated from the previous run python3 -m maxtext.inference.vllm_decode \ @@ -73,4 +80,6 @@ python3 -m maxtext.inference.vllm_decode \ scan_layers=false \ enable_single_controller=${use_pathways} \ prefuse_moe_weights=True \ - ici_tensor_parallelism=4 \ No newline at end of file + ici_tensor_parallelism=8 \ + ici_data_parallelism=8 \ + weight_dtype=bfloat16 \ No newline at end of file diff --git a/tests/end_to_end/tpu/gemma4/26b/test_gemma4_sft.sh b/tests/end_to_end/tpu/gemma4/26b/test_gemma4_sft.sh index ffb10fd63e..63d217f524 100644 --- a/tests/end_to_end/tpu/gemma4/26b/test_gemma4_sft.sh +++ b/tests/end_to_end/tpu/gemma4/26b/test_gemma4_sft.sh @@ -33,7 +33,8 @@ python3 -m maxtext.inference.vllm_decode \ hbm_utilization_vllm=0.85 \ prompt="Suggest some famous landmarks in London." \ use_chat_template=True scan_layers=false enable_single_controller=${use_pathways} \ - prefuse_moe_weights=True ici_tensor_parallelism=8 + prefuse_moe_weights=True ici_tensor_parallelism=8 \ + weight_dtype=bfloat16 # Step 2: Run SFT on the converted checkpoint python3 -m maxtext.trainers.post_train.sft.train_sft \ @@ -42,7 +43,11 @@ python3 -m maxtext.trainers.post_train.sft.train_sft \ per_device_batch_size=1 run_name=${run_id} \ steps=2 scan_layers=false \ model_name=${MODEL_NAME} enable_single_controller=${use_pathways} \ - checkpoint_storage_use_zarr3=False checkpoint_storage_use_ocdbt=False + checkpoint_storage_use_zarr3=False checkpoint_storage_use_ocdbt=False \ + ici_expert_parallelism=8 \ + weight_dtype=bfloat16 \ + opt_type=sgd \ + max_target_length=1024 # Step 3: Run inference on the checkpoint generated from the previous run python3 -m maxtext.inference.vllm_decode \ @@ -52,4 +57,5 @@ python3 -m maxtext.inference.vllm_decode \ hbm_utilization_vllm=0.85 \ prompt="Suggest some famous landmarks in London." \ use_chat_template=True scan_layers=false enable_single_controller=${use_pathways} \ - prefuse_moe_weights=True ici_tensor_parallelism=8 \ No newline at end of file + prefuse_moe_weights=True ici_tensor_parallelism=8 \ + weight_dtype=bfloat16 \ No newline at end of file diff --git a/tests/end_to_end/tpu/gpt_oss/20b/test_gpt_oss.sh b/tests/end_to_end/tpu/gpt_oss/20b/test_gpt_oss.sh index affce9de99..b922659b1d 100644 --- a/tests/end_to_end/tpu/gpt_oss/20b/test_gpt_oss.sh +++ b/tests/end_to_end/tpu/gpt_oss/20b/test_gpt_oss.sh @@ -71,7 +71,8 @@ python3 -m maxtext.trainers.pre_train.train \ checkpoint_storage_use_ocdbt=False \ model_name=${MODEL_NAME} \ scan_layers=false \ - use_multimodal=false + use_multimodal=false \ + ici_expert_parallelism=8 # Step 3: Run inference on the checkpoint generated from the previous run python3 -m maxtext.inference.decode \ diff --git a/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b.sh b/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b.sh index 6b62525015..87ff198cca 100644 --- a/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b.sh +++ b/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b.sh @@ -67,7 +67,8 @@ python3 -m maxtext.trainers.pre_train.train \ max_target_length=2048 steps=2 async_checkpointing=false \ checkpoint_storage_use_zarr3=False checkpoint_storage_use_ocdbt=False \ model_name=${MODEL_NAME} scan_layers=false use_multimodal=${USE_MULTIMODAL} \ - ici_tensor_parallelism=4 + ici_tensor_parallelism=4 \ + allow_split_physical_axes=True # Step 3: Run inference on the checkpoint generated from the previous run if [ ${USE_MULTIMODAL} == true ]; then diff --git a/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_rl.sh b/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_rl.sh index f42c8bf279..68d212ae9d 100644 --- a/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_rl.sh +++ b/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_rl.sh @@ -35,7 +35,8 @@ python3 -m maxtext.inference.vllm_decode \ prompt='Suggest some famous landmarks in London.' \ use_chat_template=True scan_layers=false enable_single_controller=${use_pathways} \ ici_tensor_parallelism=8 \ - ici_data_parallelism=4 + ici_data_parallelism=8 \ + weight_dtype=bfloat16 # Step 2: Run RL on the converted checkpoint python3 -m maxtext.trainers.post_train.rl.train_rl \ @@ -47,9 +48,10 @@ python3 -m maxtext.trainers.post_train.rl.train_rl \ enable_single_controller=${use_pathways} \ remat_policy=full \ checkpoint_storage_use_zarr3=False checkpoint_storage_use_ocdbt=False \ - rollout_tensor_parallelism=4 \ + rollout_tensor_parallelism=8 \ vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \ - vllm_additional_config='{"maxtext_config": {"model_name": "llama3.1-70b", "log_config": "false"}}' + vllm_additional_config='{"maxtext_config": {"model_name": "llama3.1-70b", "log_config": "false"}}' \ + opt_type=sgd # Step 3: Run inference on the checkpoint generated from the previous run @@ -61,5 +63,6 @@ python3 -m maxtext.inference.vllm_decode \ hbm_utilization_vllm=0.6 \ prompt='Suggest some famous landmarks in London.' \ use_chat_template=True scan_layers=true enable_single_controller=${use_pathways} \ - ici_tensor_parallelism=8 \ - ici_data_parallelism=4 + ici_tensor_parallelism=16 \ + ici_data_parallelism=4 \ + weight_dtype=bfloat16 diff --git a/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_sft.sh b/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_sft.sh index a46da4a421..f2d5639253 100644 --- a/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_sft.sh +++ b/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_sft.sh @@ -35,7 +35,8 @@ python3 -m maxtext.inference.vllm_decode \ prompt="Suggest some famous landmarks in London." \ use_chat_template=True scan_layers=false enable_single_controller=${use_pathways} \ ici_tensor_parallelism=8 \ - ici_data_parallelism=4 + ici_data_parallelism=4 \ + weight_dtype=bfloat16 # Step 2: Run SFT on the converted checkpoint python3 -m maxtext.trainers.post_train.sft.train_sft \ @@ -46,7 +47,8 @@ python3 -m maxtext.trainers.post_train.sft.train_sft \ model_name=${MODEL_NAME} tokenizer_path='meta-llama/Llama-3.1-70B-Instruct' \ enable_single_controller=${use_pathways} \ remat_policy=full \ - checkpoint_storage_use_zarr3=False checkpoint_storage_use_ocdbt=False + checkpoint_storage_use_zarr3=False checkpoint_storage_use_ocdbt=False \ + weight_dtype=bfloat16 # Step 3: Run inference on the checkpoint generated from the previous run python3 -m maxtext.inference.vllm_decode \ @@ -57,5 +59,6 @@ python3 -m maxtext.inference.vllm_decode \ hbm_utilization_vllm=0.6 \ prompt="Suggest some famous landmarks in London." \ use_chat_template=True scan_layers=true enable_single_controller=${use_pathways} \ - ici_tensor_parallelism=8 \ - ici_data_parallelism=4 + ici_tensor_parallelism=16 \ + ici_data_parallelism=2 \ + weight_dtype=bfloat16 diff --git a/tests/end_to_end/tpu/qwen3.5/35b/test_qwen3.5_rl.sh b/tests/end_to_end/tpu/qwen3.5/35b/test_qwen3.5_rl.sh index 40901d75cd..d8be7002e2 100644 --- a/tests/end_to_end/tpu/qwen3.5/35b/test_qwen3.5_rl.sh +++ b/tests/end_to_end/tpu/qwen3.5/35b/test_qwen3.5_rl.sh @@ -71,10 +71,15 @@ python3 -m maxtext.trainers.post_train.rl.train_rl \ train_micro_batch_size=1 \ num_test_batches=2 \ model_name=${MODEL_NAME} \ + ici_expert_parallelism=8 \ + ici_fsdp_parallelism=2 \ + ici_tensor_parallelism=2 \ + allow_split_physical_axes=True \ enable_single_controller=True \ checkpoint_storage_use_zarr3=False \ checkpoint_storage_use_ocdbt=False \ - rollout_tensor_parallelism=4 \ + rollout_expert_parallelism=8 \ + rollout_tensor_parallelism=2 \ vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \ vllm_additional_config='{"maxtext_config": {"model_name": "'${MODEL_NAME}'", "log_config": false, "allow_split_physical_axes": true, "prefuse_moe_weights": true}}' \ remat_policy=full \ diff --git a/tests/end_to_end/tpu/qwen3/30b/test_qwen3_rl.sh b/tests/end_to_end/tpu/qwen3/30b/test_qwen3_rl.sh index 9e0f761dd4..11735e9b63 100755 --- a/tests/end_to_end/tpu/qwen3/30b/test_qwen3_rl.sh +++ b/tests/end_to_end/tpu/qwen3/30b/test_qwen3_rl.sh @@ -48,18 +48,19 @@ python3 -m maxtext.inference.vllm_decode \ model_name=${MODEL_NAME} \ load_parameters_path=${SCANNED_CKPT_PATH} \ vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \ - hbm_utilization_vllm=0.8 \ + hbm_utilization_vllm=0.5 \ prompt='Suggest some famous landmarks in London.' \ max_target_length=256 \ max_num_batched_tokens=256 \ ici_tensor_parallelism=4 \ ici_expert_parallelism=2 \ - ici_data_parallelism=2 \ + ici_data_parallelism=8 \ allow_split_physical_axes=True \ prefuse_moe_weights=True \ use_chat_template=True \ scan_layers=True \ - enable_single_controller=True + enable_single_controller=True \ + weight_dtype=bfloat16 # Step 2: Run RL starting from the pre-converted checkpoint python3 -m maxtext.trainers.post_train.rl.train_rl \ @@ -80,7 +81,7 @@ python3 -m maxtext.trainers.post_train.rl.train_rl \ vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \ vllm_additional_config='{"maxtext_config": {"model_name": "'${MODEL_NAME}'", "log_config": false, "allow_split_physical_axes": true, "prefuse_moe_weights": true}}' \ remat_policy=full \ - hbm_utilization_vllm=0.55 \ + hbm_utilization_vllm=0.65 \ use_pathways=True \ chips_per_vm=4 \ max_target_length=512 \ @@ -90,22 +91,27 @@ python3 -m maxtext.trainers.post_train.rl.train_rl \ enable_tunix_perf_metrics=True \ rl.num_generations=16 \ debug=False \ - rl.reshard_chunk_size=1 + rl.reshard_chunk_size=1 \ + ici_expert_parallelism=8 \ + ici_fsdp_parallelism=2 \ + ici_tensor_parallelism=2 \ + allow_split_physical_axes=True # Step 3: Run inference on the checkpoint produced by the RL run python3 -m maxtext.inference.vllm_decode \ model_name=${MODEL_NAME} \ load_parameters_path=${BASE_OUTPUT_DIRECTORY}/rl/${run_id}/checkpoints/actor/2/model_params \ vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \ - hbm_utilization_vllm=0.8 \ + hbm_utilization_vllm=0.5 \ prompt='Suggest some famous landmarks in London.' \ max_target_length=256 \ max_num_batched_tokens=256 \ ici_tensor_parallelism=4 \ ici_expert_parallelism=2 \ - ici_data_parallelism=2 \ + ici_data_parallelism=8 \ allow_split_physical_axes=True \ prefuse_moe_weights=True \ use_chat_template=True \ scan_layers=True \ - enable_single_controller=True \ No newline at end of file + enable_single_controller=True \ + weight_dtype=bfloat16 \ No newline at end of file diff --git a/tests/end_to_end/tpu/qwen3/30b/test_qwen3_sft.sh b/tests/end_to_end/tpu/qwen3/30b/test_qwen3_sft.sh index c835cf9c93..e785f0fa07 100755 --- a/tests/end_to_end/tpu/qwen3/30b/test_qwen3_sft.sh +++ b/tests/end_to_end/tpu/qwen3/30b/test_qwen3_sft.sh @@ -35,9 +35,10 @@ python3 -m maxtext.inference.vllm_decode \ scan_layers=true \ enable_single_controller=True \ ici_tensor_parallelism=4 \ - ici_expert_parallelism=2 \ - ici_data_parallelism=2 \ - prompt="Suggest some famous landmarks in London." + ici_expert_parallelism=4 \ + ici_data_parallelism=1 \ + prompt="Suggest some famous landmarks in London." \ + weight_dtype=bfloat16 # Step 2: Run SFT starting from the pre-converted checkpoint python3 -m maxtext.trainers.post_train.sft.train_sft \ @@ -56,7 +57,8 @@ python3 -m maxtext.trainers.post_train.sft.train_sft \ weight_dtype=bfloat16 \ dtype=bfloat16 \ ici_expert_parallelism=8 \ - opt_type=sgd + opt_type=sgd \ + allow_split_physical_axes=True # Step 3: Run inference on the checkpoint produced by the SFT run @@ -69,6 +71,7 @@ python3 -m maxtext.inference.vllm_decode \ scan_layers=true \ enable_single_controller=True \ ici_tensor_parallelism=4 \ - ici_expert_parallelism=2 \ - ici_data_parallelism=2 \ - prompt="Suggest some famous landmarks in London." \ No newline at end of file + ici_expert_parallelism=4 \ + ici_data_parallelism=1 \ + prompt="Suggest some famous landmarks in London." \ + weight_dtype=bfloat16 \ No newline at end of file