diff --git a/tests/end_to_end/tpu/gemma3/4b/test_gemma3_lora.sh b/tests/end_to_end/tpu/gemma3/4b/test_gemma3_lora.sh index 3266545050..1e03e735fc 100755 --- a/tests/end_to_end/tpu/gemma3/4b/test_gemma3_lora.sh +++ b/tests/end_to_end/tpu/gemma3/4b/test_gemma3_lora.sh @@ -35,7 +35,8 @@ python3 -m maxtext.inference.vllm_decode \ prompt="Suggest some famous landmarks in London." \ use_chat_template=True \ scan_layers=false \ - enable_single_controller=${use_pathways} + enable_single_controller=${use_pathways} \ + weight_dtype=bfloat16 # Step 2: Run LoRA on the converted checkpoint python3 -m maxtext.trainers.post_train.sft.train_sft \ @@ -67,4 +68,5 @@ python3 -m maxtext.inference.vllm_decode \ prompt="Suggest some famous landmarks in London." \ use_chat_template=True \ scan_layers=true \ - enable_single_controller=${use_pathways} + enable_single_controller=${use_pathways} \ + weight_dtype=bfloat16 diff --git a/tests/end_to_end/tpu/gemma4/26b/test_gemma4.sh b/tests/end_to_end/tpu/gemma4/26b/test_gemma4.sh index 6989a0edbb..dc97aee22a 100644 --- a/tests/end_to_end/tpu/gemma4/26b/test_gemma4.sh +++ b/tests/end_to_end/tpu/gemma4/26b/test_gemma4.sh @@ -30,6 +30,7 @@ USE_MULTIMODAL=false # Non-Googlers please remember to point `BASE_OUTPUT_DIRECTORY` to the GCS paths where you have the scanned and unscanned checkpoints stored BASE_OUTPUT_DIRECTORY=gs://runner-maxtext-logs/${MODEL_NAME} UNSCANNED_CKPT_PATH=${BASE_OUTPUT_DIRECTORY}/to_maxtext/unscanned/${run_id}/0/items +SCANNED_CKPT_PATH=${BASE_OUTPUT_DIRECTORY}/to_maxtext/scanned/${run_id}/0/items # Non-Googlers please remember to point `DATASET_PATH` to the GCS bucket where you have your training data DATASET_PATH=gs://maxtext-dataset @@ -61,11 +62,13 @@ python3 -m maxtext.trainers.pre_train.train \ base_output_directory=${BASE_OUTPUT_DIRECTORY}/train \ dataset_type=grain grain_file_type=tfrecord \ dataset_path=${DATASET_PATH} tokenizer_type="huggingface" \ - load_parameters_path=${UNSCANNED_CKPT_PATH} \ + load_parameters_path=${SCANNED_CKPT_PATH} \ per_device_batch_size=1 run_name=${run_id} \ max_target_length=8192 steps=2 async_checkpointing=false \ checkpoint_storage_use_zarr3=False checkpoint_storage_use_ocdbt=False \ - model_name=${MODEL_NAME} scan_layers=false use_multimodal=${USE_MULTIMODAL} + model_name=${MODEL_NAME} scan_layers=true use_multimodal=${USE_MULTIMODAL} \ + ici_expert_parallelism=8 \ + weight_dtype=bfloat16 # Step 3: Run inference on the checkpoint generated from the previous run if [ ${USE_MULTIMODAL} == true ]; then @@ -74,7 +77,7 @@ if [ ${USE_MULTIMODAL} == true ]; then load_parameters_path=${BASE_OUTPUT_DIRECTORY}/train/${run_id}/checkpoints/1/items \ per_device_batch_size=1 run_name=${run_id} \ max_prefill_predict_length=272 max_target_length=300 steps=1 async_checkpointing=false \ - scan_layers=false use_multimodal=true \ + scan_layers=true use_multimodal=true \ checkpoint_storage_use_zarr3=False checkpoint_storage_use_ocdbt=False \ prompt=\'Describe\ image\ \\' image_path=\'tests/assets/test_image.jpg\' attention=\'dot_product\' else @@ -84,5 +87,5 @@ else per_device_batch_size=1 run_name=${run_id} \ max_prefill_predict_length=8 max_target_length=16 steps=1 async_checkpointing=false \ checkpoint_storage_use_zarr3=False checkpoint_storage_use_ocdbt=False \ - scan_layers=false prompt='I love to' attention=\'dot_product\' + scan_layers=true prompt='I love to' attention=\'dot_product\' fi \ No newline at end of file diff --git a/tests/end_to_end/tpu/gemma4/26b/test_gemma4_rl.sh b/tests/end_to_end/tpu/gemma4/26b/test_gemma4_rl.sh index 86eb868e1b..f391228a52 100644 --- a/tests/end_to_end/tpu/gemma4/26b/test_gemma4_rl.sh +++ b/tests/end_to_end/tpu/gemma4/26b/test_gemma4_rl.sh @@ -36,7 +36,9 @@ python3 -m maxtext.inference.vllm_decode \ scan_layers=false \ enable_single_controller=${use_pathways} \ prefuse_moe_weights=True \ - ici_tensor_parallelism=4 + ici_tensor_parallelism=8 \ + ici_data_parallelism=8 \ + weight_dtype=bfloat16 # Step 2: Run RL on the converted checkpoint python3 -m maxtext.trainers.post_train.rl.train_rl \ @@ -48,7 +50,7 @@ python3 -m maxtext.trainers.post_train.rl.train_rl \ num_batches=2 \ batch_size=4 \ train_micro_batch_size=4 \ - ici_expert_parallelism=4 \ + ici_expert_parallelism=8 \ ici_fsdp_parallelism=2 \ ici_data_parallelism=1 \ num_test_batches=2 \ @@ -56,11 +58,16 @@ python3 -m maxtext.trainers.post_train.rl.train_rl \ enable_single_controller=${use_pathways} \ checkpoint_storage_use_zarr3=False \ checkpoint_storage_use_ocdbt=False \ - rollout_tensor_parallelism=2 \ + rollout_tensor_parallelism=4 \ vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \ vllm_additional_config='{"maxtext_config": {"model_name": "gemma4-26b", "log_config": "false", "prefuse_moe_weights": "true"}}' \ remat_policy=full \ - hbm_utilization_vllm=0.55 + hbm_utilization_vllm=0.55 \ + ici_tensor_parallelism=2 \ + allow_split_physical_axes=True \ + opt_type=sgd \ + rl.num_generations=4 \ + rl.reshard_chunk_size=1 # Step 3: Run inference on the checkpoint generated from the previous run python3 -m maxtext.inference.vllm_decode \ @@ -73,4 +80,6 @@ python3 -m maxtext.inference.vllm_decode \ scan_layers=false \ enable_single_controller=${use_pathways} \ prefuse_moe_weights=True \ - ici_tensor_parallelism=4 \ No newline at end of file + ici_tensor_parallelism=8 \ + ici_data_parallelism=8 \ + weight_dtype=bfloat16 \ No newline at end of file diff --git a/tests/end_to_end/tpu/gemma4/26b/test_gemma4_sft.sh b/tests/end_to_end/tpu/gemma4/26b/test_gemma4_sft.sh index ffb10fd63e..63d217f524 100644 --- a/tests/end_to_end/tpu/gemma4/26b/test_gemma4_sft.sh +++ b/tests/end_to_end/tpu/gemma4/26b/test_gemma4_sft.sh @@ -33,7 +33,8 @@ python3 -m maxtext.inference.vllm_decode \ hbm_utilization_vllm=0.85 \ prompt="Suggest some famous landmarks in London." \ use_chat_template=True scan_layers=false enable_single_controller=${use_pathways} \ - prefuse_moe_weights=True ici_tensor_parallelism=8 + prefuse_moe_weights=True ici_tensor_parallelism=8 \ + weight_dtype=bfloat16 # Step 2: Run SFT on the converted checkpoint python3 -m maxtext.trainers.post_train.sft.train_sft \ @@ -42,7 +43,11 @@ python3 -m maxtext.trainers.post_train.sft.train_sft \ per_device_batch_size=1 run_name=${run_id} \ steps=2 scan_layers=false \ model_name=${MODEL_NAME} enable_single_controller=${use_pathways} \ - checkpoint_storage_use_zarr3=False checkpoint_storage_use_ocdbt=False + checkpoint_storage_use_zarr3=False checkpoint_storage_use_ocdbt=False \ + ici_expert_parallelism=8 \ + weight_dtype=bfloat16 \ + opt_type=sgd \ + max_target_length=1024 # Step 3: Run inference on the checkpoint generated from the previous run python3 -m maxtext.inference.vllm_decode \ @@ -52,4 +57,5 @@ python3 -m maxtext.inference.vllm_decode \ hbm_utilization_vllm=0.85 \ prompt="Suggest some famous landmarks in London." \ use_chat_template=True scan_layers=false enable_single_controller=${use_pathways} \ - prefuse_moe_weights=True ici_tensor_parallelism=8 \ No newline at end of file + prefuse_moe_weights=True ici_tensor_parallelism=8 \ + weight_dtype=bfloat16 \ No newline at end of file diff --git a/tests/end_to_end/tpu/gpt_oss/20b/test_gpt_oss.sh b/tests/end_to_end/tpu/gpt_oss/20b/test_gpt_oss.sh index affce9de99..b922659b1d 100644 --- a/tests/end_to_end/tpu/gpt_oss/20b/test_gpt_oss.sh +++ b/tests/end_to_end/tpu/gpt_oss/20b/test_gpt_oss.sh @@ -71,7 +71,8 @@ python3 -m maxtext.trainers.pre_train.train \ checkpoint_storage_use_ocdbt=False \ model_name=${MODEL_NAME} \ scan_layers=false \ - use_multimodal=false + use_multimodal=false \ + ici_expert_parallelism=8 # Step 3: Run inference on the checkpoint generated from the previous run python3 -m maxtext.inference.decode \ diff --git a/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b.sh b/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b.sh index 6b62525015..87ff198cca 100644 --- a/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b.sh +++ b/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b.sh @@ -67,7 +67,8 @@ python3 -m maxtext.trainers.pre_train.train \ max_target_length=2048 steps=2 async_checkpointing=false \ checkpoint_storage_use_zarr3=False checkpoint_storage_use_ocdbt=False \ model_name=${MODEL_NAME} scan_layers=false use_multimodal=${USE_MULTIMODAL} \ - ici_tensor_parallelism=4 + ici_tensor_parallelism=4 \ + allow_split_physical_axes=True # Step 3: Run inference on the checkpoint generated from the previous run if [ ${USE_MULTIMODAL} == true ]; then diff --git a/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_rl.sh b/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_rl.sh index f42c8bf279..68d212ae9d 100644 --- a/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_rl.sh +++ b/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_rl.sh @@ -35,7 +35,8 @@ python3 -m maxtext.inference.vllm_decode \ prompt='Suggest some famous landmarks in London.' \ use_chat_template=True scan_layers=false enable_single_controller=${use_pathways} \ ici_tensor_parallelism=8 \ - ici_data_parallelism=4 + ici_data_parallelism=8 \ + weight_dtype=bfloat16 # Step 2: Run RL on the converted checkpoint python3 -m maxtext.trainers.post_train.rl.train_rl \ @@ -47,9 +48,10 @@ python3 -m maxtext.trainers.post_train.rl.train_rl \ enable_single_controller=${use_pathways} \ remat_policy=full \ checkpoint_storage_use_zarr3=False checkpoint_storage_use_ocdbt=False \ - rollout_tensor_parallelism=4 \ + rollout_tensor_parallelism=8 \ vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \ - vllm_additional_config='{"maxtext_config": {"model_name": "llama3.1-70b", "log_config": "false"}}' + vllm_additional_config='{"maxtext_config": {"model_name": "llama3.1-70b", "log_config": "false"}}' \ + opt_type=sgd # Step 3: Run inference on the checkpoint generated from the previous run @@ -61,5 +63,6 @@ python3 -m maxtext.inference.vllm_decode \ hbm_utilization_vllm=0.6 \ prompt='Suggest some famous landmarks in London.' \ use_chat_template=True scan_layers=true enable_single_controller=${use_pathways} \ - ici_tensor_parallelism=8 \ - ici_data_parallelism=4 + ici_tensor_parallelism=16 \ + ici_data_parallelism=4 \ + weight_dtype=bfloat16 diff --git a/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_sft.sh b/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_sft.sh index a46da4a421..f2d5639253 100644 --- a/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_sft.sh +++ b/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_sft.sh @@ -35,7 +35,8 @@ python3 -m maxtext.inference.vllm_decode \ prompt="Suggest some famous landmarks in London." \ use_chat_template=True scan_layers=false enable_single_controller=${use_pathways} \ ici_tensor_parallelism=8 \ - ici_data_parallelism=4 + ici_data_parallelism=4 \ + weight_dtype=bfloat16 # Step 2: Run SFT on the converted checkpoint python3 -m maxtext.trainers.post_train.sft.train_sft \ @@ -46,7 +47,8 @@ python3 -m maxtext.trainers.post_train.sft.train_sft \ model_name=${MODEL_NAME} tokenizer_path='meta-llama/Llama-3.1-70B-Instruct' \ enable_single_controller=${use_pathways} \ remat_policy=full \ - checkpoint_storage_use_zarr3=False checkpoint_storage_use_ocdbt=False + checkpoint_storage_use_zarr3=False checkpoint_storage_use_ocdbt=False \ + weight_dtype=bfloat16 # Step 3: Run inference on the checkpoint generated from the previous run python3 -m maxtext.inference.vllm_decode \ @@ -57,5 +59,6 @@ python3 -m maxtext.inference.vllm_decode \ hbm_utilization_vllm=0.6 \ prompt="Suggest some famous landmarks in London." \ use_chat_template=True scan_layers=true enable_single_controller=${use_pathways} \ - ici_tensor_parallelism=8 \ - ici_data_parallelism=4 + ici_tensor_parallelism=16 \ + ici_data_parallelism=2 \ + weight_dtype=bfloat16 diff --git a/tests/end_to_end/tpu/qwen3.5/35b/test_qwen3.5_rl.sh b/tests/end_to_end/tpu/qwen3.5/35b/test_qwen3.5_rl.sh index 40901d75cd..d8be7002e2 100644 --- a/tests/end_to_end/tpu/qwen3.5/35b/test_qwen3.5_rl.sh +++ b/tests/end_to_end/tpu/qwen3.5/35b/test_qwen3.5_rl.sh @@ -71,10 +71,15 @@ python3 -m maxtext.trainers.post_train.rl.train_rl \ train_micro_batch_size=1 \ num_test_batches=2 \ model_name=${MODEL_NAME} \ + ici_expert_parallelism=8 \ + ici_fsdp_parallelism=2 \ + ici_tensor_parallelism=2 \ + allow_split_physical_axes=True \ enable_single_controller=True \ checkpoint_storage_use_zarr3=False \ checkpoint_storage_use_ocdbt=False \ - rollout_tensor_parallelism=4 \ + rollout_expert_parallelism=8 \ + rollout_tensor_parallelism=2 \ vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \ vllm_additional_config='{"maxtext_config": {"model_name": "'${MODEL_NAME}'", "log_config": false, "allow_split_physical_axes": true, "prefuse_moe_weights": true}}' \ remat_policy=full \ diff --git a/tests/end_to_end/tpu/qwen3/30b/test_qwen3_rl.sh b/tests/end_to_end/tpu/qwen3/30b/test_qwen3_rl.sh index 9e0f761dd4..11735e9b63 100755 --- a/tests/end_to_end/tpu/qwen3/30b/test_qwen3_rl.sh +++ b/tests/end_to_end/tpu/qwen3/30b/test_qwen3_rl.sh @@ -48,18 +48,19 @@ python3 -m maxtext.inference.vllm_decode \ model_name=${MODEL_NAME} \ load_parameters_path=${SCANNED_CKPT_PATH} \ vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \ - hbm_utilization_vllm=0.8 \ + hbm_utilization_vllm=0.5 \ prompt='Suggest some famous landmarks in London.' \ max_target_length=256 \ max_num_batched_tokens=256 \ ici_tensor_parallelism=4 \ ici_expert_parallelism=2 \ - ici_data_parallelism=2 \ + ici_data_parallelism=8 \ allow_split_physical_axes=True \ prefuse_moe_weights=True \ use_chat_template=True \ scan_layers=True \ - enable_single_controller=True + enable_single_controller=True \ + weight_dtype=bfloat16 # Step 2: Run RL starting from the pre-converted checkpoint python3 -m maxtext.trainers.post_train.rl.train_rl \ @@ -80,7 +81,7 @@ python3 -m maxtext.trainers.post_train.rl.train_rl \ vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \ vllm_additional_config='{"maxtext_config": {"model_name": "'${MODEL_NAME}'", "log_config": false, "allow_split_physical_axes": true, "prefuse_moe_weights": true}}' \ remat_policy=full \ - hbm_utilization_vllm=0.55 \ + hbm_utilization_vllm=0.65 \ use_pathways=True \ chips_per_vm=4 \ max_target_length=512 \ @@ -90,22 +91,27 @@ python3 -m maxtext.trainers.post_train.rl.train_rl \ enable_tunix_perf_metrics=True \ rl.num_generations=16 \ debug=False \ - rl.reshard_chunk_size=1 + rl.reshard_chunk_size=1 \ + ici_expert_parallelism=8 \ + ici_fsdp_parallelism=2 \ + ici_tensor_parallelism=2 \ + allow_split_physical_axes=True # Step 3: Run inference on the checkpoint produced by the RL run python3 -m maxtext.inference.vllm_decode \ model_name=${MODEL_NAME} \ load_parameters_path=${BASE_OUTPUT_DIRECTORY}/rl/${run_id}/checkpoints/actor/2/model_params \ vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \ - hbm_utilization_vllm=0.8 \ + hbm_utilization_vllm=0.5 \ prompt='Suggest some famous landmarks in London.' \ max_target_length=256 \ max_num_batched_tokens=256 \ ici_tensor_parallelism=4 \ ici_expert_parallelism=2 \ - ici_data_parallelism=2 \ + ici_data_parallelism=8 \ allow_split_physical_axes=True \ prefuse_moe_weights=True \ use_chat_template=True \ scan_layers=True \ - enable_single_controller=True \ No newline at end of file + enable_single_controller=True \ + weight_dtype=bfloat16 \ No newline at end of file diff --git a/tests/end_to_end/tpu/qwen3/30b/test_qwen3_sft.sh b/tests/end_to_end/tpu/qwen3/30b/test_qwen3_sft.sh index c835cf9c93..e785f0fa07 100755 --- a/tests/end_to_end/tpu/qwen3/30b/test_qwen3_sft.sh +++ b/tests/end_to_end/tpu/qwen3/30b/test_qwen3_sft.sh @@ -35,9 +35,10 @@ python3 -m maxtext.inference.vllm_decode \ scan_layers=true \ enable_single_controller=True \ ici_tensor_parallelism=4 \ - ici_expert_parallelism=2 \ - ici_data_parallelism=2 \ - prompt="Suggest some famous landmarks in London." + ici_expert_parallelism=4 \ + ici_data_parallelism=1 \ + prompt="Suggest some famous landmarks in London." \ + weight_dtype=bfloat16 # Step 2: Run SFT starting from the pre-converted checkpoint python3 -m maxtext.trainers.post_train.sft.train_sft \ @@ -56,7 +57,8 @@ python3 -m maxtext.trainers.post_train.sft.train_sft \ weight_dtype=bfloat16 \ dtype=bfloat16 \ ici_expert_parallelism=8 \ - opt_type=sgd + opt_type=sgd \ + allow_split_physical_axes=True # Step 3: Run inference on the checkpoint produced by the SFT run @@ -69,6 +71,7 @@ python3 -m maxtext.inference.vllm_decode \ scan_layers=true \ enable_single_controller=True \ ici_tensor_parallelism=4 \ - ici_expert_parallelism=2 \ - ici_data_parallelism=2 \ - prompt="Suggest some famous landmarks in London." \ No newline at end of file + ici_expert_parallelism=4 \ + ici_data_parallelism=1 \ + prompt="Suggest some famous landmarks in London." \ + weight_dtype=bfloat16 \ No newline at end of file