Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions tests/end_to_end/tpu/gemma3/4b/test_gemma3_lora.sh
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ python3 -m maxtext.inference.vllm_decode \
prompt="Suggest some famous landmarks in London." \
use_chat_template=True \
scan_layers=false \
enable_single_controller=${use_pathways}
enable_single_controller=${use_pathways} \
weight_dtype=bfloat16

# Step 2: Run LoRA on the converted checkpoint
python3 -m maxtext.trainers.post_train.sft.train_sft \
Expand Down Expand Up @@ -67,4 +68,5 @@ python3 -m maxtext.inference.vllm_decode \
prompt="Suggest some famous landmarks in London." \
use_chat_template=True \
scan_layers=true \
enable_single_controller=${use_pathways}
enable_single_controller=${use_pathways} \
weight_dtype=bfloat16
11 changes: 7 additions & 4 deletions tests/end_to_end/tpu/gemma4/26b/test_gemma4.sh
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ USE_MULTIMODAL=false
# Non-Googlers please remember to point `BASE_OUTPUT_DIRECTORY` to the GCS paths where you have the scanned and unscanned checkpoints stored
BASE_OUTPUT_DIRECTORY=gs://runner-maxtext-logs/${MODEL_NAME}
UNSCANNED_CKPT_PATH=${BASE_OUTPUT_DIRECTORY}/to_maxtext/unscanned/${run_id}/0/items
SCANNED_CKPT_PATH=${BASE_OUTPUT_DIRECTORY}/to_maxtext/scanned/${run_id}/0/items

# Non-Googlers please remember to point `DATASET_PATH` to the GCS bucket where you have your training data
DATASET_PATH=gs://maxtext-dataset
Expand Down Expand Up @@ -61,11 +62,13 @@ python3 -m maxtext.trainers.pre_train.train \
base_output_directory=${BASE_OUTPUT_DIRECTORY}/train \
dataset_type=grain grain_file_type=tfrecord \
dataset_path=${DATASET_PATH} tokenizer_type="huggingface" \
load_parameters_path=${UNSCANNED_CKPT_PATH} \
load_parameters_path=${SCANNED_CKPT_PATH} \
per_device_batch_size=1 run_name=${run_id} \
max_target_length=8192 steps=2 async_checkpointing=false \
checkpoint_storage_use_zarr3=False checkpoint_storage_use_ocdbt=False \
model_name=${MODEL_NAME} scan_layers=false use_multimodal=${USE_MULTIMODAL}
model_name=${MODEL_NAME} scan_layers=true use_multimodal=${USE_MULTIMODAL} \
ici_expert_parallelism=8 \
weight_dtype=bfloat16

# Step 3: Run inference on the checkpoint generated from the previous run
if [ ${USE_MULTIMODAL} == true ]; then
Expand All @@ -74,7 +77,7 @@ if [ ${USE_MULTIMODAL} == true ]; then
load_parameters_path=${BASE_OUTPUT_DIRECTORY}/train/${run_id}/checkpoints/1/items \
per_device_batch_size=1 run_name=${run_id} \
max_prefill_predict_length=272 max_target_length=300 steps=1 async_checkpointing=false \
scan_layers=false use_multimodal=true \
scan_layers=true use_multimodal=true \
checkpoint_storage_use_zarr3=False checkpoint_storage_use_ocdbt=False \
prompt=\'Describe\ image\ \<start_of_image\>\' image_path=\'tests/assets/test_image.jpg\' attention=\'dot_product\'
else
Expand All @@ -84,5 +87,5 @@ else
per_device_batch_size=1 run_name=${run_id} \
max_prefill_predict_length=8 max_target_length=16 steps=1 async_checkpointing=false \
checkpoint_storage_use_zarr3=False checkpoint_storage_use_ocdbt=False \
scan_layers=false prompt='I love to' attention=\'dot_product\'
scan_layers=true prompt='I love to' attention=\'dot_product\'
fi
19 changes: 14 additions & 5 deletions tests/end_to_end/tpu/gemma4/26b/test_gemma4_rl.sh
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,9 @@ python3 -m maxtext.inference.vllm_decode \
scan_layers=false \
enable_single_controller=${use_pathways} \
prefuse_moe_weights=True \
ici_tensor_parallelism=4
ici_tensor_parallelism=8 \
ici_data_parallelism=8 \
weight_dtype=bfloat16

# Step 2: Run RL on the converted checkpoint
python3 -m maxtext.trainers.post_train.rl.train_rl \
Expand All @@ -48,19 +50,24 @@ python3 -m maxtext.trainers.post_train.rl.train_rl \
num_batches=2 \
batch_size=4 \
train_micro_batch_size=4 \
ici_expert_parallelism=4 \
ici_expert_parallelism=8 \
ici_fsdp_parallelism=2 \
ici_data_parallelism=1 \
num_test_batches=2 \
model_name=${MODEL_NAME} \
enable_single_controller=${use_pathways} \
checkpoint_storage_use_zarr3=False \
checkpoint_storage_use_ocdbt=False \
rollout_tensor_parallelism=2 \
rollout_tensor_parallelism=4 \
vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \
vllm_additional_config='{"maxtext_config": {"model_name": "gemma4-26b", "log_config": "false", "prefuse_moe_weights": "true"}}' \
remat_policy=full \
hbm_utilization_vllm=0.55
hbm_utilization_vllm=0.55 \
ici_tensor_parallelism=2 \
allow_split_physical_axes=True \
opt_type=sgd \
rl.num_generations=4 \
rl.reshard_chunk_size=1

# Step 3: Run inference on the checkpoint generated from the previous run
python3 -m maxtext.inference.vllm_decode \
Expand All @@ -73,4 +80,6 @@ python3 -m maxtext.inference.vllm_decode \
scan_layers=false \
enable_single_controller=${use_pathways} \
prefuse_moe_weights=True \
ici_tensor_parallelism=4
ici_tensor_parallelism=8 \
ici_data_parallelism=8 \
weight_dtype=bfloat16
12 changes: 9 additions & 3 deletions tests/end_to_end/tpu/gemma4/26b/test_gemma4_sft.sh
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@ python3 -m maxtext.inference.vllm_decode \
hbm_utilization_vllm=0.85 \
prompt="Suggest some famous landmarks in London." \
use_chat_template=True scan_layers=false enable_single_controller=${use_pathways} \
prefuse_moe_weights=True ici_tensor_parallelism=8
prefuse_moe_weights=True ici_tensor_parallelism=8 \
weight_dtype=bfloat16

# Step 2: Run SFT on the converted checkpoint
python3 -m maxtext.trainers.post_train.sft.train_sft \
Expand All @@ -42,7 +43,11 @@ python3 -m maxtext.trainers.post_train.sft.train_sft \
per_device_batch_size=1 run_name=${run_id} \
steps=2 scan_layers=false \
model_name=${MODEL_NAME} enable_single_controller=${use_pathways} \
checkpoint_storage_use_zarr3=False checkpoint_storage_use_ocdbt=False
checkpoint_storage_use_zarr3=False checkpoint_storage_use_ocdbt=False \
ici_expert_parallelism=8 \
weight_dtype=bfloat16 \
opt_type=sgd \
max_target_length=1024

# Step 3: Run inference on the checkpoint generated from the previous run
python3 -m maxtext.inference.vllm_decode \
Expand All @@ -52,4 +57,5 @@ python3 -m maxtext.inference.vllm_decode \
hbm_utilization_vllm=0.85 \
prompt="Suggest some famous landmarks in London." \
use_chat_template=True scan_layers=false enable_single_controller=${use_pathways} \
prefuse_moe_weights=True ici_tensor_parallelism=8
prefuse_moe_weights=True ici_tensor_parallelism=8 \
weight_dtype=bfloat16
3 changes: 2 additions & 1 deletion tests/end_to_end/tpu/gpt_oss/20b/test_gpt_oss.sh
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,8 @@ python3 -m maxtext.trainers.pre_train.train \
checkpoint_storage_use_ocdbt=False \
model_name=${MODEL_NAME} \
scan_layers=false \
use_multimodal=false
use_multimodal=false \
ici_expert_parallelism=8

# Step 3: Run inference on the checkpoint generated from the previous run
python3 -m maxtext.inference.decode \
Expand Down
3 changes: 2 additions & 1 deletion tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b.sh
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,8 @@ python3 -m maxtext.trainers.pre_train.train \
max_target_length=2048 steps=2 async_checkpointing=false \
checkpoint_storage_use_zarr3=False checkpoint_storage_use_ocdbt=False \
model_name=${MODEL_NAME} scan_layers=false use_multimodal=${USE_MULTIMODAL} \
ici_tensor_parallelism=4
ici_tensor_parallelism=4 \
allow_split_physical_axes=True

# Step 3: Run inference on the checkpoint generated from the previous run
if [ ${USE_MULTIMODAL} == true ]; then
Expand Down
13 changes: 8 additions & 5 deletions tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_rl.sh
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ python3 -m maxtext.inference.vllm_decode \
prompt='Suggest some famous landmarks in London.' \
use_chat_template=True scan_layers=false enable_single_controller=${use_pathways} \
ici_tensor_parallelism=8 \
ici_data_parallelism=4
ici_data_parallelism=8 \
weight_dtype=bfloat16

# Step 2: Run RL on the converted checkpoint
python3 -m maxtext.trainers.post_train.rl.train_rl \
Expand All @@ -47,9 +48,10 @@ python3 -m maxtext.trainers.post_train.rl.train_rl \
enable_single_controller=${use_pathways} \
remat_policy=full \
checkpoint_storage_use_zarr3=False checkpoint_storage_use_ocdbt=False \
rollout_tensor_parallelism=4 \
rollout_tensor_parallelism=8 \
vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \
vllm_additional_config='{"maxtext_config": {"model_name": "llama3.1-70b", "log_config": "false"}}'
vllm_additional_config='{"maxtext_config": {"model_name": "llama3.1-70b", "log_config": "false"}}' \
opt_type=sgd


# Step 3: Run inference on the checkpoint generated from the previous run
Expand All @@ -61,5 +63,6 @@ python3 -m maxtext.inference.vllm_decode \
hbm_utilization_vllm=0.6 \
prompt='Suggest some famous landmarks in London.' \
use_chat_template=True scan_layers=true enable_single_controller=${use_pathways} \
ici_tensor_parallelism=8 \
ici_data_parallelism=4
ici_tensor_parallelism=16 \
ici_data_parallelism=4 \
weight_dtype=bfloat16
11 changes: 7 additions & 4 deletions tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_sft.sh
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ python3 -m maxtext.inference.vllm_decode \
prompt="Suggest some famous landmarks in London." \
use_chat_template=True scan_layers=false enable_single_controller=${use_pathways} \
ici_tensor_parallelism=8 \
ici_data_parallelism=4
ici_data_parallelism=4 \
weight_dtype=bfloat16

# Step 2: Run SFT on the converted checkpoint
python3 -m maxtext.trainers.post_train.sft.train_sft \
Expand All @@ -46,7 +47,8 @@ python3 -m maxtext.trainers.post_train.sft.train_sft \
model_name=${MODEL_NAME} tokenizer_path='meta-llama/Llama-3.1-70B-Instruct' \
enable_single_controller=${use_pathways} \
remat_policy=full \
checkpoint_storage_use_zarr3=False checkpoint_storage_use_ocdbt=False
checkpoint_storage_use_zarr3=False checkpoint_storage_use_ocdbt=False \
weight_dtype=bfloat16

# Step 3: Run inference on the checkpoint generated from the previous run
python3 -m maxtext.inference.vllm_decode \
Expand All @@ -57,5 +59,6 @@ python3 -m maxtext.inference.vllm_decode \
hbm_utilization_vllm=0.6 \
prompt="Suggest some famous landmarks in London." \
use_chat_template=True scan_layers=true enable_single_controller=${use_pathways} \
ici_tensor_parallelism=8 \
ici_data_parallelism=4
ici_tensor_parallelism=16 \
ici_data_parallelism=2 \
weight_dtype=bfloat16
7 changes: 6 additions & 1 deletion tests/end_to_end/tpu/qwen3.5/35b/test_qwen3.5_rl.sh
Original file line number Diff line number Diff line change
Expand Up @@ -71,10 +71,15 @@ python3 -m maxtext.trainers.post_train.rl.train_rl \
train_micro_batch_size=1 \
num_test_batches=2 \
model_name=${MODEL_NAME} \
ici_expert_parallelism=8 \
ici_fsdp_parallelism=2 \
ici_tensor_parallelism=2 \
allow_split_physical_axes=True \
enable_single_controller=True \
checkpoint_storage_use_zarr3=False \
checkpoint_storage_use_ocdbt=False \
rollout_tensor_parallelism=4 \
rollout_expert_parallelism=8 \
rollout_tensor_parallelism=2 \
vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \
vllm_additional_config='{"maxtext_config": {"model_name": "'${MODEL_NAME}'", "log_config": false, "allow_split_physical_axes": true, "prefuse_moe_weights": true}}' \
remat_policy=full \
Expand Down
22 changes: 14 additions & 8 deletions tests/end_to_end/tpu/qwen3/30b/test_qwen3_rl.sh
Original file line number Diff line number Diff line change
Expand Up @@ -48,18 +48,19 @@ python3 -m maxtext.inference.vllm_decode \
model_name=${MODEL_NAME} \
load_parameters_path=${SCANNED_CKPT_PATH} \
vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \
hbm_utilization_vllm=0.8 \
hbm_utilization_vllm=0.5 \
prompt='Suggest some famous landmarks in London.' \
max_target_length=256 \
max_num_batched_tokens=256 \
ici_tensor_parallelism=4 \
ici_expert_parallelism=2 \
ici_data_parallelism=2 \
ici_data_parallelism=8 \
allow_split_physical_axes=True \
prefuse_moe_weights=True \
use_chat_template=True \
scan_layers=True \
enable_single_controller=True
enable_single_controller=True \
weight_dtype=bfloat16

# Step 2: Run RL starting from the pre-converted checkpoint
python3 -m maxtext.trainers.post_train.rl.train_rl \
Expand All @@ -80,7 +81,7 @@ python3 -m maxtext.trainers.post_train.rl.train_rl \
vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \
vllm_additional_config='{"maxtext_config": {"model_name": "'${MODEL_NAME}'", "log_config": false, "allow_split_physical_axes": true, "prefuse_moe_weights": true}}' \
remat_policy=full \
hbm_utilization_vllm=0.55 \
hbm_utilization_vllm=0.65 \
use_pathways=True \
chips_per_vm=4 \
max_target_length=512 \
Expand All @@ -90,22 +91,27 @@ python3 -m maxtext.trainers.post_train.rl.train_rl \
enable_tunix_perf_metrics=True \
rl.num_generations=16 \
debug=False \
rl.reshard_chunk_size=1
rl.reshard_chunk_size=1 \
ici_expert_parallelism=8 \
ici_fsdp_parallelism=2 \
ici_tensor_parallelism=2 \
allow_split_physical_axes=True

# Step 3: Run inference on the checkpoint produced by the RL run
python3 -m maxtext.inference.vllm_decode \
model_name=${MODEL_NAME} \
load_parameters_path=${BASE_OUTPUT_DIRECTORY}/rl/${run_id}/checkpoints/actor/2/model_params \
vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \
hbm_utilization_vllm=0.8 \
hbm_utilization_vllm=0.5 \
prompt='Suggest some famous landmarks in London.' \
max_target_length=256 \
max_num_batched_tokens=256 \
ici_tensor_parallelism=4 \
ici_expert_parallelism=2 \
ici_data_parallelism=2 \
ici_data_parallelism=8 \
allow_split_physical_axes=True \
prefuse_moe_weights=True \
use_chat_template=True \
scan_layers=True \
enable_single_controller=True
enable_single_controller=True \
weight_dtype=bfloat16
17 changes: 10 additions & 7 deletions tests/end_to_end/tpu/qwen3/30b/test_qwen3_sft.sh
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,10 @@ python3 -m maxtext.inference.vllm_decode \
scan_layers=true \
enable_single_controller=True \
ici_tensor_parallelism=4 \
ici_expert_parallelism=2 \
ici_data_parallelism=2 \
prompt="Suggest some famous landmarks in London."
ici_expert_parallelism=4 \
ici_data_parallelism=1 \
prompt="Suggest some famous landmarks in London." \
weight_dtype=bfloat16

# Step 2: Run SFT starting from the pre-converted checkpoint
python3 -m maxtext.trainers.post_train.sft.train_sft \
Expand All @@ -56,7 +57,8 @@ python3 -m maxtext.trainers.post_train.sft.train_sft \
weight_dtype=bfloat16 \
dtype=bfloat16 \
ici_expert_parallelism=8 \
opt_type=sgd
opt_type=sgd \
allow_split_physical_axes=True


# Step 3: Run inference on the checkpoint produced by the SFT run
Expand All @@ -69,6 +71,7 @@ python3 -m maxtext.inference.vllm_decode \
scan_layers=true \
enable_single_controller=True \
ici_tensor_parallelism=4 \
ici_expert_parallelism=2 \
ici_data_parallelism=2 \
prompt="Suggest some famous landmarks in London."
ici_expert_parallelism=4 \
ici_data_parallelism=1 \
prompt="Suggest some famous landmarks in London." \
weight_dtype=bfloat16
Loading