From f2d599e74f28ce26e583b2dc8ecb1dc864ea05e3 Mon Sep 17 00:00:00 2001 From: EmmaLien Date: Wed, 4 Nov 2026 03:44:24 +0000 Subject: [PATCH] fix(testing): lower the HBM utilization of the Llama3.1-70B post-training vLLM decodes The post-SFT and post-RL vLLM decodes in test_llama3.1_70b_sft.sh and test_llama3.1_70b_rl.sh (TP16, scan_layers=true) set hbm_utilization_vllm=0.6. On v6e that leaves 12.04 GiB per chip after the weights (10.87 GiB) and the KV cache, and loading the compiled jit_run_model_impl program now needs 12.83 GiB, so both decodes fail with RESOURCE_EXHAUSTED (RuntimeProgramAllocationFailure). Lower hbm_utilization_vllm to 0.5 for these two decodes. The KV cache shrinks from 3411 to 2100 blocks, which is still far more than a single 2048-token prompt needs, and the program fits with about 2.4 GiB to spare. --- tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_rl.sh | 2 +- tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_sft.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_rl.sh b/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_rl.sh index 68d212ae9d..e27497dea3 100644 --- a/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_rl.sh +++ b/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_rl.sh @@ -60,7 +60,7 @@ python3 -m maxtext.inference.vllm_decode \ load_parameters_path=${BASE_OUTPUT_DIRECTORY}/rl/${run_id}/checkpoints/actor/2/model_params \ tokenizer_path='meta-llama/Llama-3.1-70B-Instruct' \ vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \ - hbm_utilization_vllm=0.6 \ + hbm_utilization_vllm=0.5 \ prompt='Suggest some famous landmarks in London.' \ use_chat_template=True scan_layers=true enable_single_controller=${use_pathways} \ ici_tensor_parallelism=16 \ diff --git a/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_sft.sh b/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_sft.sh index f2d5639253..30d0de29a6 100644 --- a/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_sft.sh +++ b/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_sft.sh @@ -56,7 +56,7 @@ python3 -m maxtext.inference.vllm_decode \ load_parameters_path=${BASE_OUTPUT_DIRECTORY}/sft/${run_id}/checkpoints/2/model_params \ tokenizer_path='meta-llama/Llama-3.1-70B-Instruct' \ vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \ - hbm_utilization_vllm=0.6 \ + hbm_utilization_vllm=0.5 \ prompt="Suggest some famous landmarks in London." \ use_chat_template=True scan_layers=true enable_single_controller=${use_pathways} \ ici_tensor_parallelism=16 \