From 824158930db1151ac85f5d1a8f90caf3041fc31d Mon Sep 17 00:00:00 2001 From: EmmaLien Date: Tue, 27 Oct 2026 07:33:17 +0000 Subject: [PATCH] fix(llama): keep the vLLM KV cache in scanned Llama decoder layers The Llama3.1-70B vLLM decodes in the RL and SFT E2E tests printed gibberish for two independent reasons. 1. LlamaDecoderLayer returned (layer_output, None) whenever scan_layers was set. When vLLM serves a scanned model, the decoder statically unrolls the scanned layers, passes each layer its own kv_cache and keeps the cache each layer returns, so every KV cache update was dropped. The first token, computed in the attention call that also writes the prompt's keys and values, was right; every later token attended to a stale cache ("Big the the the ..."). This broke the step-3 decodes, which load the scanned post-training checkpoints. Return the KV cache under scan_layers as well, as Qwen3DecoderLayer already does and as #4066 did for Gemma3. Outside vLLM, attention returns the kv_cache it was given (None), so nothing else changes. 2. vllm_decode samples with decode_sampling_temperature (1.0 by default). The tests decode the base meta-llama/Llama-3.1-70B checkpoint with the Instruct chat template, which gives incoherent text at temperature 1.0. Decode greedily in the Llama3.1-70B RL and SFT scripts, as the vllm_decode usage example does. Add a unit test that a scanned decoder layer returns the KV cache its attention returns. Bug: b/558524540 Bug: b/558524529 --- src/maxtext/models/llama2.py | 4 +- .../tpu/llama3.1/70b/test_llama3.1_70b_rl.sh | 2 + .../tpu/llama3.1/70b/test_llama3.1_70b_sft.sh | 2 + tests/unit/decoder_layer_model_mode_test.py | 41 +++++++++++++++++++ 4 files changed, 47 insertions(+), 2 deletions(-) diff --git a/src/maxtext/models/llama2.py b/src/maxtext/models/llama2.py index 1c94406763..9d6768277e 100644 --- a/src/maxtext/models/llama2.py +++ b/src/maxtext/models/llama2.py @@ -216,7 +216,7 @@ def update_cache(cache, val): stacked_kv_cache = jax.tree_util.tree_map(update_cache, stacked_kv_cache, kv_cache) # pyrefly: ignore[unbound-name] return (layer_output, stacked_kv_cache, layer_idx + 1), None # pyrefly: ignore[unbound-name] - elif cfg.scan_layers: - return layer_output, None else: + # Also under scan_layers: vLLM's static unroll over the scanned layers passes each layer its own + # `kv_cache` and keeps the cache returned here. return layer_output, kv_cache diff --git a/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_rl.sh b/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_rl.sh index e27497dea3..395fc340cd 100644 --- a/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_rl.sh +++ b/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_rl.sh @@ -33,6 +33,7 @@ python3 -m maxtext.inference.vllm_decode \ vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \ hbm_utilization_vllm=0.85 \ prompt='Suggest some famous landmarks in London.' \ + decode_sampling_temperature=0.0 \ use_chat_template=True scan_layers=false enable_single_controller=${use_pathways} \ ici_tensor_parallelism=8 \ ici_data_parallelism=8 \ @@ -62,6 +63,7 @@ python3 -m maxtext.inference.vllm_decode \ vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \ hbm_utilization_vllm=0.5 \ prompt='Suggest some famous landmarks in London.' \ + decode_sampling_temperature=0.0 \ use_chat_template=True scan_layers=true enable_single_controller=${use_pathways} \ ici_tensor_parallelism=16 \ ici_data_parallelism=4 \ diff --git a/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_sft.sh b/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_sft.sh index 30d0de29a6..7315883fd9 100644 --- a/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_sft.sh +++ b/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_sft.sh @@ -33,6 +33,7 @@ python3 -m maxtext.inference.vllm_decode \ vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \ hbm_utilization_vllm=0.85 \ prompt="Suggest some famous landmarks in London." \ + decode_sampling_temperature=0.0 \ use_chat_template=True scan_layers=false enable_single_controller=${use_pathways} \ ici_tensor_parallelism=8 \ ici_data_parallelism=4 \ @@ -58,6 +59,7 @@ python3 -m maxtext.inference.vllm_decode \ vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \ hbm_utilization_vllm=0.5 \ prompt="Suggest some famous landmarks in London." \ + decode_sampling_temperature=0.0 \ use_chat_template=True scan_layers=true enable_single_controller=${use_pathways} \ ici_tensor_parallelism=16 \ ici_data_parallelism=2 \ diff --git a/tests/unit/decoder_layer_model_mode_test.py b/tests/unit/decoder_layer_model_mode_test.py index db24b1d026..5e49d679c8 100644 --- a/tests/unit/decoder_layer_model_mode_test.py +++ b/tests/unit/decoder_layer_model_mode_test.py @@ -325,5 +325,46 @@ def test_explicit_shard_mode_is_accepted_by_config(self, layer_cls, extra_config self.assertEqual(cfg.shard_mode, ShardMode.EXPLICIT) +class DecoderLayerScannedKvCacheTest(parameterized.TestCase): + """With scan_layers=True, a decoder layer must still return the KV cache its attention returns. + + When vLLM serves a scanned model, the decoder statically unrolls the scanned layers, passes each + layer its own `kv_cache` and stores the cache the layer returns for the next step. A layer that + returns None there drops every cache update: the first token, computed in the same attention call + that writes the prompt's keys and values, is still right, but every later token attends to a stale + cache and decoding degenerates. + """ + + @parameterized.named_parameters( + ("llama2", llama2.LlamaDecoderLayer, {}, {}), + ("qwen3", qwen3.Qwen3DecoderLayer, {"decoder_block": "qwen3"}, {"quant": None}), + ) + def test_scanned_layer_returns_attention_kv_cache(self, layer_cls, extra_config, ctor_kwargs): + cfg = pyconfig.initialize([sys.argv[0], get_test_config_path()], **(_COMMON | extra_config | {"scan_layers": True})) + mesh = _mesh(cfg) + updated_kv_cache = jnp.full((2, 3), 7.0) + + def spy(_self, inputs_q, *args, **kwargs): + del args, kwargs + return jnp.zeros_like(inputs_q), updated_kv_cache + + with nn_partitioning.axis_rules(cfg.logical_axis_rules), mesh: + layer = layer_cls( + config=cfg, + model_mode=MODEL_MODE_AUTOREGRESSIVE, + mesh=mesh, + rngs=nnx.Rngs(params=0, dropout=0), + **ctor_kwargs, + ) + batch = cfg.micro_batch_size_to_train_on + inputs = jnp.zeros((batch, 1, cfg.emb_dim), dtype=jnp.float32) + positions = jnp.zeros((batch, 1), dtype=jnp.int32) + segment_ids = jnp.ones((batch, 1), dtype=jnp.int32) + with mock.patch.object(attentions.Attention, "__call__", spy): + _, kv_cache = layer(inputs, segment_ids, positions, True, MODEL_MODE_AUTOREGRESSIVE, kv_cache=jnp.zeros((2, 3))) + + self.assertIs(kv_cache, updated_kv_cache, "the scanned layer dropped the KV cache returned by its attention") + + if __name__ == "__main__": unittest.main()