diff --git a/src/maxtext/models/llama2.py b/src/maxtext/models/llama2.py index 1c94406763..9d6768277e 100644 --- a/src/maxtext/models/llama2.py +++ b/src/maxtext/models/llama2.py @@ -216,7 +216,7 @@ def update_cache(cache, val): stacked_kv_cache = jax.tree_util.tree_map(update_cache, stacked_kv_cache, kv_cache) # pyrefly: ignore[unbound-name] return (layer_output, stacked_kv_cache, layer_idx + 1), None # pyrefly: ignore[unbound-name] - elif cfg.scan_layers: - return layer_output, None else: + # Also under scan_layers: vLLM's static unroll over the scanned layers passes each layer its own + # `kv_cache` and keeps the cache returned here. return layer_output, kv_cache diff --git a/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_rl.sh b/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_rl.sh index e27497dea3..395fc340cd 100644 --- a/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_rl.sh +++ b/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_rl.sh @@ -33,6 +33,7 @@ python3 -m maxtext.inference.vllm_decode \ vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \ hbm_utilization_vllm=0.85 \ prompt='Suggest some famous landmarks in London.' \ + decode_sampling_temperature=0.0 \ use_chat_template=True scan_layers=false enable_single_controller=${use_pathways} \ ici_tensor_parallelism=8 \ ici_data_parallelism=8 \ @@ -62,6 +63,7 @@ python3 -m maxtext.inference.vllm_decode \ vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \ hbm_utilization_vllm=0.5 \ prompt='Suggest some famous landmarks in London.' \ + decode_sampling_temperature=0.0 \ use_chat_template=True scan_layers=true enable_single_controller=${use_pathways} \ ici_tensor_parallelism=16 \ ici_data_parallelism=4 \ diff --git a/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_sft.sh b/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_sft.sh index 30d0de29a6..7315883fd9 100644 --- a/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_sft.sh +++ b/tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_sft.sh @@ -33,6 +33,7 @@ python3 -m maxtext.inference.vllm_decode \ vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \ hbm_utilization_vllm=0.85 \ prompt="Suggest some famous landmarks in London." \ + decode_sampling_temperature=0.0 \ use_chat_template=True scan_layers=false enable_single_controller=${use_pathways} \ ici_tensor_parallelism=8 \ ici_data_parallelism=4 \ @@ -58,6 +59,7 @@ python3 -m maxtext.inference.vllm_decode \ vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \ hbm_utilization_vllm=0.5 \ prompt="Suggest some famous landmarks in London." \ + decode_sampling_temperature=0.0 \ use_chat_template=True scan_layers=true enable_single_controller=${use_pathways} \ ici_tensor_parallelism=16 \ ici_data_parallelism=2 \ diff --git a/tests/unit/decoder_layer_model_mode_test.py b/tests/unit/decoder_layer_model_mode_test.py index db24b1d026..5e49d679c8 100644 --- a/tests/unit/decoder_layer_model_mode_test.py +++ b/tests/unit/decoder_layer_model_mode_test.py @@ -325,5 +325,46 @@ def test_explicit_shard_mode_is_accepted_by_config(self, layer_cls, extra_config self.assertEqual(cfg.shard_mode, ShardMode.EXPLICIT) +class DecoderLayerScannedKvCacheTest(parameterized.TestCase): + """With scan_layers=True, a decoder layer must still return the KV cache its attention returns. + + When vLLM serves a scanned model, the decoder statically unrolls the scanned layers, passes each + layer its own `kv_cache` and stores the cache the layer returns for the next step. A layer that + returns None there drops every cache update: the first token, computed in the same attention call + that writes the prompt's keys and values, is still right, but every later token attends to a stale + cache and decoding degenerates. + """ + + @parameterized.named_parameters( + ("llama2", llama2.LlamaDecoderLayer, {}, {}), + ("qwen3", qwen3.Qwen3DecoderLayer, {"decoder_block": "qwen3"}, {"quant": None}), + ) + def test_scanned_layer_returns_attention_kv_cache(self, layer_cls, extra_config, ctor_kwargs): + cfg = pyconfig.initialize([sys.argv[0], get_test_config_path()], **(_COMMON | extra_config | {"scan_layers": True})) + mesh = _mesh(cfg) + updated_kv_cache = jnp.full((2, 3), 7.0) + + def spy(_self, inputs_q, *args, **kwargs): + del args, kwargs + return jnp.zeros_like(inputs_q), updated_kv_cache + + with nn_partitioning.axis_rules(cfg.logical_axis_rules), mesh: + layer = layer_cls( + config=cfg, + model_mode=MODEL_MODE_AUTOREGRESSIVE, + mesh=mesh, + rngs=nnx.Rngs(params=0, dropout=0), + **ctor_kwargs, + ) + batch = cfg.micro_batch_size_to_train_on + inputs = jnp.zeros((batch, 1, cfg.emb_dim), dtype=jnp.float32) + positions = jnp.zeros((batch, 1), dtype=jnp.int32) + segment_ids = jnp.ones((batch, 1), dtype=jnp.int32) + with mock.patch.object(attentions.Attention, "__call__", spy): + _, kv_cache = layer(inputs, segment_ids, positions, True, MODEL_MODE_AUTOREGRESSIVE, kv_cache=jnp.zeros((2, 3))) + + self.assertIs(kv_cache, updated_kv_cache, "the scanned layer dropped the KV cache returned by its attention") + + if __name__ == "__main__": unittest.main()