Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/maxtext/models/llama2.py
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,7 @@ def update_cache(cache, val):

stacked_kv_cache = jax.tree_util.tree_map(update_cache, stacked_kv_cache, kv_cache) # pyrefly: ignore[unbound-name]
return (layer_output, stacked_kv_cache, layer_idx + 1), None # pyrefly: ignore[unbound-name]
elif cfg.scan_layers:
return layer_output, None
else:
# Also under scan_layers: vLLM's static unroll over the scanned layers passes each layer its own
# `kv_cache` and keeps the cache returned here.
return layer_output, kv_cache
2 changes: 2 additions & 0 deletions tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_rl.sh
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ python3 -m maxtext.inference.vllm_decode \
vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \
hbm_utilization_vllm=0.85 \
prompt='Suggest some famous landmarks in London.' \
decode_sampling_temperature=0.0 \
use_chat_template=True scan_layers=false enable_single_controller=${use_pathways} \
ici_tensor_parallelism=8 \
ici_data_parallelism=8 \
Expand Down Expand Up @@ -62,6 +63,7 @@ python3 -m maxtext.inference.vllm_decode \
vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \
hbm_utilization_vllm=0.5 \
prompt='Suggest some famous landmarks in London.' \
decode_sampling_temperature=0.0 \
use_chat_template=True scan_layers=true enable_single_controller=${use_pathways} \
ici_tensor_parallelism=16 \
ici_data_parallelism=4 \
Expand Down
2 changes: 2 additions & 0 deletions tests/end_to_end/tpu/llama3.1/70b/test_llama3.1_70b_sft.sh
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ python3 -m maxtext.inference.vllm_decode \
vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \
hbm_utilization_vllm=0.85 \
prompt="Suggest some famous landmarks in London." \
decode_sampling_temperature=0.0 \
use_chat_template=True scan_layers=false enable_single_controller=${use_pathways} \
ici_tensor_parallelism=8 \
ici_data_parallelism=4 \
Expand All @@ -58,6 +59,7 @@ python3 -m maxtext.inference.vllm_decode \
vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \
hbm_utilization_vllm=0.5 \
prompt="Suggest some famous landmarks in London." \
decode_sampling_temperature=0.0 \
use_chat_template=True scan_layers=true enable_single_controller=${use_pathways} \
ici_tensor_parallelism=16 \
ici_data_parallelism=2 \
Expand Down
41 changes: 41 additions & 0 deletions tests/unit/decoder_layer_model_mode_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -325,5 +325,46 @@ def test_explicit_shard_mode_is_accepted_by_config(self, layer_cls, extra_config
self.assertEqual(cfg.shard_mode, ShardMode.EXPLICIT)


class DecoderLayerScannedKvCacheTest(parameterized.TestCase):
"""With scan_layers=True, a decoder layer must still return the KV cache its attention returns.

When vLLM serves a scanned model, the decoder statically unrolls the scanned layers, passes each
layer its own `kv_cache` and stores the cache the layer returns for the next step. A layer that
returns None there drops every cache update: the first token, computed in the same attention call
that writes the prompt's keys and values, is still right, but every later token attends to a stale
cache and decoding degenerates.
"""

@parameterized.named_parameters(
("llama2", llama2.LlamaDecoderLayer, {}, {}),
("qwen3", qwen3.Qwen3DecoderLayer, {"decoder_block": "qwen3"}, {"quant": None}),
)
def test_scanned_layer_returns_attention_kv_cache(self, layer_cls, extra_config, ctor_kwargs):
cfg = pyconfig.initialize([sys.argv[0], get_test_config_path()], **(_COMMON | extra_config | {"scan_layers": True}))
mesh = _mesh(cfg)
updated_kv_cache = jnp.full((2, 3), 7.0)

def spy(_self, inputs_q, *args, **kwargs):
del args, kwargs
return jnp.zeros_like(inputs_q), updated_kv_cache

with nn_partitioning.axis_rules(cfg.logical_axis_rules), mesh:
layer = layer_cls(
config=cfg,
model_mode=MODEL_MODE_AUTOREGRESSIVE,
mesh=mesh,
rngs=nnx.Rngs(params=0, dropout=0),
**ctor_kwargs,
)
batch = cfg.micro_batch_size_to_train_on
inputs = jnp.zeros((batch, 1, cfg.emb_dim), dtype=jnp.float32)
positions = jnp.zeros((batch, 1), dtype=jnp.int32)
segment_ids = jnp.ones((batch, 1), dtype=jnp.int32)
with mock.patch.object(attentions.Attention, "__call__", spy):
_, kv_cache = layer(inputs, segment_ids, positions, True, MODEL_MODE_AUTOREGRESSIVE, kv_cache=jnp.zeros((2, 3)))

self.assertIs(kv_cache, updated_kv_cache, "the scanned layer dropped the KV cache returned by its attention")


if __name__ == "__main__":
unittest.main()
Loading