Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 19 additions & 1 deletion src/maxtext/layers/nnx_decoders.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,10 @@
_FSDP_MESH_AXES = ("fsdp", "fsdp_transpose")
_GATHERED_FSDP_WEIGHT = "gathered_fsdp_weight"

# Decoder blocks scanned by NNXDecoder's generic branch whose scanned module is a block of
# `inhomogeneous_layer_cycle_interval` layers that takes vLLM's external KV caches as a tuple, one cache per layer.
_SCANNED_BLOCKS_WITH_PER_LAYER_KV_CACHES = (DecoderBlockType.GPT_OSS,)


def _drop_fsdp_mesh_axes(entry):
"""Return one PartitionSpec entry with the FSDP mesh axes removed."""
Expand Down Expand Up @@ -2448,15 +2452,29 @@ def __call__(
# Pass the kv_caches list directly to avoid copying in jnp.stack,
# which breaks vLLM PagedAttention in-place memory updates.
# The _apply_layers_sequentially function will handle it by statically unrolling.
group_kv_caches_per_block = cfg.decoder_block in _SCANNED_BLOCKS_WITH_PER_LAYER_KV_CACHES
if group_kv_caches_per_block:
# vLLM hands over one cache per decoder layer, but a scan iteration runs a whole block of
# `cycle_interval` layers: the flat list would give block i only `kv_caches[i]`. Hand each block a
# tuple of its own layers' caches instead, and write the updated caches back to the flat list below.
scanned_kv_caches = maxtext_utils.prepare_kv_caches_for_scan(
kv_caches, scan_length, cycle_interval, stack=False
)
else:
scanned_kv_caches = kv_caches
y, self.layers, _ = self._apply_layers_sequentially(
self.layers,
y,
*layer_args,
length=scan_length,
kv_caches_stacked=kv_caches,
kv_caches_stacked=scanned_kv_caches,
**layer_kwargs,
)
# kv_caches list is updated in-place inside _apply_layers_sequentially
if group_kv_caches_per_block:
maxtext_utils.update_kv_caches_after_scan(
kv_caches, scanned_kv_caches, scan_length, cycle_interval, stacked=False
)
else:
y, self.layers, _ = self._apply_layers_sequentially(
self.layers,
Expand Down
23 changes: 18 additions & 5 deletions src/maxtext/models/gpt_oss.py
Original file line number Diff line number Diff line change
Expand Up @@ -224,9 +224,9 @@ def update_cache(cache, val):

stacked_kv_cache = jax.tree_util.tree_map(update_cache, stacked_kv_cache, kv_cache)
return (layer_output, stacked_kv_cache, layer_idx + 1), None
elif cfg.scan_layers:
return layer_output, None
else:
# Also under scan_layers: when vLLM serves a scanned model, GptOssScannableBlock hands each layer its own
# `kv_cache` and keeps the cache returned here.
return layer_output, kv_cache


Expand Down Expand Up @@ -288,23 +288,36 @@ def __call__(
kv_cache=None,
attention_metadata=None,
):
"""Applies the block's decoder layers in order.

`kv_cache` holds vLLM's externally-managed KV caches, one per layer of the block in layer order, or is None
when the caches are not managed externally. Returns the block output and, when `kv_cache` is given, the
layers' updated caches as a tuple in the same order (else None).
"""
cfg = self.config

inputs = self._maybe_shard_with_logical(inputs, self.activation_axis_names)
inputs = checkpoint_name(inputs, "decoder_layer_input")
y = inputs
updated_kv_caches = []
for layer_id in range(cfg.inhomogeneous_layer_cycle_interval):
layer_name = f"layers_{layer_id}"
layer = getattr(self, layer_name)
y, kv_cache = layer(
# Each layer attends with its own cache: one cache handed down the block would leave every layer after
# the first without one.
layer_kv_cache = kv_cache[layer_id] if kv_cache is not None else None
y, layer_kv_cache = layer(
y,
decoder_segment_ids,
decoder_positions,
deterministic,
model_mode,
previous_chunk=previous_chunk,
slot=slot,
kv_cache=kv_cache,
kv_cache=layer_kv_cache,
attention_metadata=attention_metadata,
)
return y, kv_cache
updated_kv_caches.append(layer_kv_cache)
if kv_cache is None:
return y, None
return y, tuple(updated_kv_caches)
84 changes: 84 additions & 0 deletions tests/unit/nnx_decoders_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -1330,6 +1330,90 @@ def shapes(tree):
)


_GPT_OSS_CONFIG = {
"run_name": "gpt_oss_nnx_decoder_test",
"model_name": "gpt-oss-20b",
"max_target_length": 8,
"base_emb_dim": 64,
# A cycle of 2 over 4 layers gives two scanned blocks, the fewest that can mix up caches across blocks.
"base_num_decoder_layers": 4,
"base_num_query_heads": 4,
"base_num_kv_heads": 2,
"head_dim": 16,
"base_mlp_dim": 64,
"base_moe_mlp_dim": 64,
"num_experts": 4,
"num_experts_per_tok": 2,
"vocab_size": 32,
# With the TPU kernels off the sparse path lowers to ragged_dot, which runs on CPU.
"sparse_matmul": True,
"megablox": False,
"use_tokamax_gmm": False,
"dtype": "float32",
"weight_dtype": "float32",
}


class TestNNXDecoderGptOssKvCaches(unittest.TestCase):
"""Every GPT-OSS decoder layer must get its own vLLM KV cache and hand back its update.

vLLM serves the model with one externally-managed KV cache per decoder layer and keeps the caches the decoder
returns. Scanned, one scan iteration runs a block of two layers, so passing the flat list straight through gave
block i only ``kv_caches[i]``: the block's second layer attended without a cache, the layers dropped their updates
under scan_layers, and decoding a scanned checkpoint with vLLM produced gibberish.
"""

def _forward_with_kv_caches(self, scan_layers):
"""Runs one forward pass with a distinct cache per layer; returns the config and the caches handed back."""
cfg = _make_config(**{**_GPT_OSS_CONFIG, "scan_layers": scan_layers})
mesh = _make_mesh(cfg)
decoder = NNXDecoder(config=cfg, mesh=mesh, model_mode=MODEL_MODE_TRAIN, rngs=nnx.Rngs(params=0, dropout=1))
shared_embedding = Embed(
num_embeddings=cfg.vocab_size,
num_features=cfg.emb_dim,
dtype=cfg.dtype,
config=cfg,
mesh=mesh,
rngs=nnx.Rngs(params=0),
)
batch = cfg.global_batch_size_to_train_on
seq = cfg.max_target_length
ids = jax.random.randint(jax.random.PRNGKey(0), (batch, seq), 0, cfg.vocab_size)
segment_ids = jnp.full((batch, seq), DECODING_ACTIVE_SEQUENCE_INDICATOR)
positions = jnp.broadcast_to(jnp.arange(seq)[None], (batch, seq))
kv_caches = [jnp.full((2,), 10.0 * i) for i in range(cfg.num_decoder_layers)]
original_call = Attention.__call__

def bump_returned_kv_cache(attention, *args, **kwargs):
# Outside vLLM serving the attention passes its cache through; the bump stands in for vLLM's update.
out, kv_cache = original_call(attention, *args, **kwargs)
return out, None if kv_cache is None else kv_cache + 1

with mock.patch.object(Attention, "__call__", bump_returned_kv_cache):
decoder(
shared_embedding,
ids,
decoder_positions=positions,
decoder_segment_ids=segment_ids,
deterministic=True,
model_mode=MODEL_MODE_TRAIN,
kv_caches=kv_caches,
)
return cfg, kv_caches

def test_each_layer_updates_its_own_kv_cache(self):
for scan_layers in (False, True):
with self.subTest(scan_layers=scan_layers):
cfg, kv_caches = self._forward_with_kv_caches(scan_layers)

self.assertEqual(cfg.inhomogeneous_layer_cycle_interval, 2)
self.assertEqual(len(kv_caches), cfg.num_decoder_layers)
for i, kv_cache in enumerate(kv_caches):
self.assertIsNotNone(kv_cache, f"layer {i}'s KV cache update was dropped")
# Bumped exactly once, by its own layer.
np.testing.assert_array_equal(np.asarray(kv_cache), np.full((2,), 10.0 * i + 1), err_msg=f"layer {i}")


class TestNNXDecoderDeepseekAndGemma4(unittest.TestCase):
"""Tests for Deepseek and Gemma4 specific decoder logic."""

Expand Down
Loading