From dcd656cb121942640107c0c162f0a89f6341abec Mon Sep 17 00:00:00 2001 From: EmmaLien Date: Thu, 29 Oct 2026 00:13:05 +0000 Subject: [PATCH] fix(gpt-oss): give each scanned GPT-OSS layer its own vLLM KV cache vLLM serves MaxTextForCausalLM with one external KV cache per decoder layer and keeps the caches the decoder returns. With scan_layers=True, GPT-OSS scans a GptOssScannableBlock of two layers (sliding window, then global attention), but: - The generic scanned branch of NNXDecoder passed the flat cache list straight to _apply_layers_sequentially, which unrolls one iteration per block, so block i only got kv_caches[i]. Half of the caches were never used, and the block handed that one cache down its two layers. - GptOssDecoderLayer returned None instead of the updated cache under scan_layers, like LlamaDecoderLayer (#6002). So the second layer of every block attended without a cache (forward_serve_vllm returns the query as the attention output when it gets none), and decoding a scanned GPT-OSS checkpoint with vLLM, e.g. the RL checkpoint in test_gpt_oss_rl.sh, produced gibberish from the first token. For GPT-OSS, group the caches per block with prepare_kv_caches_for_scan and write them back with update_kv_caches_after_scan, as the Gemma3 and Qwen3-Next paths do; give each layer of GptOssScannableBlock its own cache and return the updated caches as a tuple; and return the updated cache from GptOssDecoderLayer under scan_layers too. Other models are unchanged. Bug: b/544614464 --- src/maxtext/layers/nnx_decoders.py | 20 ++++++- src/maxtext/models/gpt_oss.py | 23 ++++++-- tests/unit/nnx_decoders_test.py | 84 ++++++++++++++++++++++++++++++ 3 files changed, 121 insertions(+), 6 deletions(-) diff --git a/src/maxtext/layers/nnx_decoders.py b/src/maxtext/layers/nnx_decoders.py index 8d82c49c56..92f6aa6442 100644 --- a/src/maxtext/layers/nnx_decoders.py +++ b/src/maxtext/layers/nnx_decoders.py @@ -85,6 +85,10 @@ _FSDP_MESH_AXES = ("fsdp", "fsdp_transpose") _GATHERED_FSDP_WEIGHT = "gathered_fsdp_weight" +# Decoder blocks scanned by NNXDecoder's generic branch whose scanned module is a block of +# `inhomogeneous_layer_cycle_interval` layers that takes vLLM's external KV caches as a tuple, one cache per layer. +_SCANNED_BLOCKS_WITH_PER_LAYER_KV_CACHES = (DecoderBlockType.GPT_OSS,) + def _drop_fsdp_mesh_axes(entry): """Return one PartitionSpec entry with the FSDP mesh axes removed.""" @@ -2448,15 +2452,29 @@ def __call__( # Pass the kv_caches list directly to avoid copying in jnp.stack, # which breaks vLLM PagedAttention in-place memory updates. # The _apply_layers_sequentially function will handle it by statically unrolling. + group_kv_caches_per_block = cfg.decoder_block in _SCANNED_BLOCKS_WITH_PER_LAYER_KV_CACHES + if group_kv_caches_per_block: + # vLLM hands over one cache per decoder layer, but a scan iteration runs a whole block of + # `cycle_interval` layers: the flat list would give block i only `kv_caches[i]`. Hand each block a + # tuple of its own layers' caches instead, and write the updated caches back to the flat list below. + scanned_kv_caches = maxtext_utils.prepare_kv_caches_for_scan( + kv_caches, scan_length, cycle_interval, stack=False + ) + else: + scanned_kv_caches = kv_caches y, self.layers, _ = self._apply_layers_sequentially( self.layers, y, *layer_args, length=scan_length, - kv_caches_stacked=kv_caches, + kv_caches_stacked=scanned_kv_caches, **layer_kwargs, ) # kv_caches list is updated in-place inside _apply_layers_sequentially + if group_kv_caches_per_block: + maxtext_utils.update_kv_caches_after_scan( + kv_caches, scanned_kv_caches, scan_length, cycle_interval, stacked=False + ) else: y, self.layers, _ = self._apply_layers_sequentially( self.layers, diff --git a/src/maxtext/models/gpt_oss.py b/src/maxtext/models/gpt_oss.py index 218c72970d..c68ea1f42f 100644 --- a/src/maxtext/models/gpt_oss.py +++ b/src/maxtext/models/gpt_oss.py @@ -224,9 +224,9 @@ def update_cache(cache, val): stacked_kv_cache = jax.tree_util.tree_map(update_cache, stacked_kv_cache, kv_cache) return (layer_output, stacked_kv_cache, layer_idx + 1), None - elif cfg.scan_layers: - return layer_output, None else: + # Also under scan_layers: when vLLM serves a scanned model, GptOssScannableBlock hands each layer its own + # `kv_cache` and keeps the cache returned here. return layer_output, kv_cache @@ -288,15 +288,25 @@ def __call__( kv_cache=None, attention_metadata=None, ): + """Applies the block's decoder layers in order. + + `kv_cache` holds vLLM's externally-managed KV caches, one per layer of the block in layer order, or is None + when the caches are not managed externally. Returns the block output and, when `kv_cache` is given, the + layers' updated caches as a tuple in the same order (else None). + """ cfg = self.config inputs = self._maybe_shard_with_logical(inputs, self.activation_axis_names) inputs = checkpoint_name(inputs, "decoder_layer_input") y = inputs + updated_kv_caches = [] for layer_id in range(cfg.inhomogeneous_layer_cycle_interval): layer_name = f"layers_{layer_id}" layer = getattr(self, layer_name) - y, kv_cache = layer( + # Each layer attends with its own cache: one cache handed down the block would leave every layer after + # the first without one. + layer_kv_cache = kv_cache[layer_id] if kv_cache is not None else None + y, layer_kv_cache = layer( y, decoder_segment_ids, decoder_positions, @@ -304,7 +314,10 @@ def __call__( model_mode, previous_chunk=previous_chunk, slot=slot, - kv_cache=kv_cache, + kv_cache=layer_kv_cache, attention_metadata=attention_metadata, ) - return y, kv_cache + updated_kv_caches.append(layer_kv_cache) + if kv_cache is None: + return y, None + return y, tuple(updated_kv_caches) diff --git a/tests/unit/nnx_decoders_test.py b/tests/unit/nnx_decoders_test.py index aa75084488..e9d8a90b65 100644 --- a/tests/unit/nnx_decoders_test.py +++ b/tests/unit/nnx_decoders_test.py @@ -1330,6 +1330,90 @@ def shapes(tree): ) +_GPT_OSS_CONFIG = { + "run_name": "gpt_oss_nnx_decoder_test", + "model_name": "gpt-oss-20b", + "max_target_length": 8, + "base_emb_dim": 64, + # A cycle of 2 over 4 layers gives two scanned blocks, the fewest that can mix up caches across blocks. + "base_num_decoder_layers": 4, + "base_num_query_heads": 4, + "base_num_kv_heads": 2, + "head_dim": 16, + "base_mlp_dim": 64, + "base_moe_mlp_dim": 64, + "num_experts": 4, + "num_experts_per_tok": 2, + "vocab_size": 32, + # With the TPU kernels off the sparse path lowers to ragged_dot, which runs on CPU. + "sparse_matmul": True, + "megablox": False, + "use_tokamax_gmm": False, + "dtype": "float32", + "weight_dtype": "float32", +} + + +class TestNNXDecoderGptOssKvCaches(unittest.TestCase): + """Every GPT-OSS decoder layer must get its own vLLM KV cache and hand back its update. + + vLLM serves the model with one externally-managed KV cache per decoder layer and keeps the caches the decoder + returns. Scanned, one scan iteration runs a block of two layers, so passing the flat list straight through gave + block i only ``kv_caches[i]``: the block's second layer attended without a cache, the layers dropped their updates + under scan_layers, and decoding a scanned checkpoint with vLLM produced gibberish. + """ + + def _forward_with_kv_caches(self, scan_layers): + """Runs one forward pass with a distinct cache per layer; returns the config and the caches handed back.""" + cfg = _make_config(**{**_GPT_OSS_CONFIG, "scan_layers": scan_layers}) + mesh = _make_mesh(cfg) + decoder = NNXDecoder(config=cfg, mesh=mesh, model_mode=MODEL_MODE_TRAIN, rngs=nnx.Rngs(params=0, dropout=1)) + shared_embedding = Embed( + num_embeddings=cfg.vocab_size, + num_features=cfg.emb_dim, + dtype=cfg.dtype, + config=cfg, + mesh=mesh, + rngs=nnx.Rngs(params=0), + ) + batch = cfg.global_batch_size_to_train_on + seq = cfg.max_target_length + ids = jax.random.randint(jax.random.PRNGKey(0), (batch, seq), 0, cfg.vocab_size) + segment_ids = jnp.full((batch, seq), DECODING_ACTIVE_SEQUENCE_INDICATOR) + positions = jnp.broadcast_to(jnp.arange(seq)[None], (batch, seq)) + kv_caches = [jnp.full((2,), 10.0 * i) for i in range(cfg.num_decoder_layers)] + original_call = Attention.__call__ + + def bump_returned_kv_cache(attention, *args, **kwargs): + # Outside vLLM serving the attention passes its cache through; the bump stands in for vLLM's update. + out, kv_cache = original_call(attention, *args, **kwargs) + return out, None if kv_cache is None else kv_cache + 1 + + with mock.patch.object(Attention, "__call__", bump_returned_kv_cache): + decoder( + shared_embedding, + ids, + decoder_positions=positions, + decoder_segment_ids=segment_ids, + deterministic=True, + model_mode=MODEL_MODE_TRAIN, + kv_caches=kv_caches, + ) + return cfg, kv_caches + + def test_each_layer_updates_its_own_kv_cache(self): + for scan_layers in (False, True): + with self.subTest(scan_layers=scan_layers): + cfg, kv_caches = self._forward_with_kv_caches(scan_layers) + + self.assertEqual(cfg.inhomogeneous_layer_cycle_interval, 2) + self.assertEqual(len(kv_caches), cfg.num_decoder_layers) + for i, kv_cache in enumerate(kv_caches): + self.assertIsNotNone(kv_cache, f"layer {i}'s KV cache update was dropped") + # Bumped exactly once, by its own layer. + np.testing.assert_array_equal(np.asarray(kv_cache), np.full((2,), 10.0 * i + 1), err_msg=f"layer {i}") + + class TestNNXDecoderDeepseekAndGemma4(unittest.TestCase): """Tests for Deepseek and Gemma4 specific decoder logic."""