From 3d53ba2229a0aa3e2e3db8667278250f1083730f Mon Sep 17 00:00:00 2001 From: EmmaLien Date: Fri, 23 Oct 2026 02:56:59 +0000 Subject: [PATCH] fix(qwen3): route vLLM per-layer attention metadata through scanned hybrid blocks Since Qwen3.5 reuses Qwen3NextScannableBlock, a scanned Qwen3-Next/Qwen3.5 decoder raises NotImplementedError when vLLM hands it per-layer attention metadata (a dict keyed by `layer.{i}`, one entry per KV-cache group), because the sub-layers of a scanned block do not know their global layer index. vllm_decode forwards `scan_layers` to the adapter, so `vllm_decode ... scan_layers=True` -- used by the Qwen3.5 RL E2E test on its scanned checkpoints -- fails on the first forward pass. With external (vLLM) KV caches the decoder already unrolls the blocks statically, so it now resolves the dict per block instead: - NNXDecoder._apply_qwen3_hybrid_scanned_blocks looks each layer up by its global index (the same lookup as the unscanned path) and hands every block, and the remainder block, a tuple with its sub-layers' entries, ordered like the block's KV caches. - _apply_layers_sequentially takes `per_iteration_kwargs` (static-unroll path only), passed as call arguments so they survive jax.checkpoint. - Qwen3NextScannableBlock accepts that per-sub-layer tuple together with external KV caches, and still rejects an unresolved dict. --- src/maxtext/layers/nnx_decoders.py | 76 +++++++++++++++++++++-- src/maxtext/models/qwen3.py | 54 ++++++++++++++-- tests/unit/nnx_decoders_test.py | 99 +++++++++++++++++++++++++++++- 3 files changed, 217 insertions(+), 12 deletions(-) diff --git a/src/maxtext/layers/nnx_decoders.py b/src/maxtext/layers/nnx_decoders.py index 670e2f260f..2f56f103a4 100644 --- a/src/maxtext/layers/nnx_decoders.py +++ b/src/maxtext/layers/nnx_decoders.py @@ -395,6 +395,23 @@ def _qwen3_hybrid_block_class(config): return qwen3.Qwen3NextScannableBlock +def layer_attention_metadata(attention_metadata: dict[Any, Any], layer_idx: int) -> Any: + """Returns decoder layer `layer_idx`'s entry of per-layer attention metadata. + + vLLM hands hybrid models (e.g. Qwen3-Next / Qwen3.5) a mapping from layer + name (`layer.{i}`) to the metadata of that layer's KV-cache group; integer + keys are accepted too. A layer with no entry gets None. + + Args: + attention_metadata: The per-layer metadata mapping. + layer_idx: Global index of the decoder layer. + + Returns: + The layer's metadata, or None when the mapping has no entry for it. + """ + return attention_metadata.get(f"layer.{layer_idx}", attention_metadata.get(layer_idx)) + + def deepstack_process(hidden_states, bidirectional_mask, visual_embeds): """Process deepstack visual embeddings by adding them to hidden states at visual token positions. @@ -1157,6 +1174,7 @@ def _apply_layers_sequentially( unroll: int = 1, metadata_axis_name: str = "layers", forced_routed_experts_scanned=None, + per_iteration_kwargs=None, **kwargs, ): """Runs the layer stack using nnx.scan. @@ -1189,13 +1207,33 @@ def _apply_layers_sequentially( forced_routed_experts_scanned: Optional `[length, ...]` array, one slice per scan iteration, threaded through jax.lax.scan's xs alongside params/state and passed to each layer as `forced_routed_experts=`. + per_iteration_kwargs: Optional sequence of `length` dicts, the i-th + holding extra keyword arguments for iteration i only (e.g. the attention + metadata of one block's sub-layers). Requires `kv_caches_stacked`: only + its static unroll calls each iteration separately. The values are passed + as call arguments rather than closed over, so they stay correct when the + body is wrapped in jax.checkpoint. **kwargs: Keyword args forwarded to the layer (filtered by the layer signature). Returns: (final_carry, updated_layers) when kv_caches_stacked is None. (final_carry, updated_layers, returned_kv_stacked) otherwise. + + Raises: + ValueError: If `per_iteration_kwargs` is given without + `kv_caches_stacked`, or does not hold one entry per iteration. """ + if per_iteration_kwargs is not None: + if kv_caches_stacked is None: + raise ValueError( + "per_iteration_kwargs requires kv_caches_stacked: only the statically unrolled external-cache path calls " + "each iteration separately." + ) + if len(per_iteration_kwargs) != length: + raise ValueError( + f"per_iteration_kwargs must hold one entry per scan iteration ({length}); got {len(per_iteration_kwargs)}." + ) if self.config.prefetch_fsdp_weights and not fsdp_prefetch.mesh_uses_fsdp(self.mesh): max_logging.info("prefetch_fsdp_weights: no FSDP mesh axis spans more than one device; using the regular scan.") elif ( @@ -1247,15 +1285,17 @@ def layer_fn(carry, scanned_vars): # Unpack the sliced variables for THIS layer if use_kv: - current_params, current_state, kv_cache_layer = scanned_vars + current_params, current_state, kv_cache_layer, iteration_kwargs = scanned_vars forced_routed_experts_layer = None elif use_forced_routing: current_params, current_state, forced_routed_experts_layer = scanned_vars kv_cache_layer = None + iteration_kwargs = None else: current_params, current_state = scanned_vars kv_cache_layer = None forced_routed_experts_layer = None + iteration_kwargs = None if self.config.parameter_memory_host_offload: current_params = jax.tree.map( @@ -1267,12 +1307,14 @@ def layer_fn(carry, scanned_vars): layer = nnx.merge(graphdef, current_params, current_state) - # Build call kwargs, injecting per-layer kv_cache / forced routing when available + # Build call kwargs, injecting per-layer kv_cache / forced routing / per-iteration kwargs when available call_kwargs = dict(valid_kwargs) if kv_cache_layer is not None: call_kwargs["kv_cache"] = kv_cache_layer if use_forced_routing: call_kwargs["forced_routed_experts"] = forced_routed_experts_layer + if iteration_kwargs: + call_kwargs.update(iteration_kwargs) layer_out = layer(carry, *args, **call_kwargs) @@ -1320,10 +1362,11 @@ def layer_fn(carry, scanned_vars): # Statically slice the parameters and state for this layer current_params = jax.tree.map(lambda x, i=i: x[i], params) current_state = jax.tree.map(lambda x, i=i: x[i], state) + iteration_kwargs = per_iteration_kwargs[i] if per_iteration_kwargs is not None else {} # Call the layer current_carry, (_, updated_kv) = layer_fn_wrapped( - current_carry, (current_params, current_state, kv_caches_list[i]) + current_carry, (current_params, current_state, kv_caches_list[i], iteration_kwargs) ) # Update the list in-place (mutates the list passed by reference) @@ -2402,7 +2445,7 @@ def pure_layer_fn(graphdef_in, state_in, y_in, kv_in, valid_kwargs): current_kwargs = dict(layer_kwargs) if isinstance(attention_metadata, dict): - current_kwargs["attention_metadata"] = attention_metadata.get(f"layer.{lyr}", attention_metadata.get(lyr)) + current_kwargs["attention_metadata"] = layer_attention_metadata(attention_metadata, lyr) routed_experts = current_kwargs.pop("forced_routed_experts", None) if routed_experts is not None: @@ -2695,7 +2738,10 @@ def _apply_qwen3_hybrid_scanned_blocks(self, y, layer_args, layer_kwargs, kv_cac External (vLLM) caches arrive as a flat list with one entry per decoder layer, but the scan runs over blocks, so they are grouped into per-block tuples before the scan and written back to the flat list afterwards, as in - _apply_gemma4_scanned_blocks. + _apply_gemma4_scanned_blocks. vLLM's per-layer attention metadata (a dict + keyed by global layer) is regrouped the same way: a block's sub-layers do + not know their global layer index, so each block is handed a tuple with + its own sub-layers' entries, in the order of its caches. Layers past the last whole block are applied afterwards, in a short remainder block, so a layer count that is not a multiple of the cycle keeps @@ -2723,6 +2769,23 @@ def _apply_qwen3_hybrid_scanned_blocks(self, y, layer_args, layer_kwargs, kv_cac ) remainder_forced_routed_experts = per_layer[num_scanned_layers:] + per_block_kwargs = None + remainder_attention_metadata = None + attention_metadata = layer_kwargs.get("attention_metadata") + if kv_caches is not None and isinstance(attention_metadata, dict): + # Only the external-cache path can take per-block values: it unrolls the blocks statically. Without kv caches + # the dict is left in place and the block rejects it. + del layer_kwargs["attention_metadata"] + per_layer_metadata = [layer_attention_metadata(attention_metadata, lyr) for lyr in range(cfg.num_decoder_layers)] + # Grouped with the same helper as the caches below, so each block's tuple lines up with its caches. + per_block_kwargs = [ + {"attention_metadata": block_metadata} + for block_metadata in maxtext_utils.prepare_kv_caches_for_scan( + per_layer_metadata, scan_length, block_length, stack=False + ) + ] + remainder_attention_metadata = tuple(per_layer_metadata[num_scanned_layers:]) + if scan_length > 0: grouped_kv_caches = maxtext_utils.prepare_kv_caches_for_scan(kv_caches, scan_length, block_length, stack=False) y, self.layers, _ = self._apply_layers_sequentially( @@ -2733,6 +2796,7 @@ def _apply_qwen3_hybrid_scanned_blocks(self, y, layer_args, layer_kwargs, kv_cac kv_caches_stacked=grouped_kv_caches, skip_block_remat=True, forced_routed_experts_scanned=forced_routed_experts_scanned, + per_iteration_kwargs=per_block_kwargs, **layer_kwargs, ) maxtext_utils.update_kv_caches_after_scan(kv_caches, grouped_kv_caches, scan_length, block_length, stacked=False) @@ -2741,6 +2805,8 @@ def _apply_qwen3_hybrid_scanned_blocks(self, y, layer_args, layer_kwargs, kv_cac if num_remaining_layers > 0: if remainder_forced_routed_experts is not None: layer_kwargs["forced_routed_experts"] = remainder_forced_routed_experts + if remainder_attention_metadata is not None: + layer_kwargs["attention_metadata"] = remainder_attention_metadata y = self._apply_remainder_block( self.layers_remainder, y, diff --git a/src/maxtext/models/qwen3.py b/src/maxtext/models/qwen3.py index 630f809f0d..3c96c498c3 100644 --- a/src/maxtext/models/qwen3.py +++ b/src/maxtext/models/qwen3.py @@ -1690,12 +1690,30 @@ def run_global_layer(carry, xs): nnx.update(self.global_layer, final_other, intermediate_state, created_params) return y - def _forward_with_external_kv_cache(self, y, kv_cache, layer_kwargs): + def _forward_with_external_kv_cache(self, y, kv_cache, layer_kwargs, sub_layer_attention_metadata=None): """Runs the block with externally-supplied per-layer kv caches. Inference KV caches are a Python list of per-layer entries, so this path unrolls the local layers statically rather than scanning them. + + Args: + y: The block input. + kv_cache: Per-sub-layer caches, the local layers first and the + full-attention layer last. + layer_kwargs: Keyword arguments for every sub-layer call. + sub_layer_attention_metadata: Optional per-sub-layer attention metadata, + ordered like `kv_cache`; entry i replaces `layer_kwargs`'s + `attention_metadata` for sub-layer i. + + Returns: + The block output and the tuple of updated per-sub-layer caches. """ + + def sub_layer_kwargs(position): + if sub_layer_attention_metadata is None: + return layer_kwargs + return {**layer_kwargs, "attention_metadata": sub_layer_attention_metadata[position]} + updated_kvs = [] if self.local_layers is not None: graphdef, params, state = nnx.split(self.local_layers, nnx.Param, ...) @@ -1708,7 +1726,7 @@ def _forward_with_external_kv_cache(self, y, kv_cache, layer_kwargs): current_state = jax.tree.map(lambda x, i=i: x[i], state) layer = nnx.merge(graphdef, current_params, current_state) current_kv = kv_cache[i] if (kv_cache is not None and i < len(kv_cache)) else None - y, new_kv = self._run_layer(layer, y, layer_kwargs, current_kv) + y, new_kv = self._run_layer(layer, y, sub_layer_kwargs(i), current_kv) updated_kvs.append(new_kv) # Collect only non-Param state: parameters are read-only here, so stacking # them back would allocate a second copy of every layer weight. Non-Param @@ -1720,7 +1738,7 @@ def _forward_with_external_kv_cache(self, y, kv_cache, layer_kwargs): if self.global_layer is not None: global_kv = kv_cache[self.num_local] if (kv_cache is not None and self.num_local < len(kv_cache)) else None - y, new_kv = self._run_layer(self.global_layer, y, layer_kwargs, global_kv) + y, new_kv = self._run_layer(self.global_layer, y, sub_layer_kwargs(self.num_local), global_kv) updated_kvs.append(new_kv) return y, tuple(updated_kvs) @@ -1742,6 +1760,14 @@ def __call__( Args: carry: The input tensor from the previous scan iteration. + kv_cache: Optional externally-managed (vLLM) caches, one per sub-layer, + in block order (the local layers first, the full-attention layer last). + attention_metadata: Optional attention metadata: either one object shared + by every sub-layer or, together with `kv_cache`, a tuple holding one + entry per sub-layer in block order. vLLM's per-layer dict (keyed by + global layer) has to be resolved into that tuple by the caller, as + NNXDecoder._apply_qwen3_hybrid_scanned_blocks does, because the + sub-layers of a scanned block do not know their global layer index. forced_routed_experts: Optional router replay indices for this block, shaped `[num_of_layers, batch, seq, top_k]` -- one slice per sub-layer, in block order (the local layers first, the full-attention layer last). @@ -1772,13 +1798,29 @@ def __call__( ) if isinstance(attention_metadata, dict): # Such metadata is keyed by global layer index, which the sub-layers of a scanned block do not have (every local - # layer is built with layer_idx=0). The vLLM path that sends it runs unscanned (configs/inference/vllm.yml). - raise NotImplementedError("Per-layer attention_metadata (a dict keyed by layer) requires scan_layers=False.") + # layer is built with layer_idx=0), so a `layer.{i}` lookup here would read another layer's entry. With external + # kv caches the decoder resolves it and hands the block a per-sub-layer tuple instead. + raise NotImplementedError( + "Per-layer attention_metadata (a dict keyed by layer) cannot be resolved inside a scanned block; pass a " + "tuple with one entry per sub-layer together with externally-managed kv caches, or use scan_layers=False." + ) + sub_layer_attention_metadata = None + if isinstance(attention_metadata, tuple): + if kv_cache is None: + raise NotImplementedError( + "Per-sub-layer attention_metadata is only supported together with externally-managed kv caches." + ) + if len(attention_metadata) != self.num_of_layers: + raise ValueError( + f"attention_metadata must carry one entry per sub-layer of the block: expected {self.num_of_layers}, " + f"got {len(attention_metadata)}." + ) + sub_layer_attention_metadata = attention_metadata if kv_cache is not None: if forced_routed_experts is not None: raise NotImplementedError("Forced routing is not supported together with externally-managed kv caches.") - return self._forward_with_external_kv_cache(inputs, kv_cache, layer_kwargs) + return self._forward_with_external_kv_cache(inputs, kv_cache, layer_kwargs, sub_layer_attention_metadata) y = inputs if self.local_layers is not None: diff --git a/tests/unit/nnx_decoders_test.py b/tests/unit/nnx_decoders_test.py index 83a6445a6f..ddf09ddf2f 100644 --- a/tests/unit/nnx_decoders_test.py +++ b/tests/unit/nnx_decoders_test.py @@ -1138,7 +1138,7 @@ def _build(self, num_decoder_layers, **overrides): ) return cfg, decoder, shared_embedding - def _run(self, cfg, decoder, shared_embedding, kv_caches=None): + def _run(self, cfg, decoder, shared_embedding, kv_caches=None, attention_metadata=None): """Runs one TRAIN-mode forward pass and returns the logits.""" batch = cfg.global_batch_size_to_train_on seq = cfg.max_target_length @@ -1153,6 +1153,7 @@ def _run(self, cfg, decoder, shared_embedding, kv_caches=None): deterministic=True, model_mode=MODEL_MODE_TRAIN, kv_caches=kv_caches, + attention_metadata=attention_metadata, ) return logits @@ -1184,6 +1185,50 @@ def test_decoder_regroups_flat_kv_caches_per_block(self): for i, cache in enumerate(kv_caches): np.testing.assert_array_equal(np.asarray(cache), np.full((batch, seq), float(i))) + def test_decoder_routes_per_layer_attention_metadata_by_global_layer(self): + """Every layer must get its own entry of vLLM's per-layer metadata, keyed by global layer index. + + vLLM hands hybrid models a dict from ``layer.{i}`` to that layer's KV-cache + group metadata. The sub-layers of a scanned block do not know their global + index (the local ones are all built with layer_idx=0), so the decoder has to + resolve the dict for each block. Two blocks of a cycle-2 model plus a + one-layer remainder cover the scanned blocks and the remainder block alike: + a lookup by position in the block would hand layers 2..4 the entries of + layers 0..1. + """ + cfg, decoder, shared_embedding = self._build(5, inhomogeneous_layer_cycle_interval=2) + self.assertEqual(decoder.layers_remainder.num_local, 1) + batch = cfg.global_batch_size_to_train_on + seq = cfg.max_target_length + kv_caches = [jnp.full((batch, seq), float(i)) for i in range(cfg.num_decoder_layers)] + attention_metadata = {f"layer.{i}": object() for i in range(cfg.num_decoder_layers)} + + received = [] + original_call = qwen3.Qwen3NextDecoderLayer.__call__ + + def recording_call(layer, *args, attention_metadata=None, **kwargs): + received.append(attention_metadata) + # The sentinels are not real metadata, so run the layer as it runs without any. + return original_call(layer, *args, attention_metadata=None, **kwargs) + + with mock.patch.object(qwen3.Qwen3NextDecoderLayer, "__call__", recording_call): + self._run(cfg, decoder, shared_embedding, kv_caches=kv_caches, attention_metadata=attention_metadata) + + self.assertEqual(len(received), cfg.num_decoder_layers) + for lyr, metadata in enumerate(received): + self.assertIs(metadata, attention_metadata[f"layer.{lyr}"], f"layer {lyr} got another layer's metadata") + # The caches still land back on their own layers. + for i, cache in enumerate(kv_caches): + np.testing.assert_array_equal(np.asarray(cache), np.full((batch, seq), float(i))) + + def test_decoder_rejects_per_layer_attention_metadata_without_kv_caches(self): + """Without external caches the blocks scan their sub-layers, which cannot vary the metadata per layer.""" + cfg, decoder, shared_embedding = self._build(4, inhomogeneous_layer_cycle_interval=2) + attention_metadata = {f"layer.{i}": object() for i in range(cfg.num_decoder_layers)} + + with self.assertRaisesRegex(NotImplementedError, "cannot be resolved inside a scanned block"): + self._run(cfg, decoder, shared_embedding, attention_metadata=attention_metadata) + def test_decoder_keeps_layers_past_the_last_whole_block(self): """Layers left over by the block scan must still be built and applied. @@ -1869,11 +1914,15 @@ def __init__(self, *, is_full_attention_layer, **unused_kwargs): self.call_count = nnx.Intermediate(jnp.array(0, dtype=jnp.int32)) self.received_kv_cache = nnx.Intermediate(jnp.array(False)) self.received_attention_metadata = nnx.Intermediate(jnp.array(False)) + # The metadata itself when a test passes an array (to tell the sub-layers' entries apart), else -1. + self.received_attention_metadata_value = nnx.Intermediate(jnp.array(-1, dtype=jnp.int32)) def __call__(self, inputs, *unused_args, kv_cache=None, attention_metadata=None, **unused_kwargs): self.call_count.value += 1 self.received_kv_cache.value = jnp.array(kv_cache is not None) self.received_attention_metadata.value = jnp.array(attention_metadata is not None) + if isinstance(attention_metadata, jax.Array): + self.received_attention_metadata_value.value = attention_metadata.astype(jnp.int32) output = inputs + self.increment if kv_cache is None: return output @@ -1960,6 +2009,54 @@ def test_shorter_cache_does_not_index_out_of_range(self): np.testing.assert_array_equal(updated[0], 1) self.assertEqual(updated[1:], (None,) * (self.CYCLE - 1)) + @pytest.mark.cpu_only + def test_each_sublayer_gets_its_own_attention_metadata(self): + """A per-sub-layer metadata tuple must reach the sub-layers in block order, like the caches. + + vLLM's hybrid metadata is per KV-cache group, so sub-layers of one block can + need different entries; handing every sub-layer the same one would point + some of them at another group's block tables. + """ + block = self._make_block() + kv_cache = tuple(jnp.array(i) for i in range(self.CYCLE)) + attention_metadata = tuple(jnp.array(100 + i) for i in range(self.CYCLE)) + + self._call(block, kv_cache=kv_cache, attention_metadata=attention_metadata) + + np.testing.assert_array_equal( + block.local_layers.received_attention_metadata_value.value, jnp.arange(100, 100 + self.CYCLE - 1) + ) + np.testing.assert_array_equal(block.global_layer.received_attention_metadata_value.value, 100 + self.CYCLE - 1) + + @pytest.mark.cpu_only + def test_rejects_per_layer_attention_metadata_dict(self): + """A dict keyed by global layer cannot be resolved by sub-layers that do not know their global index.""" + block = self._make_block() + kv_cache = tuple(jnp.array(i) for i in range(self.CYCLE)) + attention_metadata = {f"layer.{i}": jnp.array(i) for i in range(self.CYCLE)} + + with self.assertRaisesRegex(NotImplementedError, "cannot be resolved inside a scanned block"): + self._call(block, kv_cache=kv_cache, attention_metadata=attention_metadata) + + @pytest.mark.cpu_only + def test_rejects_per_sub_layer_attention_metadata_without_external_caches(self): + """Only the external-cache path unrolls the sub-layers; the scans cannot vary metadata per sub-layer.""" + block = self._make_block() + attention_metadata = tuple(jnp.array(i) for i in range(self.CYCLE)) + + with self.assertRaisesRegex(NotImplementedError, "externally-managed kv caches"): + self._call(block, attention_metadata=attention_metadata) + + @pytest.mark.cpu_only + def test_rejects_per_sub_layer_attention_metadata_of_the_wrong_length(self): + """A tuple not covering every sub-layer would leave some sub-layer without its entry.""" + block = self._make_block() + kv_cache = tuple(jnp.array(i) for i in range(self.CYCLE)) + attention_metadata = tuple(jnp.array(i) for i in range(self.CYCLE - 1)) + + with self.assertRaisesRegex(ValueError, "one entry per sub-layer"): + self._call(block, kv_cache=kv_cache, attention_metadata=attention_metadata) + @pytest.mark.cpu_only def test_flat_per_layer_caches_survive_the_per_block_round_trip(self): """vLLM hands over one cache per layer; the scan runs over 4-layer blocks."""