Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 71 additions & 5 deletions src/maxtext/layers/nnx_decoders.py
Original file line number Diff line number Diff line change
Expand Up @@ -395,6 +395,23 @@ def _qwen3_hybrid_block_class(config):
return qwen3.Qwen3NextScannableBlock


def layer_attention_metadata(attention_metadata: dict[Any, Any], layer_idx: int) -> Any:
"""Returns decoder layer `layer_idx`'s entry of per-layer attention metadata.

vLLM hands hybrid models (e.g. Qwen3-Next / Qwen3.5) a mapping from layer
name (`layer.{i}`) to the metadata of that layer's KV-cache group; integer
keys are accepted too. A layer with no entry gets None.

Args:
attention_metadata: The per-layer metadata mapping.
layer_idx: Global index of the decoder layer.

Returns:
The layer's metadata, or None when the mapping has no entry for it.
"""
return attention_metadata.get(f"layer.{layer_idx}", attention_metadata.get(layer_idx))


def deepstack_process(hidden_states, bidirectional_mask, visual_embeds):
"""Process deepstack visual embeddings by adding them to hidden states at visual token positions.

Expand Down Expand Up @@ -1157,6 +1174,7 @@ def _apply_layers_sequentially(
unroll: int = 1,
metadata_axis_name: str = "layers",
forced_routed_experts_scanned=None,
per_iteration_kwargs=None,
**kwargs,
):
"""Runs the layer stack using nnx.scan.
Expand Down Expand Up @@ -1189,13 +1207,33 @@ def _apply_layers_sequentially(
forced_routed_experts_scanned: Optional `[length, ...]` array, one slice
per scan iteration, threaded through jax.lax.scan's xs alongside
params/state and passed to each layer as `forced_routed_experts=`.
per_iteration_kwargs: Optional sequence of `length` dicts, the i-th
holding extra keyword arguments for iteration i only (e.g. the attention
metadata of one block's sub-layers). Requires `kv_caches_stacked`: only
its static unroll calls each iteration separately. The values are passed
as call arguments rather than closed over, so they stay correct when the
body is wrapped in jax.checkpoint.
**kwargs: Keyword args forwarded to the layer (filtered by the layer
signature).

Returns:
(final_carry, updated_layers) when kv_caches_stacked is None.
(final_carry, updated_layers, returned_kv_stacked) otherwise.

Raises:
ValueError: If `per_iteration_kwargs` is given without
`kv_caches_stacked`, or does not hold one entry per iteration.
"""
if per_iteration_kwargs is not None:
if kv_caches_stacked is None:
raise ValueError(
"per_iteration_kwargs requires kv_caches_stacked: only the statically unrolled external-cache path calls "
"each iteration separately."
)
if len(per_iteration_kwargs) != length:
raise ValueError(
f"per_iteration_kwargs must hold one entry per scan iteration ({length}); got {len(per_iteration_kwargs)}."
)
if self.config.prefetch_fsdp_weights and not fsdp_prefetch.mesh_uses_fsdp(self.mesh):
max_logging.info("prefetch_fsdp_weights: no FSDP mesh axis spans more than one device; using the regular scan.")
elif (
Expand Down Expand Up @@ -1247,15 +1285,17 @@ def layer_fn(carry, scanned_vars):

# Unpack the sliced variables for THIS layer
if use_kv:
current_params, current_state, kv_cache_layer = scanned_vars
current_params, current_state, kv_cache_layer, iteration_kwargs = scanned_vars
forced_routed_experts_layer = None
elif use_forced_routing:
current_params, current_state, forced_routed_experts_layer = scanned_vars
kv_cache_layer = None
iteration_kwargs = None
else:
current_params, current_state = scanned_vars
kv_cache_layer = None
forced_routed_experts_layer = None
iteration_kwargs = None

if self.config.parameter_memory_host_offload:
current_params = jax.tree.map(
Expand All @@ -1267,12 +1307,14 @@ def layer_fn(carry, scanned_vars):

layer = nnx.merge(graphdef, current_params, current_state)

# Build call kwargs, injecting per-layer kv_cache / forced routing when available
# Build call kwargs, injecting per-layer kv_cache / forced routing / per-iteration kwargs when available
call_kwargs = dict(valid_kwargs)
if kv_cache_layer is not None:
call_kwargs["kv_cache"] = kv_cache_layer
if use_forced_routing:
call_kwargs["forced_routed_experts"] = forced_routed_experts_layer
if iteration_kwargs:
call_kwargs.update(iteration_kwargs)

layer_out = layer(carry, *args, **call_kwargs)

Expand Down Expand Up @@ -1320,10 +1362,11 @@ def layer_fn(carry, scanned_vars):
# Statically slice the parameters and state for this layer
current_params = jax.tree.map(lambda x, i=i: x[i], params)
current_state = jax.tree.map(lambda x, i=i: x[i], state)
iteration_kwargs = per_iteration_kwargs[i] if per_iteration_kwargs is not None else {}

# Call the layer
current_carry, (_, updated_kv) = layer_fn_wrapped(
current_carry, (current_params, current_state, kv_caches_list[i])
current_carry, (current_params, current_state, kv_caches_list[i], iteration_kwargs)
)

# Update the list in-place (mutates the list passed by reference)
Expand Down Expand Up @@ -2402,7 +2445,7 @@ def pure_layer_fn(graphdef_in, state_in, y_in, kv_in, valid_kwargs):

current_kwargs = dict(layer_kwargs)
if isinstance(attention_metadata, dict):
current_kwargs["attention_metadata"] = attention_metadata.get(f"layer.{lyr}", attention_metadata.get(lyr))
current_kwargs["attention_metadata"] = layer_attention_metadata(attention_metadata, lyr)

routed_experts = current_kwargs.pop("forced_routed_experts", None)
if routed_experts is not None:
Expand Down Expand Up @@ -2695,7 +2738,10 @@ def _apply_qwen3_hybrid_scanned_blocks(self, y, layer_args, layer_kwargs, kv_cac
External (vLLM) caches arrive as a flat list with one entry per decoder
layer, but the scan runs over blocks, so they are grouped into per-block
tuples before the scan and written back to the flat list afterwards, as in
_apply_gemma4_scanned_blocks.
_apply_gemma4_scanned_blocks. vLLM's per-layer attention metadata (a dict
keyed by global layer) is regrouped the same way: a block's sub-layers do
not know their global layer index, so each block is handed a tuple with
its own sub-layers' entries, in the order of its caches.

Layers past the last whole block are applied afterwards, in a short
remainder block, so a layer count that is not a multiple of the cycle keeps
Expand Down Expand Up @@ -2723,6 +2769,23 @@ def _apply_qwen3_hybrid_scanned_blocks(self, y, layer_args, layer_kwargs, kv_cac
)
remainder_forced_routed_experts = per_layer[num_scanned_layers:]

per_block_kwargs = None
remainder_attention_metadata = None
attention_metadata = layer_kwargs.get("attention_metadata")
if kv_caches is not None and isinstance(attention_metadata, dict):
# Only the external-cache path can take per-block values: it unrolls the blocks statically. Without kv caches
# the dict is left in place and the block rejects it.
del layer_kwargs["attention_metadata"]
per_layer_metadata = [layer_attention_metadata(attention_metadata, lyr) for lyr in range(cfg.num_decoder_layers)]
# Grouped with the same helper as the caches below, so each block's tuple lines up with its caches.
per_block_kwargs = [
{"attention_metadata": block_metadata}
for block_metadata in maxtext_utils.prepare_kv_caches_for_scan(
per_layer_metadata, scan_length, block_length, stack=False
)
]
remainder_attention_metadata = tuple(per_layer_metadata[num_scanned_layers:])

if scan_length > 0:
grouped_kv_caches = maxtext_utils.prepare_kv_caches_for_scan(kv_caches, scan_length, block_length, stack=False)
y, self.layers, _ = self._apply_layers_sequentially(
Expand All @@ -2733,6 +2796,7 @@ def _apply_qwen3_hybrid_scanned_blocks(self, y, layer_args, layer_kwargs, kv_cac
kv_caches_stacked=grouped_kv_caches,
skip_block_remat=True,
forced_routed_experts_scanned=forced_routed_experts_scanned,
per_iteration_kwargs=per_block_kwargs,
**layer_kwargs,
)
maxtext_utils.update_kv_caches_after_scan(kv_caches, grouped_kv_caches, scan_length, block_length, stacked=False)
Expand All @@ -2741,6 +2805,8 @@ def _apply_qwen3_hybrid_scanned_blocks(self, y, layer_args, layer_kwargs, kv_cac
if num_remaining_layers > 0:
if remainder_forced_routed_experts is not None:
layer_kwargs["forced_routed_experts"] = remainder_forced_routed_experts
if remainder_attention_metadata is not None:
layer_kwargs["attention_metadata"] = remainder_attention_metadata
y = self._apply_remainder_block(
self.layers_remainder,
y,
Expand Down
54 changes: 48 additions & 6 deletions src/maxtext/models/qwen3.py
Original file line number Diff line number Diff line change
Expand Up @@ -1690,12 +1690,30 @@ def run_global_layer(carry, xs):
nnx.update(self.global_layer, final_other, intermediate_state, created_params)
return y

def _forward_with_external_kv_cache(self, y, kv_cache, layer_kwargs):
def _forward_with_external_kv_cache(self, y, kv_cache, layer_kwargs, sub_layer_attention_metadata=None):
"""Runs the block with externally-supplied per-layer kv caches.

Inference KV caches are a Python list of per-layer entries, so this path
unrolls the local layers statically rather than scanning them.

Args:
y: The block input.
kv_cache: Per-sub-layer caches, the local layers first and the
full-attention layer last.
layer_kwargs: Keyword arguments for every sub-layer call.
sub_layer_attention_metadata: Optional per-sub-layer attention metadata,
ordered like `kv_cache`; entry i replaces `layer_kwargs`'s
`attention_metadata` for sub-layer i.

Returns:
The block output and the tuple of updated per-sub-layer caches.
"""

def sub_layer_kwargs(position):
if sub_layer_attention_metadata is None:
return layer_kwargs
return {**layer_kwargs, "attention_metadata": sub_layer_attention_metadata[position]}

updated_kvs = []
if self.local_layers is not None:
graphdef, params, state = nnx.split(self.local_layers, nnx.Param, ...)
Expand All @@ -1708,7 +1726,7 @@ def _forward_with_external_kv_cache(self, y, kv_cache, layer_kwargs):
current_state = jax.tree.map(lambda x, i=i: x[i], state)
layer = nnx.merge(graphdef, current_params, current_state)
current_kv = kv_cache[i] if (kv_cache is not None and i < len(kv_cache)) else None
y, new_kv = self._run_layer(layer, y, layer_kwargs, current_kv)
y, new_kv = self._run_layer(layer, y, sub_layer_kwargs(i), current_kv)
updated_kvs.append(new_kv)
# Collect only non-Param state: parameters are read-only here, so stacking
# them back would allocate a second copy of every layer weight. Non-Param
Expand All @@ -1720,7 +1738,7 @@ def _forward_with_external_kv_cache(self, y, kv_cache, layer_kwargs):

if self.global_layer is not None:
global_kv = kv_cache[self.num_local] if (kv_cache is not None and self.num_local < len(kv_cache)) else None
y, new_kv = self._run_layer(self.global_layer, y, layer_kwargs, global_kv)
y, new_kv = self._run_layer(self.global_layer, y, sub_layer_kwargs(self.num_local), global_kv)
updated_kvs.append(new_kv)

return y, tuple(updated_kvs)
Expand All @@ -1742,6 +1760,14 @@ def __call__(

Args:
carry: The input tensor from the previous scan iteration.
kv_cache: Optional externally-managed (vLLM) caches, one per sub-layer,
in block order (the local layers first, the full-attention layer last).
attention_metadata: Optional attention metadata: either one object shared
by every sub-layer or, together with `kv_cache`, a tuple holding one
entry per sub-layer in block order. vLLM's per-layer dict (keyed by
global layer) has to be resolved into that tuple by the caller, as
NNXDecoder._apply_qwen3_hybrid_scanned_blocks does, because the
sub-layers of a scanned block do not know their global layer index.
forced_routed_experts: Optional router replay indices for this block,
shaped `[num_of_layers, batch, seq, top_k]` -- one slice per sub-layer,
in block order (the local layers first, the full-attention layer last).
Expand Down Expand Up @@ -1772,13 +1798,29 @@ def __call__(
)
if isinstance(attention_metadata, dict):
# Such metadata is keyed by global layer index, which the sub-layers of a scanned block do not have (every local
# layer is built with layer_idx=0). The vLLM path that sends it runs unscanned (configs/inference/vllm.yml).
raise NotImplementedError("Per-layer attention_metadata (a dict keyed by layer) requires scan_layers=False.")
# layer is built with layer_idx=0), so a `layer.{i}` lookup here would read another layer's entry. With external
# kv caches the decoder resolves it and hands the block a per-sub-layer tuple instead.
raise NotImplementedError(
"Per-layer attention_metadata (a dict keyed by layer) cannot be resolved inside a scanned block; pass a "
"tuple with one entry per sub-layer together with externally-managed kv caches, or use scan_layers=False."
)
sub_layer_attention_metadata = None
if isinstance(attention_metadata, tuple):
if kv_cache is None:
raise NotImplementedError(
"Per-sub-layer attention_metadata is only supported together with externally-managed kv caches."
)
if len(attention_metadata) != self.num_of_layers:
raise ValueError(
f"attention_metadata must carry one entry per sub-layer of the block: expected {self.num_of_layers}, "
f"got {len(attention_metadata)}."
)
sub_layer_attention_metadata = attention_metadata

if kv_cache is not None:
if forced_routed_experts is not None:
raise NotImplementedError("Forced routing is not supported together with externally-managed kv caches.")
return self._forward_with_external_kv_cache(inputs, kv_cache, layer_kwargs)
return self._forward_with_external_kv_cache(inputs, kv_cache, layer_kwargs, sub_layer_attention_metadata)

y = inputs
if self.local_layers is not None:
Expand Down
Loading
Loading