From 45b0b342bad085bdf54da21c6b1e233404ac240c Mon Sep 17 00:00:00 2001 From: Doruk Arisoy <34922984+darisoy@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:29:47 +0000 Subject: [PATCH] Prefetch FSDP weights per sub-layer for inhomogeneous blocks (Qwen3.5) Adds `fsdp_prefetch_inhomogeneous`: with `prefetch_fsdp_weights=True`, a scanned block made of sub-layers with different parameter structures (Qwen3.5's `local_layers` Gated DeltaNet scan plus one global-attention `global_layer`, or any block that defines `prefetch_sublayer_names()`) is prefetched one sub-layer at a time instead of one whole block at a time. Forward: the all-gather of sub-layer j+1 overlaps the compute of sub-layer j, across block boundaries. Backward: the all-gather of sub-layer j-1 overlaps the recompute + vjp of sub-layer j, followed by a reduce-scatter of sub-layer j's weight gradients. The forward pass holds at most two sub-layers of gathered weights and the backward pass at most three. The block keeps its nested scan structure: consecutive sub-layers with the same structure form a group, and every sub-layer runs in a while loop nested in the block scan (trip-count-one loops keep `skip-simplify-while-loops_trip-count-one`). Within a group of n sub-layers, the n-1 that prefetch a sub-layer of the same group share one scan; the remaining one prefetches a sub-layer of another structure in its own trip-count-one loop. For Qwen3.5: forward: [GDN 0, GDN 1] -> [GDN 2] -> [attention] backward: [attention] -> [GDN 2, GDN 1] -> [GDN 0] The loop boundaries keep each prefetched all-gather inside the sub-layer that issues it. The gathers, reduce-scatters and remat policy handling are shared with `fsdp_prefetch`. `NNXDecoder` only takes this path when every block param lives in the listed sub-layers; homogeneous decoders and other blocks keep prefetching whole scan steps with `fsdp_prefetch`. --- src/maxtext/layers/fsdp_prefetch.py | 106 +++-- .../layers/fsdp_prefetch_inhomogeneous.py | 416 ++++++++++++++++++ src/maxtext/layers/nnx_decoders.py | 32 +- tests/unit/nnx_decoders_test.py | 328 +++++++++++++- 4 files changed, 820 insertions(+), 62 deletions(-) create mode 100644 src/maxtext/layers/fsdp_prefetch_inhomogeneous.py diff --git a/src/maxtext/layers/fsdp_prefetch.py b/src/maxtext/layers/fsdp_prefetch.py index b29ac8a8e0..7aaeb11c6f 100644 --- a/src/maxtext/layers/fsdp_prefetch.py +++ b/src/maxtext/layers/fsdp_prefetch.py @@ -519,55 +519,23 @@ def mesh_uses_fsdp(mesh) -> bool: return any(mesh.shape.get(axis, 1) > 1 for axis in sharding.FSDP_MESH_AXES) -def apply_layers_with_fsdp_prefetch( - layers, - x_in, - *args, - length: int, - mesh, - config, - remat_policy=None, - prevent_cse: bool = False, - skip_block_remat: bool = False, - **kwargs, -): - """Runs a scanned NNX layer stack with FSDP weight prefetching in the forward and backward pass. +def _layer_major(params, state, scan_axis, length): + """Moves the scan axis of `params` to the front and gives `params`/`state` a leading layer axis.""" + if scan_axis != 0: + params = jax.tree.map(lambda x: jnp.moveaxis(x, scan_axis, 0) if x.ndim > scan_axis else x, params) + params = maxtext_utils_nnx.nnx_ensure_scan_leading_axis(params, length) + state = maxtext_utils_nnx.nnx_ensure_scan_leading_axis(state, length) + return params, state - Args: - layers: The scanned (layer-stacked) NNX module. - x_in: The input activation of the first layer. - *args: Positional arguments passed to every layer call. - length: Number of layers in the stack. - mesh: The device mesh. - config: The MaxText config. - remat_policy: The `jax.checkpoint` policy of each layer (None for full remat). The forward pass - saves the activations this policy saves (in addition to each layer's input) and the backward - pass recomputes the rest; gathered weights are never saved. - prevent_cse: `prevent_cse` of the per-layer `jax.checkpoint` (only used with a remat policy). - skip_block_remat: Whether each scanned module is a block that rematerializes its own sub-layers - (as for `NNXDecoder._apply_layers_sequentially`). The pipeline keeps its per-module remat, since - without it the forward pass would save the weights the block's inner scans read, but the policy - then also saves each sub-layer's `decoder_layer_input`: the backward pass rebuilds every - sub-layer from its input once, instead of first recomputing the whole block. - **kwargs: Keyword arguments passed to every layer call (those not accepted by the layer are dropped). - Returns: - (final activation, layers, None). Non-parameter layer state is read-only in this path: state - mutated inside a layer call (e.g. sown intermediates) is not propagated back to `layers`. +def _prepare_prefetch_call(layers, args, kwargs, *, length, mesh, config, remat_policy, prevent_cse): + """Splits a scanned layer stack and its call arguments for `jax.custom_vjp`. - See `_custom_vjp_prefetch_pipeline`. + Returns (graphdef, params_leaves, state_leaves, diff_args_leaves, diff_kwargs_leaves, ctx), with + params/state layer-major (see `_layer_major`). """ - if length == 0: - return x_in, layers, None - if skip_block_remat: - remat_policy = _also_save_sublayer_inputs(remat_policy) graphdef, params, state = nnx.split(layers, nnx.Param, ...) - - scan_axis = config.param_scan_axis - if scan_axis != 0: - params = jax.tree.map(lambda x: jnp.moveaxis(x, scan_axis, 0) if x.ndim > scan_axis else x, params) - params = maxtext_utils_nnx.nnx_ensure_scan_leading_axis(params, length) - state = maxtext_utils_nnx.nnx_ensure_scan_leading_axis(state, length) + params, state = _layer_major(params, state, config.param_scan_axis, length) sig = inspect.signature(layers.__class__.__call__) valid_kwargs = {k: v for k, v in kwargs.items() if k in sig.parameters or "kwargs" in sig.parameters} @@ -606,8 +574,56 @@ def _is_diff_leaf(x): remat_policy=remat_policy, prevent_cse=prevent_cse, ) + return graphdef, tuple(params_leaves), tuple(state_leaves), diff_args_leaves, diff_kwargs_leaves, ctx + +def apply_layers_with_fsdp_prefetch( + layers, + x_in, + *args, + length: int, + mesh, + config, + remat_policy=None, + prevent_cse: bool = False, + skip_block_remat: bool = False, + **kwargs, +): + """Runs a scanned NNX layer stack with FSDP weight prefetching in the forward and backward pass. + + Args: + layers: The scanned (layer-stacked) NNX module. + x_in: The input activation of the first layer. + *args: Positional arguments passed to every layer call. + length: Number of layers in the stack. + mesh: The device mesh. + config: The MaxText config. + remat_policy: The `jax.checkpoint` policy of each layer (None for full remat). The forward pass + saves the activations this policy saves (in addition to each layer's input) and the backward + pass recomputes the rest; gathered weights are never saved. + prevent_cse: `prevent_cse` of the per-layer `jax.checkpoint` (only used with a remat policy). + skip_block_remat: Whether each scanned module is a block that rematerializes its own sub-layers + (as for `NNXDecoder._apply_layers_sequentially`). The pipeline keeps its per-module remat, since + without it the forward pass would save the weights the block's inner scans read, but the policy + then also saves each sub-layer's `decoder_layer_input`: the backward pass rebuilds every + sub-layer from its input once, instead of first recomputing the whole block. + **kwargs: Keyword arguments passed to every layer call (those not accepted by the layer are dropped). + + Returns: + (final activation, layers, None). Non-parameter layer state is read-only in this path: state + mutated inside a layer call (e.g. sown intermediates) is not propagated back to `layers`. + + See `_custom_vjp_prefetch_pipeline`. Inhomogeneous blocks (e.g. Qwen3.5) can instead prefetch one + sub-layer at a time with `fsdp_prefetch_inhomogeneous`. + """ + if length == 0: + return x_in, layers, None + if skip_block_remat: + remat_policy = _also_save_sublayer_inputs(remat_policy) + graphdef, params_leaves, state_leaves, diff_args_leaves, diff_kwargs_leaves, ctx = _prepare_prefetch_call( + layers, args, kwargs, length=length, mesh=mesh, config=config, remat_policy=remat_policy, prevent_cse=prevent_cse + ) final_y, _ = _custom_vjp_prefetch_pipeline( - graphdef, x_in, tuple(params_leaves), tuple(state_leaves), diff_args_leaves, diff_kwargs_leaves, length, ctx + graphdef, x_in, params_leaves, state_leaves, diff_args_leaves, diff_kwargs_leaves, length, ctx ) return final_y, layers, None diff --git a/src/maxtext/layers/fsdp_prefetch_inhomogeneous.py b/src/maxtext/layers/fsdp_prefetch_inhomogeneous.py new file mode 100644 index 0000000000..f09dc2d0fa --- /dev/null +++ b/src/maxtext/layers/fsdp_prefetch_inhomogeneous.py @@ -0,0 +1,416 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""FSDP weight prefetching, one sub-layer at a time, for inhomogeneous scanned blocks (e.g. Qwen3.5). + +`fsdp_prefetch` prefetches whole scan steps. When one scan step is a block of sub-layers with +different parameter structures (e.g. Qwen3.5's mix of Gated DeltaNet and global attention layers), +that gathers a whole block's weights at once. A block that defines `prefetch_sublayer_names()` is +instead prefetched one sub-layer at a time here: the all-gather of sub-layer j+1 overlaps sub-layer +j, across block boundaries, in both passes, and each sub-layer runs in a while-loop nested in the +block scan, as in the block's regular nested scan. The gathers, reduce-scatters and remat policy +handling are shared with `fsdp_prefetch`. + +See `_custom_vjp_nested_prefetch_pipeline` for the design. +""" + +# pylint: disable=protected-access + +import functools + +from flax import nnx +import jax +from jax.experimental import xla_metadata +import jax.numpy as jnp +from maxtext.layers import fsdp_prefetch +from maxtext.utils import max_logging + + +def _index_at(tree, idx): + """Layer `idx` (a traced index) of a layer-stacked pytree.""" + return jax.tree.map(lambda x: jax.lax.dynamic_index_in_dim(x, idx, axis=0, keepdims=False), tree) + + +_SKIP_TRIP_COUNT_ONE = {"skip-simplify-while-loops_trip-count-one": "true"} + + +def _sublayer_scan(body, carry, xs, reverse=False): + """`jax.lax.scan` over sub-layers that XLA keeps as a while-loop, even with a single trip. + + Each sub-layer run is its own while-loop nested in the block scan, as in the regular nested scan + of the block (see `Qwen3NextScannableBlock`). The loop boundary keeps a prefetched all-gather + inside the sub-layer that issues it: XLA cannot finish it early, ahead of that sub-layer's compute. + """ + with xla_metadata.set_xla_metadata(**_SKIP_TRIP_COUNT_ONE): + return jax.lax.scan(body, carry, xs, reverse=reverse) + + +def _concat0(*trees): + """Concatenates layer-stacked pytrees along the leading axis.""" + return jax.tree.map(lambda *xs: jnp.concatenate(xs, axis=0), *trees) + + +def _setup_groups(graphdefs, params_leaves, state_leaves, diff_args_leaves, diff_kwargs_leaves, ctxs): + """`_prefetch_setup` for each sub-layer group; returns per-group tuples (gather_fn, rs_fn, ...). + + Group params/state leaves are stacked `[length, n, ...]`; the setup sees sub-layer 0's + `[length, ...]` slice, which has the per-sub-layer shardings. + """ + setups = [ + fsdp_prefetch._prefetch_setup( + g, tuple(x[:, 0] for x in p), tuple(x[:, 0] for x in s), diff_args_leaves, diff_kwargs_leaves, ctx + ) + for g, p, s, ctx in zip(graphdefs, params_leaves, state_leaves, ctxs) + ] + _, _, gather, rs_fns, layout_fns, fwd_layers, layer_vjp_fns = zip(*setups) + params = [jax.tree_util.tree_unflatten(ctx.params_treedef, p) for p, ctx in zip(params_leaves, ctxs)] + state = [jax.tree_util.tree_unflatten(ctx.state_treedef, s) for s, ctx in zip(state_leaves, ctxs)] + return params, state, gather, rs_fns, layout_fns, fwd_layers, layer_vjp_fns + + +@functools.partial(jax.custom_vjp, nondiff_argnums=(0, 6, 7, 8)) +def _custom_vjp_nested_prefetch_pipeline( + graphdefs, x_in, params_leaves, state_leaves, diff_args_leaves, diff_kwargs_leaves, length, ctxs, sizes +): + """Runs `length` scanned blocks of sub-layer groups with per-sub-layer prefetching and nested scans. + + Used for inhomogeneous stacks (e.g. Qwen3.5), where one scan step is a block of sub-layers with + different parameter structures. A block is a sequence of groups; group `i` has `sizes[i]` + consecutive sub-layers of one structure (e.g. Qwen3.5: 3 Gated DeltaNet layers, then 1 global + attention layer). The pipeline walks the flat sequence of sub-layers: + + Forward (sub-layer j): AG(j+1) || compute(j), where AG(j+1) of the last sub-layer of a block is + sub-layer 0 of the next block. + Backward (sub-layer j): AG(j-1) || [recompute(j) + vjp(j)], then reduce-scatter dW(j). + + Each pass is a scan over the blocks whose body runs every sub-layer in a while-loop nested in it + (see `_sublayer_scan`). Within a group of n sub-layers, the n-1 that prefetch a sub-layer of the + same group (the first n-1 forward, the last n-1 backward) share one scan; the remaining one + prefetches a sub-layer of another structure and runs in its own single-trip loop. Qwen3.5 forward: + [GDN 0, GDN 1] -> [GDN 2] -> [attention]; backward: [attention] -> [GDN 2, GDN 1] -> [GDN 0]. + + Group `i` has its own graphdef, `params_leaves[i]` and `state_leaves[i]`, each stacked + `[length, sizes[i], ...]`. The forward pass holds at most two sub-layers of gathered weights and + the backward pass at most three, as in `fsdp_prefetch._custom_vjp_prefetch_pipeline`. All + sub-layer gathers but the first stay inside the block loop; the cost is one unused gather of a + sub-layer at the end of each pass. + """ + out, _ = _custom_vjp_nested_prefetch_pipeline_fwd( + graphdefs, x_in, params_leaves, state_leaves, diff_args_leaves, diff_kwargs_leaves, length, ctxs, sizes + ) + return out + + +def _custom_vjp_nested_prefetch_pipeline_fwd( + graphdefs, x_in, params_leaves, state_leaves, diff_args_leaves, diff_kwargs_leaves, length, ctxs, sizes +): + """Forward pass of `_custom_vjp_nested_prefetch_pipeline`: one scan step per block.""" + num_groups = len(graphdefs) + params, state, gather, _, layout_fns, fwd_layers, _ = _setup_groups( + graphdefs, params_leaves, state_leaves, diff_args_leaves, diff_kwargs_leaves, ctxs + ) + layouts = {} + + def fwd(i, y, w, layer_state): + if i not in layouts: + layouts[i] = layout_fns[i](y, w, layer_state) + return fwd_layers[i](y, w, layer_state, layouts[i]) + + def body(carry, xs): + y, w = carry # `w` is this block's gathered sub-layer 0. + b, b_state, b_params = xs # Group i's leaves are `[sizes[i], ...]`. + ys, saved = [], [] + for i in range(num_groups): + n = sizes[i] + if i + 1 < num_groups: + nxt_first, gather_nxt = fsdp_prefetch._slice_at(b_params[i + 1], 0), gather[i + 1] + else: + # The last block prefetches block 0's sub-layer 0, which is unused: every gather then stays + # in the block loop instead of a peeled epilogue in the parent computation. + nxt_first = fsdp_prefetch._slice_at(_index_at(params[0], (b + 1) % length), 0) + gather_nxt = gather[0] + parts_y, parts_saved = [], [] + if n >= 2: + + def same_group(carry, k, i=i, p=b_params[i], s=b_state[i]): + y, w = carry + w_next = gather[i](_index_at(p, k + 1)) # AG(j+1) || sub-layer j. + y_out, saved_k = fwd(i, y, w, _index_at(s, k)) + return (y_out, w_next), (y, saved_k) + + (y, w), (ys_k, saved_k) = _sublayer_scan(same_group, (y, w), jnp.arange(n - 1)) + parts_y.append(ys_k) + parts_saved.append(saved_k) + + def last(carry, xs_last, i=i, gather_nxt=gather_nxt): + y, w = carry + layer_state, nxt = xs_last + w_next = gather_nxt(nxt) # AG(j+1) || sub-layer j, where sub-layer j+1 is of another group. + y_out, saved_last = fwd(i, y, w, layer_state) + return (y_out, w), (w_next, saved_last) + + # The last sub-layer's state and next params ride in xs, so XLA cannot hoist them as loop invariants. + last_xs = fsdp_prefetch._expand0((fsdp_prefetch._slice_at(b_state[i], n - 1), nxt_first)) + y_last_in = y + (y, _), (w_next, saved_last) = _sublayer_scan(last, (y, w), last_xs) + w = fsdp_prefetch._slice_at(w_next, 0) + parts_y.append(fsdp_prefetch._expand0(y_last_in)) + parts_saved.append(saved_last) + ys.append(_concat0(*parts_y)) + saved.append(_concat0(*parts_saved)) + return (y, w), (tuple(ys), tuple(saved)) + + w = gather[0](fsdp_prefetch._slice_at(fsdp_prefetch._slice_at(params[0], 0), 0)) + xs = (jnp.arange(length), tuple(state), tuple(params)) + (y, _), (ys, saved) = jax.lax.scan(body, (x_in, w), xs) + residuals = (ys, saved, params_leaves, state_leaves, diff_args_leaves, diff_kwargs_leaves) + return (y, state_leaves), residuals + + +def _custom_vjp_nested_prefetch_pipeline_bwd(graphdefs, length, ctxs, sizes, res, g_out): + """Backward pass of `_custom_vjp_nested_prefetch_pipeline`: one reverse scan step per block.""" + ys, saved, params_leaves, state_leaves, d_args, d_kwargs = res + g, _ = g_out # Non-parameter layer state is not differentiated. + num_groups = len(graphdefs) + params, state, gather, rs_fns, layout_fns, _, layer_vjp_fns = _setup_groups( + graphdefs, params_leaves, state_leaves, d_args, d_kwargs, ctxs + ) + # Tie `params` to `g` so XLA cannot CSE the backward all-gathers with the forward ones or hoist + # backward weight slices before the forward pass. + g, params = jax.lax.optimization_barrier((g, tuple(params))) + layouts = {} + + def bwd(i, y, w, layer_state, g, saved_k): + if i not in layouts: + layouts[i] = layout_fns[i](y, w, layer_state) + g, dw = layer_vjp_fns[i](y, w, layer_state, g, saved_k, layouts[i]) + return g, rs_fns[i](dw) + + def body(carry, xs): + g, w = carry # `w` is this block's gathered last sub-layer. + b, b_ys, b_saved, b_state, b_params = xs # Group i's leaves are `[sizes[i], ...]`. + dws = [None] * num_groups + for i in reversed(range(num_groups)): + n = sizes[i] + if i > 0: + prv_last, gather_prv = fsdp_prefetch._slice_at(b_params[i - 1], sizes[i - 1] - 1), gather[i - 1] + else: + # Block 0 prefetches the last block's last sub-layer, which is unused (see the forward pass). + prv_last = fsdp_prefetch._slice_at(_index_at(params[-1], (b - 1) % length), sizes[-1] - 1) + gather_prv = gather[-1] + d_rest = None + if n >= 2: + + def same_group(carry, k, i=i, p=b_params[i], s=b_state[i], y_=b_ys[i], sv=b_saved[i]): + g, w = carry + w_prev = gather[i](_index_at(p, k - 1)) # AG(j-1) || bwd(j). + g, dw = bwd(i, _index_at(y_, k), w, _index_at(s, k), g, _index_at(sv, k)) + return (g, w_prev), dw + + # Iterates k = n-1 .. 1; outputs are stacked in k order. + (g, w), d_rest = _sublayer_scan(same_group, (g, w), jnp.arange(1, n), reverse=True) + + def first(carry, xs_first, i=i, gather_prv=gather_prv): + g, w = carry + y0, saved0, layer_state, prv = xs_first + w_prev = gather_prv(prv) # AG(j-1) || bwd(j), where sub-layer j-1 is of another group. + g, dw = bwd(i, y0, w, layer_state, g, saved0) + return (g, w), (w_prev, dw) + + # The first sub-layer's inputs ride in xs, so XLA cannot hoist its recompute as a loop invariant. + first_xs = fsdp_prefetch._expand0( + tuple(fsdp_prefetch._slice_at(t, 0) for t in (b_ys[i], b_saved[i], b_state[i])) + (prv_last,) + ) + (g, _), (w_prev, dw0) = _sublayer_scan(first, (g, w), first_xs) + w = fsdp_prefetch._slice_at(w_prev, 0) + dws[i] = dw0 if d_rest is None else _concat0(dw0, d_rest) + return (g, w), tuple(dws) + + w = gather[-1](fsdp_prefetch._slice_at(fsdp_prefetch._slice_at(params[-1], length - 1), sizes[-1] - 1)) + xs = (jnp.arange(length), ys, saved, tuple(state), tuple(params)) + (g, _), d_params = jax.lax.scan(body, (g, w), xs, reverse=True) + + # `d_params[i]` holds group i's grads stacked `[length, sizes[i], ...]`. + params_bar_leaves = tuple(tuple(jax.tree_util.tree_leaves(d)) for d in d_params) + diff_args_bar_leaves = tuple(jnp.zeros_like(x) for x in d_args) + diff_kwargs_bar_leaves = tuple(jnp.zeros_like(x) for x in d_kwargs) + return g, params_bar_leaves, None, diff_args_bar_leaves, diff_kwargs_bar_leaves + + +_custom_vjp_nested_prefetch_pipeline.defvjp( + _custom_vjp_nested_prefetch_pipeline_fwd, _custom_vjp_nested_prefetch_pipeline_bwd +) + + +def _prefetch_sublayer_names(layers): + """Names of the sub-layers to prefetch one at a time, or None to prefetch whole scan steps. + + A block with named sub-layer attributes opts in by defining `prefetch_sublayer_names()`, returning + its sub-layer attribute names in call order. Each sub-layer must accept the block's call arguments + and return `(x, ...)`. + """ + names_fn = getattr(layers, "prefetch_sublayer_names", None) + if names_fn is None: + return None + names = tuple(names_fn()) + return names if len(names) >= 2 else None + + +def _unstack_variable(var, num_local: int, var_scan_axis: int): + """Splits one variable of an inner-scanned sub-layer module into its `num_local` per-sub-layer slices. + + The `local_layers` axis is located from the variable's scan metadata and is dropped from its sharding + metadata. Leaves that are not `nnx.Variable`s, and variables without a `local_layers` axis of size + `num_local` (e.g. a scalar shared by every sub-layer), are passed to every sub-layer unchanged. + """ + if not isinstance(var, nnx.Variable): + return (var,) * num_local + val = var.get_value() + meta = var.get_metadata() + updates = {} + for key in ("sharding", "out_sharding", "kernel_axes", "sharding_names"): + axes = getattr(var, key, None) + if axes is None: + axes = meta.get(key) + if isinstance(axes, (jax.sharding.PartitionSpec, tuple, list)) and "local_layers" in axes: + filtered = [a for a in axes if a != "local_layers"] + updates[key] = ( + jax.sharding.PartitionSpec(*filtered) if isinstance(axes, jax.sharding.PartitionSpec) else tuple(filtered) + ) + ndim = getattr(val, "ndim", 0) + if meta.get(nnx.PARTITION_NAME) == "local_layers": + # Only stacked by the inner scan (the block scan re-labels the variables it stacks as "layers"). + local_axis = var_scan_axis if ndim > var_scan_axis else 0 + else: + inner_axis = var_scan_axis if ndim - 1 > var_scan_axis else 0 + outer_axis = var_scan_axis if ndim > var_scan_axis else 0 + local_axis = inner_axis + 1 if inner_axis >= outer_axis else inner_axis + if ndim <= local_axis or val.shape[local_axis] != num_local: + stripped = var.replace(**updates) if updates else var + return (stripped,) * num_local + slices = jnp.unstack(val, axis=local_axis) + return tuple(var.replace(value=s, **updates) for s in slices) + + +def _unstack_scanned_sublayers(sublayers, num_local: int, scan_axis: int): + """Unstacks an inner-scanned sub-layer module (`local_layers`) into `num_local` `(graphdef, params, state)` stages.""" + graphdef, params, state = nnx.split(sublayers, nnx.Param, ...) + + def unstack_state(tree, var_scan_axis): + leaves, treedef = jax.tree_util.tree_flatten(tree, is_leaf=lambda x: isinstance(x, nnx.Variable)) + if not leaves: + return [tree] * num_local + unstacked = [_unstack_variable(leaf, num_local, var_scan_axis) for leaf in leaves] + return [jax.tree_util.tree_unflatten(treedef, [u[i] for u in unstacked]) for i in range(num_local)] + + params_per_stage = unstack_state(params, scan_axis) + state_per_stage = unstack_state(state, 0) + return [(graphdef, p, s) for p, s in zip(params_per_stage, state_per_stage)] + + +def sublayer_stages(layers): + """`nnx.split(sublayer, nnx.Param, ...)` of each sub-layer to prefetch, or None. + + None means the block should prefetch whole scan steps (`fsdp_prefetch`): it does not opt in (via + `local_layers` + `global_layer` or `_prefetch_sublayer_names`) or it has params outside those + sub-layers. + """ + num_local = getattr(layers, "num_local", 0) + num_global = getattr(layers, "num_global", 0) + local_layers = getattr(layers, "local_layers", None) + global_layer = getattr(layers, "global_layer", None) + if (local_layers is not None or global_layer is not None) and num_local + num_global >= 2: + scan_axis = getattr(getattr(layers, "config", None), "param_scan_axis", 0) + stages = [] + if local_layers is not None and num_local > 0: + stages.extend(_unstack_scanned_sublayers(local_layers, num_local, scan_axis)) + if global_layer is not None and num_global == 1: + stages.append(nnx.split(global_layer, nnx.Param, ...)) + else: + names = _prefetch_sublayer_names(layers) + if names is None: + return None + stages = [nnx.split(getattr(layers, name), nnx.Param, ...) for name in names] + if len(stages) < 2: + return None + block_params = nnx.state(layers, nnx.Param) + stage_param_size = sum(x.size for _, p, _ in stages for x in jax.tree_util.tree_leaves(p)) + block_param_size = sum(x.size for x in jax.tree_util.tree_leaves(block_params)) + if stage_param_size != block_param_size: + max_logging.info("prefetch_fsdp_weights: the block has params outside its sub-layers; prefetching whole blocks.") + return None + return stages + + +def apply_layers_with_fsdp_prefetch( + layers, stages, x_in, *args, length: int, mesh, config, remat_policy=None, prevent_cse: bool = False, **kwargs +): + """Runs a scanned inhomogeneous block stack, prefetching FSDP weights one sub-layer at a time. + + Args: + layers: The scanned (block-stacked) NNX module. + stages: `sublayer_stages(layers)` (not None). + x_in: The input activation of the first block. + *args: Positional arguments passed to every sub-layer call (the block's arguments). + length: Number of blocks in the stack. + mesh: The device mesh. + config: The MaxText config. + remat_policy: The `jax.checkpoint` policy of each sub-layer (None for full remat). + prevent_cse: `prevent_cse` of the per-sub-layer `jax.checkpoint` (only used with a remat policy). + **kwargs: Keyword arguments passed to every sub-layer call (those not accepted by the block are dropped). + + Returns: + (final activation, layers, None), as `fsdp_prefetch.apply_layers_with_fsdp_prefetch`. + """ + if length == 0: + return x_in, layers, None + _, _, _, diff_args_leaves, diff_kwargs_leaves, ctx = fsdp_prefetch._prepare_prefetch_call( + layers, args, kwargs, length=length, mesh=mesh, config=config, remat_policy=remat_policy, prevent_cse=prevent_cse + ) + # Groups consecutive sub-layers of one structure (e.g. the unstacked `local_layers`), stacked + # `[length, n, ...]`, so each group can be scanned by sub-layer. + groups = [] # [graphdef, params_treedef, state_treedef, [params leaves per sub-layer], [state leaves ...]] + for stage_graphdef, p, s in stages: + p, s = fsdp_prefetch._layer_major(p, s, config.param_scan_axis, length) + p_leaves, p_treedef = jax.tree_util.tree_flatten(p) + s_leaves, s_treedef = jax.tree_util.tree_flatten(s) + shapes = [x.shape for x in p_leaves + s_leaves] + if groups: + g = groups[-1] + if ( + g[0] == stage_graphdef + and g[1] == p_treedef + and g[2] == s_treedef + and [x.shape for x in g[3][0] + g[4][0]] == shapes + ): + g[3].append(p_leaves) + g[4].append(s_leaves) + continue + groups.append([stage_graphdef, p_treedef, s_treedef, [p_leaves], [s_leaves]]) + + def stack(per_sublayer): + return tuple(jnp.stack(xs, axis=1) for xs in zip(*per_sublayer)) + + final_y, _ = _custom_vjp_nested_prefetch_pipeline( + tuple(g[0] for g in groups), + x_in, + tuple(stack(g[3]) for g in groups), + tuple(stack(g[4]) for g in groups), + diff_args_leaves, + diff_kwargs_leaves, + length, + tuple(ctx._replace(params_treedef=g[1], state_treedef=g[2]) for g in groups), + tuple(len(g[3]) for g in groups), + ) + return final_y, layers, None diff --git a/src/maxtext/layers/nnx_decoders.py b/src/maxtext/layers/nnx_decoders.py index b57a9da011..670e2f260f 100644 --- a/src/maxtext/layers/nnx_decoders.py +++ b/src/maxtext/layers/nnx_decoders.py @@ -40,7 +40,7 @@ ) from maxtext.configs.types import check_forced_routing_support from maxtext.layers import linears, mhc, moe, normalizations, quantizations -from maxtext.layers import fsdp_prefetch, nnx_scan +from maxtext.layers import fsdp_prefetch, fsdp_prefetch_inhomogeneous, nnx_scan from maxtext.layers.attentions import Attention from maxtext.layers.embeddings import Embed, PositionalEmbedding, attend_on_embedding from maxtext.layers.normalizations import RMSNorm @@ -1364,18 +1364,26 @@ def layer_fn(carry, scanned_vars): def _apply_layers_sequentially_prefetch( self, layers, x_in, *args, length: int, skip_block_remat: bool = False, **kwargs ): - """Runs the scanned layer stack with FSDP weight prefetching (see `fsdp_prefetch`).""" + """Runs the scanned layer stack with FSDP weight prefetching (see `fsdp_prefetch`). + + Inhomogeneous blocks (e.g. Qwen3.5) are prefetched one sub-layer at a time (see + `fsdp_prefetch_inhomogeneous`), each sub-layer under its own remat, so `skip_block_remat` + only applies to whole-module prefetching. + """ + prefetch_kwargs = { + "length": length, + "mesh": self.mesh, + "config": self.config, + "remat_policy": self.get_remat_policy(), + "prevent_cse": maxtext_utils.should_prevent_cse_in_remat(self.config), + } + stages = fsdp_prefetch_inhomogeneous.sublayer_stages(layers) + if stages is not None: + return fsdp_prefetch_inhomogeneous.apply_layers_with_fsdp_prefetch( + layers, stages, x_in, *args, **prefetch_kwargs, **kwargs + ) return fsdp_prefetch.apply_layers_with_fsdp_prefetch( - layers, - x_in, - *args, - length=length, - mesh=self.mesh, - config=self.config, - remat_policy=self.get_remat_policy(), - prevent_cse=maxtext_utils.should_prevent_cse_in_remat(self.config), - skip_block_remat=skip_block_remat, - **kwargs, + layers, x_in, *args, skip_block_remat=skip_block_remat, **prefetch_kwargs, **kwargs ) def get_decoder_layers(self): diff --git a/tests/unit/nnx_decoders_test.py b/tests/unit/nnx_decoders_test.py index bde9b0b4b9..83a6445a6f 100644 --- a/tests/unit/nnx_decoders_test.py +++ b/tests/unit/nnx_decoders_test.py @@ -38,7 +38,7 @@ from flax import linen as nn from flax import nnx from flax import traverse_util -from jax.sharding import Mesh, NamedSharding, PartitionSpec +from jax.sharding import AbstractMesh, AxisType, Mesh, NamedSharding, PartitionSpec from maxtext.common.common_types import ( DECODING_ACTIVE_SEQUENCE_INDICATOR, @@ -53,8 +53,10 @@ from maxtext.configs import pyconfig from maxtext.integration.tunix.weight_mapping import raiden_unscan from maxtext.layers import fsdp_prefetch +from maxtext.layers import fsdp_prefetch_inhomogeneous from maxtext.layers import linears from maxtext.layers import nnx_decoders +from maxtext.layers import nnx_scan from maxtext.layers import quantizations from maxtext.layers.attentions import Attention from maxtext.layers.embeddings import Embed @@ -63,7 +65,7 @@ from maxtext.models import gemma4, gemma4_small, qwen3, qwen3_5 from maxtext.models.gpt3 import Gpt3LayerNorm from maxtext.models.llama2 import LlamaDecoderLayer -from maxtext.utils import maxtext_utils, maxtext_utils_nnx, model_creation_utils +from maxtext.utils import maxtext_utils, maxtext_utils_nnx, model_creation_utils, sharding from tests.utils.test_helpers import get_test_config_path # --------------------------------------------------------------------------- @@ -2443,8 +2445,11 @@ def setUp(self): patcher.start() self.addCleanup(patcher.stop) - def _loss_and_grads(self, num_layers, prefetch, **overrides): - """Returns (loss, grads w.r.t. the decoder params) of a small float32 decoder, jitted.""" + def _loss_and_grads(self, num_layers, prefetch, with_grads=True, **overrides): + """Returns (loss, grads w.r.t. the decoder params) of a small float32 decoder, jitted. + + With `with_grads=False` the loss is computed without differentiation and grads is None. + """ cfg_kwargs = { "scan_layers": True, "prefetch_fsdp_weights": prefetch, @@ -2487,6 +2492,8 @@ def loss_fn(p): ) return jnp.mean(logits.astype(jnp.float32) ** 2) + if not with_grads: + return jax.jit(loss_fn)(params), None return jax.jit(jax.value_and_grad(loss_fn))(params) def _assert_parity(self, num_layers, **overrides): @@ -2578,14 +2585,325 @@ def test_blocks_with_internal_remat_save_their_sub_layer_inputs(self): pass would recompute the block and then, inside the block's own remat, every sub-layer again. Saving each sub-layer's input instead takes two residuals per Qwen3.5 block -- the local scan's stacked inputs and the global layer's -- and none of them may be a gathered weight. + Qwen3.5 blocks normally prefetch one sub-layer at a time, so whole-block prefetching is forced. """ model_config = {k: v for k, v in _QWEN3_5_CONFIG.items() if k != "base_num_decoder_layers"} - with unittest.mock.patch.object(fsdp_prefetch.max_logging, "info") as log_info: + with ( + unittest.mock.patch.object(fsdp_prefetch_inhomogeneous, "sublayer_stages", return_value=None), + unittest.mock.patch.object(fsdp_prefetch.max_logging, "info") as log_info, + ): self._assert_parity(num_layers=8, remat_policy="full", **model_config) logs = [str(c) for c in log_info.call_args_list] self.assertTrue(any("remat policy saves 2 residual(s) per layer" in log for log in logs), logs) self.assertFalse(any("shaped like gathered weights" in log for log in logs), logs) +# A tiny Qwen3.5 decoder: every scan step is a block of `inhomogeneous_layer_cycle_interval` (4) +# sub-layers mixing Gated DeltaNet and global attention, so the stack is inhomogeneous inside a block. +_QWEN3_5_TINY = { + "model_name": "qwen3.5-35b-a3b", + "base_emb_dim": 64, + "base_num_query_heads": 2, + "base_num_kv_heads": 1, + "head_dim": 32, + "base_mlp_dim": 64, + "base_moe_mlp_dim": 64, + "num_experts": 4, + "num_experts_per_tok": 2, + "shared_experts": 1, + "gdn_key_head_dim": 16, + "gdn_value_head_dim": 16, + "gdn_num_key_heads": 2, + "gdn_num_value_heads": 2, + "gdn_chunk_size": 8, + "vocab_size": 128, + "use_mrope": False, + "sparse_matmul": False, + "capacity_factor": -1, +} + + +class TestNNXDecoderPrefetchInhomogeneous(unittest.TestCase): + """Per-sub-layer prefetching of inhomogeneous blocks must match the plain scanned stack.""" + + # Reuse the parity helper without inheriting (and re-running) TestNNXDecoderPrefetchGradParity's tests. + _loss_and_grads = TestNNXDecoderPrefetchGradParity._loss_and_grads # pylint: disable=protected-access + + def _assert_parity(self, num_layers, **overrides): + """Like TestNNXDecoderPrefetchGradParity._assert_parity, with a looser gradient tolerance. + + Rematerializing the Gated DeltaNet sub-layers is not bitwise identical to the plain scan: from + three blocks on, a few elements of the GDN `in_proj_qkvz` kernel gradient differ by up to ~4e-4 + (relative ~0.5%) in float32, by the same amount with whole-block prefetching. + """ + ref_loss, ref_grads = self._loss_and_grads(num_layers, prefetch=False, **overrides) + loss, grads = self._loss_and_grads(num_layers, prefetch=True, **overrides) + np.testing.assert_allclose(loss, ref_loss, rtol=1e-5, atol=1e-6) + ref_leaves, ref_def = jax.tree_util.tree_flatten(ref_grads) + leaves, tree_def = jax.tree_util.tree_flatten(grads) + self.assertEqual(ref_def, tree_def) + for ref, got in zip(ref_leaves, leaves): + self.assertEqual(ref.shape, got.shape) + np.testing.assert_allclose(np.asarray(got), np.asarray(ref), rtol=1e-2, atol=1e-3) + + def setUp(self): + super().setUp() + # Force the prefetch path on the single-device CPU test mesh (see TestNNXDecoderPrefetching). + patcher = mock.patch.object(fsdp_prefetch, "mesh_uses_fsdp", return_value=True) + patcher.start() + self.addCleanup(patcher.stop) + + def _patch_pipelines(self): + """Wraps the per-sub-layer and whole-block prefetch pipelines to count their calls.""" + # pylint: disable=protected-access + return ( + mock.patch.object( + fsdp_prefetch_inhomogeneous, + "_custom_vjp_nested_prefetch_pipeline", + wraps=fsdp_prefetch_inhomogeneous._custom_vjp_nested_prefetch_pipeline, + ), + mock.patch.object( + fsdp_prefetch, + "_custom_vjp_prefetch_pipeline", + wraps=fsdp_prefetch._custom_vjp_prefetch_pipeline, + ), + ) + + def _assert_qwen3_5_parity(self, num_blocks, cycle=4, **overrides): + """Asserts parity for `num_blocks` Qwen3.5 blocks and that sub-layers were prefetched one at a time. + + Each block has `cycle - 1` Gated DeltaNet sub-layers and one global attention sub-layer. + """ + nested_patch, whole_patch = self._patch_pipelines() + config = {**_QWEN3_5_TINY, "inhomogeneous_layer_cycle_interval": cycle, **overrides} + with nested_patch as nested, whole_patch as whole_blocks: + self._assert_parity(num_layers=num_blocks * cycle, **config) + self.assertEqual(nested.call_count, 1) + self.assertEqual(whole_blocks.call_count, 0) + graphdefs, length, sizes = nested.call_args.args[0], nested.call_args.args[6], nested.call_args.args[8] + # The Gated DeltaNet sub-layers form one scanned group, followed by the global attention sub-layer. + self.assertEqual(len(graphdefs), 2) + self.assertEqual(sizes, (cycle - 1, 1)) + self.assertEqual(length, num_blocks) + + def test_qwen3_5_one_block(self): + self._assert_qwen3_5_parity(num_blocks=1) + + def test_qwen3_5_two_blocks(self): + self._assert_qwen3_5_parity(num_blocks=2) + + def test_qwen3_5_three_blocks(self): + self._assert_qwen3_5_parity(num_blocks=3) + + def test_qwen3_5_four_blocks(self): + # Two or more scanned blocks, so the scan body orders the sub-layers' prefetches. + self._assert_qwen3_5_parity(num_blocks=4) + + def test_qwen3_5_expert_parallelism(self): + """Per-sub-layer prefetching matches the plain scanned stack on an fsdp=2 x expert=2 mesh.""" + if jax.device_count() < 4: + self.skipTest("Needs 4 devices for an fsdp=2 x expert=2 mesh.") + self._assert_qwen3_5_parity(num_blocks=2, ici_fsdp_parallelism=2, ici_expert_parallelism=2) + + def test_qwen3_5_sub_layer_gathers_under_expert_parallelism(self): + """Under EP, each sub-layer weight is gathered over the FSDP axes `sharding.fsdp_axes_for_dim` picks. + + Non-routed weights (Gated DeltaNet, attention, router and shared expert) are sharded over + ('fsdp', 'expert') on their embed dimension and get one tagged all-gather over both axes; routed + experts keep `expert` on their expert dimension and gather only `fsdp`. The gathers are lowered on an + abstract explicit-sharding mesh, so no devices are needed. + """ + cfg = _make_config( + **{ + **_QWEN3_5_TINY, + "scan_layers": True, + "num_decoder_layers": 8, + "base_num_decoder_layers": 8, + "inhomogeneous_layer_cycle_interval": 4, + } + ) + decoder = NNXDecoder(config=cfg, mesh=_make_mesh(cfg), model_mode=MODEL_MODE_TRAIN, rngs=nnx.Rngs(0)) + stages = fsdp_prefetch_inhomogeneous.sublayer_stages(decoder.layers) + self.assertEqual(len(stages), 4) + axis_sizes = {axis: 1 for axis in cfg.mesh_axes} | {"fsdp": 4, "expert": 2} + mesh = AbstractMesh(tuple(axis_sizes.values()), tuple(axis_sizes), axis_types=(AxisType.Explicit,) * len(axis_sizes)) + + def logical_axes(var): + meta = var.get_metadata() + for key in ("sharding", "out_sharding", "sharding_names"): + axes = meta.get(key) + if isinstance(axes, (PartitionSpec, tuple, list)) and axes: + return tuple(a for a in axes if a != "layers") # Drop the block scan axis. + return None + + def lower_gather(spec, shape): + w = jax.ShapeDtypeStruct(shape, jnp.float32, sharding=NamedSharding(mesh, spec)) + + def gather(x): + return fsdp_prefetch._apply_sharding_hint(x, mesh, ShardMode.EXPLICIT) # pylint: disable=protected-access + + with jax.sharding.use_abstract_mesh(mesh): + out = jax.eval_shape(gather, w) + hlo = jax.jit(gather).trace(w).lower(lowering_platforms=("tpu",)).as_text() + return out, [line for line in hlo.splitlines() if "stablehlo.all_gather" in line] + + checked = {"routed": 0, "co_sharded": 0} + # The first Gated DeltaNet sub-layer and the global attention sub-layer. + for _, params, _ in (stages[0], stages[-1]): + for path, var in nnx.to_flat_state(params): + axes = logical_axes(var) + if not axes: + continue # Replicated (e.g. A_log, dt_bias): nothing to gather. + spec = nn.logical_to_mesh_axes(axes, cfg.logical_axis_rules) + shape = list(var.get_value().shape) + del shape[cfg.param_scan_axis] # One block's slice, as the pipeline gathers it. + out, gathers = lower_gather(spec, tuple(shape)) + name = "/".join(map(str, path)) + if not any(sharding.fsdp_axes_for_dim(entry) for entry in spec): + self.assertEqual(gathers, [], name) + continue + self.assertEqual(len(gathers), 1, name) + self.assertIn('xla_explicit_fsdp = "yes"', gathers[0], name) + self.assertIn('name = "fsdp"', gathers[0], name) + if axes[0] == "exp": + checked["routed"] += 1 + self.assertNotIn('name = "expert"', gathers[0], name) + self.assertEqual(tuple(out.sharding.spec)[0], "expert", name) + else: + checked["co_sharded"] += 1 + self.assertIn('name = "expert"', gathers[0], name) + self.assertNotIn("expert", str(out.sharding.spec), name) + # wi_0/wi_1/wo of both sub-layers; the GDN, attention, router and shared-expert kernels. + self.assertEqual(checked["routed"], 6) + self.assertGreaterEqual(checked["co_sharded"], 14) + + def test_qwen3_5_remat_minimal(self): + with unittest.mock.patch.object(fsdp_prefetch.max_logging, "info") as log_info: + self._assert_qwen3_5_parity(num_blocks=4, remat_policy="minimal") + self.assertTrue(any("remat policy saves" in str(c) for c in log_info.call_args_list)) + + def test_qwen3_5_param_scan_axis_zero(self): + self._assert_qwen3_5_parity(num_blocks=3, param_scan_axis=0) + + def test_qwen3_5_one_local_layer_per_block(self): + # A single Gated DeltaNet sub-layer per block: no same-group scan, two trip-count-one loops. + self._assert_qwen3_5_parity(num_blocks=2, cycle=2) + + def test_qwen3_5_two_local_layers_per_block(self): + # The same-group scan of the Gated DeltaNet sub-layers has a single trip. + self._assert_qwen3_5_parity(num_blocks=2, cycle=3) + + def test_qwen3_5_five_local_layers_per_block(self): + self._assert_qwen3_5_parity(num_blocks=2, cycle=6) + + def test_qwen3_5_forward_only(self): + # Without differentiation, jax.custom_vjp runs the pipeline's primal function instead of its fwd rule. + config = {**_QWEN3_5_TINY, "inhomogeneous_layer_cycle_interval": 4} + ref_loss, _ = self._loss_and_grads(8, prefetch=False, with_grads=False, **config) + nested_patch, _ = self._patch_pipelines() + with nested_patch as nested: + loss, _ = self._loss_and_grads(8, prefetch=True, with_grads=False, **config) + self.assertEqual(nested.call_count, 1) + np.testing.assert_allclose(loss, ref_loss, rtol=1e-5, atol=1e-6) + + def test_qwen3_5_global_only_blocks_prefetch_whole_blocks(self): + # With a cycle of 1 every block is a single global attention layer: nothing to split, so the + # stack keeps prefetching whole scan steps. + nested_patch, whole_patch = self._patch_pipelines() + with nested_patch as nested, whole_patch as whole_blocks: + self._assert_parity(num_layers=3, **{**_QWEN3_5_TINY, "inhomogeneous_layer_cycle_interval": 1}) + self.assertEqual(nested.call_count, 0) + self.assertEqual(whole_blocks.call_count, 1) + + def test_homogeneous_decoder_prefetches_whole_layers(self): + # A homogeneous decoder (no `prefetch_sublayer_names()`) must keep using `fsdp_prefetch`. + nested_patch, whole_patch = self._patch_pipelines() + with nested_patch as nested, whole_patch as whole_layers: + self._loss_and_grads(3, prefetch=True) + self.assertEqual(nested.call_count, 0) + self.assertEqual(whole_layers.call_count, 1) + + def test_sublayer_stages_require_all_params_in_sublayers(self): + class Block(nnx.Module): + + def __init__(self, extra): + self.layer_0 = nnx.Linear(2, 2, rngs=nnx.Rngs(0)) + self.layer_1 = nnx.Linear(2, 3, rngs=nnx.Rngs(1)) + if extra: + self.extra = nnx.Param(jnp.ones((2,))) + + def prefetch_sublayer_names(self): + return ("layer_0", "layer_1") + + # pylint: disable=protected-access + block = Block(extra=False) + stages = fsdp_prefetch_inhomogeneous.sublayer_stages(block) + self.assertEqual(len(stages), 2) + self.assertEqual(stages[1][1]["kernel"].shape, (2, 3)) + block = Block(extra=True) + with unittest.mock.patch.object(fsdp_prefetch_inhomogeneous.max_logging, "info") as log_info: + self.assertIsNone(fsdp_prefetch_inhomogeneous.sublayer_stages(block)) + self.assertTrue(any("params outside its sub-layers" in str(c) for c in log_info.call_args_list)) + + def test_single_sublayer_block_prefetches_whole_blocks(self): + block = SimpleNamespace(prefetch_sublayer_names=lambda: ("layer_0",)) + # pylint: disable=protected-access + self.assertIsNone(fsdp_prefetch_inhomogeneous._prefetch_sublayer_names(block)) + self.assertIsNone(fsdp_prefetch_inhomogeneous._prefetch_sublayer_names(SimpleNamespace())) + + def test_sublayer_stages_local_layers_without_global_layer(self): + class Block(nnx.Module): + + def __init__(self): + self.num_local, self.num_global, self.global_layer = 2, 0, None + self.config = SimpleNamespace(param_scan_axis=1) + self.local_layers = nnx_scan.create_scanned_layers( + lambda r: nnx.Linear(2, 3, rngs=r), + length=2, + param_scan_axis=1, + metadata_axis_name="local_layers", + rngs=nnx.Rngs(0), + ) + + stages = fsdp_prefetch_inhomogeneous.sublayer_stages(Block()) + self.assertEqual(len(stages), 2) + self.assertEqual(stages[0][1]["kernel"].shape, (2, 3)) + + def test_sublayer_stages_with_fewer_than_two_sublayers_prefetch_whole_blocks(self): + class Block(nnx.Module): + + def __init__(self): + # Claims two local layers but has none: only the global layer can be split off. + self.num_local, self.num_global, self.local_layers = 2, 1, None + self.global_layer = nnx.Linear(2, 2, rngs=nnx.Rngs(0)) + + self.assertIsNone(fsdp_prefetch_inhomogeneous.sublayer_stages(Block())) + + def test_unstack_inner_scanned_sublayers(self): + # pylint: disable=protected-access + for scan_axis in (0, 1): + local_layers = nnx_scan.create_scanned_layers( + lambda r: nnx.Linear(2, 3, rngs=r), + length=3, + param_scan_axis=scan_axis, + metadata_axis_name="local_layers", + rngs=nnx.Rngs(0), + ) + stacked = nnx.state(local_layers, nnx.Param)["kernel"].get_value() + stages = fsdp_prefetch_inhomogeneous._unstack_scanned_sublayers(local_layers, 3, scan_axis) + self.assertEqual(len(stages), 3) + for i, (_, params, _) in enumerate(stages): + kernel = params["kernel"] + np.testing.assert_array_equal(kernel.get_value(), jnp.take(stacked, i, axis=scan_axis)) + self.assertNotIn("local_layers", tuple(kernel.get_metadata().get("sharding_names") or ())) + + def test_unstack_variable_passes_unstacked_leaves_to_every_sublayer(self): + # pylint: disable=protected-access + raw = jnp.ones((3, 4)) + self.assertEqual(fsdp_prefetch_inhomogeneous._unstack_variable(raw, 3, 0), (raw, raw, raw)) + shared = nnx.Variable(jnp.zeros(())) + self.assertEqual(fsdp_prefetch_inhomogeneous._unstack_variable(shared, 3, 0), (shared, shared, shared)) + + if __name__ == "__main__": unittest.main()