From c107fed33857a334420a0a6b3222521bf5eed5d7 Mon Sep 17 00:00:00 2001 From: Yuchen Yao Date: Tue, 17 Nov 2026 00:33:34 +0000 Subject: [PATCH] Add explicit FSDP weight prefetching for m3 ScannedLayers Inside scanned m3 decoder layers, XLA latency-hiding flags alone produce byte-identical HLOs and leave per-layer FSDP weight all-gathers serialized or split across windowed einsums on TPU7x. Adding double-buffered FSDP weight prefetching hides layer k+1's weight all-gather (and optionally layer k+1's gradient reduce-scatter) behind layer k's compute while keeping parameter paths unchanged and leaving compiled HLO byte-identical when disabled. - Add `src/maxtext/m3/infra/fsdp_prefetch.py`: path-transparent `FsdpPrefetchRunner`, `apply_fsdp_prefetch`, and `maybe_apply_fsdp_prefetch` implementing a `jax.custom_vjp` `unroll=2` double-buffered scan pipeline supporting explicit and auto meshes, multi-axis sharding (2D FSDP, FSDP+TP, FSDP+DP), SparseCore 1 pinning on multi-trip TPU7x scan loops (`num_layers >= 6`), host-stack residual streaming for `minimal_offloaded` remat policies, fail-closed rejection of `remat_policy='none'`, and safe fallback for `num_layers <= 1`, non-FSDP meshes, decode mode, and `parameter_memory_host_offload=True`. - Add optional `fsdp_prefetch` hook on `ScannedLayers` in `src/maxtext/m3/infra/scan.py` (preserving `scan -> fsdp_prefetch` and `fsdp_prefetch -> scan` order invariance and zero new imports). - Add `prefetch_fsdp_weights` and `prefetch_fsdp_defer_grad_reduce_scatter` (both default `False`) to `TrainerConfig` in `src/maxtext/m3/configs/config.py`, and wire `maybe_apply_fsdp_prefetch` into both production drivers (`src/maxtext/utils/model_creation_utils.py` and `src/maxtext/m3/train/loop.py`). - Add `tests/unit/m3/fsdp_prefetch_test.py` covering helper logic, forward and backward parity across explicit/auto/multi-axis/stateful/minimal_offloaded configurations with perturbation negative controls, fail-closed checks, dual-driver wiring, compiled-HLO byte-identity when disabled, and compiled-HLO overlap structure when enabled. --- src/maxtext/m3/configs/config.py | 16 + src/maxtext/m3/infra/__init__.py | 3 +- src/maxtext/m3/infra/fsdp_prefetch.py | 1040 +++++++++++++++++++++ src/maxtext/m3/infra/scan.py | 30 +- src/maxtext/m3/train/loop.py | 2 + src/maxtext/utils/model_creation_utils.py | 2 + tests/unit/m3/fsdp_prefetch_test.py | 741 +++++++++++++++ 7 files changed, 1827 insertions(+), 7 deletions(-) create mode 100644 src/maxtext/m3/infra/fsdp_prefetch.py create mode 100644 tests/unit/m3/fsdp_prefetch_test.py diff --git a/src/maxtext/m3/configs/config.py b/src/maxtext/m3/configs/config.py index 2e3f04c342..35b669e646 100644 --- a/src/maxtext/m3/configs/config.py +++ b/src/maxtext/m3/configs/config.py @@ -184,6 +184,22 @@ class TrainerConfig(BaseModel): remat_policy: str = "NONE" names_which_can_be_saved: list[str] = Field(default_factory=list) names_which_can_be_offloaded: list[str] = Field(default_factory=list) + prefetch_fsdp_weights: bool = Field( + default=False, + description=( + "Overlap next scanned layer's FSDP weight all-gather with current layer compute (requires" + " scan_layers=True and non-'none' remat_policy). Known limitation: remat_policy='minimal_offloaded'" + " compiles and matches numerics, but synchronous host-stack DMA stalls PCIe on TPU7x; pair with" + " remat_policy='full' or device-resident savepoints for peak throughput." + ), + ) + prefetch_fsdp_defer_grad_reduce_scatter: bool = Field( + default=False, + description=( + "With prefetch_fsdp_weights and shard_mode='explicit', defer each layer's weight-gradient" + " reduce-scatter to the next layer's backward step (extra HBM for one layer of unreduced grads)." + ), + ) enable_tile_search: bool = False tile_search_mode: str = "smart" tile_search_iters: int = 10 diff --git a/src/maxtext/m3/infra/__init__.py b/src/maxtext/m3/infra/__init__.py index 197fff3c00..2c963c2072 100644 --- a/src/maxtext/m3/infra/__init__.py +++ b/src/maxtext/m3/infra/__init__.py @@ -17,6 +17,7 @@ Modules: * `scan.py` — layer stacking via `jax.lax.scan` + `maybe_apply_scan`. * `remat.py` — path-transparent remat policies, `maybe_apply_remat`, and `check_tags`. + * `fsdp_prefetch.py` — double-buffered FSDP weight prefetch & deferred reduce-scatter for `ScannedLayers`. * `quantization.py` — `maybe_quantize`: qwix int8 / fp8 by interception, over the projection layers of the decoder stack. * `named_scopes.py` — `jax.named_scope` per submodule, so profiles and HLO @@ -27,7 +28,7 @@ Composition order is fixed, and `maxtext.m3.train.loop.setup_model` is the reference for it: -`construct -> scan -> remat -> quantize -> named scopes -> optimizer -> host offload`. +`construct -> scan -> remat -> fsdp_prefetch -> quantize -> named scopes -> optimizer -> host offload`. None of these wrap the model in another module, and quantization leaves the variable tree untouched (asserted). Model files contain only zero-cost diff --git a/src/maxtext/m3/infra/fsdp_prefetch.py b/src/maxtext/m3/infra/fsdp_prefetch.py new file mode 100644 index 0000000000..186306541f --- /dev/null +++ b/src/maxtext/m3/infra/fsdp_prefetch.py @@ -0,0 +1,1040 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Path-transparent FSDP weight prefetching for m3 `ScannedLayers`. + +Enabled with `prefetch_fsdp_weights=True` (requires `scan_layers=True` and a +non-`none` remat policy). Replaces the default `unroll=1` scan execution with a +`jax.custom_vjp` `unroll=2` double-buffered pipeline that overlaps: + + * Forward layer k: all-gather W(k+1) || compute layer k + * Backward layer k: all-gather W(k-1) || (recompute + vjp layer k), + plus reduce-scatter dW(k) (or deferred dW(k+1) when + `prefetch_fsdp_defer_grad_reduce_scatter=True` under + `shard_mode=explicit`). + +The transform attaches a static runner to each `ScannedLayers` instance in place +without inserting wrapper modules, keeping `nnx.state(model, nnx.Param)` paths +identical and leaving compiled HLO byte-identical when disabled. + +Known limitation: `remat_policy='minimal_offloaded'` (and offloaded `custom` +residuals) compile and match numerics via explicit pinned-host stack streaming, +but synchronous host-stack updates currently stall PCIe DMA on TPU7x (`~3.3x` +slower than unprefetched `minimal_offloaded`); pair `prefetch_fsdp_weights=True` +with `remat_policy='full'` or device-resident savepoints for peak throughput. +""" + +import collections +from collections.abc import Callable +import dataclasses +import functools +from typing import Any + +from flax import nnx +import jax +from jax import ad_checkpoint +from jax import checkpoint_policies as cp +from jax.experimental.layout import Layout, with_layout_constraint +import jax.numpy as jnp +from jax.sharding import AxisType, Mesh, NamedSharding +from jax.sharding import PartitionSpec as P + +from maxtext.m3.core import sharding as m3_sharding +from maxtext.m3.infra import scan as m3_scan + +_TRAIN_MODE = "train" +FSDP_MESH_AXES = ("fsdp", "fsdp_transpose") +FSDP_CO_AXES = FSDP_MESH_AXES + ("context", "expert") +# Activation contracting axes besides FSDP that can contribute to unreduced weight grads. +_EXTRA_BATCH_AXES = ("data", "context", "expert") + +_PASSTHROUGH_PRIMITIVES = frozenset( + {"name", "copy", "copy_p", "convert_element_type", "sharding_constraint", "reshard", "mesh_cast"} +) + + +def fsdp_axes_for_dim(entry: Any) -> tuple[str, ...]: + """Returns FSDP and trailing co-sharded axes on one PartitionSpec dimension.""" + axes = () if entry is None else (tuple(entry) if isinstance(entry, (tuple, list)) else (entry,)) + first_fsdp = next((i for i, a in enumerate(axes) if a in FSDP_MESH_AXES), None) + if first_fsdp is None: + return () + return tuple(a for a in axes[first_fsdp:] if a in FSDP_CO_AXES) + + +def remove_fsdp_from_spec(spec: P) -> P: + """Strips FSDP and trailing co-sharded axes from every dimension of `spec`.""" + out = [] + for entry in tuple(spec): + axes = () if entry is None else (tuple(entry) if isinstance(entry, (tuple, list)) else (entry,)) + drop = set(fsdp_axes_for_dim(axes)) + kept = tuple(a for a in axes if a not in drop) + out.append(kept if len(kept) > 1 else (kept[0] if kept else None)) + return P(*out) + + +def mesh_uses_fsdp(mesh: Any) -> bool: + """Whether `mesh` has an active FSDP axis (`fsdp` or `fsdp_transpose` > 1).""" + if mesh is None or getattr(mesh, "empty", False): + return False + shape = getattr(mesh, "shape", None) or {} + return any(int(shape.get(axis, 1)) > 1 for axis in FSDP_MESH_AXES) + + +def _is_explicit_mesh(mesh: Any) -> bool: + axis_types = tuple(getattr(mesh, "axis_types", ())) + return bool(axis_types) and all(t == AxisType.Explicit for t in axis_types) + + +def _default_sparse_core_id(mesh: Any) -> int | None: + """Pins FSDP prefetch collectives to SparseCore 1 on dual-SparseCore TPU7x.""" + devices = getattr(mesh, "devices", None) + if devices is None or devices.size == 0: + return None + device_kind = getattr(devices.flat[0], "device_kind", "").lower() + if any(tag in device_kind for tag in ("7x", "tpu7", "v7")): + return 1 + return None + + +def _has_multi_trip_loop(length: int) -> bool: + """Returns True when the `unroll=2` middle scan has >= 2 trips (`length >= 6`). + + The pipeline peels layer 0 and layer `length - 1` outside an `unroll=2` scan + over the remaining `length - 2` layers. When `(length - 2) // 2 < 2` + (`length < 6`), XLA unrolls/inlines the body into `ENTRY`, where TPU7x + `compute_on("tpu_sparsecore")` hits `UNIMPLEMENTED: ComputeEmitter` on + `all-gather(%convert)`. Gating SparseCore pinning, intra-loop barriers, and + host-stack offloading on `length >= 6` avoids that crash. + """ + return length > 2 and (length - 2) // 2 >= 2 + + +def _on_sparse_core(fn: Callable[..., Any], core_id: int | None, x: jax.Array | None = None) -> Callable[..., Any]: + """Pins `fn` to SparseCore `core_id` when supported by the operand shape.""" + if core_id is None or core_id < 0: + return fn + if x is not None and (x.ndim < 2 or x.shape[-1] < 128): + return fn + from jax.experimental.compute_on import compute_on # pylint: disable=import-outside-toplevel + + return compute_on( + fn, + compute_type="tpu_sparsecore", + out_memory_spaces=jax.memory.Space.Device, + compiler_options={"sparse_core_config": {"core_ids": [core_id]}}, + ) + + +def _register_overlap_transform(mesh: Any) -> None: + """Registers the TPU post-scheduler async overlap pass when running on TPU.""" + devices = getattr(mesh, "devices", None) + if devices is None or devices.size == 0 or getattr(devices.flat[0], "platform", None) != "tpu": + return + from maxtext.layers import fsdp_prefetch_overlap_transform # pylint: disable=import-outside-toplevel + + fsdp_prefetch_overlap_transform.register_transform(platforms="tpu") + + +def _extra_batch_axes(mesh: Any, spec: P, fsdp_axes: tuple[str, ...]) -> tuple[str, ...]: + used = {a for entry in tuple(spec) for a in ((entry,) if isinstance(entry, str) else (entry or ()))} + return tuple(a for a in _EXTRA_BATCH_AXES if mesh.shape.get(a, 1) > 1 and a not in used and a not in fsdp_axes) + + +def _expected_unreduced_axes(mesh: Any, src_spec: P, fsdp_axes: tuple[str, ...]) -> frozenset[str]: + return frozenset(fsdp_axes) | frozenset(_extra_batch_axes(mesh, src_spec, fsdp_axes)) + + +def _fsdp_dims(ndim: int, mesh: Any, src_spec: P) -> list[tuple[int, tuple[str, ...]]] | None: + """Returns `(dim, fsdp_axes)` pairs where FSDP axes are minor-most, or None if non-minor.""" + spec = tuple(src_spec) + (None,) * (ndim - len(tuple(src_spec))) + dims: list[tuple[int, tuple[str, ...]]] = [] + for i, entry in enumerate(spec): + axes = () if entry is None else (tuple(entry) if isinstance(entry, (tuple, list)) else (entry,)) + axes = tuple(a for a in axes if mesh.shape.get(a, 1) > 1) + fsdp = fsdp_axes_for_dim(axes) + if not fsdp: + continue + if axes[len(axes) - len(fsdp) :] != fsdp: + return None + dims.append((i, fsdp)) + return dims + + +def _fsdp_scatter_dim(ndim: int, mesh: Any, src_spec: P) -> tuple[int, tuple[str, ...]] | None: + dims = _fsdp_dims(ndim, mesh, src_spec) + if not dims or len(dims) != 1: + return None + return dims[0] + + +def _is_unreduced_over(g: jax.Array, axes: frozenset[str]) -> bool: + g_spec = getattr(getattr(jax.typeof(g), "sharding", None), "spec", None) + return g_spec is not None and frozenset(g_spec.unreduced) == frozenset(axes) + + +def _pin_scatter_dim_major(g: jax.Array, dim: int) -> jax.Array: + """Places `dim` major-most so SparseCore can emit a direct reduce-scatter.""" + return with_layout_constraint(g, Layout((dim,) + tuple(i for i in range(g.ndim) if i != dim))) + + +def _explicit_fsdp_all_gather( + w: jax.Array, + mesh: Mesh, + src_spec: P, + gathered_spec: P, + *, + reduced: bool = False, + sparse_core_id: int | None = None, +) -> jax.Array | None: + """All-gathers `w` over its FSDP axes with an explicit `shard_map` collective.""" + dims = _fsdp_dims(w.ndim, mesh, src_spec) + if not dims: + return None + spec = tuple(src_spec) + (None,) * (w.ndim - len(tuple(src_spec))) + + if reduced and len(dims) == 1: + dim, ax = dims[0] + gathered = tuple(gathered_spec) + (None,) * (w.ndim - len(tuple(gathered_spec))) + in_spec_list = [] + for entry in spec: + entry_axes = () if entry is None else (tuple(entry) if isinstance(entry, (tuple, list)) else (entry,)) + stripped = set(fsdp_axes_for_dim(entry_axes)) + kept = tuple(a for a in entry_axes if a not in stripped or mesh.shape.get(a, 1) > 1) + in_spec_list.append(kept if len(kept) > 1 else (kept[0] if kept else None)) + in_spec = P(*in_spec_list) + if tuple(in_spec) != spec: + w = jax.sharding.reshard(w, NamedSharding(mesh, in_spec)) + extra = _extra_batch_axes(mesh, in_spec, ax) + + def all_gather_reduced(x): + return jax.lax.all_gather(x, ax if len(ax) > 1 else ax[0], axis=dim, tiled=True, to="reduced") + + def gather_reduced(x): + x = _on_sparse_core(all_gather_reduced, sparse_core_id, x)(x) + if extra: + x = jax.lax.pcast(x, extra if len(extra) > 1 else extra[0], to="reduced") + return x + + return jax.shard_map( + gather_reduced, + mesh=mesh, + in_specs=in_spec, + out_specs=P(*gathered, reduced=frozenset(ax) | frozenset(extra)), + )(w) + + def gather_all(x): + for i, ax in dims: + x = jax.lax.all_gather(x, ax if len(ax) > 1 else ax[0], axis=i, tiled=True) + return x + + def gather(x): + return _on_sparse_core(gather_all, sparse_core_id, x)(x) + + return jax.shard_map(gather, mesh=mesh, in_specs=P(*spec), out_specs=gathered_spec, check_vma=False)(w) + + +def _explicit_fsdp_reduce_scatter( + g: jax.Array, + mesh: Mesh, + src_spec: P, + gathered_spec: P, + *, + sparse_core_id: int | None = None, +) -> jax.Array | None: + """Reduce-scatters an `unreduced` gathered-weight gradient back to `src_spec`.""" + scatter = _fsdp_scatter_dim(g.ndim, mesh, src_spec) + if scatter is None or not _is_unreduced_over(g, _expected_unreduced_axes(mesh, src_spec, scatter[1])): + return None + dim, ax = scatter + extra = _extra_batch_axes(mesh, src_spec, ax) + spec = tuple(src_spec) + (None,) * (g.ndim - len(tuple(src_spec))) + gathered = tuple(gathered_spec) + (None,) * (g.ndim - len(tuple(gathered_spec))) + g_spec = jax.typeof(g).sharding.spec + g = _pin_scatter_dim_major(g, dim) + + def psum_scatter(x): + if dim != 0 and x.ndim == 2 and sparse_core_id is not None and sparse_core_id >= 0: + # `compute_on` outlines its call boundary with row-major `{1, 0}` layout; moving `dim` + # to axis 0 keeps the scatter axis major-most for direct SparseCore reduce-scatter. + x = jnp.moveaxis(x, dim, 0) + x = _on_sparse_core( + functools.partial( + jax.lax.psum_scatter, axis_name=ax if len(ax) > 1 else ax[0], scatter_dimension=0, tiled=True + ), + sparse_core_id, + x, + )(x) + x = jnp.moveaxis(x, 0, dim) + else: + x = _on_sparse_core( + functools.partial( + jax.lax.psum_scatter, axis_name=ax if len(ax) > 1 else ax[0], scatter_dimension=dim, tiled=True + ), + sparse_core_id, + x, + )(x) + if extra: + x = jax.lax.psum(x, extra if len(extra) > 1 else extra[0]) + return x + + return jax.shard_map( + psum_scatter, + mesh=mesh, + in_specs=P(*gathered, unreduced=frozenset(ax) | frozenset(extra), unreduced_kind=g_spec.unreduced_kind), + out_specs=P(*spec), + )(g) + + +def _needs_minor_dim_layout_pin(per_layer_ndim: int, layer_spec: P | None, mesh: Any) -> bool: + if per_layer_ndim < 3 or layer_spec is None or len(layer_spec) < per_layer_ndim: + return False + entry = layer_spec[per_layer_ndim - 1] + axes = () if entry is None else (tuple(entry) if isinstance(entry, (tuple, list)) else (entry,)) + axes = tuple(a for a in axes if mesh.shape.get(a, 1) > 1) + return bool(fsdp_axes_for_dim(axes)) + + +def _is_literal(v: Any) -> bool: + return hasattr(v, "val") + + +def _is_passthrough(eqn: Any) -> bool: + """Returns True when `eqn` preserves value identity for residual origin tracking.""" + if len(eqn.invars) != 1 or len(eqn.outvars) != 1: + return False + name = eqn.primitive.name + if name in _PASSTHROUGH_PRIMITIVES: + return True + if name == "reduce_precision": + dtype = eqn.invars[0].aval.dtype + if not jnp.issubdtype(dtype, jnp.floating): + return False + finfo = jnp.finfo(dtype) + return eqn.params.get("exponent_bits") == finfo.nexp and eqn.params.get("mantissa_bits") == finfo.nmant + return False + + +class _ResidualLayout(collections.namedtuple("_ResidualLayout", ["treedef", "src", "saved_avals", "saved_offload_mask"])): + """Origin map for each residual leaf of `jax.vjp(remat(layer), carry, w)`.""" + + +def _vjp_residual_layout(f: Callable[..., Any], carry: Any, w: Any, layer_rest: Any, consts: Any) -> _ResidualLayout: + """Traces `jax.vjp(f, carry, w)` once and classifies which leaves come from inputs vs policy.""" + y_leaves, y_treedef = jax.tree_util.tree_flatten(carry) + w_leaves, w_treedef = jax.tree_util.tree_flatten(w) + s_leaves, s_treedef = jax.tree_util.tree_flatten(layer_rest) + c_leaves, c_treedef = jax.tree_util.tree_flatten(consts) + treedef_box: list[Any] = [] + + def residual_leaves(yl, wl, sl, cl): + y_ = jax.tree_util.tree_unflatten(y_treedef, yl) + w_ = jax.tree_util.tree_unflatten(w_treedef, wl) + s_ = jax.tree_util.tree_unflatten(s_treedef, sl) + c_ = jax.tree_util.tree_unflatten(c_treedef, cl) + _, vjp_fn, _ = jax.vjp(lambda y2, w2: f(y2, w2, s_, c_), y_, w_, has_aux=True) + leaves, treedef = jax.tree_util.tree_flatten(vjp_fn) + treedef_box.append(treedef) + return leaves + + closed = jax.make_jaxpr(residual_leaves)(y_leaves, w_leaves, s_leaves, c_leaves) + jaxpr = closed.jaxpr + n_y = len(y_leaves) + n_w = len(w_leaves) + n_s = len(s_leaves) + origin: dict[Any, tuple[str, int]] = {} + for i, v in enumerate(jaxpr.invars): + if i < n_y: + origin[v] = ("y", i) + elif i < n_y + n_w: + origin[v] = ("w", i - n_y) + elif i < n_y + n_w + n_s: + origin[v] = ("s", i - n_y - n_w) + else: + origin[v] = ("c", i - n_y - n_w - n_s) + for eqn in jaxpr.eqns: + if _is_passthrough(eqn): + src_var, out_var = eqn.invars[0], eqn.outvars[0] + if ( + not _is_literal(src_var) + and src_var in origin + and src_var.aval.shape == out_var.aval.shape + and src_var.aval.dtype == out_var.aval.dtype + ): + origin[out_var] = origin[src_var] + + src: list[tuple[str, int] | None] = [] + saved_avals: list[Any] = [] + saved_offload_mask: list[bool] = [] + host_space = getattr(getattr(jax, "memory", None), "Space", None) + host_space_val = getattr(host_space, "Host", None) + for v in jaxpr.outvars: + s = None if _is_literal(v) else origin.get(v) + src.append(s) + if s is None: + saved_avals.append(v.aval) + mem_space = getattr(v.aval, "memory_space", None) + mem_kind = getattr(getattr(v.aval, "sharding", None), "memory_kind", None) + is_host = (host_space_val is not None and mem_space == host_space_val) or mem_kind == "pinned_host" + saved_offload_mask.append(bool(is_host)) + return _ResidualLayout(treedef_box[0], tuple(src), tuple(saved_avals), tuple(saved_offload_mask)) + + +def _split_residuals(vjp_fn: Any, layout: _ResidualLayout) -> tuple[Any, ...]: + leaves = jax.tree_util.tree_leaves(vjp_fn) + return tuple(leaf for leaf, s in zip(leaves, layout.src) if s is None) + + +def _rebuild_vjp(layout: _ResidualLayout, saved: tuple[Any, ...], carry: Any, w: Any, layer_rest: Any, consts: Any): + groups = { + "y": jax.tree_util.tree_leaves(carry), + "w": jax.tree_util.tree_leaves(w), + "s": jax.tree_util.tree_leaves(layer_rest), + "c": jax.tree_util.tree_leaves(consts), + } + saved_iter = iter(saved) + leaves = [next(saved_iter) if s is None else groups[s[0]][s[1]] for s in layout.src] + return jax.tree_util.tree_unflatten(layout.treedef, leaves) + + +def _slice_at(tree: Any, idx: Any) -> Any: + return jax.tree.map(lambda x: x[idx], tree) + + +def _slice_range(tree: Any, start: int, end: int) -> Any: + return jax.tree.map(lambda x: x[start:end], tree) + + +def _with_memory_kind(x: jax.Array, kind: str) -> jax.Array: + s = jax.typeof(x).sharding + if not isinstance(s, NamedSharding): + return x + return jax.device_put(x, s.with_memory_kind(kind)) + + +def _host_stack_buffer(carry: Any, n: int) -> Any: + def _alloc(y): + return _with_memory_kind(jnp.zeros_like(jnp.broadcast_to(y[None], (n,) + y.shape)), "pinned_host") + + return jax.tree.map(_alloc, carry) + + +def _host_stack_write(buf: Any, carry: Any, idx: Any) -> Any: + def _write(b, y): + return jax.lax.dynamic_update_slice(b, _with_memory_kind(y[None], "pinned_host"), (idx,) + (0,) * y.ndim) + + return jax.tree.map(_write, buf, carry) + + +def _host_stack_read(buf: Any, idx: Any) -> Any: + def _read(b): + s = jax.lax.dynamic_slice(b, (idx,) + (0,) * (b.ndim - 1), (1,) + b.shape[1:]) + return jnp.squeeze(_with_memory_kind(s, "device"), axis=0) + + return jax.tree.map(_read, buf) + + +def _keep_layer_inputs_on_device(remat_policy: Any) -> Any: + """Keeps `decoder_layer_input` on device inside `jax.checkpoint` when streamed via host stack.""" + if remat_policy is None: + return None + + def policy(prim, *args, **params): + decision = remat_policy(prim, *args, **params) + if params.get("name") == "decoder_layer_input" and isinstance(decision, ad_checkpoint.Offloadable): + return ad_checkpoint.Saveable + return decision + + return policy + + +def _device_only_policy(remat_policy: Any) -> Any: + """Converts `Offloadable` decisions to `Saveable` so the prefetch loop handles host DMA.""" + if remat_policy is None: + return None + + def policy(prim, *args, **params): + decision = remat_policy(prim, *args, **params) + if isinstance(decision, ad_checkpoint.Offloadable): + return ad_checkpoint.Saveable + return decision + + return policy + + +def _merge_offload_residual_layout(raw_layout: _ResidualLayout, dev_layout: _ResidualLayout) -> _ResidualLayout: + """Attaches `raw_layout.saved_offload_mask` to `dev_layout`, failing closed on count mismatch.""" + if len(dev_layout.saved_avals) != len(raw_layout.saved_offload_mask): + raise ValueError( + "Device-only remat rewrite changed the saved residual leaf count " + f"({len(dev_layout.saved_avals)} vs {len(raw_layout.saved_offload_mask)}); " + "cannot safely stream offloaded residuals through host-stack buffers." + ) + return dev_layout._replace(saved_offload_mask=raw_layout.saved_offload_mask) + + +_PrefetchCtx = collections.namedtuple( + "_PrefetchCtx", + [ + "mesh", + "explicit", + "leaf_scan_axes", + "layer_specs", + "params_treedef", + "rest_treedef", + "has_rest", + "kw_treedef", + "static_kw", + "remat_policy", + "prevent_cse", + "defer_grad_reduce_scatter", + "offload_layer_inputs", + "sparse_core_id", + "layout_cache", + ], +) + + +def _prefetch_setup(graphdef, params_leaves, rest_leaves, dyn_kw_leaves, ctx: _PrefetchCtx): + """Builds per-layer gather, reduce-scatter, forward, and backward callables.""" + moved_params = [ + jnp.moveaxis(p, ax, 0) if ax != 0 and getattr(p, "ndim", 0) > ax else p + for p, ax in zip(params_leaves, ctx.leaf_scan_axes) + ] + params = jax.tree_util.tree_unflatten(ctx.params_treedef, moved_params) + rest = jax.tree_util.tree_unflatten(ctx.rest_treedef, rest_leaves) + layer_specs = ctx.layer_specs + layer_shardings = [NamedSharding(ctx.mesh, s) if s is not None else None for s in layer_specs] + pin_layout_flags = [ + _needs_minor_dim_layout_pin(getattr(p, "ndim", 1) - 1, s, ctx.mesh) for p, s in zip(moved_params, layer_specs) + ] + consts = tuple(dyn_kw_leaves) + + def gather_fn(layer_params): + leaves, treedef = jax.tree_util.tree_flatten(layer_params) + out = [] + for x, src_sharding, pin in zip(leaves, layer_shardings, pin_layout_flags): + if src_sharding is not None: + if not ctx.explicit: + x = jax.lax.with_sharding_constraint(x, src_sharding) + gathered_spec = remove_fsdp_from_spec(src_sharding.spec) + gathered = _explicit_fsdp_all_gather( + x, + ctx.mesh, + src_sharding.spec, + gathered_spec, + reduced=ctx.explicit, + sparse_core_id=ctx.sparse_core_id, + ) + if gathered is not None: + x = gathered + elif ctx.explicit: + x = jax.sharding.reshard(x, NamedSharding(ctx.mesh, gathered_spec)) + else: + x = jax.lax.with_sharding_constraint(x, NamedSharding(ctx.mesh, gathered_spec)) + if pin: + x = with_layout_constraint(x, Layout(tuple(range(x.ndim)))) + out.append(x) + return jax.tree_util.tree_unflatten(treedef, out) + + def layer_pure(carry_in, layer_weights, layer_rest, layer_consts): + dyn_kw = jax.tree_util.tree_unflatten(ctx.kw_treedef, layer_consts) + layer = nnx.merge(graphdef, layer_weights, layer_rest) + carry_out = super(m3_scan.ScannedLayers, layer).__call__(carry_in, **dyn_kw, **ctx.static_kw) + _, _, new_rest = nnx.split(layer, nnx.Param, ...) + return carry_out, new_rest + + def rs_one(g, src_sharding, tag): + if src_sharding is None: + return g + gathered_spec = remove_fsdp_from_spec(src_sharding.spec) + if tag: + out = _explicit_fsdp_reduce_scatter( + g, ctx.mesh, src_sharding.spec, gathered_spec, sparse_core_id=ctx.sparse_core_id + ) + if out is not None: + return out + scatter = _fsdp_scatter_dim(g.ndim, ctx.mesh, src_sharding.spec) + if scatter is not None and _is_unreduced_over(g, _expected_unreduced_axes(ctx.mesh, src_sharding.spec, scatter[1])): + out = _explicit_fsdp_reduce_scatter( + g, ctx.mesh, src_sharding.spec, gathered_spec, sparse_core_id=ctx.sparse_core_id + ) + if out is not None: + return out + g = _pin_scatter_dim_major(g, scatter[0]) + if ctx.explicit: + return jax.sharding.reshard(g, src_sharding) + return jax.lax.with_sharding_constraint(g, src_sharding) + + tag_cache: dict[str, list[bool]] = {} + + def tag_flags(leaves): + if "flags" not in tag_cache: + flags = [] + for g, s in zip(leaves, layer_shardings): + scatter = None if s is None else _fsdp_scatter_dim(g.ndim, ctx.mesh, s.spec) + eligible = scatter is not None and _is_unreduced_over(g, _expected_unreduced_axes(ctx.mesh, s.spec, scatter[1])) + flags.append(bool(ctx.defer_grad_reduce_scatter and eligible)) + tag_cache["flags"] = flags + return tag_cache["flags"] + + def rs_fn(layer_grads, which="all"): + leaves, treedef = jax.tree_util.tree_flatten(layer_grads) + flags = tag_flags(leaves) + out = [ + rs_one(g, s, tag) if which == "all" or (which == "tagged") == tag else g + for g, s, tag in zip(leaves, layer_shardings, flags) + ] + return jax.tree_util.tree_unflatten(treedef, out) + + rs_fn.tag_flags = tag_flags + + raw_remat_layer = jax.checkpoint(layer_pure, policy=ctx.remat_policy, prevent_cse=ctx.prevent_cse) + dev_remat_layer = jax.checkpoint(layer_pure, policy=_device_only_policy(ctx.remat_policy), prevent_cse=ctx.prevent_cse) + + def layout_fn(carry_in, w, layer_rest): + key = ( + id(graphdef), + ctx.params_treedef, + ctx.rest_treedef, + tuple((jax.typeof(p).shape, jax.typeof(p).dtype) for p in jax.tree_util.tree_leaves(w)), + ) + + def _build(): + raw_layout = _vjp_residual_layout(raw_remat_layer, carry_in, w, layer_rest, consts) + if any(raw_layout.saved_offload_mask): + dev_layout = _vjp_residual_layout(dev_remat_layer, carry_in, w, layer_rest, consts) + return _merge_offload_residual_layout(raw_layout, dev_layout) + return raw_layout + + return ctx.layout_cache(key, _build) + + def fwd_layer(carry_in, w, layer_rest, layout): + if ctx.remat_policy is cp.nothing_saveable and not layout.saved_avals: + carry_out, new_rest = layer_pure(carry_in, w, layer_rest, consts) + return carry_out, new_rest, () + active_remat = dev_remat_layer if any(layout.saved_offload_mask) else raw_remat_layer + carry_out, vjp_fn, new_rest = jax.vjp( + lambda c_, w_: active_remat(c_, w_, layer_rest, consts), carry_in, w, has_aux=True + ) + return carry_out, new_rest, _split_residuals(vjp_fn, layout) + + def layer_vjp_fn(carry_in, w, layer_rest, g_y, saved, layout): + vjp_fn = _rebuild_vjp(layout, saved, carry_in, w, layer_rest, consts) + return vjp_fn(g_y) + + return params, rest, gather_fn, rs_fn, layout_fn, fwd_layer, layer_vjp_fn + + +@functools.partial(jax.custom_vjp, nondiff_argnums=(0, 5, 6)) +def _custom_vjp_prefetch_pipeline(graphdef, carry_in, params_leaves, rest_leaves, dyn_kw_leaves, length, ctx): + """Runs `length >= 2` layers with double-buffered forward & backward weight prefetch.""" + out, _ = _custom_vjp_prefetch_pipeline_fwd(graphdef, carry_in, params_leaves, rest_leaves, dyn_kw_leaves, length, ctx) + return out + + +def _custom_vjp_prefetch_pipeline_fwd(graphdef, carry_in, params_leaves, rest_leaves, dyn_kw_leaves, length, ctx): + """Forward pass of the double-buffered FSDP weight-prefetch scan pipeline.""" + params, rest, gather_fn, _, layout_fn, fwd_layer, _ = _prefetch_setup( + graphdef, params_leaves, rest_leaves, dyn_kw_leaves, ctx + ) + w_0 = gather_fn(_slice_at(params, 0)) + layout = layout_fn(carry_in, w_0, _slice_at(rest, 0)) + w_curr = gather_fn(_slice_at(params, 1)) + y, new_rest_0, saved_0 = fwd_layer(carry_in, w_0, _slice_at(rest, 0), layout) + ordered = _has_multi_trip_loop(length) + offload_mask = layout.saved_offload_mask + offload_saved = ordered and any(offload_mask) + ys_mid = None + saved_mid = None + saved_host_bufs = None + rest_mid = None + if length > 2: + offload_mid = ctx.offload_layer_inputs and ordered + + def body(carry, xs): + y_k, w_k, ys_buf, s_bufs = carry + rest_k, next_idx = xs + params_next = _slice_at(params, next_idx) + if ordered: + y_k, params_next = jax.lax.optimization_barrier((y_k, params_next)) + w_next = gather_fn(params_next) + y_out, new_rest_k, saved_k = fwd_layer(y_k, w_k, rest_k, layout) + rest_out = new_rest_k if ctx.has_rest else None + if offload_saved: + new_s_bufs = [] + dev_saved_k = [] + for s_leaf, buf, is_off in zip(saved_k, s_bufs, offload_mask): + if is_off: + new_s_bufs.append(_host_stack_write(buf, s_leaf, next_idx - 2)) + dev_saved_k.append(None) + else: + new_s_bufs.append(None) + dev_saved_k.append(s_leaf) + s_bufs = tuple(new_s_bufs) + saved_out = tuple(dev_saved_k) + else: + saved_out = saved_k + if offload_mid: + ys_buf = _host_stack_write(ys_buf, y_k, next_idx - 2) + return (y_out, w_next, ys_buf, s_bufs), (None, saved_out, rest_out) + return (y_out, w_next, ys_buf, s_bufs), (y_k, saved_out, rest_out) + + ys_buf = _host_stack_buffer(y, length - 2) if offload_mid else None + s_bufs_init = ( + tuple(_host_stack_buffer(s_leaf, length - 2) if is_off else None for s_leaf, is_off in zip(saved_0, offload_mask)) + if offload_saved + else None + ) + (y, w_curr, ys_buf, saved_host_bufs), (ys_mid, saved_mid, rest_mid) = jax.lax.scan( + body, + (y, w_curr, ys_buf, s_bufs_init), + (_slice_range(rest, 1, length - 1), jnp.arange(2, length, dtype=jnp.int32)), + unroll=2, + ) + if offload_mid: + ys_mid = ys_buf + y_last_in = y + final_y, new_rest_last, saved_last = fwd_layer(y_last_in, w_curr, _slice_at(rest, length - 1), layout) + if offload_saved: + saved_0 = tuple(_with_memory_kind(s, "pinned_host") if is_off else s for s, is_off in zip(saved_0, offload_mask)) + saved_last = tuple( + _with_memory_kind(s, "pinned_host") if is_off else s for s, is_off in zip(saved_last, offload_mask) + ) + saved_mid = (saved_mid, saved_host_bufs) + if ctx.has_rest: + if length > 2: + new_rest_full = jax.tree.map( + lambda r0, rm, rl: jnp.concatenate([r0[None], rm, rl[None]], axis=0), + new_rest_0, + rest_mid, + new_rest_last, + ) + else: + new_rest_full = jax.tree.map(lambda r0, rl: jnp.stack([r0, rl], axis=0), new_rest_0, new_rest_last) + else: + new_rest_full = rest + residuals = ( + carry_in, + ys_mid, + y_last_in, + (saved_0, saved_mid, saved_last), + params_leaves, + rest_leaves, + dyn_kw_leaves, + ) + return (final_y, new_rest_full), residuals + + +def _custom_vjp_prefetch_pipeline_bwd(graphdef, length, ctx, res, g_out): + """Backward pass overlapping layer k-1 weight all-gather with layer k recompute and VJP.""" + carry_in, ys_mid, y_last_in, saved, params_leaves, rest_leaves, dyn_kw_leaves = res + saved_0, saved_mid, saved_last = saved + g, _ = g_out + params, rest, gather_fn, rs_fn, layout_fn, _, layer_vjp_fn = _prefetch_setup( + graphdef, params_leaves, rest_leaves, dyn_kw_leaves, ctx + ) + + last = length - 1 + params_last = _slice_at(params, last) + params_prev_init = _slice_at(params, last - 1) + g, params_last, params_prev_init = jax.lax.optimization_barrier((g, params_last, params_prev_init)) + w_last = gather_fn(params_last) + layout = layout_fn(y_last_in, w_last, _slice_at(rest, last)) + ordered = _has_multi_trip_loop(length) + offload_mask = layout.saved_offload_mask + offload_saved = ordered and any(offload_mask) + if offload_saved: + saved_mid, saved_host_bufs = saved_mid + saved_last = tuple(_with_memory_kind(s, "device") if is_off else s for s, is_off in zip(saved_last, offload_mask)) + else: + saved_host_bufs = None + w_prev = gather_fn(params_prev_init) + g, dw = layer_vjp_fn(y_last_in, w_last, _slice_at(rest, last), g, saved_last, layout) + + def write(d_acc, dw_tree, idx, which): + acc_leaves, treedef = jax.tree_util.tree_flatten(d_acc) + d_leaves = jax.tree_util.tree_leaves(dw_tree) + flags = rs_fn.tag_flags(d_leaves) if which != "all" else [None] * len(d_leaves) + out = [ + jax.lax.dynamic_update_index_in_dim(a, d, idx, axis=0) if which == "all" or (which == "tagged") == f else a + for a, d, f in zip(acc_leaves, d_leaves, flags) + ] + return jax.tree_util.tree_unflatten(treedef, out) + + def deferred(dw_tree): + leaves, treedef = jax.tree_util.tree_flatten(dw_tree) + flags = rs_fn.tag_flags(leaves) + return jax.tree_util.tree_unflatten(treedef, [x if f else jnp.zeros((0,), x.dtype) for x, f in zip(leaves, flags)]) + + d_params = jax.tree.map(jnp.zeros_like, params) + d_params = write(d_params, rs_fn(dw, "untagged"), last, "untagged") + pending = deferred(dw) + if length > 2: + offload_mid = ctx.offload_layer_inputs and ordered + + def body(carry, xs): + g_k, w_k, d_acc, pending_rs = carry + y_k, saved_k, rest_k, k_idx = xs + if offload_mid: + y_k = _host_stack_read(ys_mid, k_idx - 1) + if offload_saved: + saved_k = tuple( + _host_stack_read(buf, k_idx - 1) if is_off else s_leaf + for s_leaf, buf, is_off in zip(saved_k, saved_host_bufs, offload_mask) + ) + params_prev = _slice_at(params, k_idx - 1) + if ordered: + g_k, y_k, saved_k, params_prev = jax.lax.optimization_barrier((g_k, y_k, saved_k, params_prev)) + d_acc = write(d_acc, rs_fn(pending_rs, "tagged"), k_idx + 1, "tagged") + w_km1 = gather_fn(params_prev) + g_km1, dw_k = layer_vjp_fn(y_k, w_k, rest_k, g_k, saved_k, layout) + dw_k = rs_fn(dw_k, "untagged") + if ordered: + g_km1, dw_k = jax.lax.optimization_barrier((g_km1, dw_k)) + d_acc = write(d_acc, dw_k, k_idx, "untagged") + return (g_km1, w_km1, d_acc, deferred(dw_k)), None + + ys_xs = jnp.zeros((length - 2, 0), jnp.int8) if offload_mid else ys_mid + (g, w_prev, d_params, pending), _ = jax.lax.scan( + body, + (g, w_prev, d_params, pending), + (ys_xs, saved_mid, _slice_range(rest, 1, length - 1), jnp.arange(1, length - 1, dtype=jnp.int32)), + reverse=True, + unroll=2, + ) + d_params = write(d_params, rs_fn(pending, "tagged"), 1, "tagged") + if offload_saved: + saved_0 = tuple(_with_memory_kind(s, "device") if is_off else s for s, is_off in zip(saved_0, offload_mask)) + g, dw0 = layer_vjp_fn(carry_in, w_prev, _slice_at(rest, 0), g, saved_0, layout) + d_params = write(d_params, rs_fn(dw0), 0, "all") + + params_bar_leaves = tuple( + jnp.moveaxis(dp, 0, ax) if ax != 0 and getattr(dp, "ndim", 0) > ax else dp + for dp, ax in zip(jax.tree_util.tree_leaves(d_params), ctx.leaf_scan_axes) + ) + dyn_kw_bar_leaves = tuple(jnp.zeros_like(x) for x in dyn_kw_leaves) + return g, params_bar_leaves, None, dyn_kw_bar_leaves + + +_custom_vjp_prefetch_pipeline.defvjp(_custom_vjp_prefetch_pipeline_fwd, _custom_vjp_prefetch_pipeline_bwd) + + +def _resolve_mesh(explicit_mesh: Any, params: Any) -> Any: + """Resolves the active mesh from explicit argument, parameter shardings, or ambient abstract mesh.""" + if explicit_mesh is not None and not getattr(explicit_mesh, "empty", False): + return explicit_mesh + for leaf in jax.tree_util.tree_leaves(params): + s = getattr(jax.typeof(leaf), "sharding", None) or getattr(leaf, "sharding", None) + m = getattr(s, "mesh", None) + if m is not None and not getattr(m, "empty", False): + return m + ambient = jax.sharding.get_abstract_mesh() + if not ambient.empty: + return ambient + return None + + +def _extract_layer_specs(params: Any, param_scan_axis: int, mesh: Any, explicit: bool) -> tuple[P | None, ...]: + """Extracts each parameter leaf's per-layer mesh PartitionSpec (with scan axis removed).""" + specs: list[P | None] = [] + + def _visit(var): + raw = var.get_value() + ax = min(param_scan_axis, raw.ndim - 1) if raw.ndim >= 1 else 0 + spec: P | None = None + if explicit: + s = getattr(jax.typeof(raw), "sharding", None) or getattr(raw, "sharding", None) + if isinstance(s, NamedSharding) and s.spec is not None: + entries = list(s.spec) + entries.extend([None] * max(0, raw.ndim - len(entries))) + if raw.ndim >= 1: + del entries[ax] + spec = P(*entries) + if spec is None: + meta_spec = var.get_metadata().get("out_sharding") + if isinstance(meta_spec, P): + entries = list(meta_spec) + entries.extend([None] * max(0, raw.ndim - len(entries))) + if raw.ndim >= 1: + del entries[ax] + spec = m3_sharding.project_spec_to_mesh(P(*entries), mesh, drop_explicit_unit_axes=explicit) + if spec is not None and not any( + fsdp_axes_for_dim(tuple(a for a in ((e,) if isinstance(e, str) else (e or ())) if mesh.shape.get(a, 1) > 1)) + for e in tuple(spec) + ): + spec = None + specs.append(spec) + return var + + jax.tree.map(_visit, params, is_leaf=lambda x: isinstance(x, nnx.Variable)) + return tuple(specs) + + +def _resolve_remat_policy(remat_spec: Any) -> Callable[..., Any]: + """Extracts a JAX checkpoint policy callable from a `Remat` wrapper or raw callable.""" + if callable(remat_spec): + return remat_spec + pol = getattr(remat_spec, "policy", None) + return cp.nothing_saveable if pol is None else pol + + +@dataclasses.dataclass(frozen=True) +class FsdpPrefetchRunner: + """Static, path-transparent prefetch runner attached to `ScannedLayers.fsdp_prefetch`.""" + + defer_grad_reduce_scatter: bool = False + offload_layer_inputs: bool = False + sparse_core_id: int | None = None + enable_overlap_transform: bool = True + mesh: Mesh | None = None + + def __call__(self, scanned: m3_scan.ScannedLayers, carry: Any, **kwargs: Any) -> Any: + remat_cfg = getattr(scanned, "remat", None) + if remat_cfg is None: + raise ValueError( + "remat_policy='none' is not supported with prefetch_fsdp_weights=True; the FSDP weight-prefetch " + "custom_vjp pipeline requires per-layer gradient checkpointing so gathered weights are not retained " + "between forward and backward." + ) + if scanned.num_layers <= 1: + return scanned._run_regular_scan(carry, **kwargs) # pylint: disable=protected-access + + graphdef, params, rest = nnx.split(scanned, nnx.Param, ...) + mesh = _resolve_mesh(self.mesh, params) + if not mesh_uses_fsdp(mesh): + return scanned._run_regular_scan(carry, **kwargs) # pylint: disable=protected-access + + explicit = _is_explicit_mesh(mesh) + if self.enable_overlap_transform: + _register_overlap_transform(mesh) + if self.sparse_core_id is not None: + sc_id = self.sparse_core_id + else: + # Only pin collectives to SparseCore 1 when `unroll=2` emits a multi-trip loop (`>= 6` layers). + sc_id = _default_sparse_core_id(mesh) if _has_multi_trip_loop(scanned.num_layers) else None + + axis = scanned._param_scan_axis # pylint: disable=protected-access + layer_specs = _extract_layer_specs(params, axis, mesh, explicit) + if not any(s is not None for s in layer_specs): + return scanned._run_regular_scan(carry, **kwargs) # pylint: disable=protected-access + + stripped_params = m3_scan.strip_layer_axis(params, axis) + stacked_rest = rest + stripped_rest = m3_scan.strip_layer_axis(rest, 0) + params_leaves, params_treedef = jax.tree_util.tree_flatten(stripped_params) + rest_leaves, rest_treedef = jax.tree_util.tree_flatten(stripped_rest) + leaf_scan_axes = tuple( + min(axis, getattr(p, "ndim", 1) - 1) if getattr(p, "ndim", 0) >= 1 else 0 for p in params_leaves + ) + + mro = type(scanned).__mro__ + layer_cls = mro[mro.index(m3_scan.ScannedLayers) + 1] + filtered_kw = m3_scan.filter_layer_kwargs(layer_cls, kwargs) + traced_kw = {k: v for k, v in filtered_kw.items() if isinstance(v, jax.Array)} + static_kw = {k: v for k, v in filtered_kw.items() if k not in traced_kw} + dyn_kw_leaves, kw_treedef = jax.tree_util.tree_flatten(traced_kw) + + remat_policy = _resolve_remat_policy(remat_cfg) + if self.offload_layer_inputs: + remat_policy = _keep_layer_inputs_on_device(remat_policy) + + layout_box: dict[Any, _ResidualLayout] = {} + + def layout_cache(key, build): + if key not in layout_box: + layout_box[key] = build() + return layout_box[key] + + ctx = _PrefetchCtx( + mesh=mesh, + explicit=explicit, + leaf_scan_axes=leaf_scan_axes, + layer_specs=layer_specs, + params_treedef=params_treedef, + rest_treedef=rest_treedef, + has_rest=bool(rest_leaves), + kw_treedef=kw_treedef, + static_kw=static_kw, + remat_policy=remat_policy, + prevent_cse=False, + defer_grad_reduce_scatter=self.defer_grad_reduce_scatter, + offload_layer_inputs=self.offload_layer_inputs, + sparse_core_id=sc_id, + layout_cache=layout_cache, + ) + with jax.named_scope("fsdp_prefetch"): + final_carry, new_rest = _custom_vjp_prefetch_pipeline( + graphdef, + carry, + tuple(params_leaves), + tuple(rest_leaves), + tuple(dyn_kw_leaves), + scanned.num_layers, + ctx, + ) + if rest_leaves: + restored_rest = jax.tree.map( + lambda original, updated: original.replace(updated[...]), + stacked_rest, + new_rest, + is_leaf=lambda x: isinstance(x, nnx.Variable), + ) + nnx.update(scanned, restored_rest) + return final_carry + + +def apply_fsdp_prefetch( + model: nnx.Module, + *, + defer_grad_reduce_scatter: bool = False, + offload_layer_inputs: bool = False, + sparse_core_id: int | None = None, + enable_overlap_transform: bool = True, + mesh: Mesh | None = None, +) -> nnx.Module: + """Attaches `FsdpPrefetchRunner` to every `ScannedLayers` (or `nnx.Sequential`) in `model`.""" + runner = FsdpPrefetchRunner( + defer_grad_reduce_scatter=defer_grad_reduce_scatter, + offload_layer_inputs=offload_layer_inputs, + sparse_core_id=sparse_core_id, + enable_overlap_transform=enable_overlap_transform, + mesh=mesh, + ) + for _, node in nnx.iter_graph(model): + if isinstance(node, (m3_scan.ScannedLayers, nnx.Sequential)): + node.fsdp_prefetch = runner + return model + + +def maybe_apply_fsdp_prefetch( + model: nnx.Module, + config: Any, + *, + mesh: Any = None, + model_mode: str = _TRAIN_MODE, +) -> nnx.Module: + """Applies FSDP weight prefetching when `config.prefetch_fsdp_weights` is enabled for training.""" + if model_mode != _TRAIN_MODE: + return model + if not getattr(config, "prefetch_fsdp_weights", False): + return model + if getattr(config, "parameter_memory_host_offload", False): + return model + return apply_fsdp_prefetch( + model, + defer_grad_reduce_scatter=bool(getattr(config, "prefetch_fsdp_defer_grad_reduce_scatter", False)), + offload_layer_inputs=getattr(config, "decoder_layer_input", "device") == "offload", + mesh=mesh if isinstance(mesh, Mesh) else None, + ) diff --git a/src/maxtext/m3/infra/scan.py b/src/maxtext/m3/infra/scan.py index dd3c84f857..1860161469 100644 --- a/src/maxtext/m3/infra/scan.py +++ b/src/maxtext/m3/infra/scan.py @@ -115,6 +115,10 @@ def _filter_layer_kwargs(layer_cls: type[Any], kwargs: dict[str, Any]) -> dict[s return {k: v for k, v in kwargs.items() if k in accepted} +strip_layer_axis = _strip_layer_axis +filter_layer_kwargs = _filter_layer_kwargs + + class ScannedLayers(nnx.Module): """Structurally identical layers with stacked state, executed via `jax.lax.scan`. @@ -130,6 +134,7 @@ def __init__( *, param_scan_axis: int = 1, remat: Any = None, + fsdp_prefetch: Any = None, ): # pylint: disable=super-init-not-called if not isinstance(param_scan_axis, int) or param_scan_axis < 0: raise ValueError("param_scan_axis must be a non-negative integer.") @@ -150,8 +155,16 @@ def __init__( self._param_scan_axis = param_scan_axis self.num_layers = len(layers) self.remat = remat + self.fsdp_prefetch = fsdp_prefetch def __call__(self, carry: Any, **kwargs: Any) -> Any: + prefetch_runner = getattr(self, "fsdp_prefetch", None) + if prefetch_runner is not None: + return prefetch_runner(self, carry, **kwargs) + return self._run_regular_scan(carry, **kwargs) + + def _run_regular_scan(self, carry: Any, **kwargs: Any) -> Any: + """Runs standard `jax.lax.scan` over stacked layer parameters without weight prefetch.""" axis = self._param_scan_axis graphdef, params, rest = nnx.split(self, nnx.Param, ...) params = jax.tree.map(lambda x: jnp.moveaxis(x, min(axis, x.ndim - 1), 0), params) @@ -237,11 +250,14 @@ def scan_run( *, param_scan_axis: int, remat: Any = None, + fsdp_prefetch: Any = None, ) -> ScannedLayers: """Stacks `layers` (all of one class) into a `Scanned` instance.""" if not layers: raise ValueError("Cannot scan an empty run of layers.") - return _scanned_class(type(layers[0]))(layers, param_scan_axis=param_scan_axis, remat=remat) + return _scanned_class(type(layers[0]))( + layers, param_scan_axis=param_scan_axis, remat=remat, fsdp_prefetch=fsdp_prefetch + ) def _identical_runs(layers: Sequence[nnx.Module]) -> list[list[nnx.Module]]: @@ -265,17 +281,19 @@ def scan_layers(layers: nnx.Sequential, *, param_scan_axis: int) -> nnx.Module: become an `nnx.Sequential` of `ScannedLayers`, one per run. """ prior_remat = getattr(layers, "remat", None) + prior_prefetch = getattr(layers, "fsdp_prefetch", None) runs = _identical_runs(list(layers.layers)) scanned = [] for run in runs: if isinstance(run[0], ScannedLayers): - if prior_remat is not None: - for item in run: - if getattr(item, "remat", None) is None: - item.remat = prior_remat + for item in run: + if prior_remat is not None and getattr(item, "remat", None) is None: + item.remat = prior_remat + if prior_prefetch is not None and getattr(item, "fsdp_prefetch", None) is None: + item.fsdp_prefetch = prior_prefetch scanned.extend(run) continue - scanned.append(scan_run(run, param_scan_axis=param_scan_axis, remat=prior_remat)) + scanned.append(scan_run(run, param_scan_axis=param_scan_axis, remat=prior_remat, fsdp_prefetch=prior_prefetch)) if len(scanned) == 1: return scanned[0] return nnx.Sequential(*scanned) diff --git a/src/maxtext/m3/train/loop.py b/src/maxtext/m3/train/loop.py index 42fa237eb0..2cf179623c 100644 --- a/src/maxtext/m3/train/loop.py +++ b/src/maxtext/m3/train/loop.py @@ -93,6 +93,7 @@ project_spec_to_mesh, sharding_from_config, ) +from maxtext.m3.infra import fsdp_prefetch as m3_fsdp_prefetch from maxtext.m3.infra import quantization from maxtext.m3.infra import remat as m3_remat from maxtext.m3.infra import scan as m3_scan @@ -127,6 +128,7 @@ def setup_model(config, mesh: Mesh, sharding: Sharding | None = None) -> tuple[n model = m3_remat.maybe_apply_remat(model, config) sharding_obj = sharding or sharding_from_config(config) m3_remat.check_model_tags(model, config, mesh=mesh, sharding=sharding_obj) + model = m3_fsdp_prefetch.maybe_apply_fsdp_prefetch(model, config, mesh=mesh) if getattr(config, "quantization", "") and _ambient_mesh_is_explicit(): raise ValueError( "quantization is not supported with shard_mode='explicit' yet: qwix drops out_sharding on quantized matmuls. " diff --git a/src/maxtext/utils/model_creation_utils.py b/src/maxtext/utils/model_creation_utils.py index 14c48ca304..b90c2e4866 100644 --- a/src/maxtext/utils/model_creation_utils.py +++ b/src/maxtext/utils/model_creation_utils.py @@ -738,6 +738,7 @@ def create_model( from maxtext.m3.core import kernels as m3_kernels # pylint: disable=import-outside-toplevel from maxtext.m3.core import moe as m3_moe # pylint: disable=import-outside-toplevel from maxtext.m3.core import sharding as m3_sharding # pylint: disable=import-outside-toplevel + from maxtext.m3.infra import fsdp_prefetch as m3_fsdp_prefetch # pylint: disable=import-outside-toplevel from maxtext.m3.infra import remat as m3_remat # pylint: disable=import-outside-toplevel from maxtext.m3.infra import scan as m3_scan # pylint: disable=import-outside-toplevel from maxtext.m3.infra.named_scopes import add_named_scopes # pylint: disable=import-outside-toplevel @@ -753,6 +754,7 @@ def create_model( model = m3_remat.maybe_apply_remat(model, config, model_mode=model_mode) if model_mode == MODEL_MODE_TRAIN: m3_remat.check_model_tags(model, config, mesh=mesh) + model = m3_fsdp_prefetch.maybe_apply_fsdp_prefetch(model, config, mesh=mesh, model_mode=model_mode) if getattr(config, "quantization", "") and m3_sharding._ambient_mesh_is_explicit(): # pylint: disable=protected-access raise ValueError( "quantization is not supported with shard_mode='explicit' yet: qwix drops out_sharding on quantized " diff --git a/tests/unit/m3/fsdp_prefetch_test.py b/tests/unit/m3/fsdp_prefetch_test.py new file mode 100644 index 0000000000..c8ae2cdaaf --- /dev/null +++ b/tests/unit/m3/fsdp_prefetch_test.py @@ -0,0 +1,741 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit and compiled-HLO tests for m3 FSDP weight prefetching (`m3/infra/fsdp_prefetch.py`).""" + +import gc +import re +from types import SimpleNamespace + +from absl.testing import absltest, parameterized +from flax import nnx +import jax +import jax.numpy as jnp +from jax.sharding import NamedSharding +from jax.sharding import PartitionSpec as P +import numpy as np + +from maxtext.m3.core.sharding import create_mesh +from maxtext.m3.infra import fsdp_prefetch, remat, scan +from maxtext.m3.models.qwen3.modeling_qwen3 import create_qwen3_model +from maxtext.m3.train import loop as m3_loop +from maxtext.utils import model_creation_utils + + +def _tiny_qwen3_config( + *, + num_layers=6, + scan_layers=True, + param_scan_axis=1, + remat_policy="full", + prefetch_fsdp_weights=False, + prefetch_fsdp_defer_grad_reduce_scatter=False, + shard_mode="explicit", +): + """Minimal Qwen3 config exercised by CPU unit tests.""" + return SimpleNamespace( + use_m3_model=True, + model_name="qwen3-0.6b", + scan_layers=scan_layers, + param_scan_axis=param_scan_axis, + remat_policy=remat_policy, + prefetch_fsdp_weights=prefetch_fsdp_weights, + prefetch_fsdp_defer_grad_reduce_scatter=prefetch_fsdp_defer_grad_reduce_scatter, + shard_mode=shard_mode, + attention="dot_product", + quantization="", + parameter_memory_host_offload=False, + decoder_layer_input="device", + num_decoder_layers=num_layers, + emb_dim=16, + mlp_dim=32, + num_query_heads=4, + num_kv_heads=2, + head_dim=4, + vocab_size=32, + dtype=jnp.float32, + weight_dtype=jnp.float32, + normalization_layer_epsilon=1e-6, + use_qk_norm=True, + rope_max_timescale=1000000, + float32_logits=True, + logits_dot_in_fp32=True, + normalize_embedding_logits=False, + final_logits_soft_cap=0, + cast_logits_to_fp32=True, + max_target_length=4, + global_batch_size_to_train_on=4, + micro_batch_size_to_train_on=4, + per_device_batch_size=1, + init_weights_seed=0, + learning_rate=1e-3, + gradient_clipping_threshold=0.0, + opt_type="adamw", + adam_b1=0.9, + adam_b2=0.999, + adam_eps=1e-8, + adam_eps_root=0.0, + adam_weight_decay=0.0, + adamw_mask=[], + trainable_parameters_mask=[], + skip_step_on_spikes=False, + mu_dtype=None, + use_iota_embed=False, + logical_axis_rules=(), + ) + + +def _flat_params(mod): + return { + tuple(k.key for k in path): np.asarray(v[...]) + for path, v in jax.tree_util.tree_flatten_with_path( + nnx.state(mod, nnx.Param), is_leaf=lambda x: isinstance(x, nnx.Variable) + )[0] + } + + +def _make_batch(mesh, batch_axes="fsdp", seq_len=4): + """Creates deterministic sharded token and position batches.""" + spec = P(batch_axes, None) if batch_axes is not None else P(None, None) + sharding = NamedSharding(mesh, spec) + tokens = jax.device_put( + jnp.array([[1, 2, 3, 4], [4, 3, 2, 1], [5, 6, 7, 8], [8, 7, 6, 5]], dtype=jnp.int32)[:, :seq_len], + sharding, + ) + positions = jax.device_put( + jnp.broadcast_to(jnp.arange(seq_len, dtype=jnp.int32), (4, seq_len)), + sharding, + ) + return tokens, positions + + +def _build_qwen3(cfg, mesh): + model = create_qwen3_model(cfg, mesh=mesh, rngs=nnx.Rngs(0)) + model = scan.maybe_apply_scan(model, cfg) + model = remat.maybe_apply_remat(model, cfg) + model = fsdp_prefetch.maybe_apply_fsdp_prefetch(model, cfg, mesh=mesh) + return model + + +def _loss_and_grad_fn(explicit: bool, batch_axes="fsdp"): + """Returns a `(loss, grads)` callable compatible with explicit and auto meshes.""" + out_spec = P(batch_axes, None, None) + + def _run(model, tokens, positions): + def loss_fn(m): + logits = m(tokens, positions) + if explicit: + log_probs = jax.sharding.auto_axes(jax.nn.log_softmax, out_sharding=out_spec)(logits) + else: + log_probs = jax.nn.log_softmax(logits) + return -jnp.mean(log_probs[..., 0]) + + return nnx.value_and_grad(loss_fn)(model) + + return _run + + +def _parse_hlo_computations(hlo_text: str) -> dict[str, list[str]]: + """Splits compiled XLA HLO text into `{computation_name: [instruction_lines]}`.""" + comps: dict[str, list[str]] = {} + cur_name = None + for line in hlo_text.splitlines(): + m = re.match(r"^(?:ENTRY\s+)?(%[\w.\-]+)\s*\(", line) + if m and line.rstrip().endswith("{"): + cur_name = m.group(1) + comps[cur_name] = [] + elif line.strip() == "}" and cur_name is not None: + cur_name = None + elif cur_name is not None and "=" in line: + comps[cur_name].append(line.strip()) + return comps + + +_HLO_DEBUG_SECTIONS = frozenset({"FileNames", "FunctionNames", "FileLocations", "StackFrames"}) +_HLO_COMP_DEF_RE = re.compile(r"^(ENTRY\s+)?(%[\w.\-]+)\s*\((.*)\)\s*->\s*(.*\{\s*)$") +_HLO_PARAM_DECL_RE = re.compile(r"(?:^|(?<=,\s)|(?<=\*/))([A-Za-z_][\w.\-]*)(\s*:)") +_HLO_STR_OR_IDENT_RE = re.compile(r'"[^"]*"|%([A-Za-z_][\w.\-]*)') + + +def _hlo_comp_stem(name: str) -> str: + stem = re.sub(r"\.\d+(?=\.|_spmd\b|$)", "", name) + return re.sub(r"\bregion_\d+", "region", stem) + + +def _hlo_ssa_stem(name: str) -> str: + return re.sub(r"\.\d+(?=\.|$)", "", name) + + +def _normalize_hlo(hlo_text: str) -> str: + """Canonicalizes debug tables and monotonic XLA SSA/subcomputation counters while preserving graph wiring.""" + lines = hlo_text.splitlines() + comp_map: dict[str, str] = {} + comp_counts: dict[str, int] = {} + for line in lines: + m = _HLO_COMP_DEF_RE.match(line.strip()) + if m: + raw_comp = m.group(2)[1:] + if raw_comp not in comp_map: + stem = _hlo_comp_stem(raw_comp) + idx = comp_counts.get(stem, 0) + comp_counts[stem] = idx + 1 + comp_map[raw_comp] = f"{stem}#{idx}" + + out_lines: list[str] = [] + in_debug_section = False + in_comp = False + local_map: dict[str, str] = {} + local_counts: dict[str, int] = {} + + def _map_local(raw_name: str) -> str: + if raw_name in comp_map: + return comp_map[raw_name] + if raw_name not in local_map: + stem = _hlo_ssa_stem(raw_name) + idx = local_counts.get(stem, 0) + local_counts[stem] = idx + 1 + local_map[raw_name] = f"{stem}#{idx}" + return local_map[raw_name] + + def _replace_param(pm: re.Match[str]) -> str: + return f"{_map_local(pm.group(1))}{pm.group(2)}" + + def _replace_ident(im: re.Match[str]) -> str: + return f"%{_map_local(im.group(1))}" if im.group(1) is not None else im.group(0) + + for line in lines: + stripped = line.strip() + if not in_comp: + if stripped in _HLO_DEBUG_SECTIONS: + in_debug_section = True + continue + if in_debug_section: + if not stripped or re.match(r"^\d+\s+", stripped): + continue + in_debug_section = False + if stripped.startswith("HloModule "): + line = re.sub(r"^(\s*HloModule\s+)\S+?,", r"\1,", line, count=1) + out_lines.append(line) + continue + + m = _HLO_COMP_DEF_RE.match(stripped) + if m: + in_comp = True + local_map.clear() + local_counts.clear() + entry_prefix = m.group(1) or "" + comp_canon = comp_map[m.group(2)[1:]] + params_str = _HLO_PARAM_DECL_RE.sub(_replace_param, m.group(3)) + out_lines.append(f"{entry_prefix}%{comp_canon} ({params_str}) -> {m.group(4).rstrip()}") + continue + + if in_comp and (stripped == "}" or stripped.startswith("},")): + in_comp = False + out_lines.append(line) + continue + + if in_comp: + line = re.sub(r"\s*\bstack_frame_id=\d+", "", line) + line = re.sub(r",?\s*metadata=\{\s*\}", "", line) + line = _HLO_STR_OR_IDENT_RE.sub(_replace_ident, line) + if stripped or (out_lines and out_lines[-1] != ""): + out_lines.append(line) + + return "\n".join(out_lines) + + +class _TupleCarryStatefulLayer(nnx.Module): + """Toy layer with tuple carry `(hidden, aux_sum)` and mutable non-Param state.""" + + def __init__(self, dim: int, mesh, *, rngs: nnx.Rngs): + spec = P("fsdp", None) + w = jax.random.normal(rngs.params(), (dim, dim), dtype=jnp.float32) * 0.2 + w = jax.device_put(w, NamedSharding(mesh, spec)) + self.weight = nnx.Param(w, out_sharding=spec, eager_sharding=True) + self.calls = nnx.Variable(jnp.array(0, dtype=jnp.int32), out_sharding=P()) + + def __call__(self, carry, *, scale: float = 1.0): + x, aux = carry + self.calls[...] = self.calls[...] + 1 + y = jnp.tanh(jnp.einsum("bd,df->bf", x, self.weight[...])) * scale + return (x + y, aux + jnp.mean(y**2)) + + +class FsdpPrefetchHelpersTest(absltest.TestCase): + """Tests for pure-Python FSDP axis and SparseCore helpers.""" + + def test_fsdp_axes_and_spec_removal(self): + self.assertEqual(fsdp_prefetch.fsdp_axes_for_dim(("fsdp", "expert")), ("fsdp", "expert")) + self.assertEqual(fsdp_prefetch.fsdp_axes_for_dim(("expert", "fsdp")), ("fsdp",)) + self.assertEqual(fsdp_prefetch.fsdp_axes_for_dim("expert"), ()) + self.assertEqual( + fsdp_prefetch.remove_fsdp_from_spec(P(("fsdp", "fsdp_transpose", "context", "expert"), "tensor")), + P(None, "tensor"), + ) + self.assertEqual( + fsdp_prefetch.remove_fsdp_from_spec(P("expert", ("fsdp", "context"), "tensor")), + P("expert", None, "tensor"), + ) + + def test_has_multi_trip_loop_threshold(self): + for n in range(1, 6): + self.assertFalse(fsdp_prefetch._has_multi_trip_loop(n)) # pylint: disable=protected-access + for n in (6, 8, 28): + self.assertTrue(fsdp_prefetch._has_multi_trip_loop(n)) # pylint: disable=protected-access + + def test_merge_offload_residual_layout_fails_closed_on_count_mismatch(self): + layout_cls = fsdp_prefetch._ResidualLayout # pylint: disable=protected-access + merge_fn = fsdp_prefetch._merge_offload_residual_layout # pylint: disable=protected-access + raw = layout_cls(treedef=None, src=(None, None), saved_avals=("a", "b"), saved_offload_mask=(True, False)) + dev_ok = layout_cls(treedef=None, src=(None, None), saved_avals=("a_dev", "b_dev"), saved_offload_mask=(False, False)) + merged = merge_fn(raw, dev_ok) + self.assertEqual(merged.saved_avals, ("a_dev", "b_dev")) + self.assertEqual(merged.saved_offload_mask, (True, False)) + dev_bad = layout_cls(treedef=None, src=(None,), saved_avals=("a_dev",), saved_offload_mask=(False,)) + with self.assertRaisesRegex(ValueError, "saved residual leaf count"): + merge_fn(raw, dev_bad) + + def test_default_sparse_core_id_targets_tpu7x_only(self): + fake_tpu7x = SimpleNamespace(devices=np.array([SimpleNamespace(device_kind="TPU7x")])) + fake_v5p = SimpleNamespace(devices=np.array([SimpleNamespace(device_kind="TPU v5")])) + fake_cpu = SimpleNamespace(devices=np.array([SimpleNamespace(device_kind="cpu")])) + self.assertEqual(fsdp_prefetch._default_sparse_core_id(fake_tpu7x), 1) # pylint: disable=protected-access + self.assertIsNone(fsdp_prefetch._default_sparse_core_id(fake_v5p)) # pylint: disable=protected-access + self.assertIsNone(fsdp_prefetch._default_sparse_core_id(fake_cpu)) # pylint: disable=protected-access + + def test_normalize_hlo_canonicalizes_counters_and_detects_real_diffs(self): + hlo_a = """HloModule jit__step, is_scheduled=true, num_partitions=4 + +FileNames +1 "/a/model.py" + +FunctionNames +1 "fn_a" + +FileLocations +1 {file_name_id=1 function_name_id=1 line=10 end_line=10 column=1 end_column=1} + +StackFrames +1 {file_location_id=1 parent_frame_id=0} + +%region_1.8_spmd.clone (Arg_0.6: f32[], Arg_1.7: f32[]) -> f32[] { + %Arg_0.6 = f32[] parameter(0) + %Arg_1.7 = f32[] parameter(1) + ROOT %add.8 = f32[] add(%Arg_0.6, %Arg_1.7), metadata={op_name="jit(_step)/add" stack_frame_id=1} +} + +%bitcast_copy_fusion_computation (param_0: f32[4]) -> f32[4] { + %param_0 = f32[4]{0} parameter(0) + ROOT %bitcast = f32[4]{0} bitcast(%param_0), metadata={stack_frame_id=1} +} + +ENTRY %main.37_spmd (Arg_0.1: f32[4], /*index=5*/Arg_5.2: f32[4]) -> f32[4] { + %Arg_0.1 = f32[4]{0} parameter(0) + %Arg_5.2 = f32[4]{0} parameter(1) + %ws = f32[4]{0} fusion(%Arg_0.1), kind=kLoop, calls=%bitcast_copy_fusion_computation + ROOT %ar = f32[4]{0} all-reduce(%ws), to_apply=%region_1.8_spmd.clone, metadata={op_name="ar" stack_frame_id=1} +} +""" + hlo_b = """HloModule jit__step.42, is_scheduled=true, num_partitions=4 + +FileNames +1 "/x/other.py" +2 "/y/extra.py" + +FunctionNames +1 "other_fn" + +FileLocations +1 {file_name_id=2 function_name_id=1 line=99 end_line=99 column=4 end_column=4} + +StackFrames +1 {file_location_id=1 parent_frame_id=0} +2 {file_location_id=1 parent_frame_id=1} + +%region_18.408_spmd.clone (Arg_0.906: f32[], Arg_1.907: f32[]) -> f32[] { + %Arg_0.906 = f32[] parameter(0) + %Arg_1.907 = f32[] parameter(1) + ROOT %add.908 = f32[] add(%Arg_0.906, %Arg_1.907), metadata={op_name="jit(_step)/add" stack_frame_id=2} +} + +%bitcast_copy_fusion_computation.77 (param_0.312: f32[4]) -> f32[4] { + %param_0.312 = f32[4]{0} parameter(0) + ROOT %bitcast.88 = f32[4]{0} bitcast(%param_0.312) +} + +ENTRY %main.537_spmd (Arg_0.501: f32[4], /*index=5*/Arg_5.502: f32[4]) -> f32[4] { + %Arg_0.501 = f32[4]{0} parameter(0) + %Arg_5.502 = f32[4]{0} parameter(1) + %ws.19 = f32[4]{0} fusion(%Arg_0.501), kind=kLoop, calls=%bitcast_copy_fusion_computation.77 + ROOT %ar.70 = f32[4]{0} all-reduce(%ws.19), to_apply=%region_18.408_spmd.clone, metadata={op_name="ar" stack_frame_id=2} +} +""" + self.assertEqual(_normalize_hlo(hlo_a), _normalize_hlo(hlo_b)) + # Positive controls: rewiring operands or inserting an extra barrier must fail equality. + hlo_rewired = hlo_b.replace("add(%Arg_0.906, %Arg_1.907)", "add(%Arg_1.907, %Arg_0.906)") + self.assertNotEqual(_normalize_hlo(hlo_a), _normalize_hlo(hlo_rewired)) + hlo_barrier = hlo_b.replace( + "ROOT %ar.70 = f32[4]{0} all-reduce(%ws.19)", + "%opt-barrier.1 = f32[4]{0} opt-barrier(%ws.19)\n" " ROOT %ar.70 = f32[4]{0} all-reduce(%opt-barrier.1)", + ) + self.assertNotEqual(_normalize_hlo(hlo_a), _normalize_hlo(hlo_barrier)) + + +class FsdpPrefetchCorrectnessTest(parameterized.TestCase): + """Numerical parity, path transparency, and edge-case tests on 4-device CPU.""" + + def setUp(self): + super().setUp() + if jax.device_count() < 4: + self.skipTest("Requires at least 4 host devices (XLA_FLAGS=--xla_force_host_platform_device_count=4).") + + @parameterized.named_parameters( + ("explicit_full_pf", "explicit", "full", False, 1, 6), + ("explicit_full_pf_drs", "explicit", "full", True, 1, 6), + ("auto_full_pf", "auto", "full", False, 1, 6), + ("explicit_minimal_pf", "explicit", "minimal", False, 1, 6), + ("explicit_minimal_pf_drs", "explicit", "minimal", True, 1, 6), + ("explicit_axis0_pf_drs", "explicit", "full", True, 0, 6), + ("explicit_2layers_pf_drs", "explicit", "full", True, 1, 2), + ("explicit_3layers_pf_drs", "explicit", "full", True, 1, 3), + ("explicit_1layer_full", "explicit", "full", True, 1, 1), + ("explicit_1layer_minoff", "explicit", "minimal_offloaded", True, 1, 1), + ) + def test_qwen3_prefetch_matches_baseline_loss_grads_and_paths( + self, mode, remat_policy, defer_rs, scan_axis, num_layers + ): + explicit = mode == "explicit" + mesh = create_mesh(devices=jax.devices()[:4], ici={"fsdp": 4}, explicit=explicit) + self.enter_context(jax.set_mesh(mesh)) + tokens, positions = _make_batch(mesh) + + cfg_base = _tiny_qwen3_config( + num_layers=num_layers, + param_scan_axis=scan_axis, + remat_policy=remat_policy, + prefetch_fsdp_weights=False, + shard_mode=mode, + ) + cfg_pf = _tiny_qwen3_config( + num_layers=num_layers, + param_scan_axis=scan_axis, + remat_policy=remat_policy, + prefetch_fsdp_weights=True, + prefetch_fsdp_defer_grad_reduce_scatter=defer_rs, + shard_mode=mode, + ) + m_base = _build_qwen3(cfg_base, mesh) + m_pf = _build_qwen3(cfg_pf, mesh) + + # Path-transparency check: identical parameter keys, shapes, and dtypes. + p_base, p_pf = _flat_params(m_base), _flat_params(m_pf) + self.assertEqual(set(p_base.keys()), set(p_pf.keys())) + for k in p_base: + self.assertEqual(p_base[k].shape, p_pf[k].shape, msg=str(k)) + self.assertEqual(p_base[k].dtype, p_pf[k].dtype, msg=str(k)) + + step = nnx.jit(_loss_and_grad_fn(explicit)) + loss_base, grad_base = step(m_base, tokens, positions) + loss_pf, grad_pf = step(m_pf, tokens, positions) + + np.testing.assert_allclose(loss_pf, loss_base, rtol=1e-6, atol=1e-6) + gb, gp = _flat_params(grad_base), _flat_params(grad_pf) + self.assertEqual(set(gb.keys()), set(gp.keys())) + for k in gb: + np.testing.assert_allclose(gp[k], gb[k], rtol=1e-5, atol=1e-5, err_msg=str(k)) + + # Negative control: perturbing one scanned weight must break loss parity. + perturbed = _build_qwen3(cfg_pf, mesh) + perturbed.decoder.layers.mlp.wi_0.kernel[...] = perturbed.decoder.layers.mlp.wi_0.kernel[...] + 0.5 + loss_perturbed, _ = step(perturbed, tokens, positions) + self.assertGreater(abs(float(loss_perturbed) - float(loss_base)), 1e-4) + + @parameterized.named_parameters( + ("fsdp2_tp2", {"fsdp": 2, "tensor": 2}, "fsdp"), + ("fsdp2_fsdp_t2", {"fsdp": 2, "fsdp_transpose": 2}, ("fsdp", "fsdp_transpose")), + ("fsdp2_dp2", {"data": 2, "fsdp": 2}, ("data", "fsdp")), + ) + def test_multiaxis_sharding_matches_baseline(self, ici, batch_axes): + mesh = create_mesh(devices=jax.devices()[:4], ici=ici, explicit=True) + self.enter_context(jax.set_mesh(mesh)) + tokens, positions = _make_batch(mesh, batch_axes=batch_axes) + + cfg_base = _tiny_qwen3_config(num_layers=4, remat_policy="full", prefetch_fsdp_weights=False) + cfg_pf = _tiny_qwen3_config( + num_layers=4, + remat_policy="full", + prefetch_fsdp_weights=True, + prefetch_fsdp_defer_grad_reduce_scatter=True, + ) + m_base = _build_qwen3(cfg_base, mesh) + m_pf = _build_qwen3(cfg_pf, mesh) + step = nnx.jit(_loss_and_grad_fn(True, batch_axes=batch_axes)) + loss_base, grad_base = step(m_base, tokens, positions) + loss_pf, grad_pf = step(m_pf, tokens, positions) + np.testing.assert_allclose(loss_pf, loss_base, rtol=1e-6, atol=1e-6) + gb, gp = _flat_params(grad_base), _flat_params(grad_pf) + for k in gb: + np.testing.assert_allclose(gp[k], gb[k], rtol=1e-5, atol=1e-5, err_msg=str(k)) + + def test_tuple_carry_and_stateful_rest_match_regular_scan(self): + mesh = create_mesh(devices=jax.devices()[:4], ici={"fsdp": 4}, explicit=True) + self.enter_context(jax.set_mesh(mesh)) + + def make_stack(enable_prefetch: bool): + seq = nnx.Sequential(*[_TupleCarryStatefulLayer(8, mesh, rngs=nnx.Rngs(i)) for i in range(6)]) + scanned = scan.scan_layers(seq, param_scan_axis=1) + scanned.remat = remat.Remat(policy=jax.checkpoint_policies.nothing_saveable) + if enable_prefetch: + fsdp_prefetch.apply_fsdp_prefetch(scanned, defer_grad_reduce_scatter=True, mesh=mesh) + return scanned + + s_base = make_stack(False) + s_pf = make_stack(True) + x = jax.device_put(jnp.ones((4, 8), dtype=jnp.float32) * 0.3, NamedSharding(mesh, P("fsdp", None))) + aux0 = jax.device_put(jnp.array(0.0, dtype=jnp.float32), NamedSharding(mesh, P())) + + def loss_fn(mod): + y, aux = mod((x, aux0), scale=0.5) + return jnp.mean(y) + aux + + loss_base, grad_base = nnx.value_and_grad(loss_fn)(s_base) + loss_pf, grad_pf = nnx.value_and_grad(loss_fn)(s_pf) + np.testing.assert_allclose(loss_pf, loss_base, rtol=1e-6, atol=1e-6) + np.testing.assert_allclose(grad_pf.weight[...], grad_base.weight[...], rtol=1e-5, atol=1e-5) + np.testing.assert_array_equal(s_pf.calls[...], s_base.calls[...]) + + def test_remat_none_fails_closed(self): + mesh = create_mesh(devices=jax.devices()[:4], ici={"fsdp": 4}, explicit=True) + self.enter_context(jax.set_mesh(mesh)) + cfg = _tiny_qwen3_config(num_layers=4, remat_policy="none", prefetch_fsdp_weights=True) + model = _build_qwen3(cfg, mesh) + tokens, positions = _make_batch(mesh) + with self.assertRaisesRegex(ValueError, "remat_policy='none'"): + model(tokens, positions) + + def test_non_fsdp_mesh_and_disabled_modes_fall_back_cleanly(self): + mesh_dp = create_mesh(devices=jax.devices()[:4], ici={"data": 4}, explicit=True) + self.enter_context(jax.set_mesh(mesh_dp)) + cfg_base = _tiny_qwen3_config(num_layers=4, remat_policy="full", prefetch_fsdp_weights=False) + cfg_pf = _tiny_qwen3_config(num_layers=4, remat_policy="full", prefetch_fsdp_weights=True) + m_base = _build_qwen3(cfg_base, mesh_dp) + m_pf = _build_qwen3(cfg_pf, mesh_dp) + tokens, positions = _make_batch(mesh_dp, batch_axes="data") + step = nnx.jit(_loss_and_grad_fn(True, batch_axes="data")) + l_base, _ = step(m_base, tokens, positions) + l_pf, _ = step(m_pf, tokens, positions) + np.testing.assert_allclose(l_pf, l_base, rtol=1e-6, atol=1e-6) + + # Decode mode and host parameter offload leave model untouched. + m_dec = fsdp_prefetch.maybe_apply_fsdp_prefetch(m_base, cfg_pf, mesh=mesh_dp, model_mode="decode") + self.assertIsNone(m_dec.decoder.layers.fsdp_prefetch) + cfg_offload = _tiny_qwen3_config(num_layers=4, remat_policy="full", prefetch_fsdp_weights=True) + cfg_offload.parameter_memory_host_offload = True + m_off = fsdp_prefetch.maybe_apply_fsdp_prefetch(m_base, cfg_offload, mesh=mesh_dp) + self.assertIsNone(m_off.decoder.layers.fsdp_prefetch) + + def test_transform_order_invariance_and_dual_driver_wiring(self): + mesh = create_mesh(devices=jax.devices()[:4], ici={"fsdp": 4}, explicit=True) + self.enter_context(jax.set_mesh(mesh)) + cfg = _tiny_qwen3_config( + num_layers=4, + remat_policy="full", + prefetch_fsdp_weights=True, + prefetch_fsdp_defer_grad_reduce_scatter=True, + ) + # Apply prefetch before scan and verify ScannedLayers inherits it. + m_before = create_qwen3_model(cfg, mesh=mesh, rngs=nnx.Rngs(0)) + m_before = fsdp_prefetch.maybe_apply_fsdp_prefetch(m_before, cfg, mesh=mesh) + m_before = scan.maybe_apply_scan(m_before, cfg) + m_before = remat.maybe_apply_remat(m_before, cfg) + self.assertIsInstance(m_before.decoder.layers.fsdp_prefetch, fsdp_prefetch.FsdpPrefetchRunner) + + # Both production driver (`model_creation_utils.from_config`) and `m3/train/loop.py:setup_model` wire it. + m_mcu = model_creation_utils.from_config(cfg, mesh=mesh, rngs=nnx.Rngs(0)) + self.assertIsInstance(m_mcu.decoder.layers.fsdp_prefetch, fsdp_prefetch.FsdpPrefetchRunner) + self.assertTrue(m_mcu.decoder.layers.fsdp_prefetch.defer_grad_reduce_scatter) + + m_loop, _ = m3_loop.setup_model(cfg, mesh) + self.assertIsInstance(m_loop.decoder.layers.fsdp_prefetch, fsdp_prefetch.FsdpPrefetchRunner) + self.assertTrue(m_loop.decoder.layers.fsdp_prefetch.defer_grad_reduce_scatter) + + def test_minimal_offloaded_remat_six_layers_parity(self): + """6-layer `minimal_offloaded` exercises multi-trip `unroll=2` host-stack residual offload.""" + mesh = create_mesh(devices=jax.devices()[:4], ici={"fsdp": 4}, explicit=True) + self.enter_context(jax.set_mesh(mesh)) + cfg_base = _tiny_qwen3_config(num_layers=6, remat_policy="minimal_offloaded", prefetch_fsdp_weights=False) + cfg_pf = _tiny_qwen3_config(num_layers=6, remat_policy="minimal_offloaded", prefetch_fsdp_weights=True) + cfg_drs = _tiny_qwen3_config( + num_layers=6, + remat_policy="minimal_offloaded", + prefetch_fsdp_weights=True, + prefetch_fsdp_defer_grad_reduce_scatter=True, + ) + m_base = _build_qwen3(cfg_base, mesh) + m_pf = _build_qwen3(cfg_pf, mesh) + m_drs = _build_qwen3(cfg_drs, mesh) + tokens, positions = _make_batch(mesh) + step = nnx.jit(_loss_and_grad_fn(True)) + loss_base, grads_base = step(m_base, tokens, positions) + for m in (m_pf, m_drs): + loss_m, grads_m = step(m, tokens, positions) + np.testing.assert_allclose(loss_m, loss_base, rtol=1e-6, atol=1e-6) + for k, ref_g in _flat_params(grads_base).items(): + np.testing.assert_allclose(_flat_params(grads_m)[k], ref_g, rtol=1e-5, atol=1e-5, err_msg=f"grad mismatch {k}") + + +class FsdpPrefetchHloStructureTest(absltest.TestCase): + """Compiled-HLO structural and byte-identity tests (with positive & negative controls).""" + + def setUp(self): + super().setUp() + if jax.device_count() < 4: + self.skipTest("Requires at least 4 host devices (XLA_FLAGS=--xla_force_host_platform_device_count=4).") + + def _compile_model_hlo(self, model, cfg, mesh, logits_transform=None): + """Compiles one train step and returns post-optimization HLO text.""" + tokens, positions = _make_batch(mesh) + graphdef, state = nnx.split(model) + explicit = cfg.shard_mode == "explicit" + out_spec = P("fsdp", None, None) + + def _step(st, tok, pos): + def loss_fn(s): + logits = nnx.merge(graphdef, s)(tok, pos) + if logits_transform is not None: + logits = logits_transform(logits) + if explicit: + lp = jax.sharding.auto_axes(jax.nn.log_softmax, out_sharding=out_spec)(logits) + else: + lp = jax.nn.log_softmax(logits) + return -jnp.mean(lp[..., 0]) + + return jax.value_and_grad(loss_fn)(st) + + compiled = jax.jit(_step).lower(state, tokens, positions).compile() + return compiled.as_text() + + def _compile_train_step_hlo(self, cfg, mesh): + return self._compile_model_hlo(_build_qwen3(cfg, mesh), cfg, mesh) + + def test_disabled_hlo_is_byte_identical_to_untransformed_baseline(self): + mesh = create_mesh(devices=jax.devices()[:4], ici={"fsdp": 4}, explicit=True) + self.enter_context(jax.set_mesh(mesh)) + + cfg_off = _tiny_qwen3_config(num_layers=6, remat_policy="full", prefetch_fsdp_weights=False) + # Build without calling `maybe_apply_fsdp_prefetch` at all. + m_plain = create_qwen3_model(cfg_off, mesh=mesh, rngs=nnx.Rngs(0)) + m_plain = scan.maybe_apply_scan(m_plain, cfg_off) + m_plain = remat.maybe_apply_remat(m_plain, cfg_off) + m_disabled = _build_qwen3(cfg_off, mesh) + hlo_untransformed, hlo_disabled = [self._compile_model_hlo(m, cfg_off, mesh) for m in (m_plain, m_disabled)] + norm_untransformed = _normalize_hlo(hlo_untransformed) + norm_disabled = _normalize_hlo(hlo_disabled) + self.assertEqual(norm_disabled, norm_untransformed) + + # Also verify stability across inner trace-cache eviction (reproduces xdist memory-pressure frame shifts). + jax.clear_caches() + gc.collect() + hlo_disabled_evicted = self._compile_model_hlo(_build_qwen3(cfg_off, mesh), cfg_off, mesh) + self.assertEqual(_normalize_hlo(hlo_disabled_evicted), norm_untransformed) + + # Positive controls: enabling prefetch, inserting a single barrier, or inserting an extra collective + # must each fail `_normalize_hlo` equality. + cfg_on = _tiny_qwen3_config(num_layers=6, remat_policy="full", prefetch_fsdp_weights=True) + hlo_enabled = self._compile_model_hlo(_build_qwen3(cfg_on, mesh), cfg_on, mesh) + self.assertNotEqual(_normalize_hlo(hlo_enabled), norm_disabled) + + hlo_with_barrier = self._compile_model_hlo(m_plain, cfg_off, mesh, logits_transform=jax.lax.optimization_barrier) + self.assertNotEqual(_normalize_hlo(hlo_with_barrier), norm_untransformed) + + hlo_with_extra_ag = self._compile_model_hlo( + m_plain, + cfg_off, + mesh, + logits_transform=lambda x: jax.lax.with_sharding_constraint(x, P(None, None, None)), + ) + self.assertNotEqual(_normalize_hlo(hlo_with_extra_ag), norm_untransformed) + + def test_compiled_hlo_prefetches_next_layer_weight_before_current_layer_compute_ends(self): + mesh = create_mesh(devices=jax.devices()[:4], ici={"fsdp": 4}, explicit=True) + self.enter_context(jax.set_mesh(mesh)) + + cfg_off = _tiny_qwen3_config(num_layers=6, remat_policy="full", prefetch_fsdp_weights=False) + cfg_pf = _tiny_qwen3_config(num_layers=6, remat_policy="full", prefetch_fsdp_weights=True) + cfg_drs = _tiny_qwen3_config( + num_layers=6, + remat_policy="full", + prefetch_fsdp_weights=True, + prefetch_fsdp_defer_grad_reduce_scatter=True, + ) + + hlo_off = self._compile_train_step_hlo(cfg_off, mesh) + hlo_pf = self._compile_train_step_hlo(cfg_pf, mesh) + hlo_drs = self._compile_train_step_hlo(cfg_drs, mesh) + + prefetch_scope_re = re.compile(r'op_name="[^"]*fsdp_prefetch') + # Negative control: disabled HLO has no `fsdp_prefetch` op scope. + self.assertIsNone(prefetch_scope_re.search(hlo_off)) + # Positive control: enabled HLO carries `fsdp_prefetch` op scope and explicit collectives. + self.assertIsNotNone(prefetch_scope_re.search(hlo_pf)) + self.assertIn("all-gather(", hlo_pf) + self.assertIn("reduce-scatter(", hlo_pf) + + # Find the forward `while` loop body inside `hlo_pf`: with `num_layers=6` and `unroll=2`, + # layers 1..4 execute in a 2-trip unrolled loop (2 layers/iter), carrying prefetched weights + # and issuing layer k+1's all-gather BEFORE layer k's compute finishes. + comps_pf = _parse_hlo_computations(hlo_pf) + fwd_bodies = [ + (name, lines) + for name, lines in comps_pf.items() + if any(prefetch_scope_re.search(ln) for ln in lines) + and any("all-gather(" in ln for ln in lines) + and not any("reduce-scatter(" in ln for ln in lines) + ] + self.assertNotEmpty(fwd_bodies, "Expected a compiled forward prefetch while-loop body with all-gather ops.") + dot_comps = {name for name, lines in comps_pf.items() if any(" dot(" in ln or "= dot(" in ln for ln in lines)} + + def _is_dot_op(line: str) -> bool: + if " dot(" in line or "= dot(" in line: + return True + m = re.search(r"\bcalls=(%[\w.\-]+)", line) + return bool(m and m.group(1) in dot_comps) + + _, fwd_lines = fwd_bodies[0] + ag_indices = [i for i, ln in enumerate(fwd_lines) if "all-gather(" in ln] + dot_indices = [i for i, ln in enumerate(fwd_lines) if _is_dot_op(ln)] + self.assertNotEmpty(ag_indices, "Expected weight all-gather instructions in forward prefetch while body.") + self.assertNotEmpty(dot_indices, "Expected matmul dot instructions in forward prefetch while body.") + # Prefetched layer k+1's weight all-gather is produced before layer k's compute finishes. + self.assertLess(min(ag_indices), max(dot_indices)) + # Unroll=2 body has 2 layers of weight all-gathers (7 FSDP weights/layer * 2 = 14) vs 7 in non-prefetch. + self.assertGreaterEqual(len(ag_indices), 12) + + # Negative control: non-prefetch forward while body has only 1 layer of weight all-gathers (7). + comps_off = _parse_hlo_computations(hlo_off) + off_fwd_bodies = [ + lines + for lines in comps_off.values() + if any("all-gather(" in ln for ln in lines) and not any("reduce-scatter(" in ln for ln in lines) + ] + self.assertNotEmpty(off_fwd_bodies) + self.assertLess(sum(1 for ln in off_fwd_bodies[0] if "all-gather(" in ln), len(ag_indices)) + + # Deferred RS keeps backward reduce-scatters in the backward while body and changes the carry layout. + self.assertIn("reduce-scatter(", hlo_drs) + self.assertNotEqual(hlo_drs, hlo_pf) + + +if __name__ == "__main__": + absltest.main()