From 33a3932abe2b8e4531bfa9f84a17833d6f40b2f9 Mon Sep 17 00:00:00 2001 From: Doruk Arisoy <34922984+darisoy@users.noreply.github.com> Date: Thu, 22 Oct 2026 10:10:22 -0700 Subject: [PATCH] Avoid MoE weight relayouts with FSDP prefetch via moe_flat_fsdp_weights Allows moe_flat_fsdp_weights=True without the explicit fp8 weight all-gather (bf16 and prefetch_fsdp_weights=True), so routed-expert weights are stored as 2D [E * rows, cols] and FSDP-gathered on their major-most dim. The gathered weight is then a free bitcast of the [E_local, rows, cols] gmm operand instead of a 3.5 GiB transpose per layer. fsdp_prefetch: pin the gathered-weight layout only for 3D+ weights gathered on their minor-most dim, and drop the per-layer sub-stack slicing / concatenation and the up-front moveaxis that added full-stack copies. Tests: unit test for _needs_minor_dim_layout_pin (now module level), and an abstract-model/sharding test for unquantized moe_flat_fsdp_weights with prefetch_fsdp_weights. fsdp_prefetch_inhomogeneous: its stage params are already layer-major, so the shared setup no longer moves the scan axis for them (scan_axis=0); flat routed-expert weights are covered by new Qwen3.5 parity tests. Qwen3.5 tests check that flat routed-expert weights are gathered per sub-layer over fsdp only (tagged, on dim 0, no layout pin) under expert parallelism. --- src/maxtext/configs/types.py | 9 +- src/maxtext/layers/fsdp_prefetch.py | 130 ++++++++++++------ .../layers/fsdp_prefetch_inhomogeneous.py | 3 +- src/maxtext/layers/moe.py | 16 +-- tests/unit/fsdp_prefetch_test.py | 28 ++++ tests/unit/moe_test.py | 73 +++++++--- tests/unit/nnx_decoders_test.py | 106 ++++++++++++++ 7 files changed, 294 insertions(+), 71 deletions(-) diff --git a/src/maxtext/configs/types.py b/src/maxtext/configs/types.py index 752ebf64b2..116ab7a48a 100644 --- a/src/maxtext/configs/types.py +++ b/src/maxtext/configs/types.py @@ -6362,14 +6362,13 @@ def _stream_chunked(tokenize_data: bool) -> bool: if self.moe_flat_fsdp_weights: if not ( self.sparse_matmul - and self.shard_embed_moe_on_fsdp - and self.use_tokamax_gmm - and self.use_gmm_v2 + and (self.shard_embed_moe_on_fsdp or not self.quantization) + and ((self.use_tokamax_gmm and self.use_gmm_v2) or self.prefetch_fsdp_weights) and not self.prefuse_moe_weights ): raise ValueError( - "moe_flat_fsdp_weights=True requires sparse_matmul=True, shard_embed_moe_on_fsdp=True," - " use_tokamax_gmm=True, use_gmm_v2=True and prefuse_moe_weights=False." + "moe_flat_fsdp_weights=True requires sparse_matmul=True, (shard_embed_moe_on_fsdp=True or unquantized)," + " ((use_tokamax_gmm=True and use_gmm_v2=True) or prefetch_fsdp_weights=True) and prefuse_moe_weights=False." ) # These either pick their own 3D expert weight layout or carry per-expert params that assume it. if ( diff --git a/src/maxtext/layers/fsdp_prefetch.py b/src/maxtext/layers/fsdp_prefetch.py index 7aaeb11c6f..88967502e5 100644 --- a/src/maxtext/layers/fsdp_prefetch.py +++ b/src/maxtext/layers/fsdp_prefetch.py @@ -108,12 +108,24 @@ def _per_layer_sharding(stacked, mesh): return NamedSharding(mesh, jax.sharding.PartitionSpec(*tuple(s.spec)[1:])) +def _needs_minor_dim_layout_pin(stacked, layer_sharding, mesh): + """Whether a per-layer weight is 3D+ and FSDP-sharded on its minor-most dimension.""" + ndim = getattr(stacked, "ndim", 0) - 1 + if ndim < 3 or layer_sharding is None or layer_sharding.spec is None or len(layer_sharding.spec) < ndim: + return False + entry = layer_sharding.spec[ndim - 1] + axes = () if entry is None else (tuple(entry) if isinstance(entry, (tuple, list)) else (entry,)) + axes = tuple(a for a in axes if mesh.shape.get(a, 1) > 1) + return bool(sharding.fsdp_axes_for_dim(axes)) + + _PrefetchCtx = collections.namedtuple( "_PrefetchCtx", [ "mesh", "shard_mode", "debug_sharding", + "scan_axis", "static_args_leaves", "static_kwargs_leaves", "params_treedef", @@ -265,23 +277,31 @@ def _prefetch_setup(graphdef, params_leaves, state_leaves, diff_args_leaves, dif layer_state, g, saved, layout) -> (dy, dw)` differentiates one layer in the backward pass from those activations and the gathered weights `w`, recomputing everything else. """ + if ctx.scan_axis != 0: + params_leaves = [ + jnp.moveaxis(p, ctx.scan_axis, 0) if getattr(p, "ndim", 0) > ctx.scan_axis else p for p in params_leaves + ] params = jax.tree_util.tree_unflatten(ctx.params_treedef, params_leaves) state = jax.tree_util.tree_unflatten(ctx.state_treedef, state_leaves) layer_shardings = [_per_layer_sharding(p, ctx.mesh) for p in params_leaves] consts = (tuple(diff_args_leaves), tuple(diff_kwargs_leaves)) - def gather_one(x): - x = _apply_sharding_hint(x, ctx.mesh, ctx.shard_mode, ctx.debug_sharding) - if getattr(x, "ndim", 0) >= 2: - # Pin the gathered weight to the default layout. Otherwise XLA may pick a different layout for - # the gathered buffer that rides in the loop carry and then relayout it with a full-size copy - # on the TensorCore before its consumer; pinned, any relayout happens on the gather output. - x = with_layout_constraint(x, Layout(tuple(range(x.ndim)))) - return x + pin_layout_flags = [_needs_minor_dim_layout_pin(p, s, ctx.mesh) for p, s in zip(params_leaves, layer_shardings)] def gather_fn(layer_params): """All-gathers one layer's FSDP-sharded params (bf16[E, D/fsdp, F] -> bf16[E, D, F]).""" - return jax.tree.map(gather_one, layer_params) + leaves, treedef = jax.tree_util.tree_flatten(layer_params) + out = [] + for x, pin in zip(leaves, pin_layout_flags): + x = _apply_sharding_hint(x, ctx.mesh, ctx.shard_mode, ctx.debug_sharding) + if pin: + # 3D+ weights gathered on their minor-most dimension (e.g. [E, F, D/fsdp] where D/fsdp is not a + # multiple of 128) exit all-gather in layout {1, 0, 2}. Pinning only those weights to default + # major-to-minor layout keeps their relayout on the gather output (offloaded to SparseCore) + # without forcing unnecessary SparseCore data-format copies on weights gathered on major dims. + x = with_layout_constraint(x, Layout(tuple(range(x.ndim)))) + out.append(x) + return jax.tree_util.tree_unflatten(treedef, out) def layer_pure(y, layer_weights, layer_state, layer_consts): """One layer call with every array input explicit (the layer arguments come from `layer_consts`).""" @@ -397,20 +417,21 @@ def _custom_vjp_prefetch_pipeline_fwd( def body(carry, xs): y_k, w_k = carry - state_k, params_next = xs + state_k, next_idx = xs + params_next = _slice_at(params, next_idx) if ordered: # In the 2nd unrolled step of each iteration, waits for the 1st step's layer_fn(y_k, w_k) # so W(k) is dead before AG(k+2) starts and AG(k+2) can reuse W(k)'s buffer. - y_k, w_k, params_next = jax.lax.optimization_barrier((y_k, w_k, params_next)) + y_k, params_next = jax.lax.optimization_barrier((y_k, params_next)) w_next = gather_fn(params_next) # AG(k+1) || layer k. y_out, saved_k = fwd_layer(y_k, w_k, state_k, layout) return (y_out, w_next), (y_k, saved_k) - # Iterates k = 1 .. length-2; element j of xs has state[j+1] and params[j+2]. + # Iterates k = 1 .. length-2; element j of xs has state[j+1] and next_idx = j + 2. (y, w_curr), (ys_mid, saved_mid) = jax.lax.scan( body, (y, w_curr), - (_slice_range(state, 1, length - 1), _slice_range(params, 2, length)), + (_slice_range(state, 1, length - 1), jnp.arange(2, length, dtype=jnp.int32)), unroll=2, ) y_last_in = y if length > 1 else x_in @@ -439,50 +460,71 @@ def _custom_vjp_prefetch_pipeline_bwd(graphdef, length, ctx, res, g_out): params, state, gather_fn, rs_fn, layout_fn, _, layer_vjp_fn = _prefetch_setup( graphdef, params_leaves, state_leaves, diff_args_leaves, diff_kwargs_leaves, ctx ) - # Tie `params` to `g` so XLA cannot CSE the backward all-gathers with peeled forward slices or - # hoist backward weight slices before the forward pass. - g, params = jax.lax.optimization_barrier((g, params)) last = length - 1 - w_last = gather_fn(_slice_at(params, last)) + # Tie peeled backward slices to `g` so XLA cannot CSE backward all-gathers with peeled forward + # slices or hoist backward weight slices before the forward pass, without placing an + # optimization_barrier on the full 12-layer parameter stack. + params_last = _slice_at(params, last) + params_prev_init = _slice_at(params, last - 1) if length > 1 else None + if length > 1: + g, params_last, params_prev_init = jax.lax.optimization_barrier((g, params_last, params_prev_init)) + else: + g, params_last = jax.lax.optimization_barrier((g, params_last)) + w_last = gather_fn(params_last) layout = layout_fn(y_last_in, w_last, _slice_at(state, last)) - w_prev = gather_fn(_slice_at(params, last - 1)) if length > 1 else None # AG(L-2) || bwd(L-1). + w_prev = gather_fn(params_prev_init) if length > 1 else None # AG(L-2) || bwd(L-1). g, dw = layer_vjp_fn(y_last_in, w_last, _slice_at(state, last), g, saved_last, layout) - parts = [_expand0(rs_fn(dw))] + d_params = jax.tree.map( + lambda p, d: jax.lax.dynamic_update_index_in_dim(jnp.zeros_like(p), d, last, axis=0), + params, + rs_fn(dw), + ) if length > 1: if length > 2: ordered = (length - 2) // 2 >= 2 def body(carry, xs): - g_k, w_k = carry - y_k, saved_k, state_k, params_prev = xs + g_k, w_k, d_acc = carry + y_k, saved_k, state_k, k_idx = xs + params_prev = _slice_at(params, k_idx - 1) if ordered: - # In the 2nd unrolled step of each iteration, waits for the 1st step's g_k and w_km1 so - # W(k) is dead before AG(k-2) starts and XLA does not interleave the two unrolled layers' - # recomputes and dW buffers. - g_k, w_k, y_k, saved_k, params_prev = jax.lax.optimization_barrier((g_k, w_k, y_k, saved_k, params_prev)) + # In the 2nd unrolled step of each iteration, waits for the 1st step's g_k (which is + # barrier-tied to dw_k) so W(k) and dW(k) are dead before AG(k-2) and recompute(k-1) start. + g_k, y_k, saved_k, params_prev = jax.lax.optimization_barrier((g_k, y_k, saved_k, params_prev)) w_km1 = gather_fn(params_prev) # AG(k-1) || bwd(k). g_km1, dw_k = layer_vjp_fn(y_k, w_k, state_k, g_k, saved_k, layout) dw_k = rs_fn(dw_k) if ordered: g_km1, dw_k = jax.lax.optimization_barrier((g_km1, dw_k)) - return (g_km1, w_km1), dw_k - - # Iterates k = length-2 .. 1; element j of xs is layer k = j + 1, and outputs are stacked in j order. - (g, w_prev), d_mid = jax.lax.scan( + d_acc = jax.tree.map( + lambda acc, d: jax.lax.dynamic_update_index_in_dim(acc, d, k_idx, axis=0), + d_acc, + dw_k, + ) + return (g_km1, w_km1, d_acc), None + + # Iterates k = length-2 .. 1; element j of xs is layer k = j + 1. + (g, w_prev, d_params), _ = jax.lax.scan( body, - (g, w_prev), - (ys_mid, saved_mid, _slice_range(state, 1, length - 1), _slice_range(params, 0, length - 2)), + (g, w_prev, d_params), + (ys_mid, saved_mid, _slice_range(state, 1, length - 1), jnp.arange(1, length - 1, dtype=jnp.int32)), reverse=True, unroll=2, ) - parts.insert(0, d_mid) g, dw0 = layer_vjp_fn(x_in, w_prev, _slice_at(state, 0), g, saved_0, layout) - parts.insert(0, _expand0(rs_fn(dw0))) + d_params = jax.tree.map( + lambda acc, d: jax.lax.dynamic_update_index_in_dim(acc, d, 0, axis=0), + d_params, + rs_fn(dw0), + ) - # Per-layer grads stacked in layer order: [dW0, dW1 .. dW(L-2), dW(L-1)]. - d_params = parts[0] if len(parts) == 1 else jax.tree.map(lambda *xs: jnp.concatenate(xs, axis=0), *parts) - params_bar_leaves = tuple(jax.tree_util.tree_leaves(d_params)) + params_bar_leaves = jax.tree_util.tree_leaves(d_params) + if ctx.scan_axis != 0: + params_bar_leaves = [ + jnp.moveaxis(p, 0, ctx.scan_axis) if getattr(p, "ndim", 0) > ctx.scan_axis else p for p in params_bar_leaves + ] + params_bar_leaves = tuple(params_bar_leaves) # diff_*_leaves hold only arrays (non-array leaves were replaced by a dummy array), and the layer # arguments are treated as non-differentiable, so their cotangents are zeros. @@ -519,9 +561,13 @@ def mesh_uses_fsdp(mesh) -> bool: return any(mesh.shape.get(axis, 1) > 1 for axis in sharding.FSDP_MESH_AXES) -def _layer_major(params, state, scan_axis, length): - """Moves the scan axis of `params` to the front and gives `params`/`state` a leading layer axis.""" - if scan_axis != 0: +def _layer_major(params, state, scan_axis, length, move_scan_axis=True): + """Gives `params`/`state` a leading layer axis, moving the scan axis of `params` to the front. + + With `move_scan_axis=False` the scan axis stays in place; `_prefetch_setup` then moves it inside + the custom vjp (`ctx.scan_axis`), so the transposed stack is not materialized around the pipeline. + """ + if move_scan_axis and scan_axis != 0: params = jax.tree.map(lambda x: jnp.moveaxis(x, scan_axis, 0) if x.ndim > scan_axis else x, params) params = maxtext_utils_nnx.nnx_ensure_scan_leading_axis(params, length) state = maxtext_utils_nnx.nnx_ensure_scan_leading_axis(state, length) @@ -532,10 +578,11 @@ def _prepare_prefetch_call(layers, args, kwargs, *, length, mesh, config, remat_ """Splits a scanned layer stack and its call arguments for `jax.custom_vjp`. Returns (graphdef, params_leaves, state_leaves, diff_args_leaves, diff_kwargs_leaves, ctx), with - params/state layer-major (see `_layer_major`). + the scan axis of params still at `ctx.scan_axis` (`_prefetch_setup` moves it to the front). """ graphdef, params, state = nnx.split(layers, nnx.Param, ...) - params, state = _layer_major(params, state, config.param_scan_axis, length) + scan_axis = config.param_scan_axis + params, state = _layer_major(params, state, scan_axis, length, move_scan_axis=False) sig = inspect.signature(layers.__class__.__call__) valid_kwargs = {k: v for k, v in kwargs.items() if k in sig.parameters or "kwargs" in sig.parameters} @@ -565,6 +612,7 @@ def _is_diff_leaf(x): mesh=mesh, shard_mode=config.shard_mode, debug_sharding=config.debug_sharding, + scan_axis=scan_axis, static_args_leaves=static_args_leaves, static_kwargs_leaves=static_kwargs_leaves, params_treedef=params_treedef, diff --git a/src/maxtext/layers/fsdp_prefetch_inhomogeneous.py b/src/maxtext/layers/fsdp_prefetch_inhomogeneous.py index f09dc2d0fa..d3f2d7e66c 100644 --- a/src/maxtext/layers/fsdp_prefetch_inhomogeneous.py +++ b/src/maxtext/layers/fsdp_prefetch_inhomogeneous.py @@ -410,7 +410,8 @@ def stack(per_sublayer): diff_args_leaves, diff_kwargs_leaves, length, - tuple(ctx._replace(params_treedef=g[1], state_treedef=g[2]) for g in groups), + # The stage params are already layer-major (`_layer_major` above), so the setup must not move the scan axis again. + tuple(ctx._replace(params_treedef=g[1], state_treedef=g[2], scan_axis=0) for g in groups), tuple(len(g[3]) for g in groups), ) return final_y, layers, None diff --git a/src/maxtext/layers/moe.py b/src/maxtext/layers/moe.py index e634a7af86..1c8e2c4c22 100644 --- a/src/maxtext/layers/moe.py +++ b/src/maxtext/layers/moe.py @@ -2623,9 +2623,14 @@ def get_routed_moe_shardings(is_batch_sharded_by_expert, has_input_ids): wo_pspec = self._logical_to_mesh_axes(("exp", "mlp_no_fsdp", "embed_moe")) else: # Tell XLA to automatically gather the D dimension (e.g. for dynamic absmax scaling) - w0_pspec = self._logical_to_mesh_axes(("exp", None, "mlp_no_fsdp")) - w1_pspec = self._logical_to_mesh_axes(("exp", None, "mlp_no_fsdp")) - wo_pspec = self._logical_to_mesh_axes(("exp", "mlp_no_fsdp", None)) + if self.config.moe_flat_fsdp_weights: + w0_pspec = self._logical_to_mesh_axes((CompoundLogicalAxis(("exp", "embed_moe")), "mlp_no_fsdp")) + w1_pspec = self._logical_to_mesh_axes((CompoundLogicalAxis(("exp", "embed_moe")), "mlp_no_fsdp")) + wo_pspec = self._logical_to_mesh_axes((CompoundLogicalAxis(("exp", "embed_moe")), None)) + else: + w0_pspec = self._logical_to_mesh_axes(("exp", None, "mlp_no_fsdp")) + w1_pspec = self._logical_to_mesh_axes(("exp", None, "mlp_no_fsdp")) + wo_pspec = self._logical_to_mesh_axes(("exp", "mlp_no_fsdp", None)) # Update kernel pspec for FSDP AG w0_pspec = remove_mesh_axes_from_partition_spec(w0_pspec, ("fsdp",)) w1_pspec = remove_mesh_axes_from_partition_spec(w1_pspec, ("fsdp",)) @@ -2646,11 +2651,6 @@ def get_routed_moe_shardings(is_batch_sharded_by_expert, has_input_ids): is_batch_sharded_by_expert = is_batch_sharded_by_ep(inputs) weight_gather = explicitly_weight_ag() - if self.config.moe_flat_fsdp_weights and not weight_gather: - raise NotImplementedError( - "moe_flat_fsdp_weights needs the explicit fp8 weight all-gather" - " (shard_embed_moe_on_fsdp + fixed weight calibration)." - ) ( batch_logical_axis, input_partition_pspec, diff --git a/tests/unit/fsdp_prefetch_test.py b/tests/unit/fsdp_prefetch_test.py index 9266c9d70c..7087a56b44 100644 --- a/tests/unit/fsdp_prefetch_test.py +++ b/tests/unit/fsdp_prefetch_test.py @@ -47,6 +47,34 @@ def test_slice_and_expand0(self): np.testing.assert_array_equal(sub["w"], np.arange(4.0, 12.0).reshape(2, 4)) self.assertEqual(fsdp_prefetch._expand0(layer)["w"].shape, (1, 4)) # pylint: disable=protected-access + def test_needs_minor_dim_layout_pin(self): + mesh = _abstract_mesh((4, 2, 2), ("fsdp", "expert", "tensor")) + w_3d = jnp.ones((2, 4, 8, 16)) # [layers, E, F, D] -> 3D per layer + w_2d = jnp.ones((2, 32, 16)) # [layers, E*F, D] -> 2D per layer + # 3D weight FSDP-sharded on minor-most dim (dim 2) needs layout pinning. + self.assertTrue( + fsdp_prefetch._needs_minor_dim_layout_pin( # pylint: disable=protected-access + w_3d, NamedSharding(mesh, PartitionSpec("expert", "tensor", "fsdp")), mesh + ) + ) + # 3D weight FSDP-sharded on a non-minor dim (dim 1) does not need layout pinning. + self.assertFalse( + fsdp_prefetch._needs_minor_dim_layout_pin( # pylint: disable=protected-access + w_3d, NamedSharding(mesh, PartitionSpec("expert", "fsdp", "tensor")), mesh + ) + ) + # 2D flat weight FSDP-sharded on dim 0 or dim 1 does not need layout pinning. + self.assertFalse( + fsdp_prefetch._needs_minor_dim_layout_pin( # pylint: disable=protected-access + w_2d, NamedSharding(mesh, PartitionSpec(("expert", "fsdp"), "tensor")), mesh + ) + ) + self.assertFalse( + fsdp_prefetch._needs_minor_dim_layout_pin( # pylint: disable=protected-access + w_2d, NamedSharding(mesh, PartitionSpec("tensor", "fsdp")), mesh + ) + ) + def test_explicit_all_gather_without_fsdp_sharding_returns_none(self): mesh = Mesh(np.array(jax.devices()[:1]).reshape(1, 1), ("fsdp", "tensor")) w = jnp.ones((4, 8)) diff --git a/tests/unit/moe_test.py b/tests/unit/moe_test.py index c14632c27d..97b089fb84 100644 --- a/tests/unit/moe_test.py +++ b/tests/unit/moe_test.py @@ -4817,25 +4817,27 @@ def fake_ra2a(operand, output, input_offsets, send_sizes, output_offsets, recv_s def _flat_fsdp_config(**overrides): """Tiny DeepSeek config on the tokamax gmm_v2 fp8 path with the explicit weight all-gather over FSDP.""" - return _tiny_deepseek_config( - "flat_fsdp_" + "_".join(f"{k}{v}" for k, v in sorted(overrides.items())), - ici_expert_parallelism=2, - ici_fsdp_parallelism=jax.device_count() // 2, - shard_embed_moe_on_fsdp=True, - use_tokamax_gmm=True, - use_gmm_v2=True, - quantization="fp8_full", - use_qwix_quantization=True, - weight_quantization_calibration_method="fixed,-224,224", - act_quantization_calibration_method="fixed,-224,224", - bwd_quantization_calibration_method="fixed,-1,1", + defaults = { + "ici_expert_parallelism": 2, + "ici_fsdp_parallelism": jax.device_count() // 2, + "shard_embed_moe_on_fsdp": True, + "use_tokamax_gmm": True, + "use_gmm_v2": True, + "quantization": "fp8_full", + "use_qwix_quantization": True, + "weight_quantization_calibration_method": "fixed,-224,224", + "act_quantization_calibration_method": "fixed,-224,224", + "bwd_quantization_calibration_method": "fixed,-1,1", # The tiny config sets float32_gate_logits=True, which is rejected with a quantized router projection. - quantize_router_proj=False, + "quantize_router_proj": False, # gmm_v2 only quantizes the lhs (which the fixed act scale requires) when the contraction dim is at least the # MXU width; with smaller K it dequantizes the rhs instead and rejects the lhs scale. - base_emb_dim=256, - base_moe_mlp_dim=256, - **overrides, + "base_emb_dim": 256, + "base_moe_mlp_dim": 256, + } + # Overrides replace the defaults above (e.g. quantization="" for an unquantized config). + return _tiny_deepseek_config( + "flat_fsdp_" + "_".join(f"{k}{v}" for k, v in sorted(overrides.items())), **{**defaults, **overrides} ) @@ -4877,6 +4879,45 @@ def test_abstract_model_and_shardings(self): self.assertEqual(abs_state[name].shape, shape, name) self.assertEqual(shardings[name].get_value().spec[0], ("expert", "fsdp"), name) + def test_unquantized_prefetch_abstract_model_and_shardings(self): + if jax.device_count() < 4 or jax.device_count() % 2: + self.skipTest("Needs an even number (>= 4) of devices for expert parallelism 2 and FSDP >= 2.") + cfg = _flat_fsdp_config( + moe_flat_fsdp_weights=True, + prefetch_fsdp_weights=True, + quantization="", + use_qwix_quantization=False, + shard_embed_moe_on_fsdp=False, + use_tokamax_gmm=False, + use_gmm_v2=False, + ) + mesh = Mesh(maxtext_utils.create_device_mesh(cfg), cfg.mesh_axes) + with nn_partitioning.axis_rules(cfg.logical_axis_rules): + model = nnx.eval_shape( + lambda: moe.RoutedMoE( + config=cfg, + num_experts=cfg.num_experts, + num_experts_per_tok=cfg.num_experts_per_tok, + mesh=mesh, + kernel_init=nd_dense_init(1.0, "fan_in", "truncated_normal"), + kernel_axes=("embed", "mlp"), + intermediate_dim=cfg.moe_mlp_dim, + dtype=cfg.dtype, + rngs=nnx.Rngs(0), + ) + ) + _, abs_state, _ = nnx.split(model, nnx.Param, ...) + shardings = sharding.nnx_construct_named_sharding(abs_state, mesh) + + num_experts, emb, mlp = cfg.num_experts, cfg.base_emb_dim, cfg.moe_mlp_dim + for name, shape in ( + ("wi_0", (num_experts * emb, mlp)), + ("wi_1", (num_experts * emb, mlp)), + ("wo", (num_experts * mlp, emb)), + ): + self.assertEqual(abs_state[name].shape, shape, name) + self.assertEqual(shardings[name].get_value().spec[0], ("expert", "fsdp"), name) + @pytest.mark.tpu_only class FlatFsdpWeightsParityTest(unittest.TestCase): diff --git a/tests/unit/nnx_decoders_test.py b/tests/unit/nnx_decoders_test.py index 83a6445a6f..1d12c49814 100644 --- a/tests/unit/nnx_decoders_test.py +++ b/tests/unit/nnx_decoders_test.py @@ -2777,6 +2777,73 @@ def gather(x): self.assertEqual(checked["routed"], 6) self.assertGreaterEqual(checked["co_sharded"], 14) + def test_qwen3_5_flat_routed_expert_gathers_under_expert_parallelism(self): + """With `moe_flat_fsdp_weights`, each sub-layer's flat routed-expert weight is gathered over `fsdp` only. + + The flat [E * rows, cols] weights are sharded over the compound ('expert', 'fsdp') on dim 0. The + per-sub-layer gather strips only `fsdp`, as one tagged all-gather on that major-most dim, and keeps + `expert`; the gathered weight is then the [E_local * rows, cols] block the gmm reshapes for free. + """ + cfg = _make_config( + **{ + **_QWEN3_5_TINY, + "scan_layers": True, + "num_decoder_layers": 8, + "base_num_decoder_layers": 8, + "inhomogeneous_layer_cycle_interval": 4, + "sparse_matmul": True, + "prefetch_fsdp_weights": True, + "moe_flat_fsdp_weights": True, + } + ) + decoder = NNXDecoder(config=cfg, mesh=_make_mesh(cfg), model_mode=MODEL_MODE_TRAIN, rngs=nnx.Rngs(0)) + stages = fsdp_prefetch_inhomogeneous.sublayer_stages(decoder.layers) + self.assertEqual(len(stages), 4) + axis_sizes = {axis: 1 for axis in cfg.mesh_axes} | {"fsdp": 4, "expert": 2} + mesh = AbstractMesh(tuple(axis_sizes.values()), tuple(axis_sizes), axis_types=(AxisType.Explicit,) * len(axis_sizes)) + + checked = 0 + for _, params, _ in stages: + for path, var in nnx.to_flat_state(params): + axes = None + for key in ("sharding", "out_sharding", "sharding_names"): + meta_axes = var.get_metadata().get(key) + if isinstance(meta_axes, (PartitionSpec, tuple, list)) and meta_axes: + axes = tuple(a for a in meta_axes if a != "layers") + break + if not axes or not isinstance(axes[0], sharding.CompoundLogicalAxis): + continue + name = "/".join(map(str, path)) + spec = sharding.logical_to_mesh_axes(axes, mesh, rules=cfg.logical_axis_rules) + self.assertEqual(spec[0], ("expert", "fsdp"), name) + shape = list(var.get_value().shape) + del shape[cfg.param_scan_axis] # One block's slice, as the pipeline gathers it. + self.assertEqual(len(shape), 2, name) + w = jax.ShapeDtypeStruct(tuple(shape), jnp.float32, sharding=NamedSharding(mesh, spec)) + + def gather(x): + return fsdp_prefetch._apply_sharding_hint(x, mesh, ShardMode.EXPLICIT) # pylint: disable=protected-access + + with jax.sharding.use_abstract_mesh(mesh): + out = jax.eval_shape(gather, w) + hlo = jax.jit(gather).trace(w).lower(lowering_platforms=("tpu",)).as_text() + gathers = [line for line in hlo.splitlines() if "stablehlo.all_gather" in line] + self.assertEqual(len(gathers), 1, name) + self.assertIn('xla_explicit_fsdp = "yes"', gathers[0], name) + self.assertIn("all_gather_dim = 0", gathers[0], name) + self.assertNotIn('name = "expert"', gathers[0], name) + self.assertEqual(tuple(out.sharding.spec)[0], "expert", name) + # The flat weight must not get the minor-dim layout pin that 3D `wo` needs. + self.assertFalse( + fsdp_prefetch._needs_minor_dim_layout_pin( # pylint: disable=protected-access + jnp.zeros((1, *shape)), NamedSharding(mesh, spec), mesh + ), + name, + ) + checked += 1 + # wi_0, wi_1 and wo of each of the 4 sub-layers. + self.assertEqual(checked, 12) + def test_qwen3_5_remat_minimal(self): with unittest.mock.patch.object(fsdp_prefetch.max_logging, "info") as log_info: self._assert_qwen3_5_parity(num_blocks=4, remat_policy="minimal") @@ -2785,6 +2852,45 @@ def test_qwen3_5_remat_minimal(self): def test_qwen3_5_param_scan_axis_zero(self): self._assert_qwen3_5_parity(num_blocks=3, param_scan_axis=0) + def test_qwen3_5_moe_flat_fsdp_weights(self): + """Flat 2D routed-expert weights ([E * rows, cols]) prefetch one sub-layer at a time like 3D ones.""" + self._assert_qwen3_5_flat_weights_parity(num_blocks=2) + + def test_qwen3_5_moe_flat_fsdp_weights_param_scan_axis_zero(self): + self._assert_qwen3_5_flat_weights_parity(num_blocks=2, param_scan_axis=0) + + def _assert_qwen3_5_flat_weights_parity(self, num_blocks, cycle=4, **overrides): + """Like `_assert_qwen3_5_parity`, with `moe_flat_fsdp_weights=True`. + + Without `gmm_v2`, `moe_flat_fsdp_weights` needs `prefetch_fsdp_weights=True`, so the reference keeps + the flag and instead takes the plain scanned stack by reporting a mesh without FSDP. + """ + config = { + **_QWEN3_5_TINY, + "inhomogeneous_layer_cycle_interval": cycle, + "sparse_matmul": True, + "moe_flat_fsdp_weights": True, + **overrides, + } + num_layers = num_blocks * cycle + with mock.patch.object(fsdp_prefetch, "mesh_uses_fsdp", return_value=False): + ref_loss, ref_grads = self._loss_and_grads(num_layers, prefetch=True, **config) + nested_patch, whole_patch = self._patch_pipelines() + with nested_patch as nested, whole_patch as whole_blocks: + loss, grads = self._loss_and_grads(num_layers, prefetch=True, **config) + self.assertEqual(nested.call_count, 1) + self.assertEqual(whole_blocks.call_count, 0) + np.testing.assert_allclose(loss, ref_loss, rtol=1e-5, atol=1e-6) + ref_leaves, ref_def = jax.tree_util.tree_flatten(ref_grads) + leaves, tree_def = jax.tree_util.tree_flatten(grads) + self.assertEqual(ref_def, tree_def) + # The routed-expert weights really are flat: some gradient leaf has an [E * embed] dimension. + flat_rows = _QWEN3_5_TINY["num_experts"] * _QWEN3_5_TINY["base_emb_dim"] + self.assertTrue(any(flat_rows in g.shape for g in leaves)) + for ref, got in zip(ref_leaves, leaves): + self.assertEqual(ref.shape, got.shape) + np.testing.assert_allclose(np.asarray(got), np.asarray(ref), rtol=1e-2, atol=1e-3) + def test_qwen3_5_one_local_layer_per_block(self): # A single Gated DeltaNet sub-layer per block: no same-group scan, two trip-count-one loops. self._assert_qwen3_5_parity(num_blocks=2, cycle=2)