From 4d7dd82a72fa9d75f42691ad5b34429c581ce368 Mon Sep 17 00:00:00 2001 From: Yuchen Yao Date: Fri, 13 Nov 2026 23:40:35 +0000 Subject: [PATCH] feat(m3): adopt shared MoEExecutor for GraniteMoE and multi-rank EP GLM-5.3 - Route GraniteMoeExperts compound ('fsdp', 'expert') and non-compound paths through shared m3_moe.grouped_matmul and MoEExecutor.fused when sparse_matmul=True, preserving zero expert-weight all-gather. - Route Glm5Experts and Glm5MoE through shared MoEExecutor.fused when ep_size > 1 and sparse_matmul=True (keeping routing weights in float32 and sharding batch activations across ('data', 'fsdp', 'expert')), while keeping ep_size == 1 on native ragged_dot. - Mask unassigned tail rows [sum(group_sizes):M] in forward and backward across m3_moe.grouped_matmul backends and add grouped_unsharded_fused fallback when no Mesh is in scope. FIXES: b/580120934 --- src/maxtext/m3/configs/glm5_3_flash_config.py | 17 +- src/maxtext/m3/core/moe.py | 65 ++++- .../glm5_3_flash/modeling_glm5_3_flash.py | 238 +++++++++++----- .../models/granitemoe/modeling_granitemoe.py | 170 ++++++++++-- tests/unit/m3/glm5_3_flash_config_test.py | 35 +++ tests/unit/m3/glm5_3_flash_moe_test.py | 204 +++++++++++++- tests/unit/m3/granitemoe_test.py | 262 +++++++++++++++++- tests/unit/m3/moe_test.py | 132 ++++++++- 8 files changed, 1008 insertions(+), 115 deletions(-) diff --git a/src/maxtext/m3/configs/glm5_3_flash_config.py b/src/maxtext/m3/configs/glm5_3_flash_config.py index b076cd621e..5211bba743 100644 --- a/src/maxtext/m3/configs/glm5_3_flash_config.py +++ b/src/maxtext/m3/configs/glm5_3_flash_config.py @@ -51,7 +51,7 @@ def validate_no_packing(decoder_segment_ids: Any) -> None: " linear_conv_kernel_dim kda_implementation" " dsa_attention_implementation rope_parameters index_kpool" " index_kpool_always_select_tail linear_lower_bound mhc_eps" - " enable_debug_stats router_param_dtype" + " enable_debug_stats router_param_dtype sparse_matmul" ).split() ) @@ -136,6 +136,7 @@ class Glm5Config: remat_policy: str = "full" enable_debug_stats: bool = False num_vocab_tiling: int = 1 + sparse_matmul: bool = True def __post_init__(self) -> None: if isinstance(self.dtype, (np.dtype, type)): @@ -218,6 +219,8 @@ def __post_init__(self) -> None: kd, vd = self.linear_key_head_dim, self.linear_value_head_dim if kd != vd: raise ValueError(f"linear_key_head_dim ({kd}) must equal linear_value_head_dim ({vd}).") + if not isinstance(self.sparse_matmul, bool): + raise ValueError(f"sparse_matmul must be a bool; got {self.sparse_matmul!r}.") @property def qk_head_dim(self) -> int: @@ -339,7 +342,7 @@ def from_mesh(cls, mesh: jax.sharding.Mesh) -> "Glm5Sharding": ) use_tp = mesh.shape["tensor"] > 1 ep_size = int(mesh.shape["expert"]) - return cls(use_tp=use_tp, ep_size=ep_size) + return cls(use_tp=use_tp, ep_in_batch=ep_size > 1, ep_size=ep_size) def map_axis( self, @@ -351,13 +354,17 @@ def map_axis( act = tensor_type == m3_sharding.TensorType.ACTIVATION if axis == "batch": if act: - return (self.dp_axis, self.fsdp_axis, self.ep_axis) if self.ep_in_batch else (self.dp_axis, self.fsdp_axis) + if self.ep_in_batch or self.ep_size > 1 or tensor_name in ("tokens", "moe_tokens"): + return (self.dp_axis, self.fsdp_axis, self.ep_axis) + return (self.dp_axis, self.fsdp_axis) return None - if axis == "expert": + if axis in {"expert", "experts"}: return None if act else self.ep_axis + if axis == "expert_embed": + return None if axis in {"embed", "q_lora", "kv_lora"}: return None if act else self.fsdp_axis - if axis in {"vocab", "heads", "kv", "mlp"}: + if axis in {"vocab", "heads", "kv", "mlp", "expert_mlp"}: return self.tp_axis if self.use_tp else None if axis == "norm" or (axis == "length" and act): return None diff --git a/src/maxtext/m3/core/moe.py b/src/maxtext/m3/core/moe.py index d9826906ee..d2832d2b94 100644 --- a/src/maxtext/m3/core/moe.py +++ b/src/maxtext/m3/core/moe.py @@ -229,6 +229,8 @@ def _pallas_grouped_matmul_bwd( if backend_kind in ("gmm_v2", "tokamax"): if not is_tpu: + lhs_safe = jnp.where(valid_rows, lhs, jnp.zeros((), dtype=lhs.dtype)) + grad_safe = jnp.where(valid_rows, grad.astype(lhs.dtype), jnp.zeros((), dtype=lhs.dtype)) _, vjp_fn = jax.vjp( lambda l, r: jax.lax.ragged_dot( l, @@ -827,6 +829,8 @@ def _grouped_unsharded( activation: Callable[[jax.Array], jax.Array], swiglu_limit: float | None, executor: MoEExecutor, + backend: str = "ragged_dot", + is_tpu: bool = False, ) -> jax.Array: """Portable `ragged_dot` grouped execution without `shard_map`.""" if executor.bwd_wgrad_overlap: @@ -866,7 +870,7 @@ def _grouped_unsharded( sorted_x = _permute_rows(slot_rows, perm) sorted_x = jnp.where((jnp.arange(num_slots, dtype=jnp.int32) < valid_count)[:, None], sorted_x, 0) - matmul_fn = functools.partial(grouped_matmul, backend="ragged_dot", is_tpu=False) + matmul_fn = functools.partial(grouped_matmul, backend=backend, is_tpu=is_tpu) expert_out = _run_local_expert_mlp( sorted_x, projections, @@ -1235,6 +1239,65 @@ def routed_experts( return checkpoint_name(out.astype(x.dtype), "moe_routed_output") +def grouped_unsharded_fused( + x: jax.Array, + weights: jax.Array, + indices: jax.Array, + gate_up_proj: jax.Array, + down_proj: jax.Array, + *, + executor: MoEExecutor = RAGGED_DOT, + activation: Callable[[jax.Array], jax.Array] = jax.nn.silu, + swiglu_limit: float | None = None, + is_tpu: bool = False, +) -> jax.Array: + """Executes fused routed experts on a single device when no `Mesh` is in scope.""" + if gate_up_proj.ndim not in (3, 4) or down_proj.ndim != 3: + raise ValueError( + f"Expected 3D or 4D gate_up_proj and 3D down_proj; got gate_up_proj={gate_up_proj.shape}, " + f"down_proj={down_proj.shape}." + ) + num_experts = gate_up_proj.shape[0] + _validate_routed_inputs(x, weights, indices, num_experts) + projections = (gate_up_proj.astype(x.dtype), down_proj.astype(x.dtype)) + backend = executor.kind if executor.kind in SHARDED_EXECUTOR_KINDS else "ragged_dot" + if is_tpu and backend in SHARDED_EXECUTOR_KINDS: + mesh_1d = jax.sharding.Mesh(jax.local_devices()[:1], ("_moe_1d",)) + return jax.shard_map( + lambda x_l, w_l, idx_l, gu_l, dn_l: _grouped_unsharded( + x_l, + w_l, + idx_l, + (gu_l, dn_l), + fused=True, + num_experts=num_experts, + activation=activation, + swiglu_limit=swiglu_limit, + executor=executor, + backend=backend, + is_tpu=True, + ).astype(x_l.dtype), + mesh=mesh_1d, + in_specs=(P(), P(), P(), P(), P()), + out_specs=P(), + check_vma=False, + )(x, weights, indices, projections[0], projections[1]) + out = _grouped_unsharded( + x, + weights, + indices, + projections, + fused=True, + num_experts=num_experts, + activation=activation, + swiglu_limit=swiglu_limit, + executor=executor, + backend=backend, + is_tpu=is_tpu, + ) + return out.astype(x.dtype) + + def routed_experts_fused( x: jax.Array, weights: jax.Array, diff --git a/src/maxtext/m3/models/glm5_3_flash/modeling_glm5_3_flash.py b/src/maxtext/m3/models/glm5_3_flash/modeling_glm5_3_flash.py index e3e92417fa..42fe537f9c 100644 --- a/src/maxtext/m3/models/glm5_3_flash/modeling_glm5_3_flash.py +++ b/src/maxtext/m3/models/glm5_3_flash/modeling_glm5_3_flash.py @@ -22,13 +22,18 @@ from flax import nnx import jax +from jax.ad_checkpoint import checkpoint_name import jax.numpy as jnp from jax.sharding import Mesh from jax.typing import DTypeLike -from maxtext.m3.core import sharding as m3_sharding from maxtext.m3.configs import glm5_3_flash_config as glm5_config +from maxtext.m3.core import kernels as m3_kernels +from maxtext.m3.core import moe as m3_moe +from maxtext.m3.core import sharding as m3_sharding import optax +# pylint: disable=too-many-positional-arguments,abstract-method,too-many-function-args,no-name-in-module,import-outside-toplevel + def _make_lin(dt: DTypeLike, p_dt: DTypeLike, sharding, rngs: nnx.Rngs): """Constructs a Linear layer factory closure with sharding metadata.""" @@ -956,31 +961,56 @@ def __init__( self.num_experts, self.top_k = e, config.num_experts_per_tok self.swiglu_limit = config.mlp_clamp_limit self.sharding = sharding + self.sparse_matmul = config.sparse_matmul + self.executor = m3_moe.moe_from_config(config, sharding) self.ep_axis = sharding.ep_axis if isinstance(sharding, glm5_config.Glm5Sharding) else "expert" - if isinstance(sharding, glm5_config.Glm5Sharding) and sharding.ep_size > 1: - raise NotImplementedError( - f"Glm5Experts expert parallelism ({self.ep_axis}={sharding.ep_size} >" - " 1) requires shard_map EP dispatch (follow-up F34)." - ) - mesh = jax.sharding.get_abstract_mesh() - if not mesh.empty and self.ep_axis in mesh.shape and mesh.shape[self.ep_axis] > 1: - raise NotImplementedError( - "Glm5Experts expert parallelism" - f" ({self.ep_axis}={mesh.shape[self.ep_axis]} > 1) requires shard_map" - " EP dispatch (follow-up F34)." - ) + if self.executor.kind not in m3_moe.SHARDED_EXECUTOR_KINDS: + if isinstance(sharding, glm5_config.Glm5Sharding) and sharding.ep_size > 1: + raise NotImplementedError( + f"Glm5Experts expert parallelism ({self.ep_axis}={sharding.ep_size} >" + " 1) requires shard_map EP dispatch (follow-up F34)." + ) + mesh = jax.sharding.get_abstract_mesh() + if not mesh.empty and self.ep_axis in mesh.shape and mesh.shape[self.ep_axis] > 1: + raise NotImplementedError( + "Glm5Experts expert parallelism" + f" ({self.ep_axis}={mesh.shape[self.ep_axis]} > 1) requires shard_map" + " EP dispatch (follow-up F34)." + ) p_dt = getattr(jnp, config.weight_dtype) gu_axes = ("expert", "embed", None, "mlp") d_axes = ("expert", "mlp", "embed") - gu_spec = sharding("gate_up_proj", gu_axes) if sharding else None - d_spec = sharding("down_proj", d_axes) if sharding else None + gu_kw = m3_sharding.param_metadata(sharding, "gate_up_proj", gu_axes) if sharding else {} + d_kw = m3_sharding.param_metadata(sharding, "down_proj", d_axes) if sharding else {} gu_init = nnx.initializers.normal(0.02)(rngs.params(), (e, h, 2, inter), p_dt) - self.gate_up_proj = nnx.Param(gu_init, sharding=gu_spec) + self.gate_up_proj = nnx.Param(gu_init, **gu_kw) d_init = nnx.initializers.normal(0.02)(rngs.params(), (e, inter, h), p_dt) - self.down_proj = nnx.Param(d_init, out_sharding=d_spec) + self.down_proj = nnx.Param(d_init, **d_kw) + + def _resolve_ep_size_and_mesh( + self, + hidden_states: jax.Array, + ) -> tuple[int, Any, jax.sharding.Mesh | None]: + """Resolves active/inferred mesh and expert-parallelism size for `hidden_states`.""" + shd_ep_size = self.sharding.ep_size if isinstance(self.sharding, glm5_config.Glm5Sharding) else 1 + amb_mesh = jax.sharding.get_abstract_mesh() + inferred_mesh = None + if amb_mesh.empty: + for arr in (hidden_states, self.gate_up_proj[...]): + arr_shd = jax.typeof(arr).sharding + if isinstance(arr_shd, jax.sharding.NamedSharding) and not arr_shd.mesh.empty: + inferred_mesh = arr_shd.mesh + break + active_mesh = amb_mesh if not amb_mesh.empty else (inferred_mesh.abstract_mesh if inferred_mesh is not None else None) + mesh_ep_size = ( + int(active_mesh.shape[self.ep_axis]) + if active_mesh is not None and not active_mesh.empty and self.ep_axis in active_mesh.shape + else 1 + ) + return max(shd_ep_size, mesh_ep_size), active_mesh, inferred_mesh def __call__( self, @@ -988,66 +1018,146 @@ def __call__( topk_indices: jax.Array, topk_weights: jax.Array, ) -> jax.Array: + orig_shape = hidden_states.shape + dt = hidden_states.dtype + gu = self.gate_up_proj[...].astype(dt) + d_w = self.down_proj[...].astype(dt) + ep_size, active_mesh, inferred_mesh = self._resolve_ep_size_and_mesh(hidden_states) + + if ep_size > 1 and self.executor.kind in m3_moe.SHARDED_EXECUTOR_KINDS: + if hidden_states.ndim == 2: + x_3d = hidden_states[:, None, :] + idx_3d = topk_indices[:, None, :] + w_3d = topk_weights[:, None, :].astype(jnp.float32) + elif hidden_states.ndim == 3: + x_3d = hidden_states + idx_3d = topk_indices + w_3d = topk_weights.astype(jnp.float32) + else: + raise ValueError(f"Expected 2D or 3D hidden_states; got {hidden_states.shape}.") + + def _run_with_mesh(mesh_obj: Any) -> jax.Array: + exec_cfg = self.executor + x_in, w_in, idx_in = x_3d, w_3d, idx_3d + reshaped_to_tokens = False + batch_ax = m3_kernels.resolve_mesh_axes(mesh_obj, exec_cfg.batch) + batch_shards = m3_kernels.mesh_axis_size(mesh_obj, batch_ax) + mesh_ep = m3_kernels.mesh_axis_size(mesh_obj, m3_kernels.resolve_mesh_axes(mesh_obj, exec_cfg.experts)) + if x_in.shape[0] % batch_shards != 0: + total_tokens = x_in.shape[0] * x_in.shape[1] + if total_tokens % batch_shards == 0: + x_in = x_in.reshape(total_tokens, 1, x_in.shape[2]) + w_in = w_in.reshape(total_tokens, 1, w_in.shape[2]) + idx_in = idx_in.reshape(total_tokens, 1, idx_in.shape[2]) + reshaped_to_tokens = True + elif mesh_ep == 1: + exec_cfg = dataclasses.replace(exec_cfg, batch=None) + elif total_tokens % mesh_ep == 0: + x_in = x_in.reshape(total_tokens, 1, x_in.shape[2]) + w_in = w_in.reshape(total_tokens, 1, w_in.shape[2]) + idx_in = idx_in.reshape(total_tokens, 1, idx_in.shape[2]) + reshaped_to_tokens = True + exec_cfg = dataclasses.replace(exec_cfg, batch=exec_cfg.experts) + else: + pad_tok = mesh_ep - (total_tokens % mesh_ep) + x_in = jnp.pad(x_in.reshape(total_tokens, 1, x_in.shape[2]), ((0, pad_tok), (0, 0), (0, 0))) + w_in = jnp.pad(w_in.reshape(total_tokens, 1, w_in.shape[2]), ((0, pad_tok), (0, 0), (0, 0))) + idx_in = jnp.pad( + idx_in.reshape(total_tokens, 1, idx_in.shape[2]), + ((0, pad_tok), (0, 0), (0, 0)), + constant_values=-1, + ) + reshaped_to_tokens = True + exec_cfg = dataclasses.replace(exec_cfg, batch=exec_cfg.experts) + y = exec_cfg.fused( + x_in, + w_in, + idx_in, + gu, + d_w, + swiglu_limit=self.swiglu_limit, + ) + if reshaped_to_tokens: + y = y[: x_3d.shape[0] * x_3d.shape[1]].reshape(x_3d.shape) + return y + + if inferred_mesh is not None and jax.sharding.get_abstract_mesh().empty: + with jax.sharding.use_abstract_mesh(inferred_mesh.abstract_mesh): + out_3d = _run_with_mesh(inferred_mesh.abstract_mesh) + elif active_mesh is not None and not active_mesh.empty: + out_3d = _run_with_mesh(active_mesh) + else: + out_3d = m3_moe.grouped_unsharded_fused( + x_3d, + w_3d, + idx_3d, + gu, + d_w, + executor=self.executor, + swiglu_limit=self.swiglu_limit, + is_tpu=(jax.default_backend() == "tpu"), + ) + return out_3d.reshape(orig_shape).astype(dt) + if isinstance(self.sharding, glm5_config.Glm5Sharding) and self.sharding.ep_size > 1: raise NotImplementedError( f"Glm5Experts.__call__ with {self.ep_axis} > 1 requires shard_map EP" f" dispatch (follow-up F34); got ep_size={self.sharding.ep_size}." ) - mesh = None - x_shd = jax.typeof(hidden_states).sharding - param_shd = jax.typeof(self.gate_up_proj.value).sharding - if isinstance(x_shd, jax.sharding.NamedSharding): - mesh = x_shd.mesh - elif isinstance(param_shd, jax.sharding.NamedSharding): - mesh = param_shd.mesh - elif not jax.sharding.get_abstract_mesh().empty: - mesh = jax.sharding.get_abstract_mesh() - - if mesh is not None and not mesh.empty: - if self.ep_axis in mesh.shape and mesh.shape[self.ep_axis] > 1: + if active_mesh is not None and not active_mesh.empty: + if self.ep_axis in active_mesh.shape and active_mesh.shape[self.ep_axis] > 1: raise NotImplementedError( f"Glm5Experts.__call__ with {self.ep_axis} > 1 requires shard_map" " EP dispatch (follow-up F34); got mesh" - f" {self.ep_axis}={mesh.shape[self.ep_axis]}." + f" {self.ep_axis}={active_mesh.shape[self.ep_axis]}." ) - flat_idx = topk_indices.reshape(-1) - flat_w = topk_weights.reshape(-1, 1).astype(jnp.float32) - perm = jnp.argsort(flat_idx) - sorted_h = hidden_states[perm // self.top_k] + h_2d = hidden_states.reshape(-1, orig_shape[-1]) + flat_idx = topk_indices.reshape(-1).astype(jnp.int32) + is_valid = (flat_idx >= 0) & (flat_idx < self.num_experts) + bucket = jnp.where(is_valid, flat_idx, self.num_experts) + flat_w = jnp.where(is_valid, topk_weights.reshape(-1), 0).reshape(-1, 1).astype(jnp.float32) + perm = jnp.argsort(bucket, stable=True) inv_perm = jnp.argsort(perm) - group_sizes = jnp.bincount(flat_idx, length=self.num_experts) - - dt = hidden_states.dtype - gu = self.gate_up_proj[...].astype(dt) + sorted_valid = is_valid[perm][:, None] + zero_scalar = jnp.zeros((), dtype=dt) + sorted_h = jnp.where(sorted_valid, h_2d[perm // self.top_k], zero_scalar) + group_sizes = jnp.bincount(bucket, length=self.num_experts + 1)[: self.num_experts] + + is_tpu = ( + m3_kernels.is_tpu_mesh(active_mesh) + if active_mesh is not None and not active_mesh.empty + else (jax.default_backend() == "tpu") + ) + rdot_prec = jax.lax.Precision.DEFAULT if is_tpu else jax.lax.Precision.HIGHEST gate = jax.lax.ragged_dot( sorted_h, gu[:, :, 0, :], group_sizes=group_sizes, preferred_element_type=dt, - precision=jax.lax.Precision.HIGHEST, + precision=rdot_prec, ) up = jax.lax.ragged_dot( sorted_h, gu[:, :, 1, :], group_sizes=group_sizes, preferred_element_type=dt, - precision=jax.lax.Precision.HIGHEST, + precision=rdot_prec, ) gate = jnp.minimum(gate, self.swiglu_limit) up = jnp.clip(up, -self.swiglu_limit, self.swiglu_limit) - act = jax.nn.silu(gate) * up - d_w = self.down_proj[...].astype(dt) + act = jnp.where(sorted_valid, jax.nn.silu(gate) * up, zero_scalar) sorted_out = jax.lax.ragged_dot( act, d_w, group_sizes=group_sizes, preferred_element_type=dt, - precision=jax.lax.Precision.HIGHEST, + precision=rdot_prec, ) - n = hidden_states.shape[0] + sorted_out = jnp.where(sorted_valid, sorted_out, zero_scalar) + n = h_2d.shape[0] weighted = (sorted_out[inv_perm].astype(jnp.float32) * flat_w).reshape(n, self.top_k, -1) - return jnp.sum(weighted, axis=1).astype(dt) + return checkpoint_name(jnp.sum(weighted, axis=1).reshape(orig_shape).astype(dt), "moe_routed_output") class Glm5MoE(nnx.Module): @@ -1062,20 +1172,21 @@ def __init__( ): self.config = config self.ep_axis = sharding.ep_axis if isinstance(sharding, glm5_config.Glm5Sharding) else "expert" - if isinstance(sharding, glm5_config.Glm5Sharding) and sharding.ep_size > 1: - raise NotImplementedError( - f"Glm5MoE expert parallelism ({self.ep_axis}={sharding.ep_size} > 1)" - " requires all-to-all dispatch (planned for G8); single-host bringup" - f" requires {self.ep_axis}=1." - ) - mesh = jax.sharding.get_abstract_mesh() - if not mesh.empty and self.ep_axis in mesh.shape and mesh.shape[self.ep_axis] > 1: - raise NotImplementedError( - "Glm5MoE expert parallelism" - f" ({self.ep_axis}={mesh.shape[self.ep_axis]} > 1) requires" - " all-to-all dispatch (planned for G8); single-host bringup requires" - f" {self.ep_axis}=1." - ) + if not config.sparse_matmul: + if isinstance(sharding, glm5_config.Glm5Sharding) and sharding.ep_size > 1: + raise NotImplementedError( + f"Glm5MoE expert parallelism ({self.ep_axis}={sharding.ep_size} > 1)" + " requires all-to-all dispatch (planned for G8); single-host bringup" + f" requires {self.ep_axis}=1." + ) + mesh = jax.sharding.get_abstract_mesh() + if not mesh.empty and self.ep_axis in mesh.shape and mesh.shape[self.ep_axis] > 1: + raise NotImplementedError( + "Glm5MoE expert parallelism" + f" ({self.ep_axis}={mesh.shape[self.ep_axis]} > 1) requires" + " all-to-all dispatch (planned for G8); single-host bringup requires" + f" {self.ep_axis}=1." + ) self.experts = Glm5Experts(config, rngs=rngs, sharding=sharding) self.gate = Glm5TopkRouter(config, rngs=rngs, sharding=sharding) self.shared_experts = Glm5MLP( @@ -1089,12 +1200,15 @@ def __call__(self, hidden_states: jax.Array) -> tuple[jax.Array, tuple[jax.Array orig_shape = hidden_states.shape h_flat = hidden_states.reshape(-1, orig_shape[-1]) router_logits, topk_weights, topk_indices, router_probs = self.gate(h_flat) - expert_out = self.experts(h_flat, topk_indices, topk_weights) - expert_out = expert_out.reshape(orig_shape) - shared_out = self.shared_experts(hidden_states) topk_reshaped = topk_indices.reshape(orig_shape[:-1] + (self.config.num_experts_per_tok,)) weights_reshaped = topk_weights.reshape(orig_shape[:-1] + (self.config.num_experts_per_tok,)) probs_reshaped = router_probs.reshape(orig_shape[:-1] + (self.config.n_routed_experts,)) + ep_size, _, _ = self.experts._resolve_ep_size_and_mesh(hidden_states) # pylint: disable=protected-access + if ep_size > 1 and self.experts.executor.kind in m3_moe.SHARDED_EXECUTOR_KINDS and len(orig_shape) == 3: + expert_out = self.experts(hidden_states, topk_reshaped, weights_reshaped) + else: + expert_out = self.experts(h_flat, topk_indices, topk_weights).reshape(orig_shape) + shared_out = self.shared_experts(hidden_states) return expert_out + shared_out, ( router_logits, topk_reshaped, diff --git a/src/maxtext/m3/models/granitemoe/modeling_granitemoe.py b/src/maxtext/m3/models/granitemoe/modeling_granitemoe.py index e5a2cbc77b..a41da26fb1 100644 --- a/src/maxtext/m3/models/granitemoe/modeling_granitemoe.py +++ b/src/maxtext/m3/models/granitemoe/modeling_granitemoe.py @@ -26,6 +26,8 @@ import jax import jax.numpy as jnp from jax.sharding import PartitionSpec as P +from maxtext.m3.core import kernels as m3_kernels +from maxtext.m3.core import moe as m3_moe from maxtext.m3.core import rope as m3_rope from maxtext.m3.core import sharding as m3_sharding import numpy as np @@ -100,6 +102,7 @@ class GraniteMoeConfig: scan_layers: bool = True remat_policy: str = "none" matmul_precision: str = "default" + sparse_matmul: bool = True def __post_init__(self) -> None: if self.hidden_size <= 0 or self.hidden_size % self.num_attention_heads != 0: @@ -124,6 +127,8 @@ def __post_init__(self) -> None: raise ValueError(f"GraniteMoe supports hidden_act='silu', got {self.hidden_act!r}.") if self.remat_policy not in ("none", "full", "minimal"): raise ValueError(f"Unsupported remat_policy {self.remat_policy!r}; expected 'none', 'full', or 'minimal'.") + if not isinstance(self.sparse_matmul, bool): + raise ValueError(f"sparse_matmul must be a bool; got {self.sparse_matmul!r}.") @property def head_dim(self) -> int: @@ -133,7 +138,12 @@ def head_dim(self) -> int: def from_config(cls, config: Any, **overrides: Any) -> GraniteMoeConfig: """Builds `GraniteMoeConfig` from `GraniteMoeConfig`, M3 `BaseConfig`, `MaxTextConfig`, HF config, or dict.""" if isinstance(config, cls): - return dataclasses.replace(config, **overrides) if overrides else config + if not overrides: + return config + rep = dict(overrides) + if "sparse_matmul" in rep and not isinstance(rep["sparse_matmul"], bool): + raise ValueError(f"sparse_matmul must be a bool; got {rep['sparse_matmul']!r}.") + return dataclasses.replace(config, **rep) if hasattr(config, "use_m3_model") and not getattr(config, "use_m3_model"): raise ValueError("GraniteMoe requires use_m3_model=True.") @@ -165,6 +175,11 @@ def _get(primary: str, *fallbacks: str, default: Any = dataclasses.MISSING) -> A mp_val = _get("matmul_precision", default=_GRANITEMOE_ONLY_DEFAULTS["matmul_precision"]) mp_str = str(getattr(mp_val, "value", mp_val)).lower() + raw_sparse = ( + config["sparse_matmul"] + if isinstance(config, Mapping) and "sparse_matmul" in config + else (getattr(config, "sparse_matmul") if hasattr(config, "sparse_matmul") else True) + ) kwargs: dict[str, Any] = { "vocab_size": int(_get("vocab_size", default=49155)), "hidden_size": int(_get("hidden_size", "emb_dim", "base_emb_dim", default=1024)), @@ -198,8 +213,11 @@ def _get(primary: str, *fallbacks: str, default: Any = dataclasses.MISSING) -> A "scan_layers": bool(_get("scan_layers", default=True)), "remat_policy": str(remat_val), "matmul_precision": mp_str, + "sparse_matmul": raw_sparse, } kwargs.update(overrides) + if not isinstance(kwargs.get("sparse_matmul", True), bool): + raise ValueError(f"sparse_matmul must be a bool; got {kwargs.get('sparse_matmul')!r}.") return cls(**kwargs) from_maxtext_config = from_config @@ -222,6 +240,8 @@ def from_hf_config_json( allowed = {f.name for f in dataclasses.fields(cls)} filtered = {k: v for k, v in raw.items() if k in allowed and v is not None} filtered.update(overrides) + if "sparse_matmul" in filtered and not isinstance(filtered["sparse_matmul"], bool): + raise ValueError(f"sparse_matmul must be a bool; got {filtered['sparse_matmul']!r}.") return cls(**filtered) @@ -307,10 +327,10 @@ def map_axis( return None case "experts", m3_sharding.TensorType.WEIGHT: return ("fsdp", "expert") if self.compound_ep else "expert" - case "embed_exp", m3_sharding.TensorType.WEIGHT: + case ("embed_exp" | "expert_embed"), m3_sharding.TensorType.WEIGHT: return None if self.compound_ep else "fsdp" - case ("heads" | "kv_heads" | "mlp"), _: - return "tensor" if self.tp_size > 1 else "tensor" + case ("heads" | "kv_heads" | "mlp" | "expert_mlp"), _: + return "tensor" case "vocab", _: return "tensor" if self.vocab_tp else None case ("length" | "head_dim" | "norm"), _: @@ -599,8 +619,10 @@ def _local_ragged_swiglu( num_local_experts: int, top_k: int, precision: jax.lax.Precision, + backend: str = "ragged_dot", + is_tpu: bool = False, ) -> jax.Array: - """Executes SwiGLU `jax.lax.ragged_dot` for experts `[expert_offset, expert_offset + num_local_experts)`.""" + """Executes SwiGLU grouped matmul for experts `[expert_offset, expert_offset + num_local_experts)`.""" dt = x_tokens.dtype num_tokens, hidden_size = x_tokens.shape total_slots = num_tokens * top_k @@ -624,28 +646,35 @@ def _local_ragged_swiglu( group_sizes = jnp.bincount(bucket, length=num_local_experts + 1)[:num_local_experts] gu = gu_w.astype(dt) - gate = jax.lax.ragged_dot( - sorted_x, - gu[:, :, 0, :], - group_sizes=group_sizes, - preferred_element_type=dt, - precision=precision, - ) - up = jax.lax.ragged_dot( - sorted_x, - gu[:, :, 1, :], - group_sizes=group_sizes, - preferred_element_type=dt, - precision=precision, - ) - act = jnp.where(sorted_valid, jax.nn.silu(gate) * up, zero_scalar) - sorted_out = jax.lax.ragged_dot( - act, - down_w.astype(dt), - group_sizes=group_sizes, - preferred_element_type=dt, - precision=precision, - ) + dn = down_w.astype(dt) + if backend in m3_moe.SHARDED_EXECUTOR_KINDS: + gate = m3_moe.grouped_matmul(sorted_x, gu[:, :, 0, :], group_sizes, backend=backend, is_tpu=is_tpu) + up = m3_moe.grouped_matmul(sorted_x, gu[:, :, 1, :], group_sizes, backend=backend, is_tpu=is_tpu) + act = jnp.where(sorted_valid, jax.nn.silu(gate) * up, zero_scalar) + sorted_out = m3_moe.grouped_matmul(act, dn, group_sizes, backend=backend, is_tpu=is_tpu) + else: + gate = jax.lax.ragged_dot( + sorted_x, + gu[:, :, 0, :], + group_sizes=group_sizes, + preferred_element_type=dt, + precision=precision, + ) + up = jax.lax.ragged_dot( + sorted_x, + gu[:, :, 1, :], + group_sizes=group_sizes, + preferred_element_type=dt, + precision=precision, + ) + act = jnp.where(sorted_valid, jax.nn.silu(gate) * up, zero_scalar) + sorted_out = jax.lax.ragged_dot( + act, + dn, + group_sizes=group_sizes, + preferred_element_type=dt, + precision=precision, + ) sorted_out = jnp.where(sorted_valid, sorted_out, zero_scalar) unsorted_out = sorted_out[inv_perm] * masked_w[:, None] return jnp.sum(unsorted_out.reshape(num_tokens, top_k, hidden_size), axis=1) @@ -709,6 +738,8 @@ def __init__( self.top_k = config.num_experts_per_tok self.precision = _resolve_precision(config.matmul_precision) self.sharding = sharding + self.sparse_matmul = config.sparse_matmul + self.executor = m3_moe.moe_from_config(config, sharding) w_dt = _resolve_jnp_dtype(config.weight_dtype) gu_spec = sharding("gate_up_proj", ("experts", "embed_exp", None, "mlp")) if sharding else None @@ -737,15 +768,86 @@ def __call__( down_w = self.down_proj[...] sh = self.sharding + backend = self.executor.kind abs_mesh = jax.sharding.get_abstract_mesh() + inferred_mesh = None + if abs_mesh.empty and backend in m3_moe.SHARDED_EXECUTOR_KINDS: + for arr in (x_3d, gu_w): + arr_shd = jax.typeof(arr).sharding + if isinstance(arr_shd, jax.sharding.NamedSharding) and not arr_shd.mesh.empty: + inferred_mesh = arr_shd.mesh + abs_mesh = inferred_mesh.abstract_mesh + break + if ( + sh is None + and not abs_mesh.empty + and backend in m3_moe.SHARDED_EXECUTOR_KINDS + and {"data", "fsdp", "tensor", "expert"}.issubset(set(abs_mesh.axis_names)) + ): + sh = GraniteMoeSharding( + num_experts=self.num_experts, + dp_size=int(abs_mesh.shape["data"]), + fsdp_size=int(abs_mesh.shape["fsdp"]), + tp_size=int(abs_mesh.shape["tensor"]), + ep_size=int(abs_mesh.shape["expert"]), + ) + is_tpu = m3_kernels.is_tpu_mesh(abs_mesh) if not abs_mesh.empty else (jax.default_backend() == "tpu") use_compound_shard_map = ( isinstance(sh, GraniteMoeSharding) and sh.compound_ep and not abs_mesh.empty - and (sh.fsdp_size * sh.ep_size > 1 or sh.tp_size > 1) + and (backend in m3_moe.SHARDED_EXECUTOR_KINDS or sh.fsdp_size * sh.ep_size > 1 or sh.tp_size > 1) ) if not use_compound_shard_map: + if backend in m3_moe.SHARDED_EXECUTOR_KINDS: + exec_cfg = dataclasses.replace(self.executor, experts="expert") + if not abs_mesh.empty: + batch_ax = m3_kernels.resolve_mesh_axes(abs_mesh, exec_cfg.batch) + batch_shards = m3_kernels.mesh_axis_size(abs_mesh, batch_ax) + ep_size = m3_kernels.mesh_axis_size(abs_mesh, m3_kernels.resolve_mesh_axes(abs_mesh, exec_cfg.experts)) + x_in, w_in, idx_in = x_3d, top_k_weights_3d, top_k_indices_3d + reshaped = False + if x_in.shape[0] % batch_shards != 0: + if num_tokens % batch_shards == 0: + x_in = x_in.reshape(num_tokens, 1, hidden_size) + w_in = w_in.reshape(num_tokens, 1, self.top_k) + idx_in = idx_in.reshape(num_tokens, 1, self.top_k) + reshaped = True + elif ep_size == 1: + exec_cfg = dataclasses.replace(exec_cfg, batch=None) + elif num_tokens % ep_size == 0: + x_in = x_in.reshape(num_tokens, 1, hidden_size) + w_in = w_in.reshape(num_tokens, 1, self.top_k) + idx_in = idx_in.reshape(num_tokens, 1, self.top_k) + reshaped = True + exec_cfg = dataclasses.replace(exec_cfg, batch=exec_cfg.experts) + else: + pad_tok = ep_size - (num_tokens % ep_size) + x_in = jnp.pad(x_in.reshape(num_tokens, 1, hidden_size), ((0, pad_tok), (0, 0), (0, 0))) + w_in = jnp.pad(w_in.reshape(num_tokens, 1, self.top_k), ((0, pad_tok), (0, 0), (0, 0))) + idx_in = jnp.pad( + idx_in.reshape(num_tokens, 1, self.top_k), + ((0, pad_tok), (0, 0), (0, 0)), + constant_values=-1, + ) + reshaped = True + exec_cfg = dataclasses.replace(exec_cfg, batch=exec_cfg.experts) + if inferred_mesh is not None: + with jax.sharding.use_abstract_mesh(inferred_mesh.abstract_mesh): + out_3d = exec_cfg.fused(x_in, w_in, idx_in, gu_w, down_w) + else: + out_3d = exec_cfg.fused(x_in, w_in, idx_in, gu_w, down_w) + return out_3d[:num_tokens].reshape(bsz, seq_len, hidden_size) if reshaped else out_3d + return m3_moe.grouped_unsharded_fused( + x_3d, + top_k_weights_3d, + top_k_indices_3d, + gu_w, + down_w, + executor=exec_cfg, + is_tpu=is_tpu, + ) if isinstance(sh, GraniteMoeSharding) and not sh.compound_ep and sh.ep_size > 1: raise NotImplementedError( f"Non-compound EP (num_experts={self.num_experts}, fsdp={sh.fsdp_size}, expert={sh.ep_size}) " @@ -761,6 +863,8 @@ def __call__( num_local_experts=self.num_experts, top_k=self.top_k, precision=self.precision, + backend=backend, + is_tpu=is_tpu, ) return out_2d.reshape(bsz, seq_len, hidden_size) @@ -817,6 +921,8 @@ def _ep_tp_kernel( num_local_experts=local_experts, top_k=top_k, precision=precision, + backend=backend, + is_tpu=is_tpu, ).reshape(b_slice, s_slice, h_dim) if tp_active: @@ -827,12 +933,16 @@ def _ep_tp_kernel( return jax.lax.psum(y_partial, ("fsdp", "expert")) return y_partial - return jax.shard_map( + smap_fn = jax.shard_map( _ep_tp_kernel, in_specs=(act_spec, act_spec, act_spec, gu_spec, down_spec), out_specs=act_spec, check_vma=False, - )(x_3d, top_k_indices_3d, top_k_weights_3d, gu_w, down_w) + ) + if inferred_mesh is not None: + with jax.sharding.use_abstract_mesh(inferred_mesh.abstract_mesh): + return smap_fn(x_3d, top_k_indices_3d, top_k_weights_3d, gu_w, down_w) + return smap_fn(x_3d, top_k_indices_3d, top_k_weights_3d, gu_w, down_w) class GraniteMoeMoE(nnx.Module): diff --git a/tests/unit/m3/glm5_3_flash_config_test.py b/tests/unit/m3/glm5_3_flash_config_test.py index 51130ae5e2..b66276992c 100644 --- a/tests/unit/m3/glm5_3_flash_config_test.py +++ b/tests/unit/m3/glm5_3_flash_config_test.py @@ -372,6 +372,18 @@ def test_from_mesh(self): } s_ctx_1 = Glm5Sharding.from_mesh(mesh_ctx_1) self.assertEqual(s_ctx_1.ep_size, 1) + self.assertFalse(s_ctx_1.ep_in_batch) + + mesh_ep4 = mock.create_autospec(jax.sharding.Mesh, instance=True) + mesh_ep4.axis_names = ("data", "fsdp", "tensor", "expert") + mesh_ep4.shape = {"data": 1, "fsdp": 2, "tensor": 1, "expert": 4} + s_ep4 = Glm5Sharding.from_mesh(mesh_ep4) + self.assertEqual(s_ep4.ep_size, 4) + self.assertTrue(s_ep4.ep_in_batch) + self.assertEqual( + s_ep4("kda_head_act", ("heads", "batch", None, None), m3_sharding.TensorType.ACTIVATION), + P(None, ("data", "fsdp", "expert"), None, None), + ) mesh_ctx_2 = mock.create_autospec(jax.sharding.Mesh, instance=True) mesh_ctx_2.axis_names = ("data", "fsdp", "tensor", "expert", "context") @@ -460,6 +472,29 @@ def test_pyconfig_integration(self): with self.assertRaisesRegex(ValueError, "requires use_m3_model=True"): Glm5Config.from_maxtext_config(cfg_no_m3) + def test_sparse_matmul_validation_and_sharding_axes(self): + self.assertTrue(Glm5Config().sparse_matmul) + self.assertFalse(Glm5Config(sparse_matmul=False).sparse_matmul) + base_cfg = pyconfig.initialize(["", "src/maxtext/configs/base.yml", "model_name=glm5.3-flash"]) + self.assertTrue(Glm5Config.from_maxtext_config(base_cfg).sparse_matmul) + self.assertFalse(Glm5Config.from_maxtext_config(base_cfg, sparse_matmul=False).sparse_matmul) + self.assertFalse(Glm5Config.from_maxtext_config(Glm5Config(sparse_matmul=True), sparse_matmul=False).sparse_matmul) + + for bad_bool in ("false", "true", 1, 0, None): + with self.assertRaisesRegex(ValueError, "sparse_matmul must be a bool"): + Glm5Config(sparse_matmul=cast(Any, bad_bool)) + with self.assertRaisesRegex(ValueError, "sparse_matmul must be a bool"): + Glm5Config.from_maxtext_config(base_cfg, sparse_matmul=bad_bool) + with self.assertRaisesRegex(ValueError, "sparse_matmul must be a bool"): + Glm5Config.from_maxtext_config(Glm5Config(), sparse_matmul=bad_bool) + + shd = Glm5Sharding(use_tp=True) + self.assertEqual(shd("gate_up_proj", ("experts", "expert_embed", "expert_mlp")), P("expert", None, "tensor")) + self.assertEqual( + shd("moe_tokens", ("batch", "length", "embed"), m3_sharding.TensorType.ACTIVATION), + P(("data", "fsdp", "expert"), None, None), + ) + if __name__ == "__main__": absltest.main() diff --git a/tests/unit/m3/glm5_3_flash_moe_test.py b/tests/unit/m3/glm5_3_flash_moe_test.py index 26a3de9a8d..fb2f305c41 100644 --- a/tests/unit/m3/glm5_3_flash_moe_test.py +++ b/tests/unit/m3/glm5_3_flash_moe_test.py @@ -20,6 +20,7 @@ import itertools import math import os +import re from typing import Any import unittest from unittest import mock @@ -45,16 +46,12 @@ raise HAS_TORCH = False -pytestmark = [ - pytest.mark.scheduled_only, - pytest.mark.skipif(not HAS_TORCH, reason="Torch or transformers not available"), -] - def _to_np(x): return x.detach().cpu().numpy() +@pytest.mark.scheduled_only @unittest.skipUnless(HAS_TORCH, "Torch or transformers not available") class Glm5MoeTest(absltest.TestCase): """Numerical parity and invariant tests for Clamped-SwiGLU MLP and MoE.""" @@ -386,24 +383,24 @@ def test_moe_sharding_and_guards(self): self._assert_allclose(out_exp, ref_out, atol=1e-5) def test_ep_sharding_guard(self): - """Verifies ep>1 construction-time and runtime guards on a real 4-device mesh.""" + """Verifies ep>1 construction-time and runtime guards when sparse_matmul=False.""" if len(jax.devices()) < 4: if "TEST_SRCDIR" in os.environ: self.fail("Expected >= 4 JAX devices under Blaze XLA_FLAGS, got" f" {len(jax.devices())}") self.skipTest(f"Requires >= 4 JAX devices, got {len(jax.devices())}") - cfg = self._mini_cfg() + cfg = dataclasses.replace(self._mini_cfg(), sparse_matmul=False) mesh_ep4 = m3_sharding.create_mesh(ici={"expert": 4}, devices=jax.devices()[:4]) shd_ep4 = glm5_config.Glm5Sharding.from_mesh(mesh_ep4) - # 1. Verify Glm5MoE constructor raises NotImplementedError + # 1. Verify Glm5MoE constructor raises NotImplementedError when sparse_matmul=False with self.assertRaisesRegex(NotImplementedError, "expert parallelism"): modeling_glm5_3_flash.Glm5MoE(cfg, sharding=shd_ep4, rngs=nnx.Rngs(0)) - # 2. Verify Glm5Experts constructor raises NotImplementedError + # 2. Verify Glm5Experts constructor raises NotImplementedError when sparse_matmul=False with self.assertRaisesRegex(NotImplementedError, "expert parallelism"): modeling_glm5_3_flash.Glm5Experts(cfg, sharding=shd_ep4, rngs=nnx.Rngs(0)) - # 3. Construct Glm5Experts without sharding and place x on mesh_ep4 + # 3. Construct Glm5Experts without sharding (sparse_matmul=False) and place x on mesh_ep4 exp_guard = modeling_glm5_3_flash.Glm5Experts(cfg, rngs=nnx.Rngs(0)) x_test = jax.random.normal(jax.random.PRNGKey(0), (4, cfg.hidden_size), dtype=jnp.float32) topk_idx = jnp.array([[0, 2], [1, 3], [2, 0], [3, 1]], dtype=jnp.int32) @@ -598,5 +595,192 @@ def test_router_param_dtype_independent_of_weight_dtype(self): self.assertEqual(moe.shared_experts.down_proj.kernel[...].dtype, jnp.bfloat16) +class Glm5MoeTokamaxExecutionTest(absltest.TestCase): + """Torch-free single-device and multi-device TP/EP parity tests for Glm5Experts on MoEExecutor.""" + + def setUp(self): + super().setUp() + self.enter_context(nnx.use_eager_sharding(True)) + + def _sync_nnx_vars(self, src_mod: Any, dst_mod: Any) -> None: + src_leaves = jax.tree.leaves(nnx.state(src_mod), is_leaf=lambda x: isinstance(x, nnx.Variable)) + dst_leaves = jax.tree.leaves(nnx.state(dst_mod), is_leaf=lambda x: isinstance(x, nnx.Variable)) + for s_var, d_var in zip(src_leaves, dst_leaves, strict=True): + shd = getattr(d_var.get_value(), "sharding", None) + d_var.set_value(jax.device_put(s_var.get_value(), shd) if shd is not None else jnp.asarray(s_var.get_value())) + + def test_single_device_and_small_token_tokamax_forward_and_vjp_parity(self): + cfg_ref = glm5_config.Glm5Config.mini_config( + hidden_size=64, + intermediate_size=128, + moe_intermediate_size=32, + n_routed_experts=8, + num_experts_per_tok=2, + dtype="float32", + weight_dtype="float32", + sparse_matmul=False, + ) + cfg_tok = dataclasses.replace(cfg_ref, sparse_matmul=True) + exp_ref = modeling_glm5_3_flash.Glm5Experts(cfg_ref, rngs=nnx.Rngs(7)) + with m3_sharding.allow_unsharded_construction(): + exp_tok = modeling_glm5_3_flash.Glm5Experts(cfg_tok, rngs=nnx.Rngs(7), sharding=glm5_config.Glm5Sharding(ep_size=2)) + self._sync_nnx_vars(exp_ref, exp_tok) + + # Small token count (6 tokens * top_k=2 -> M=12 < 128) + sentinel -1 unassigned slot (F6, F8). + x = jax.random.normal(jax.random.PRNGKey(11), (6, 64), dtype=jnp.float32) + idx = jnp.array([[0, 1], [2, -1], [4, 5], [1, 3], [6, 7], [0, -1]], dtype=jnp.int32) + w = jnp.array( + [[0.6, 0.4], [1.0, 0.9], [0.5, 0.5], [0.7, 0.3], [0.2, 0.8], [1.0, 0.5]], + dtype=jnp.float32, + ) + cot = jax.random.normal(jax.random.PRNGKey(12), x.shape, dtype=jnp.float32) + + out_ref = exp_ref(x, idx, w) + out_tok = exp_tok(x, idx, w) + np.testing.assert_allclose(np.asarray(out_tok), np.asarray(out_ref), rtol=1e-5, atol=1e-5) + + def _loss(mod, x_in): + return jnp.sum(mod(x_in, idx, w) * cot) + + _, (gmod_ref, gx_ref) = nnx.value_and_grad(_loss, argnums=(0, 1))(exp_ref, x) + _, (gmod_tok, gx_tok) = nnx.value_and_grad(_loss, argnums=(0, 1))(exp_tok, x) + np.testing.assert_allclose(np.asarray(gx_tok), np.asarray(gx_ref), rtol=1e-5, atol=1e-5) + np.testing.assert_allclose( + np.asarray(gmod_tok.gate_up_proj.get_value()), + np.asarray(gmod_ref.gate_up_proj.get_value()), + rtol=1e-5, + atol=1e-5, + ) + np.testing.assert_allclose( + np.asarray(gmod_tok.down_proj.get_value()), + np.asarray(gmod_ref.down_proj.get_value()), + rtol=1e-5, + atol=1e-5, + ) + + # Negative control: perturbing gate_up_proj must break output parity. + with m3_sharding.allow_unsharded_construction(): + exp_bad = modeling_glm5_3_flash.Glm5Experts(cfg_tok, rngs=nnx.Rngs(7), sharding=glm5_config.Glm5Sharding(ep_size=2)) + self._sync_nnx_vars(exp_ref, exp_bad) + exp_bad.gate_up_proj.set_value(exp_bad.gate_up_proj.get_value() + 2.0) + self.assertGreater(float(jnp.max(jnp.abs(exp_bad(x, idx, w) - out_ref))), 1e-3) + + def test_multidevice_tp_and_ep_tokamax_execution_and_no_weight_all_gather(self): + devices = jax.devices() + if len(devices) < 4: + self.skipTest(f"Requires >= 4 devices; got {len(devices)}") + + cfg_ref = glm5_config.Glm5Config.mini_config( + hidden_size=64, + intermediate_size=128, + moe_intermediate_size=32, + n_routed_experts=8, + num_experts_per_tok=2, + dtype="float32", + weight_dtype="float32", + sparse_matmul=False, + ) + cfg_tok = dataclasses.replace(cfg_ref, sparse_matmul=True) + moe_ref = modeling_glm5_3_flash.Glm5MoE(cfg_ref, rngs=nnx.Rngs(19)) + + x = jax.random.normal(jax.random.PRNGKey(23), (4, 4, 64), dtype=jnp.float32) + cot = jax.random.normal(jax.random.PRNGKey(29), x.shape, dtype=jnp.float32) + x_dec = jax.random.normal(jax.random.PRNGKey(31), (1, 1, 64), dtype=jnp.float32) + + @nnx.jit + def _ref_step(m, inp, c): + def _l(mod, z): + y, _ = mod(z) + return jnp.sum(y * c), y + + (loss_v, y_v), (gmod_v, dx_v) = nnx.value_and_grad(_l, argnums=(0, 1), has_aux=True)(m, inp) + return loss_v, y_v, gmod_v, dx_v + + _, out_ref, ref_gmod, ref_dx = _ref_step(moe_ref, x, cot) + ref_gmod_flat = list(nnx.to_flat_state(ref_gmod)) + out_dec_ref, _ = nnx.jit(lambda m, inp: m(inp))(moe_ref, x_dec) + + # Verify ep_size > 1 still raises NotImplementedError when sparse_matmul=False. + mesh_ep = m3_sharding.create_mesh(ici={"expert": 4}, devices=devices[:4]) + shd_ep = glm5_config.Glm5Sharding.from_mesh(mesh_ep) + with self.assertRaisesRegex(NotImplementedError, "expert=1"): + with jax.sharding.set_mesh(mesh_ep): + modeling_glm5_3_flash.Glm5MoE(cfg_ref, rngs=nnx.Rngs(19), sharding=shd_ep) + + full_expert_ag_re = rf"f32\[{cfg_tok.n_routed_experts},[0-9,]+\].*all-gather\(" + h_dim = cfg_tok.hidden_size + i_dim = cfg_tok.moe_intermediate_size + local_expert_wt_ag_re = rf"f32\[[0-9]+,{h_dim},2,{i_dim}\].*all-gather\(|f32\[[0-9]+,{i_dim},{h_dim}\].*all-gather\(" + + for ici in ({"fsdp": 2, "tensor": 2}, {"fsdp": 2, "expert": 2}, {"expert": 4}, {"expert": 2, "tensor": 2}): + mesh = m3_sharding.create_mesh(ici=ici, devices=devices[:4]) + shd = glm5_config.Glm5Sharding.from_mesh(mesh) + with jax.sharding.set_mesh(mesh): + moe_tok = modeling_glm5_3_flash.Glm5MoE(cfg_tok, rngs=nnx.Rngs(19), sharding=shd) + self._sync_nnx_vars(moe_ref, moe_tok) + x_shd = jax.device_put( + x, + jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec(("data", "fsdp", "expert"), None, None)), + ) + cot_shd = jax.device_put(cot, x_shd.sharding) + + @nnx.jit + def _step(m, inp, c): + def _l(mod, z): + y, _ = mod(z) + return jnp.sum(y * c), y + + (loss_v, y_v), (gmod_v, dx_v) = nnx.value_and_grad(_l, argnums=(0, 1), has_aux=True)(m, inp) + return loss_v, y_v, gmod_v, dx_v + + hlo = _step.lower(moe_tok, x_shd, cot_shd).compile().as_text() + full_expert_ag = re.findall(full_expert_ag_re, hlo) + self.assertEmpty( + full_expert_ag, + msg=f"Found forbidden full-expert weight all-gather on ici={ici}: {full_expert_ag}", + ) + if ici.get("fsdp", 1) == 1: + expert_wt_ag = re.findall(local_expert_wt_ag_re, hlo) + self.assertEmpty( + expert_wt_ag, + msg=f"Expected zero expert-weight all-gather when fsdp=1 on ici={ici}; got {expert_wt_ag}", + ) + + if ici == {"expert": 2, "tensor": 2}: + replicated_s = jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec(None, None, None, None)) + hlo_pos = ( + jax.jit(lambda v, s=replicated_s: jax.lax.with_sharding_constraint(v, s)) + .lower(moe_tok.experts.gate_up_proj.get_value()) + .compile() + .as_text() + ) + self.assertNotEmpty( + re.findall(full_expert_ag_re, hlo_pos), + msg=f"Positive control failed: expected full-expert all-gather in HLO; got: {hlo_pos}", + ) + + _, out_tok, gmod_tok, dx_tok = _step(moe_tok, x_shd, cot_shd) + np.testing.assert_allclose(np.asarray(out_tok), np.asarray(out_ref), rtol=1e-4, atol=1e-4) + np.testing.assert_allclose(np.asarray(dx_tok), np.asarray(ref_dx), rtol=1e-4, atol=1e-4) + tok_gmod_flat = list(nnx.to_flat_state(gmod_tok)) + self.assertLen(tok_gmod_flat, len(ref_gmod_flat)) + for (p_ref, g_ref), (p_tok, g_tok) in zip(ref_gmod_flat, tok_gmod_flat): + self.assertEqual(p_tok, p_ref) + np.testing.assert_allclose( + np.asarray(g_tok), + np.asarray(g_ref), + rtol=1e-4, + atol=1e-4, + err_msg=f"Param grad mismatch for {p_tok} on ici={ici}", + ) + + if ici == {"fsdp": 2, "expert": 2}: + x_dec_shd = jax.device_put( + x_dec, jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec(None, None, None)) + ) + out_dec_tok, _ = nnx.jit(lambda m, inp: m(inp))(moe_tok, x_dec_shd) + np.testing.assert_allclose(np.asarray(out_dec_tok), np.asarray(out_dec_ref), rtol=1e-4, atol=1e-4) + + if __name__ == "__main__": absltest.main() diff --git a/tests/unit/m3/granitemoe_test.py b/tests/unit/m3/granitemoe_test.py index f094e1a045..2b4cd2ddcd 100644 --- a/tests/unit/m3/granitemoe_test.py +++ b/tests/unit/m3/granitemoe_test.py @@ -31,6 +31,7 @@ import re import subprocess import tempfile +import types from typing import Any import unittest @@ -231,13 +232,14 @@ def _assert_zero_expert_all_gather(hlo_text: str, forbidden: Mapping[tuple[int, """Asserts that compiled HLO contains zero all-gather ops producing expert weight shapes.""" for line in hlo_text.splitlines(): stripped = line.strip() - if stripped.startswith("//"): + if stripped.startswith("//") or "=" not in stripped: continue - m = _ALL_GATHER_RE.search(stripped) + rhs = stripped.split("=", 1)[1] + m = _ALL_GATHER_RE.search(rhs) if m is None: continue - lhs = stripped[: m.start()] - for dims_str in _HLO_SHAPE_RE.findall(lhs): + shape_prefix = rhs[: m.start()] + for dims_str in _HLO_SHAPE_RE.findall(shape_prefix): shape = tuple(int(d) for d in dims_str.split(",") if d.strip()) if shape in forbidden: raise AssertionError(f"[{tag}] Forbidden expert weight all-gather of {forbidden[shape]} {shape}: {stripped}") @@ -1258,6 +1260,258 @@ def test_extract_hf_router_logits_handles_legacy_and_v5_22_tuple_layouts(self): self.assertIs(_extract_hf_router_logits(expected_logits, num_experts), expected_logits) +class GraniteMoeTokamaxExecutionTest(absltest.TestCase): + """Config validation, single-device, and multi-device Tokamax tests for GraniteMoe.""" + + def setUp(self): + super().setUp() + # Keep eager sharding on across full-suite collection (e.g., wan modules flip flax_always_shard_variable=False). + self.enter_context(nnx.use_eager_sharding(True)) + + def _sync_nnx_vars(self, src_mod: Any, dst_mod: Any) -> None: + for (_, s_var), (_, d_var) in zip( + _flatten_nnx_vars(nnx.state(src_mod)), + _flatten_nnx_vars(nnx.state(dst_mod)), + strict=True, + ): + shd = getattr(d_var.get_value(), "sharding", None) + d_var.set_value(jax.device_put(s_var.get_value(), shd) if shd is not None else jnp.asarray(s_var.get_value())) + + def test_sparse_matmul_config_validation_and_sharding_axes(self): + self.assertTrue(modeling_granitemoe.GraniteMoeConfig().sparse_matmul) + self.assertFalse(_make_mini_config(sparse_matmul=False).sparse_matmul) + for bad_bool in ("yes", "false", 1, 0, None): + with self.assertRaisesRegex(ValueError, "sparse_matmul must be a bool"): + _make_mini_config(sparse_matmul=bad_bool) + with self.assertRaisesRegex(ValueError, "sparse_matmul must be a bool"): + modeling_granitemoe.GraniteMoeConfig.from_maxtext_config(types.SimpleNamespace(sparse_matmul=bad_bool)) + with self.assertRaisesRegex(ValueError, "sparse_matmul must be a bool"): + modeling_granitemoe.GraniteMoeConfig.from_maxtext_config( + modeling_granitemoe.GraniteMoeConfig(), sparse_matmul=bad_bool + ) + + ns = types.SimpleNamespace(sparse_matmul=True) + self.assertTrue(modeling_granitemoe.GraniteMoeConfig.from_maxtext_config(ns).sparse_matmul) + self.assertFalse(modeling_granitemoe.GraniteMoeConfig.from_maxtext_config(ns, sparse_matmul=False).sparse_matmul) + + shd = modeling_granitemoe.GraniteMoeSharding() + self.assertEqual( + shd("input_linear", ("experts", "expert_embed", "expert_mlp")), + jax.sharding.PartitionSpec(("fsdp", "expert"), None, "tensor"), + ) + + def test_single_and_multidevice_tokamax_forward_vjp_parity_and_no_weight_all_gather(self): + cfg_ref = _make_mini_config(sparse_matmul=False) + cfg_tok = _make_mini_config(sparse_matmul=True) + exp_ref = modeling_granitemoe.GraniteMoeExperts(cfg_ref, rngs=nnx.Rngs(5)) + exp_tok = modeling_granitemoe.GraniteMoeExperts(cfg_tok, rngs=nnx.Rngs(5)) + self._sync_nnx_vars(exp_ref, exp_tok) + + # Small token count (2 * 3 = 6 tokens * top_k=2 -> M=12 < 128) + sentinel -1 unassigned slot (F6, F8). + x = jax.random.normal(jax.random.PRNGKey(13), (2, 3, cfg_ref.hidden_size), dtype=jnp.float32) + idx = jnp.array([[[0, 1], [2, -1], [4, 5]], [[1, 3], [6, 7], [0, -1]]], dtype=jnp.int32) + gates = jnp.array( + [[[0.6, 0.4], [1.0, 0.9], [0.5, 0.5]], [[0.7, 0.3], [0.2, 0.8], [1.0, 0.5]]], + dtype=jnp.float32, + ) + cot = jax.random.normal(jax.random.PRNGKey(17), x.shape, dtype=jnp.float32) + + out_ref = exp_ref(x, idx, gates) + out_tok = exp_tok(x, idx, gates) + np.testing.assert_allclose(np.asarray(out_tok), np.asarray(out_ref), rtol=1e-5, atol=1e-5) + + def _loss(mod, x_in): + return jnp.sum(mod(x_in, idx, gates) * cot) + + _, (gmod_ref, gx_ref) = nnx.value_and_grad(_loss, argnums=(0, 1))(exp_ref, x) + _, (gmod_tok, gx_tok) = nnx.value_and_grad(_loss, argnums=(0, 1))(exp_tok, x) + np.testing.assert_allclose(np.asarray(gx_tok), np.asarray(gx_ref), rtol=1e-5, atol=1e-5) + np.testing.assert_allclose( + np.asarray(gmod_tok.gate_up_proj.get_value()), + np.asarray(gmod_ref.gate_up_proj.get_value()), + rtol=1e-5, + atol=1e-5, + ) + np.testing.assert_allclose( + np.asarray(gmod_tok.down_proj.get_value()), + np.asarray(gmod_ref.down_proj.get_value()), + rtol=1e-5, + atol=1e-5, + ) + + # Negative control: perturbing gate_up_proj breaks output parity. + exp_bad = modeling_granitemoe.GraniteMoeExperts(cfg_tok, rngs=nnx.Rngs(5)) + self._sync_nnx_vars(exp_ref, exp_bad) + exp_bad.gate_up_proj.set_value(exp_bad.gate_up_proj.get_value() + 2.0) + self.assertGreater(float(jnp.max(jnp.abs(exp_bad(x, idx, gates) - out_ref))), 1e-3) + + def _moe_forbidden_shapes(cfg: modeling_granitemoe.GraniteMoeConfig, tp_size: int) -> dict[tuple[int, ...], str]: + n_exp = cfg.num_local_experts + h_dim = cfg.hidden_size + i_dim = cfg.intermediate_size + forbidden: dict[tuple[int, ...], str] = { + (n_exp, h_dim, 2, i_dim): "experts.gate_up_proj", + (n_exp, h_dim, 2 * i_dim): "experts.gate_up_proj[collapsed]", + (n_exp, h_dim * 2, i_dim): "experts.gate_up_proj[collapsed]", + (n_exp, i_dim, h_dim): "experts.down_proj", + } + if tp_size > 1 and i_dim % tp_size == 0: + i_tp = i_dim // tp_size + forbidden[(n_exp, h_dim, 2, i_tp)] = "experts.gate_up_proj[tp_shard]" + forbidden[(n_exp, h_dim, 2 * i_tp)] = "experts.gate_up_proj[tp_shard_collapsed]" + forbidden[(n_exp, h_dim * 2, i_tp)] = "experts.gate_up_proj[tp_shard_collapsed]" + forbidden[(n_exp, i_tp, h_dim)] = "experts.down_proj[tp_shard]" + return forbidden + + devices = jax.devices() + if len(devices) >= 4: + moe_ref = modeling_granitemoe.GraniteMoeMoE(cfg_ref, rngs=nnx.Rngs(31)) + x3d = jax.random.normal(jax.random.PRNGKey(37), (4, 4, cfg_ref.hidden_size), dtype=jnp.float32) + cot3d = jax.random.normal(jax.random.PRNGKey(41), x3d.shape, dtype=jnp.float32) + + @nnx.jit + def _ref_moe_step(m, inp, c): + def _l(mod, z): + y, _, _ = mod(z) + return jnp.sum(y * c), y + + (loss_v, y_v), (gmod_v, dx_v) = nnx.value_and_grad(_l, argnums=(0, 1), has_aux=True)(m, inp) + return loss_v, y_v, gmod_v, dx_v + + _, ref_y, ref_gmod, ref_dx = _ref_moe_step(moe_ref, x3d, cot3d) + ref_gmod_flat = list(nnx.to_flat_state(ref_gmod)) + + full_expert_ag_re = rf"f32\[{cfg_tok.num_local_experts},[0-9,]+\].*all-gather\(" + h_dim = cfg_tok.hidden_size + i_dim = cfg_tok.intermediate_size + local_expert_wt_ag_re = ( + rf"f32\[[0-9]+,{h_dim},2,{i_dim}\].*all-gather\(|f32\[[0-9]+,{i_dim},{h_dim}\].*all-gather\(" + ) + + for ici in ({"data": 4}, {"fsdp": 2, "expert": 2}, {"fsdp": 2, "tensor": 2}): + mesh = m3_sharding.create_mesh(ici=ici, devices=devices[:4]) + shd = modeling_granitemoe.GraniteMoeSharding.from_mesh(mesh, cfg_tok) + with jax.set_mesh(mesh): + moe_tok = modeling_granitemoe.GraniteMoeMoE(cfg_tok, sharding=shd, rngs=nnx.Rngs(31)) + self._sync_nnx_vars(moe_ref, moe_tok) + x_shd = jax.device_put( + x3d, + jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec(("data", "fsdp", "expert"), None, None)), + ) + cot_shd = jax.device_put(cot3d, x_shd.sharding) + + @nnx.jit + def _step(m, inp, c): + def _l(mod, z): + y, _, _ = mod(z) + return jnp.sum(y * c), y + + (loss_v, y_v), (gmod_v, dx_v) = nnx.value_and_grad(_l, argnums=(0, 1), has_aux=True)(m, inp) + return loss_v, y_v, gmod_v, dx_v + + hlo = _step.lower(moe_tok, x_shd, cot_shd).compile().as_text() + forbidden = _moe_forbidden_shapes(cfg_tok, tp_size=int(mesh.shape["tensor"])) + _assert_zero_expert_all_gather(hlo, forbidden, tag=f"granite_tokamax_{ici}") + full_expert_ag = re.findall(full_expert_ag_re, hlo) + self.assertEmpty( + full_expert_ag, + msg=f"Found forbidden full-expert weight all-gather on ici={ici}: {full_expert_ag}", + ) + local_wt_ag = re.findall(local_expert_wt_ag_re, hlo) + self.assertEmpty( + local_wt_ag, + msg=f"Expected zero expert-weight all-gather on compound ici={ici}; got {local_wt_ag}", + ) + + if ici == {"fsdp": 2, "expert": 2}: + self.assertIsInstance(moe_tok.experts.gate_up_proj.get_value().sharding, jax.sharding.NamedSharding) + replicated_s = jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec(None, None, None, None)) + hlo_pos = ( + jax.jit(lambda v, s=replicated_s: jax.lax.with_sharding_constraint(v, s)) + .lower(moe_tok.experts.gate_up_proj.get_value()) + .compile() + .as_text() + ) + with self.assertRaisesRegex(AssertionError, "Forbidden expert weight all-gather"): + _assert_zero_expert_all_gather(hlo_pos, forbidden, tag="granite_tokamax_pos_control") + self.assertNotEmpty(re.findall(full_expert_ag_re, hlo_pos)) + + _, y_tok, gmod_tok, dx_tok = _step(moe_tok, x_shd, cot_shd) + np.testing.assert_allclose(np.asarray(y_tok), np.asarray(ref_y), rtol=1e-4, atol=1e-4) + np.testing.assert_allclose(np.asarray(dx_tok), np.asarray(ref_dx), rtol=1e-4, atol=1e-4) + tok_gmod_flat = list(nnx.to_flat_state(gmod_tok)) + self.assertLen(tok_gmod_flat, len(ref_gmod_flat)) + for (p_ref, g_ref), (p_tok, g_tok) in zip(ref_gmod_flat, tok_gmod_flat): + self.assertEqual(p_tok, p_ref) + np.testing.assert_allclose( + np.asarray(g_tok), + np.asarray(g_ref), + rtol=1e-4, + atol=1e-4, + err_msg=f"Param grad mismatch for {p_tok} on ici={ici}", + ) + + # Non-compound EP > 1 layout (6 experts on fsdp=2, expert=2: 6 % (2*2) != 0, 6 % 2 == 0). + cfg_nc_ref = _make_mini_config(num_local_experts=6, sparse_matmul=False) + cfg_nc_tok = _make_mini_config(num_local_experts=6, sparse_matmul=True) + moe_nc_ref = modeling_granitemoe.GraniteMoeMoE(cfg_nc_ref, rngs=nnx.Rngs(43)) + x_dec = jax.random.normal(jax.random.PRNGKey(47), (1, 1, cfg_nc_ref.hidden_size), dtype=jnp.float32) + _, ref_nc_y, ref_nc_gmod, ref_nc_dx = _ref_moe_step(moe_nc_ref, x3d, cot3d) + ref_nc_gmod_flat = list(nnx.to_flat_state(ref_nc_gmod)) + ref_nc_dec_y, _, _ = nnx.jit(lambda m, inp: m(inp))(moe_nc_ref, x_dec) + + mesh_nc = m3_sharding.create_mesh(ici={"fsdp": 2, "expert": 2}, devices=devices[:4]) + shd_nc = modeling_granitemoe.GraniteMoeSharding.from_mesh(mesh_nc, cfg_nc_tok) + self.assertFalse(shd_nc.compound_ep) + with jax.set_mesh(mesh_nc): + moe_nc_tok = modeling_granitemoe.GraniteMoeMoE(cfg_nc_tok, sharding=shd_nc, rngs=nnx.Rngs(43)) + self._sync_nnx_vars(moe_nc_ref, moe_nc_tok) + x_nc_shd = jax.device_put( + x3d, + jax.sharding.NamedSharding(mesh_nc, jax.sharding.PartitionSpec(("data", "fsdp", "expert"), None, None)), + ) + cot_nc_shd = jax.device_put(cot3d, x_nc_shd.sharding) + + @nnx.jit + def _step_nc(m, inp, c): + def _l(mod, z): + y, _, _ = mod(z) + return jnp.sum(y * c), y + + (loss_v, y_v), (gmod_v, dx_v) = nnx.value_and_grad(_l, argnums=(0, 1), has_aux=True)(m, inp) + return loss_v, y_v, gmod_v, dx_v + + hlo_nc = _step_nc.lower(moe_nc_tok, x_nc_shd, cot_nc_shd).compile().as_text() + forbidden_nc = _moe_forbidden_shapes(cfg_nc_tok, tp_size=int(mesh_nc.shape["tensor"])) + _assert_zero_expert_all_gather(hlo_nc, forbidden_nc, tag="granite_tokamax_non_compound_fsdp2_ep2") + full_nc_ag = re.findall(rf"f32\[{cfg_nc_tok.num_local_experts},[0-9,]+\].*all-gather\(", hlo_nc) + self.assertEmpty( + full_nc_ag, + msg=f"Found forbidden full-expert weight all-gather on non-compound EP=2: {full_nc_ag}", + ) + + _, nc_y_tok, nc_gmod_tok, nc_dx_tok = _step_nc(moe_nc_tok, x_nc_shd, cot_nc_shd) + np.testing.assert_allclose(np.asarray(nc_y_tok), np.asarray(ref_nc_y), rtol=1e-4, atol=1e-4) + np.testing.assert_allclose(np.asarray(nc_dx_tok), np.asarray(ref_nc_dx), rtol=1e-4, atol=1e-4) + nc_tok_gmod_flat = list(nnx.to_flat_state(nc_gmod_tok)) + self.assertLen(nc_tok_gmod_flat, len(ref_nc_gmod_flat)) + for (p_ref, g_ref), (p_tok, g_tok) in zip(ref_nc_gmod_flat, nc_tok_gmod_flat): + self.assertEqual(p_tok, p_ref) + np.testing.assert_allclose( + np.asarray(g_tok), + np.asarray(g_ref), + rtol=1e-4, + atol=1e-4, + err_msg=f"Non-compound EP param grad mismatch for {p_tok}", + ) + + x_dec_shd = jax.device_put( + x_dec, jax.sharding.NamedSharding(mesh_nc, jax.sharding.PartitionSpec(None, None, None)) + ) + nc_dec_y_tok, _, _ = nnx.jit(lambda m, inp: m(inp))(moe_nc_tok, x_dec_shd) + np.testing.assert_allclose(np.asarray(nc_dec_y_tok), np.asarray(ref_nc_dec_y), rtol=1e-4, atol=1e-4) + + # ============================================================================== # Suite 3: Live PyTorch Weight-Copy Parity & `PerturbSpec` Negative Controls # ============================================================================== diff --git a/tests/unit/m3/moe_test.py b/tests/unit/m3/moe_test.py index c0c6be5fca..40374d72ce 100644 --- a/tests/unit/m3/moe_test.py +++ b/tests/unit/m3/moe_test.py @@ -16,6 +16,7 @@ import re from types import SimpleNamespace +import contextlib from unittest import mock from absl.testing import absltest @@ -387,6 +388,131 @@ def test_capacity_factor_and_ragged_buffer_dropping_with_negative_controls(self) np.testing.assert_allclose(tok_rbf, ragged_rbf, rtol=1e-5, atol=1e-5) self.assertGreater(float(jnp.max(jnp.abs(tok_rbf - dropless))), 1e-2) + @parameterized.named_parameters( + ("ragged_dot", "ragged_dot"), + ("tokamax", "tokamax"), + ) + def test_grouped_matmul_unassigned_tail_rows_zeroed_in_fwd_and_bwd(self, backend): + key = jax.random.key(314) + k1, k2, k3 = jax.random.split(key, 3) + m, valid_n, k, n = 16, 10, 12, 8 + group_sizes = jnp.array([4, 0, 6, 0], dtype=jnp.int32) + lhs_valid = jax.random.normal(k1, (valid_n, k), dtype=jnp.float32) + rhs = jax.random.normal(k2, (4, k, n), dtype=jnp.float32) + cot_valid = jax.random.normal(k3, (valid_n, n), dtype=jnp.float32) + + lhs_padded = jnp.concatenate([lhs_valid, jnp.full((m - valid_n, k), 500.0, dtype=jnp.float32)], axis=0) + cot_padded = jnp.concatenate([cot_valid, jnp.full((m - valid_n, n), -700.0, dtype=jnp.float32)], axis=0) + + ref_out, ref_vjp = jax.vjp( + lambda l, r: m3_moe.grouped_matmul(l, r, group_sizes, backend="ragged_dot", is_tpu=False), + lhs_valid, + rhs, + ) + ref_dlhs, ref_drhs = ref_vjp(cot_valid) + + use_tpu_path = backend == "tokamax" + with contextlib.ExitStack() as stack: + if use_tpu_path: + + def _fake_gmm_v2(lhs, rhs, group_sizes, *, transpose_rhs=False, preferred_element_type=None, **kw): + del kw + rhs_eff = rhs.swapaxes(1, 2) if transpose_rhs else rhs + res = jax.lax.ragged_dot(lhs, rhs_eff, group_sizes, preferred_element_type=jnp.float32) + return res.astype(preferred_element_type) if preferred_element_type is not None else res + + def _fake_tgmm_v2(lhs, rhs, group_sizes, *, num_actual_groups=None, preferred_element_type=None, **kw): + del kw + res = jax.lax.ragged_dot_general( + lhs, + rhs, + group_sizes, + ragged_dot_dimension_numbers=jax.lax.RaggedDotDimensionNumbers( + dot_dimension_numbers=(((0,), (0,)), ((), ())), + lhs_ragged_dimensions=(0,), + rhs_group_dimensions=(), + ), + preferred_element_type=jnp.float32, + ) + if num_actual_groups is not None: + res = res[:num_actual_groups] + return res.astype(preferred_element_type) if preferred_element_type is not None else res + + stack.enter_context(mock.patch.object(gmm_v2, "gmm_v2", side_effect=_fake_gmm_v2)) + stack.enter_context(mock.patch.object(tgmm_v2, "tgmm_v2", side_effect=_fake_tgmm_v2)) + out, vjp_fn = jax.vjp( + lambda l, r: m3_moe.grouped_matmul( + l, + r, + group_sizes, + backend="gmm_v2" if use_tpu_path else backend, + is_tpu=use_tpu_path, + ), + lhs_padded, + rhs, + ) + dlhs, drhs = vjp_fn(cot_padded) + + np.testing.assert_allclose(out[:valid_n], ref_out, rtol=1e-5, atol=1e-5) + np.testing.assert_array_equal(np.asarray(out[valid_n:]), np.zeros((m - valid_n, n), dtype=np.float32)) + np.testing.assert_allclose(dlhs[:valid_n], ref_dlhs, rtol=1e-5, atol=1e-5) + np.testing.assert_array_equal(np.asarray(dlhs[valid_n:]), np.zeros((m - valid_n, k), dtype=np.float32)) + np.testing.assert_allclose(drhs, ref_drhs, rtol=1e-5, atol=1e-5) + + if backend == "tokamax": + lhs_nan = jnp.concatenate([lhs_valid, jnp.full((m - valid_n, k), jnp.nan, dtype=jnp.float32)], axis=0) + cot_nan = jnp.concatenate([cot_valid, jnp.full((m - valid_n, n), jnp.nan, dtype=jnp.float32)], axis=0) + captured_pads = {} + + def mock_gmm(*args, **kwargs): + lhs_arg = kwargs.get("lhs", args[0] if args else None) + rhs_arg = kwargs.get("rhs", args[1] if len(args) > 1 else None) + rhs_eff = rhs_arg.swapaxes(1, 2) if kwargs.get("transpose_rhs", False) else rhs_arg + out_dtype = kwargs.get("preferred_element_type", lhs_arg.dtype) + if "fwd_lhs_pad" not in captured_pads: + captured_pads["fwd_lhs_pad"] = np.asarray(lhs_arg) + return jnp.zeros((lhs_arg.shape[0], rhs_eff.shape[2]), dtype=out_dtype) + captured_pads["bwd_grad_pad"] = np.asarray(lhs_arg) + return jnp.zeros((lhs_arg.shape[0], rhs_eff.shape[2]), dtype=out_dtype) + + def mock_tgmm(*args, **kwargs): + lhs_arg = kwargs.get("lhs", args[0] if args else None) + rhs_arg = kwargs.get("rhs", args[1] if len(args) > 1 else None) + num_groups = kwargs.get("num_actual_groups", rhs.shape[0]) + captured_pads["bwd_lhs_pad"] = np.asarray(lhs_arg) + return jnp.zeros((num_groups, lhs_arg.shape[1], rhs_arg.shape[1]), dtype=rhs.dtype) + + with ( + mock.patch.object(gmm_v2, "gmm_v2", side_effect=mock_gmm), + mock.patch.object(tgmm_v2, "tgmm_v2", side_effect=mock_tgmm), + ): + tpu_out, tpu_vjp = jax.vjp( + lambda l, r: m3_moe.grouped_matmul(l, r, group_sizes, backend="gmm_v2", is_tpu=True), + lhs_nan, + rhs, + ) + tpu_dlhs, tpu_drhs = tpu_vjp(cot_nan) + + self.assertFalse(np.isnan(captured_pads["fwd_lhs_pad"]).any()) + self.assertFalse(np.isnan(captured_pads["bwd_lhs_pad"]).any()) + self.assertFalse(np.isnan(captured_pads["bwd_grad_pad"]).any()) + self.assertFalse(np.isnan(np.asarray(tpu_out)).any()) + self.assertFalse(np.isnan(np.asarray(tpu_dlhs)).any()) + self.assertFalse(np.isnan(np.asarray(tpu_drhs)).any()) + + def test_grouped_unsharded_fused_without_active_mesh(self): + x, weights, indices, _ = _make_routing_inputs(batch=2, length=3, embed=16, num_experts=4, top_k=2) + indices = indices.at[0, 0, 1].set(-1) + wi_0, wi_1, wo = _make_expert_weights(num_experts=4, embed=16, mlp=12) + w_gu_4d = jnp.stack([wi_0, wi_1], axis=2) + ref = m3_moe.grouped_unsharded_fused( + x, weights, indices, w_gu_4d, wo, swiglu_limit=1.2, executor=m3_moe.MoEExecutor("ragged_dot"), is_tpu=False + ) + tok = m3_moe.grouped_unsharded_fused( + x, weights, indices, w_gu_4d, wo, swiglu_limit=1.2, executor=m3_moe.MoEExecutor("tokamax"), is_tpu=False + ) + np.testing.assert_allclose(tok, ref, rtol=1e-5, atol=1e-5) + class MultiDeviceEPAndTPTest(parameterized.TestCase): """Verifies multi-device `(FSDP >= 1, EP > 1, TP > 1)` parity, HLO collectives, and `ScannedLayers`.""" @@ -591,7 +717,7 @@ def test_ep_and_tp_forward_and_all_leaf_grad_parity_and_zero_expert_all_gather( ref_exec=ref_exec, ) - def test_scanned_layers_preserve_paths_and_match_unscanned_under_ep_and_tp(self): + def test_scan_layers_preserve_paths_and_match_unscanned_under_ep_and_tp(self): ici = self._ep_tp_layouts()[0] mesh = create_mesh(ici=ici, devices=jax.devices()[: (ici["fsdp"] * ici["expert"] * ici["tensor"])]) sharding = DefaultSharding() @@ -930,7 +1056,7 @@ def _mini_qwen3_8_cfg(self, *, scan_layers: bool, use_tokamax_gmm: bool = True) @parameterized.named_parameters( ("unscanned_tokamax", False, True), - ("scanned_tokamax", True, True), + ("scan_layers_tokamax", True, True), ) def test_qwen3_8_sparse_moe_block_ep_tp_forward_and_grad_parity(self, scan_layers, use_tokamax_gmm): n = jax.device_count() @@ -1039,7 +1165,7 @@ def _stack_loss(stack, h): @parameterized.named_parameters( ("unscanned_tokamax", False, True), - ("scanned_tokamax", True, True), + ("scan_layers_tokamax", True, True), ) def test_qwen3_8_full_model_ep_forward_and_all_leaf_grad_parity(self, scan_layers, use_tokamax_gmm): n = jax.device_count()