From 26df15b31d3a8fc7b6c43ae97adb79a0a453c0af Mon Sep 17 00:00:00 2001 From: Yuchen Yao Date: Thu, 12 Nov 2026 22:30:09 +0000 Subject: [PATCH] feat(m3): support forced_routed_experts router replay in m3/core/moe.py - Thread optional forced_routed_experts (shape [..., num_experts_per_tok], UNSET_ROUTED_EXPERT = -1 for padded/inactive slots) through route_top_k, apply_forced_routed_experts, MoEExecutor, routed_experts, routed_experts_fused, and grouped_unsharded_fused. - Preserve 100% identical HLO when forced_routed_experts is None. - Validate shape, integer dtype, and concrete ID range [-1, num_experts) failing closed on invalid shapes/dtypes/IDs, while clamping -1 slots before take_along_axis and zeroing inactive slot weights and denominators so -1 slots contribute exact 0.0 to outputs, group_sizes, and gradients. - Partition forced_routed_experts along (batch, length) inside shard_map across EP == 1 and EP > 1 (ragged_all_to_all + dropless overflow buffer). - Add 30 4-CPU unit tests in tests/unit/m3/moe_router_replay_test.py covering zero-overhead HLO parity, exact 0.0 fwd+bwd equivalence on natural top_k indices, non-natural forced overrides, -1 partial and all--1 row padding, and fail-closed validation. --- src/maxtext/m3/core/moe.py | 718 ++++++++++++++++- tests/unit/m3/moe_router_replay_test.py | 980 ++++++++++++++++++++++++ 2 files changed, 1666 insertions(+), 32 deletions(-) create mode 100644 tests/unit/m3/moe_router_replay_test.py diff --git a/src/maxtext/m3/core/moe.py b/src/maxtext/m3/core/moe.py index 4478678e09..34a21a4e5c 100644 --- a/src/maxtext/m3/core/moe.py +++ b/src/maxtext/m3/core/moe.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Shared routed-expert execution, load-balancing statistics, and overlap hooks. +"""Shared routed-expert execution, router replay, load-balancing stats, and overlap hooks. Model modules own router score computation (softmax, sigmoid, group-limited top-k, routing bias, shared experts) and pass selected `(weights, indices)` to @@ -24,6 +24,12 @@ # or for fused (E, D, 2*M) / (E, D, 2, M) gate_up projections: out = self.executor.fused(x, weights, indices, gate_up_proj, down_proj) +For RL router replay and teacher-forced routing verification, optional +`forced_routed_experts` (integer shape `[..., num_experts_per_tok]`, where +`UNSET_ROUTED_EXPERT = -1` marks a padded/inactive slot) can be passed to +`route_top_k`, `apply_forced_routed_experts`, `MoEExecutor`, `routed_experts`, +or `routed_experts_fused`. + Resolving `PartitionSpec` axes from `Sharding` at construction avoids reading `jax.typeof(x).sharding` at trace time, which reports replicated `P(None, ...)` on `Auto` mesh axes inside `jax.jit` and `ScannedLayers`. Because `MoEExecutor` @@ -49,6 +55,9 @@ ExecutorKind = Literal["dense", "ragged_dot", "gmm_v2", "tokamax"] SHARDED_EXECUTOR_KINDS: tuple[str, ...] = ("gmm_v2", "tokamax") +RouterScoring = Literal["softmax", "topk_softmax", "sigmoid"] + +UNSET_ROUTED_EXPERT = -1 _COMBINE_DTYPE = jnp.float32 _TOKAMAX_M_ALIGN = 256 @@ -60,6 +69,237 @@ def _align_up(value: int, multiple: int) -> int: return max(multiple, ((int(value) + multiple - 1) // multiple) * multiple) +# ============================================================================== +# Router replay & forced_routed_experts helpers +# ============================================================================== + + +def valid_expert_mask(indices: jax.Array, num_experts: int) -> jax.Array: + """Returns boolean mask where expert IDs are active (`0 <= id < num_experts`).""" + return (indices >= 0) & (indices < int(num_experts)) + + +def _validate_forced_routed_experts( + forced_routed_experts: jax.Array, + token_prefix: tuple[int, ...], + top_k: int, + num_experts: int, +) -> None: + """Validates `forced_routed_experts` shape, integer dtype, and concrete ID range.""" + if num_experts < 1: + raise ValueError(f"num_experts must be >= 1; got {num_experts}.") + if top_k < 1: + raise ValueError(f"top_k (num_experts_per_tok) must be >= 1; got {top_k}.") + expected_shape = (*token_prefix, int(top_k)) + if forced_routed_experts.ndim != len(expected_shape) or tuple(forced_routed_experts.shape) != expected_shape: + raise ValueError( + f"forced_routed_experts shape {forced_routed_experts.shape} must match " + f"(*token_prefix, num_experts_per_tok)={expected_shape}." + ) + if jnp.issubdtype(forced_routed_experts.dtype, jnp.bool_) or not jnp.issubdtype( + forced_routed_experts.dtype, jnp.integer + ): + raise ValueError(f"forced_routed_experts must have an integer dtype (not bool); got {forced_routed_experts.dtype}.") + if hasattr(forced_routed_experts, "min"): + try: + if forced_routed_experts.size > 0: + min_id = int(forced_routed_experts.min()) + max_id = int(forced_routed_experts.max()) + if min_id < UNSET_ROUTED_EXPERT or max_id >= int(num_experts): + raise ValueError( + f"forced_routed_experts entries must be in [-1, {num_experts}); got min={min_id}, max={max_id}." + ) + except (jax.errors.ConcretizationTypeError, TypeError): + pass + + +def apply_forced_routed_experts( + scores: jax.Array, + forced_routed_experts: jax.Array, + *, + norm_topk_prob: bool = False, + routed_scaling_factor: float = 1.0, + scores_are_logits: bool = False, + eps: float = 0.0, + dtype: Any | None = None, +) -> tuple[jax.Array, jax.Array]: + """Gathers and normalizes routing weights at `forced_routed_experts` indices. + + Args: + scores: Full router probabilities or pre-softmax router logits of shape + `[*token_prefix, num_experts]`. + forced_routed_experts: Integer tensor of shape + `[*token_prefix, num_experts_per_tok]` where `-1` (`UNSET_ROUTED_EXPERT`) + denotes a padded/inactive expert slot. + norm_topk_prob: Whether to normalize active slot weights by their sum. + routed_scaling_factor: Multiplicative scaling factor applied after normalization. + scores_are_logits: If True, applies masked `softmax` over gathered top-k + logits (excluding `-1` padded slots from the softmax denominator). + eps: Optional additive epsilon in the normalization denominator. + dtype: Output dtype for `routing_weights` (defaults to `scores.dtype`). + + Returns: + Tuple `(routing_weights, effective_indices)` of shape + `[*token_prefix, num_experts_per_tok]`, where inactive slots have weight + `0.0` and index `-1`. + """ + if scores.ndim < 2: + raise ValueError(f"Expected scores with rank >= 2 ([..., num_experts]); got shape {scores.shape}.") + num_experts = int(scores.shape[-1]) + token_prefix = tuple(scores.shape[:-1]) + if forced_routed_experts.ndim != scores.ndim: + raise ValueError( + f"forced_routed_experts rank ({forced_routed_experts.ndim}, shape={forced_routed_experts.shape}) " + f"must match scores rank ({scores.ndim}, shape={scores.shape})." + ) + top_k = int(forced_routed_experts.shape[-1]) + _validate_forced_routed_experts(forced_routed_experts, token_prefix, top_k, num_experts) + + forced_idx = forced_routed_experts.astype(jnp.int32) + valid_mask = valid_expert_mask(forced_idx, num_experts) + gather_idx = jnp.where(valid_mask, jnp.maximum(forced_idx, 0), 0) + scores_f32 = scores.astype(jnp.float32) + + if scores_are_logits: + gathered_logits = jnp.take_along_axis(scores_f32, gather_idx, axis=-1) + neg_inf = jnp.asarray(jnp.finfo(jnp.float32).min / 2.0, dtype=jnp.float32) + masked_logits = jnp.where(valid_mask, gathered_logits, neg_inf) + weights = jax.nn.softmax(masked_logits, axis=-1) + weights = jnp.where(valid_mask, weights, jnp.zeros_like(weights)) + else: + gathered_probs = jnp.take_along_axis(scores_f32, gather_idx, axis=-1) + weights = jnp.where(valid_mask, gathered_probs, jnp.zeros_like(gathered_probs)) + + if norm_topk_prob: + denom = jnp.sum(weights, axis=-1, keepdims=True) + denom_with_eps = denom + jnp.asarray(eps, dtype=denom.dtype) if eps > 0.0 else denom + safe_denom = jnp.where(denom > 0, denom_with_eps, jnp.ones_like(denom)) + weights = weights / safe_denom + + if routed_scaling_factor != 1.0: + weights = weights * jnp.asarray(routed_scaling_factor, dtype=weights.dtype) + + out_dtype = scores.dtype if dtype is None else dtype + eff_indices = jnp.where(valid_mask, forced_idx, UNSET_ROUTED_EXPERT) + return weights.astype(out_dtype), eff_indices + + +def route_top_k( + router_logits: jax.Array, + top_k: int, + *, + scoring: RouterScoring = "softmax", + norm_topk_prob: bool = False, + routed_scaling_factor: float = 1.0, + e_score_correction_bias: jax.Array | None = None, + num_group: int = 1, + topk_group: int = 1, + eps: float = 0.0, + forced_routed_experts: jax.Array | None = None, + dtype: Any | None = None, +) -> tuple[jax.Array, jax.Array, jax.Array]: + """Computes `(routing_weights, router_indices, router_probs)` with optional router replay. + + Supports `"softmax"` (post-softmax top-k, e.g. Qwen3.8 / Gemma4), + `"topk_softmax"` (pre-softmax top-k, e.g. GraniteMoE / Mixtral), and + `"sigmoid"` (sigmoid with optional `e_score_correction_bias` and group-limited + top-k, e.g. GLM-5 / DeepSeek-V4). + """ + if router_logits.ndim < 2: + raise ValueError(f"Expected router_logits with rank >= 2 ([..., num_experts]); got {router_logits.shape}.") + num_experts = int(router_logits.shape[-1]) + if top_k < 1 or top_k > num_experts: + raise ValueError(f"top_k ({top_k}) must be in [1, num_experts ({num_experts})].") + if forced_routed_experts is not None: + _validate_forced_routed_experts( + forced_routed_experts, + tuple(router_logits.shape[:-1]), + top_k, + num_experts, + ) + + out_dtype = router_logits.dtype if dtype is None else dtype + logits_f32 = router_logits.astype(jnp.float32) + + if scoring == "softmax": + router_probs = jax.nn.softmax(logits_f32, axis=-1) + if forced_routed_experts is not None: + weights, indices = apply_forced_routed_experts( + router_probs, + forced_routed_experts, + norm_topk_prob=norm_topk_prob, + routed_scaling_factor=routed_scaling_factor, + scores_are_logits=False, + eps=eps, + dtype=out_dtype, + ) + return weights, indices, router_probs + top_vals, top_idx = jax.lax.top_k(router_probs, top_k) + elif scoring == "topk_softmax": + router_probs = jax.nn.softmax(logits_f32, axis=-1) + if forced_routed_experts is not None: + weights, indices = apply_forced_routed_experts( + logits_f32, + forced_routed_experts, + norm_topk_prob=norm_topk_prob, + routed_scaling_factor=routed_scaling_factor, + scores_are_logits=True, + eps=eps, + dtype=out_dtype, + ) + return weights, indices, router_probs + top_logits, top_idx = jax.lax.top_k(logits_f32, top_k) + top_vals = jax.nn.softmax(top_logits, axis=-1) + elif scoring == "sigmoid": + router_probs = jax.nn.sigmoid(logits_f32) + if forced_routed_experts is not None: + weights, indices = apply_forced_routed_experts( + router_probs, + forced_routed_experts, + norm_topk_prob=norm_topk_prob, + routed_scaling_factor=routed_scaling_factor, + scores_are_logits=False, + eps=eps, + dtype=out_dtype, + ) + return weights, indices, router_probs + if e_score_correction_bias is not None: + scores_for_choice = jax.lax.stop_gradient(router_probs + e_score_correction_bias.astype(jnp.float32)) + else: + scores_for_choice = router_probs + if num_group > 1: + if num_experts % num_group != 0: + raise ValueError(f"num_experts ({num_experts}) must be divisible by num_group ({num_group}).") + if topk_group < 1 or topk_group > num_group: + raise ValueError(f"topk_group ({topk_group}) must be in [1, num_group ({num_group})].") + exp_per_group = num_experts // num_group + prefix_shape = scores_for_choice.shape[:-1] + grouped = jnp.reshape(scores_for_choice, (*prefix_shape, num_group, exp_per_group)) + group_k = min(2, exp_per_group) if e_score_correction_bias is not None else 1 + group_scores = jnp.sum(jax.lax.top_k(grouped, group_k)[0], axis=-1) + _, group_idx = jax.lax.top_k(group_scores, topk_group) + g_arange = jnp.arange(num_group, dtype=jnp.int32) + bcast_shape = (1,) * len(prefix_shape) + (num_group, 1) + group_mask = jnp.any(jnp.reshape(g_arange, bcast_shape) == group_idx[..., None, :], axis=-1) + score_mask = jnp.repeat(group_mask, exp_per_group, axis=-1) + masked_scores = jnp.where(score_mask, scores_for_choice, jnp.finfo(jnp.float32).min) + else: + masked_scores = scores_for_choice + _, top_idx = jax.lax.top_k(masked_scores, top_k) + top_vals = jnp.take_along_axis(router_probs, top_idx, axis=-1) + else: + raise ValueError(f"Unknown router scoring mode {scoring!r}; expected 'softmax', 'topk_softmax', or 'sigmoid'.") + + if norm_topk_prob: + denom = jnp.sum(top_vals, axis=-1, keepdims=True) + if eps > 0.0: + denom = denom + jnp.asarray(eps, dtype=denom.dtype) + top_vals = top_vals / denom + if routed_scaling_factor != 1.0: + top_vals = top_vals * jnp.asarray(routed_scaling_factor, dtype=top_vals.dtype) + return top_vals.astype(out_dtype), top_idx.astype(jnp.int32), router_probs + + # ============================================================================== # Custom-VJP scheduling barriers & backward wgrad overlap hook # ============================================================================== @@ -352,6 +592,7 @@ class MoEExecutor: dropless_buffer_factor: float = -1.0 use_ragged_sort: bool = False bwd_wgrad_overlap: bool = False + length: MeshAxes = None def __post_init__(self) -> None: if ( @@ -372,13 +613,19 @@ def __call__( self, x: jax.Array, weights: jax.Array, - indices: jax.Array, + indices: jax.Array | None, wi_0: jax.Array, wi_1: jax.Array, wo: jax.Array, + forced_routed_experts: jax.Array | None = None, *, activation: Callable[[jax.Array], jax.Array] = jax.nn.silu, swiglu_limit: float | None = None, + router_probs: jax.Array | None = None, + norm_topk_prob: bool = False, + routed_scaling_factor: float = 1.0, + scores_are_logits: bool = False, + eps: float = 0.0, ) -> jax.Array: """Executes routed experts with separate `(wi_0, wi_1, wo)` projections.""" return routed_experts( @@ -388,21 +635,33 @@ def __call__( wi_0, wi_1, wo, + forced_routed_experts=forced_routed_experts, executor=self, activation=activation, swiglu_limit=swiglu_limit, + router_probs=router_probs, + norm_topk_prob=norm_topk_prob, + routed_scaling_factor=routed_scaling_factor, + scores_are_logits=scores_are_logits, + eps=eps, ) def fused( self, x: jax.Array, weights: jax.Array, - indices: jax.Array, + indices: jax.Array | None, gate_up_proj: jax.Array, down_proj: jax.Array, + forced_routed_experts: jax.Array | None = None, *, activation: Callable[[jax.Array], jax.Array] = jax.nn.silu, swiglu_limit: float | None = None, + router_probs: jax.Array | None = None, + norm_topk_prob: bool = False, + routed_scaling_factor: float = 1.0, + scores_are_logits: bool = False, + eps: float = 0.0, ) -> jax.Array: """Executes routed experts with 3D `(E, D, 2*M)` or 4D `(E, D, 2, M)` `gate_up_proj`.""" return routed_experts_fused( @@ -411,9 +670,15 @@ def fused( indices, gate_up_proj, down_proj, + forced_routed_experts=forced_routed_experts, executor=self, activation=activation, swiglu_limit=swiglu_limit, + router_probs=router_probs, + norm_topk_prob=norm_topk_prob, + routed_scaling_factor=routed_scaling_factor, + scores_are_logits=scores_are_logits, + eps=eps, ) @@ -582,6 +847,7 @@ def moe_from_config(config: Any, sharding: Sharding | None = None) -> MoEExecuto dropless_buffer_factor=dbf, use_ragged_sort=use_ragged_sort, bwd_wgrad_overlap=bwd_wgrad_overlap, + length=tokens_spec[1], ) @@ -935,6 +1201,8 @@ def _grouped_unsharded( activation: Callable[[jax.Array], jax.Array], swiglu_limit: float | None, executor: MoEExecutor, + backend: str = "ragged_dot", + is_tpu: bool = False, ) -> jax.Array: """Portable `ragged_dot` grouped execution without `shard_map`.""" if executor.bwd_wgrad_overlap: @@ -974,7 +1242,7 @@ def _grouped_unsharded( sorted_x = _permute_rows(slot_rows, perm) sorted_x = jnp.where((jnp.arange(num_slots, dtype=jnp.int32) < valid_count)[:, None], sorted_x, 0) - matmul_fn = functools.partial(grouped_matmul, backend="ragged_dot", is_tpu=False) + matmul_fn = functools.partial(grouped_matmul, backend=backend, is_tpu=is_tpu) expert_out = _run_local_expert_mlp( sorted_x, projections, @@ -992,6 +1260,81 @@ def _grouped_unsharded( return jnp.reshape(combined, (batch, length, embed)) +def _validate_forced_routed_inputs( + x: jax.Array, + weights: jax.Array, + indices: jax.Array | None, + forced_routed_experts: jax.Array, + num_experts: int, + *, + router_probs: jax.Array | None = None, +) -> None: + """Validates 3D inputs when `forced_routed_experts` overrides routing indices.""" + if x.ndim != 3: + raise ValueError(f"Expected 3D x (B, S, D); got x={x.shape}.") + if forced_routed_experts.ndim != 3: + raise ValueError( + f"Expected 3D forced_routed_experts (B, S, K); got forced_routed_experts={forced_routed_experts.shape}." + ) + top_k = int(forced_routed_experts.shape[2]) + _validate_forced_routed_experts(forced_routed_experts, (x.shape[0], x.shape[1]), top_k, num_experts) + if indices is not None and (indices.ndim != 3 or indices.shape != forced_routed_experts.shape): + raise ValueError( + f"indices shape {indices.shape} must match forced_routed_experts shape {forced_routed_experts.shape}." + ) + if router_probs is not None: + if router_probs.ndim != 3 or router_probs.shape[:2] != x.shape[:2] or router_probs.shape[2] != num_experts: + raise ValueError( + f"router_probs shape {router_probs.shape} must match (B, S, num_experts)=" + f"({x.shape[0]}, {x.shape[1]}, {num_experts})." + ) + else: + if weights.ndim != 3 or weights.shape[:2] != x.shape[:2]: + raise ValueError(f"Expected 3D weights matching x batch/length {x.shape[:2]}; got weights={weights.shape}.") + if weights.shape[2] not in (top_k, num_experts): + raise ValueError( + f"weights last dim ({weights.shape[2]}) must match top_k ({top_k}) or num_experts ({num_experts}); " + f"got weights={weights.shape}." + ) + + +def _apply_forced_routed_inputs( + weights_or_scores: jax.Array, + forced_routed_experts: jax.Array, + num_experts: int, + *, + gather_from_scores: bool, + norm_topk_prob: bool = False, + routed_scaling_factor: float = 1.0, + scores_are_logits: bool = False, + eps: float = 0.0, +) -> tuple[jax.Array, jax.Array]: + """Resolves per-slot `(weights, indices)` under `forced_routed_experts`.""" + if gather_from_scores: + return apply_forced_routed_experts( + weights_or_scores, + forced_routed_experts, + norm_topk_prob=norm_topk_prob, + routed_scaling_factor=routed_scaling_factor, + scores_are_logits=scores_are_logits, + eps=eps, + dtype=_COMBINE_DTYPE, + ) + + forced_idx = forced_routed_experts.astype(jnp.int32) + valid_mask = valid_expert_mask(forced_idx, num_experts) + eff_w = jnp.where(valid_mask, weights_or_scores.astype(_COMBINE_DTYPE), jnp.zeros((), dtype=_COMBINE_DTYPE)) + if norm_topk_prob: + denom = jnp.sum(eff_w, axis=-1, keepdims=True) + denom_with_eps = denom + jnp.asarray(eps, dtype=denom.dtype) if eps > 0.0 else denom + safe_denom = jnp.where(denom > 0, denom_with_eps, jnp.ones_like(denom)) + eff_w = eff_w / safe_denom + if routed_scaling_factor != 1.0: + eff_w = eff_w * jnp.asarray(routed_scaling_factor, dtype=eff_w.dtype) + eff_idx = jnp.where(valid_mask, forced_idx, UNSET_ROUTED_EXPERT) + return eff_w, eff_idx + + def _sharded_grouped( x: jax.Array, weights: jax.Array, @@ -1003,16 +1346,24 @@ def _sharded_grouped( activation: Callable[[jax.Array], jax.Array], swiglu_limit: float | None, executor: MoEExecutor, + forced_routed_experts: jax.Array | None = None, + gather_from_scores: bool = False, + norm_topk_prob: bool = False, + routed_scaling_factor: float = 1.0, + scores_are_logits: bool = False, + eps: float = 0.0, ) -> jax.Array: """Distributed `shard_map` execution supporting `EP >= 1`, `TP >= 1`, and `FSDP >= 1`.""" mesh = kernels.mesh_in_scope() is_tpu = kernels.is_tpu_mesh(mesh) batch_ax = kernels.resolve_mesh_axes(mesh, executor.batch) + length_ax = kernels.resolve_mesh_axes(mesh, executor.length) experts_ax = kernels.resolve_mesh_axes(mesh, executor.experts) mlp_ax = kernels.resolve_mesh_axes(mesh, executor.mlp) batch_shards = kernels.mesh_axis_size(mesh, batch_ax) + length_shards = kernels.mesh_axis_size(mesh, length_ax) ep_size = kernels.mesh_axis_size(mesh, experts_ax) tp_size = kernels.mesh_axis_size(mesh, mlp_ax) ep_axes = kernels.mesh_axis_names(experts_ax) @@ -1022,6 +1373,8 @@ def _sharded_grouped( raise NotImplementedError(f"Expert sharding supports at most one mesh axis; got {ep_axes}.") if x.shape[0] % batch_shards != 0: raise ValueError(f"Batch {x.shape[0]} does not divide over the {batch_shards} token shards of {batch_ax!r}.") + if x.shape[1] % length_shards != 0: + raise ValueError(f"Length {x.shape[1]} does not divide over the {length_shards} sequence shards of {length_ax!r}.") if num_experts % ep_size != 0: raise ValueError( f"num_experts ({num_experts}) must be divisible by expert parallelism size ({ep_size}) on {experts_ax!r}." @@ -1056,7 +1409,7 @@ def _sharded_grouped( if mlp_dim % tp_size != 0: raise ValueError(f"Expert MLP dim ({mlp_dim}) must be divisible by tensor parallelism size ({tp_size}).") - tokens_spec = P(batch_ax, None, None) + tokens_spec = P(batch_ax, length_ax, None) matmul_fn = functools.partial(grouped_matmul, backend=executor.kind, is_tpu=is_tpu) e_loc = num_experts // ep_size m_tp = mlp_dim // tp_size @@ -1353,18 +1706,48 @@ def _ov_scan_body(state, _): combined = jax.lax.psum(combined, tp_axes) return jnp.reshape(combined, (b_loc, seq_len, d_dim)) - sharded_fn = jax.shard_map( - per_device, + x_in = kernels.ensure_explicit_spec(x, tokens_spec, mesh) + weights_in = kernels.ensure_explicit_spec(weights, tokens_spec, mesh) + proj_in = tuple(kernels.ensure_explicit_spec(p, spec, mesh) for p, spec in zip(projections, proj_specs)) + + if forced_routed_experts is None: + sharded_fn = jax.shard_map( + per_device, + mesh=mesh, + in_specs=(tokens_spec, tokens_spec, tokens_spec, *proj_specs), + out_specs=tokens_spec, + check_vma=False, + ) + indices_in = kernels.ensure_explicit_spec(indices, tokens_spec, mesh) + return sharded_fn(x_in, weights_in, indices_in, *proj_in) + + def per_device_forced( + x_sh: jax.Array, + weights_sh: jax.Array, + forced_sh: jax.Array, + *proj_sh: jax.Array, + ) -> jax.Array: + eff_w_sh, eff_idx_sh = _apply_forced_routed_inputs( + weights_sh, + forced_sh, + num_experts, + gather_from_scores=gather_from_scores, + norm_topk_prob=norm_topk_prob, + routed_scaling_factor=routed_scaling_factor, + scores_are_logits=scores_are_logits, + eps=eps, + ) + return per_device(x_sh, eff_w_sh, eff_idx_sh, *proj_sh) + + sharded_forced_fn = jax.shard_map( + per_device_forced, mesh=mesh, in_specs=(tokens_spec, tokens_spec, tokens_spec, *proj_specs), out_specs=tokens_spec, check_vma=False, ) - x_in = kernels.ensure_explicit_spec(x, tokens_spec, mesh) - weights_in = kernels.ensure_explicit_spec(weights, tokens_spec, mesh) - indices_in = kernels.ensure_explicit_spec(indices, tokens_spec, mesh) - proj_in = tuple(kernels.ensure_explicit_spec(p, spec, mesh) for p, spec in zip(projections, proj_specs)) - return sharded_fn(x_in, weights_in, indices_in, *proj_in) + forced_in = kernels.ensure_explicit_spec(forced_routed_experts, tokens_spec, mesh) + return sharded_forced_fn(x_in, weights_in, forced_in, *proj_in) # ============================================================================== @@ -1395,31 +1778,103 @@ def _validate_routed_inputs( def routed_experts( x: jax.Array, weights: jax.Array, - indices: jax.Array, + indices: jax.Array | None, wi_0: jax.Array, wi_1: jax.Array, wo: jax.Array, + forced_routed_experts: jax.Array | None = None, *, executor: MoEExecutor = DENSE, activation: Callable[[jax.Array], jax.Array] = jax.nn.silu, swiglu_limit: float | None = None, + router_probs: jax.Array | None = None, + norm_topk_prob: bool = False, + routed_scaling_factor: float = 1.0, + scores_are_logits: bool = False, + eps: float = 0.0, ) -> jax.Array: """Executes selected experts with separate `(wi_0, wi_1, wo)` projections.""" if wi_0.ndim != 3 or wi_1.ndim != 3 or wo.ndim != 3: raise ValueError(f"Expected 3D wi_0, wi_1, wo; got {wi_0.shape}, {wi_1.shape}, {wo.shape}.") num_experts = wi_0.shape[0] - _validate_routed_inputs(x, weights, indices, num_experts) + if forced_routed_experts is None: + if indices is None: + raise ValueError("indices must be provided when forced_routed_experts is None.") + _validate_routed_inputs(x, weights, indices, num_experts) + if wi_0.shape[1] != x.shape[2] or wo.shape[0] != num_experts or wo.shape[2] != x.shape[2]: + raise ValueError(f"Incompatible expert weight shapes for x={x.shape}: wi_0={wi_0.shape}, wo={wo.shape}.") + weights = weights.astype(_COMBINE_DTYPE) + projections = (wi_0.astype(x.dtype), wi_1.astype(x.dtype), wo.astype(x.dtype)) + match executor.kind: + case "dense": + out = _dense_separate( + x, + weights, + indices, + *projections, + num_experts=num_experts, + activation=activation, + swiglu_limit=swiglu_limit, + capacity_factor=executor.capacity_factor, + ) + case "ragged_dot": + out = _grouped_unsharded( + x, + weights, + indices, + projections, + fused=False, + num_experts=num_experts, + activation=activation, + swiglu_limit=swiglu_limit, + executor=executor, + ) + case "gmm_v2" | "tokamax": + out = _sharded_grouped( + x, + weights, + indices, + projections, + fused=False, + num_experts=num_experts, + activation=activation, + swiglu_limit=swiglu_limit, + executor=executor, + ) + case other: + raise ValueError(f"Unknown MoE executor kind {other!r}; expected 'dense', 'ragged_dot', 'gmm_v2', or 'tokamax'.") + return checkpoint_name(out.astype(x.dtype), "moe_routed_output") + + _validate_forced_routed_inputs( + x, + weights, + indices, + forced_routed_experts, + num_experts, + router_probs=router_probs, + ) if wi_0.shape[1] != x.shape[2] or wo.shape[0] != num_experts or wo.shape[2] != x.shape[2]: raise ValueError(f"Incompatible expert weight shapes for x={x.shape}: wi_0={wi_0.shape}, wo={wo.shape}.") - - weights = weights.astype(_COMBINE_DTYPE) projections = (wi_0.astype(x.dtype), wi_1.astype(x.dtype), wo.astype(x.dtype)) + weights_source = router_probs if router_probs is not None else weights + gather_from_scores = router_probs is not None or weights.shape[2] != forced_routed_experts.shape[2] + match executor.kind: case "dense": + eff_weights, eff_indices = _apply_forced_routed_inputs( + weights_source, + forced_routed_experts, + num_experts, + gather_from_scores=gather_from_scores, + norm_topk_prob=norm_topk_prob, + routed_scaling_factor=routed_scaling_factor, + scores_are_logits=scores_are_logits, + eps=eps, + ) out = _dense_separate( x, - weights, - indices, + eff_weights, + eff_indices, *projections, num_experts=num_experts, activation=activation, @@ -1427,10 +1882,20 @@ def routed_experts( capacity_factor=executor.capacity_factor, ) case "ragged_dot": + eff_weights, eff_indices = _apply_forced_routed_inputs( + weights_source, + forced_routed_experts, + num_experts, + gather_from_scores=gather_from_scores, + norm_topk_prob=norm_topk_prob, + routed_scaling_factor=routed_scaling_factor, + scores_are_logits=scores_are_logits, + eps=eps, + ) out = _grouped_unsharded( x, - weights, - indices, + eff_weights, + eff_indices, projections, fused=False, num_experts=num_experts, @@ -1441,30 +1906,134 @@ def routed_experts( case "gmm_v2" | "tokamax": out = _sharded_grouped( x, - weights, - indices, + weights_source.astype(_COMBINE_DTYPE), + forced_routed_experts, projections, fused=False, num_experts=num_experts, activation=activation, swiglu_limit=swiglu_limit, executor=executor, + forced_routed_experts=forced_routed_experts, + gather_from_scores=gather_from_scores, + norm_topk_prob=norm_topk_prob, + routed_scaling_factor=routed_scaling_factor, + scores_are_logits=scores_are_logits, + eps=eps, ) case other: raise ValueError(f"Unknown MoE executor kind {other!r}; expected 'dense', 'ragged_dot', 'gmm_v2', or 'tokamax'.") return checkpoint_name(out.astype(x.dtype), "moe_routed_output") +def grouped_unsharded_fused( + x: jax.Array, + weights: jax.Array, + indices: jax.Array | None, + gate_up_proj: jax.Array, + down_proj: jax.Array, + forced_routed_experts: jax.Array | None = None, + *, + executor: MoEExecutor = RAGGED_DOT, + activation: Callable[[jax.Array], jax.Array] = jax.nn.silu, + swiglu_limit: float | None = None, + is_tpu: bool = False, + router_probs: jax.Array | None = None, + norm_topk_prob: bool = False, + routed_scaling_factor: float = 1.0, + scores_are_logits: bool = False, + eps: float = 0.0, +) -> jax.Array: + """Executes fused routed experts on a single device when no `Mesh` is in scope.""" + if gate_up_proj.ndim not in (3, 4) or down_proj.ndim != 3: + raise ValueError( + f"Expected 3D or 4D gate_up_proj and 3D down_proj; got gate_up_proj={gate_up_proj.shape}, " + f"down_proj={down_proj.shape}." + ) + num_experts = gate_up_proj.shape[0] + if forced_routed_experts is None: + if indices is None: + raise ValueError("indices must be provided when forced_routed_experts is None.") + _validate_routed_inputs(x, weights, indices, num_experts) + eff_weights = weights.astype(_COMBINE_DTYPE) + eff_indices = indices + else: + _validate_forced_routed_inputs( + x, + weights, + indices, + forced_routed_experts, + num_experts, + router_probs=router_probs, + ) + weights_source = router_probs if router_probs is not None else weights + gather_from_scores = router_probs is not None or weights.shape[2] != forced_routed_experts.shape[2] + eff_weights, eff_indices = _apply_forced_routed_inputs( + weights_source, + forced_routed_experts, + num_experts, + gather_from_scores=gather_from_scores, + norm_topk_prob=norm_topk_prob, + routed_scaling_factor=routed_scaling_factor, + scores_are_logits=scores_are_logits, + eps=eps, + ) + + projections = (gate_up_proj.astype(x.dtype), down_proj.astype(x.dtype)) + backend = executor.kind if executor.kind in ("ragged_dot", "tokamax") else "ragged_dot" + if is_tpu and backend == "tokamax": + mesh_1d = jax.sharding.Mesh(jax.devices()[:1], ("_moe_1d",)) + return jax.shard_map( + lambda x_l, w_l, idx_l, gu_l, dn_l: _grouped_unsharded( + x_l, + w_l, + idx_l, + (gu_l, dn_l), + fused=True, + num_experts=num_experts, + activation=activation, + swiglu_limit=swiglu_limit, + executor=executor, + backend=backend, + is_tpu=True, + ).astype(x_l.dtype), + mesh=mesh_1d, + in_specs=(P(), P(), P(), P(), P()), + out_specs=P(), + check_vma=False, + )(x, eff_weights, eff_indices, projections[0], projections[1]) + out = _grouped_unsharded( + x, + eff_weights, + eff_indices, + projections, + fused=True, + num_experts=num_experts, + activation=activation, + swiglu_limit=swiglu_limit, + executor=executor, + backend=backend, + is_tpu=is_tpu, + ) + return out.astype(x.dtype) + + def routed_experts_fused( x: jax.Array, weights: jax.Array, - indices: jax.Array, + indices: jax.Array | None, gate_up_proj: jax.Array, down_proj: jax.Array, + forced_routed_experts: jax.Array | None = None, *, executor: MoEExecutor = DENSE, activation: Callable[[jax.Array], jax.Array] = jax.nn.silu, swiglu_limit: float | None = None, + router_probs: jax.Array | None = None, + norm_topk_prob: bool = False, + routed_scaling_factor: float = 1.0, + scores_are_logits: bool = False, + eps: float = 0.0, ) -> jax.Array: """Executes selected experts with fused 3D `(E, D, 2*M)` or 4D `(E, D, 2, M)` `gate_up_proj`.""" if gate_up_proj.ndim not in (3, 4) or down_proj.ndim != 3: @@ -1478,21 +2047,90 @@ def routed_experts_fused( raise ValueError(f"4D gate_up_proj axis 2 must have size 2 (gate, up); got {gate_up_proj.shape}.") num_experts = gate_up_proj.shape[0] - _validate_routed_inputs(x, weights, indices, num_experts) + if forced_routed_experts is None: + if indices is None: + raise ValueError("indices must be provided when forced_routed_experts is None.") + _validate_routed_inputs(x, weights, indices, num_experts) + if gate_up_proj.shape[1] != x.shape[2] or down_proj.shape[0] != num_experts or down_proj.shape[2] != x.shape[2]: + raise ValueError( + f"Incompatible fused expert weight shapes for x={x.shape}: " + f"gate_up_proj={gate_up_proj.shape}, down_proj={down_proj.shape}." + ) + weights = weights.astype(_COMBINE_DTYPE) + projections = (gate_up_proj.astype(x.dtype), down_proj.astype(x.dtype)) + match executor.kind: + case "dense": + out = _dense_fused( + x, + weights, + indices, + *projections, + num_experts=num_experts, + activation=activation, + swiglu_limit=swiglu_limit, + capacity_factor=executor.capacity_factor, + ) + case "ragged_dot": + out = _grouped_unsharded( + x, + weights, + indices, + projections, + fused=True, + num_experts=num_experts, + activation=activation, + swiglu_limit=swiglu_limit, + executor=executor, + ) + case "gmm_v2" | "tokamax": + out = _sharded_grouped( + x, + weights, + indices, + projections, + fused=True, + num_experts=num_experts, + activation=activation, + swiglu_limit=swiglu_limit, + executor=executor, + ) + case other: + raise ValueError(f"Unknown MoE executor kind {other!r}; expected 'dense', 'ragged_dot', 'gmm_v2', or 'tokamax'.") + return checkpoint_name(out.astype(x.dtype), "moe_routed_output") + + _validate_forced_routed_inputs( + x, + weights, + indices, + forced_routed_experts, + num_experts, + router_probs=router_probs, + ) if gate_up_proj.shape[1] != x.shape[2] or down_proj.shape[0] != num_experts or down_proj.shape[2] != x.shape[2]: raise ValueError( f"Incompatible fused expert weight shapes for x={x.shape}: " f"gate_up_proj={gate_up_proj.shape}, down_proj={down_proj.shape}." ) - - weights = weights.astype(_COMBINE_DTYPE) projections = (gate_up_proj.astype(x.dtype), down_proj.astype(x.dtype)) + weights_source = router_probs if router_probs is not None else weights + gather_from_scores = router_probs is not None or weights.shape[2] != forced_routed_experts.shape[2] + match executor.kind: case "dense": + eff_weights, eff_indices = _apply_forced_routed_inputs( + weights_source, + forced_routed_experts, + num_experts, + gather_from_scores=gather_from_scores, + norm_topk_prob=norm_topk_prob, + routed_scaling_factor=routed_scaling_factor, + scores_are_logits=scores_are_logits, + eps=eps, + ) out = _dense_fused( x, - weights, - indices, + eff_weights, + eff_indices, *projections, num_experts=num_experts, activation=activation, @@ -1500,10 +2138,20 @@ def routed_experts_fused( capacity_factor=executor.capacity_factor, ) case "ragged_dot": + eff_weights, eff_indices = _apply_forced_routed_inputs( + weights_source, + forced_routed_experts, + num_experts, + gather_from_scores=gather_from_scores, + norm_topk_prob=norm_topk_prob, + routed_scaling_factor=routed_scaling_factor, + scores_are_logits=scores_are_logits, + eps=eps, + ) out = _grouped_unsharded( x, - weights, - indices, + eff_weights, + eff_indices, projections, fused=True, num_experts=num_experts, @@ -1514,14 +2162,20 @@ def routed_experts_fused( case "gmm_v2" | "tokamax": out = _sharded_grouped( x, - weights, - indices, + weights_source.astype(_COMBINE_DTYPE), + forced_routed_experts, projections, fused=True, num_experts=num_experts, activation=activation, swiglu_limit=swiglu_limit, executor=executor, + forced_routed_experts=forced_routed_experts, + gather_from_scores=gather_from_scores, + norm_topk_prob=norm_topk_prob, + routed_scaling_factor=routed_scaling_factor, + scores_are_logits=scores_are_logits, + eps=eps, ) case other: raise ValueError(f"Unknown MoE executor kind {other!r}; expected 'dense', 'ragged_dot', 'gmm_v2', or 'tokamax'.") diff --git a/tests/unit/m3/moe_router_replay_test.py b/tests/unit/m3/moe_router_replay_test.py new file mode 100644 index 0000000000..c634b98427 --- /dev/null +++ b/tests/unit/m3/moe_router_replay_test.py @@ -0,0 +1,980 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit and 4-CPU multi-device tests for `forced_routed_experts` router replay in `maxtext.m3.core.moe`.""" + +import functools +from types import SimpleNamespace +from unittest import mock + +from absl.testing import absltest +from absl.testing import parameterized +import jax +import jax.numpy as jnp +from jax.sharding import NamedSharding +import numpy as np + +from maxtext.kernels.megablox import pallas_mosaic_tpu_v2_gmm_kernel as gmm_v2 +# pylint: disable=too-many-positional-arguments +from maxtext.kernels.megablox import pallas_mosaic_tpu_v2_tgmm_kernel as tgmm_v2 +from maxtext.m3.core import kernels as m3_kernels +from maxtext.m3.core import moe as m3_moe +from maxtext.m3.core.sharding import DefaultSharding, create_mesh + + +def _make_expert_weights( + num_experts: int = 4, + embed: int = 16, + mlp: int = 12, + *, + dtype=jnp.float32, + seed: int = 10, +) -> tuple[jax.Array, jax.Array, jax.Array]: + """Builds deterministic `(wi_0, wi_1, wo)` expert weight tensors.""" + k0, k1, k2 = jax.random.split(jax.random.key(seed), 3) + scale = 0.15 + wi_0 = (jax.random.normal(k0, (num_experts, embed, mlp), dtype=jnp.float32) * scale).astype(dtype) + wi_1 = (jax.random.normal(k1, (num_experts, embed, mlp), dtype=jnp.float32) * scale).astype(dtype) + wo = (jax.random.normal(k2, (num_experts, mlp, embed), dtype=jnp.float32) * scale).astype(dtype) + return wi_0, wi_1, wo + + +def _cpu_ragged_all_to_all_ref( + operand: jax.Array, + output: jax.Array, + input_offsets: jax.Array, + send_sizes: jax.Array, + output_offsets: jax.Array, + recv_sizes: jax.Array, + *, + axis_name: str, +) -> jax.Array: + """Pure-JAX collective reference matching `jax.lax.ragged_all_to_all` semantics on CPU.""" + del output_offsets + ep_size = send_sizes.shape[0] + in_len = operand.shape[0] + out_len = output.shape[0] + cap = max(1, min(int(in_len), int(out_len))) + col = jnp.arange(cap, dtype=jnp.int32)[None, :] + gather_idx = input_offsets[:, None] + col + valid_send = col < send_sizes[:, None] + safe_gather = jnp.clip(gather_idx, 0, max(int(in_len) - 1, 0)) + staged = jnp.where(valid_send[:, :, None], operand[safe_gather], jnp.zeros((), dtype=operand.dtype)) + recvd = jax.lax.all_to_all(staged, axis_name, split_axis=0, concat_axis=0) + + recv_ends = jnp.cumsum(recv_sizes) + recv_starts = recv_ends - recv_sizes + row_idx = jnp.arange(out_len, dtype=jnp.int32) + src_idx = jnp.searchsorted(recv_ends, row_idx, side="right") + valid_row = row_idx < recv_ends[-1] + safe_src = jnp.minimum(src_idx, ep_size - 1) + pos_in_src = jnp.clip(row_idx - recv_starts[safe_src], 0, cap - 1) + return jnp.where(valid_row[:, None], recvd[safe_src, pos_in_src], output) + + +class NoneReplayZeroOverheadTest(parameterized.TestCase): + """Verifies `forced_routed_experts=None` is a zero-overhead no-op with identical HLO.""" + + @parameterized.named_parameters( + ("dense", "dense"), + ("ragged_dot", "ragged_dot"), + ("tokamax", "tokamax"), + ) + def test_none_forced_routed_experts_matches_base_hlo_and_grads(self, kind): + mesh = create_mesh( + ici={"fsdp": 1, "expert": 2, "tensor": 2} if jax.device_count() >= 4 else None, + devices=jax.devices()[:4] if jax.device_count() >= 4 else jax.devices()[:1], + ) + x = jax.random.normal(jax.random.key(0), (4, 4, 16), dtype=jnp.float32) + logits = jax.random.normal(jax.random.key(1), (4, 4, 4), dtype=jnp.float32) + weights, indices, _ = m3_moe.route_top_k(logits, 2, scoring="softmax", norm_topk_prob=True) + wi_0, wi_1, wo = _make_expert_weights(num_experts=4, embed=16, mlp=12) + w_gu = jnp.stack([wi_0, wi_1], axis=2) if kind == "tokamax" else jnp.concatenate([wi_0, wi_1], axis=-1) + + executor = m3_moe.MoEExecutor( + kind=kind, + batch=("data", "fsdp") if kind == "tokamax" else None, + experts="expert" if kind == "tokamax" else None, + mlp="tensor" if kind == "tokamax" else None, + length="context" if kind == "tokamax" else None, + ) + + with jax.set_mesh(mesh): + fn_default = jax.jit(executor) + fn_none = jax.jit(functools.partial(executor, forced_routed_experts=None)) + hlo_default = fn_default.lower(x, weights, indices, wi_0, wi_1, wo).as_text("hlo") + hlo_none = fn_none.lower(x, weights, indices, wi_0, wi_1, wo).as_text("hlo") + # Strip module header name before comparing lowered HLO instructions. + body_default = "\n".join(hlo_default.splitlines()[1:]) + body_none = "\n".join(hlo_none.splitlines()[1:]) + self.assertEqual(body_default, body_none) + + out_default = fn_default(x, weights, indices, wi_0, wi_1, wo) + out_none = fn_none(x, weights, indices, wi_0, wi_1, wo) + self.assertEqual(float(jnp.max(jnp.abs(out_default - out_none))), 0.0) + + f_default = executor.fused(x, weights, indices, w_gu, wo) + f_none = executor.fused(x, weights, indices, w_gu, wo, forced_routed_experts=None) + self.assertEqual(float(jnp.max(jnp.abs(f_default - f_none))), 0.0) + + +class NaturalTopKExactEquivalenceTest(parameterized.TestCase): + """Verifies 0.0 max abs diff forward & backward when `forced_routed_experts == natural_top_k`.""" + + @parameterized.named_parameters( + ("softmax_unnorm", "softmax", False, 1.0, False, 0.0), + ("softmax_norm", "softmax", True, 1.5, False, 0.0), + ("topk_softmax_unnorm", "topk_softmax", False, 1.0, False, 0.0), + ("topk_softmax_norm", "topk_softmax", True, 1.25, False, 0.0), + ("sigmoid_norm_bias_groups", "sigmoid", True, 2.5, True, 1e-20), + ("sigmoid_unnorm_plain", "sigmoid", False, 1.0, False, 0.0), + ) + def test_route_top_k_natural_replay_exact_zero_diff_fwd_and_bwd( + self, scoring, norm_topk_prob, scale, use_bias_groups, eps + ): + k0, k1, k2 = jax.random.split(jax.random.key(42), 3) + num_experts = 8 + top_k = 2 + logits = jax.random.normal(k0, (3, 4, num_experts), dtype=jnp.float32) + # Add small spacing so top-k order is strictly tie-free. + logits = logits + jnp.arange(num_experts, dtype=jnp.float32) * 1e-2 + bias = jax.random.normal(k1, (num_experts,), dtype=jnp.float32) * 0.1 if use_bias_groups else None + num_group = 4 if use_bias_groups else 1 + topk_group = 2 if use_bias_groups else 1 + cot_w = jax.random.normal(k2, (3, 4, top_k), dtype=jnp.float32) + + nat_w, nat_idx, nat_probs = m3_moe.route_top_k( + logits, + top_k, + scoring=scoring, + norm_topk_prob=norm_topk_prob, + routed_scaling_factor=scale, + e_score_correction_bias=bias, + num_group=num_group, + topk_group=topk_group, + eps=eps, + forced_routed_experts=None, + ) + rep_w, rep_idx, rep_probs = m3_moe.route_top_k( + logits, + top_k, + scoring=scoring, + norm_topk_prob=norm_topk_prob, + routed_scaling_factor=scale, + e_score_correction_bias=bias, + num_group=num_group, + topk_group=topk_group, + eps=eps, + forced_routed_experts=nat_idx, + ) + + self.assertEqual(float(jnp.max(jnp.abs(rep_w - nat_w))), 0.0) + self.assertTrue(bool(jnp.array_equal(rep_idx, nat_idx))) + self.assertEqual(float(jnp.max(jnp.abs(rep_probs - nat_probs))), 0.0) + + def loss_nat(l_in): + w, _, _ = m3_moe.route_top_k( + l_in, + top_k, + scoring=scoring, + norm_topk_prob=norm_topk_prob, + routed_scaling_factor=scale, + e_score_correction_bias=bias, + num_group=num_group, + topk_group=topk_group, + eps=eps, + forced_routed_experts=None, + ) + return jnp.sum(w * cot_w) + + def loss_rep(l_in): + w, _, _ = m3_moe.route_top_k( + l_in, + top_k, + scoring=scoring, + norm_topk_prob=norm_topk_prob, + routed_scaling_factor=scale, + e_score_correction_bias=bias, + num_group=num_group, + topk_group=topk_group, + eps=eps, + forced_routed_experts=nat_idx, + ) + return jnp.sum(w * cot_w) + + grad_nat = jax.jit(jax.grad(loss_nat))(logits) + grad_rep = jax.jit(jax.grad(loss_rep))(logits) + self.assertEqual(float(jnp.max(jnp.abs(grad_rep - grad_nat))), 0.0) + + @parameterized.named_parameters( + ("dense_sep", "dense", False), + ("dense_fused3d", "dense", True), + ("ragged_dot_sep", "ragged_dot", False), + ("ragged_dot_fused4d", "ragged_dot", True), + ("tokamax_ep1_sep", "tokamax_ep1", False), + ("tokamax_ep1_fused3d", "tokamax_ep1", True), + ("tokamax_ep2_tp2_sep", "tokamax_ep2", False), + ("tokamax_ep2_tp2_fused4d", "tokamax_ep2", True), + ("tokamax_ep4_overflow_sep", "tokamax_ep4_ov", False), + ("tokamax_ep4_overflow_fused3d", "tokamax_ep4_ov", True), + ) + def test_executor_natural_top_k_exact_zero_diff_fwd_and_bwd(self, mode, fused): + if mode in ("tokamax_ep2", "tokamax_ep4_ov") and jax.device_count() < 4: + self.skipTest(f"Requires >= 4 CPU devices; got {jax.device_count()}.") + + num_experts = 8 + top_k = 2 + batch, length, embed, mlp = 4, 4, 16, 12 + k0, k1, k2 = jax.random.split(jax.random.key(101), 3) + x = jax.random.normal(k0, (batch, length, embed), dtype=jnp.float32) + logits = jax.random.normal(k1, (batch, length, num_experts), dtype=jnp.float32) + logits = logits + jnp.arange(num_experts, dtype=jnp.float32) * 1e-2 + cotangent = jax.random.normal(k2, x.shape, dtype=jnp.float32) + wi_0, wi_1, wo = _make_expert_weights(num_experts=num_experts, embed=embed, mlp=mlp, seed=17) + w_gu = ( + jnp.stack([wi_0, wi_1], axis=2) + if mode in ("ragged_dot", "tokamax_ep2") + else jnp.concatenate([wi_0, wi_1], axis=-1) + ) + + if mode == "dense": + mesh = create_mesh(devices=jax.devices()[:1]) + executor = m3_moe.DENSE + elif mode == "ragged_dot": + mesh = create_mesh(devices=jax.devices()[:1]) + executor = m3_moe.RAGGED_DOT + elif mode == "tokamax_ep1": + mesh = create_mesh(devices=jax.devices()[:1]) + executor = m3_moe.MoEExecutor(kind="tokamax", bwd_wgrad_overlap=True) + elif mode == "tokamax_ep2": + mesh = create_mesh(ici={"fsdp": 1, "expert": 2, "tensor": 2}, devices=jax.devices()[:4]) + executor = m3_moe.MoEExecutor( + kind="tokamax", + batch=("data", "fsdp"), + experts="expert", + mlp="tensor", + length="context", + bwd_wgrad_overlap=True, + ) + else: + mesh = create_mesh(ici={"fsdp": 1, "expert": 4, "tensor": 1}, devices=jax.devices()[:4]) + executor = m3_moe.MoEExecutor( + kind="tokamax", + batch=("data", "fsdp", "expert"), + experts="expert", + mlp="tensor", + dropless_buffer_factor=1.25, + bwd_wgrad_overlap=True, + ) + + nat_w, nat_idx, _ = m3_moe.route_top_k( + logits, top_k, scoring="softmax", norm_topk_prob=True, routed_scaling_factor=1.5 + ) + + with jax.set_mesh(mesh): + if not fused: + + def nat_fwd(x_in, l_in, w0, w1, w_out): + w, idx, _ = m3_moe.route_top_k(l_in, top_k, scoring="softmax", norm_topk_prob=True, routed_scaling_factor=1.5) + return executor(x_in, w, idx, w0, w1, w_out) + + def rep_router_fwd(x_in, l_in, fi_in, w0, w1, w_out): + w, idx, _ = m3_moe.route_top_k( + l_in, + top_k, + scoring="softmax", + norm_topk_prob=True, + routed_scaling_factor=1.5, + forced_routed_experts=fi_in, + ) + return executor(x_in, w, idx, w0, w1, w_out) + + def rep_exec_fwd(x_in, l_in, fi_in, w0, w1, w_out): + probs = jax.nn.softmax(l_in, axis=-1) + return executor( + x_in, + nat_w, + fi_in, + w0, + w1, + w_out, + forced_routed_experts=fi_in, + router_probs=probs, + norm_topk_prob=True, + routed_scaling_factor=1.5, + ) + + out_nat = jax.jit(nat_fwd)(x, logits, wi_0, wi_1, wo) + out_rep_r = jax.jit(rep_router_fwd)(x, logits, nat_idx, wi_0, wi_1, wo) + out_rep_e = jax.jit(rep_exec_fwd)(x, logits, nat_idx, wi_0, wi_1, wo) + grads_nat = jax.jit( + jax.grad( + lambda x_in, l_in, w0, w1, w_out: jnp.sum(nat_fwd(x_in, l_in, w0, w1, w_out) * cotangent), + argnums=(0, 1, 2, 3, 4), + ) + )(x, logits, wi_0, wi_1, wo) + grads_rep_r = jax.jit( + jax.grad( + lambda x_in, l_in, fi_in, w0, w1, w_out: jnp.sum( + rep_router_fwd(x_in, l_in, fi_in, w0, w1, w_out) * cotangent + ), + argnums=(0, 1, 3, 4, 5), + ) + )(x, logits, nat_idx, wi_0, wi_1, wo) + grads_rep_e = jax.jit( + jax.grad( + lambda x_in, l_in, fi_in, w0, w1, w_out: jnp.sum( + rep_exec_fwd(x_in, l_in, fi_in, w0, w1, w_out) * cotangent + ), + argnums=(0, 1, 3, 4, 5), + ) + )(x, logits, nat_idx, wi_0, wi_1, wo) + else: + + def nat_fwd_f(x_in, l_in, gu, dn): + w, idx, _ = m3_moe.route_top_k(l_in, top_k, scoring="softmax", norm_topk_prob=True, routed_scaling_factor=1.5) + return executor.fused(x_in, w, idx, gu, dn) + + def rep_router_fwd_f(x_in, l_in, fi_in, gu, dn): + w, idx, _ = m3_moe.route_top_k( + l_in, + top_k, + scoring="softmax", + norm_topk_prob=True, + routed_scaling_factor=1.5, + forced_routed_experts=fi_in, + ) + return executor.fused(x_in, w, idx, gu, dn) + + def rep_exec_fwd_f(x_in, l_in, fi_in, gu, dn): + probs = jax.nn.softmax(l_in, axis=-1) + return executor.fused( + x_in, + probs, + None, + gu, + dn, + forced_routed_experts=fi_in, + norm_topk_prob=True, + routed_scaling_factor=1.5, + ) + + out_nat = jax.jit(nat_fwd_f)(x, logits, w_gu, wo) + out_rep_r = jax.jit(rep_router_fwd_f)(x, logits, nat_idx, w_gu, wo) + out_rep_e = jax.jit(rep_exec_fwd_f)(x, logits, nat_idx, w_gu, wo) + grads_nat = jax.jit( + jax.grad(lambda x_in, l_in, gu, dn: jnp.sum(nat_fwd_f(x_in, l_in, gu, dn) * cotangent), argnums=(0, 1, 2, 3)) + )(x, logits, w_gu, wo) + grads_rep_r = jax.jit( + jax.grad( + lambda x_in, l_in, fi_in, gu, dn: jnp.sum(rep_router_fwd_f(x_in, l_in, fi_in, gu, dn) * cotangent), + argnums=(0, 1, 3, 4), + ) + )(x, logits, nat_idx, w_gu, wo) + grads_rep_e = jax.jit( + jax.grad( + lambda x_in, l_in, fi_in, gu, dn: jnp.sum(rep_exec_fwd_f(x_in, l_in, fi_in, gu, dn) * cotangent), + argnums=(0, 1, 3, 4), + ) + )(x, logits, nat_idx, w_gu, wo) + + self.assertEqual(float(jnp.max(jnp.abs(out_rep_r - out_nat))), 0.0) + self.assertEqual(float(jnp.max(jnp.abs(out_rep_e - out_nat))), 0.0) + for g_r, g_e, g_n in zip(grads_rep_r, grads_rep_e, grads_nat): + self.assertEqual(float(jnp.max(jnp.abs(g_r - g_n))), 0.0) + # Multi-device mesh parallel reductions introduce up to 1 ULP (~6e-8) float32 roundoff. + self.assertLessEqual(float(jnp.max(jnp.abs(g_e - g_n))), 1e-6) + + def test_explicit_mesh_and_sequence_sharding_partition_forced_routed_experts(self): + if jax.device_count() < 4: + self.skipTest(f"Requires >= 4 CPU devices; got {jax.device_count()}.") + ici = {"context": 2, "expert": 2} + mesh_exp = create_mesh(ici=ici, devices=jax.devices()[:4], explicit=True) + sharding = DefaultSharding() + cfg = SimpleNamespace(num_experts=4, sparse_matmul=True, moe_bwd_wgrad_overlap=True) + exec_exp = m3_moe.moe_from_config(cfg, sharding) + self.assertEqual(exec_exp.length, "context") + + x = jax.random.normal(jax.random.key(7), (4, 4, 16), dtype=jnp.float32) + logits = jax.random.normal(jax.random.key(8), (4, 4, 4), dtype=jnp.float32) + logits = logits + jnp.arange(4, dtype=jnp.float32) * 1e-2 + cotangent = jax.random.normal(jax.random.key(9), x.shape, dtype=jnp.float32) + wi_0, wi_1, wo = _make_expert_weights(num_experts=4, embed=16, mlp=12, seed=21) + nat_w, nat_idx, nat_probs = m3_moe.route_top_k(logits, 2, scoring="softmax", norm_topk_prob=True) + + with jax.set_mesh(mesh_exp): + tok_s = NamedSharding(mesh_exp, sharding("tokens", ("batch", "length", "embed"), m3_moe.TensorType.ACTIVATION)) + w0_s = NamedSharding(mesh_exp, sharding("wi_0", ("experts", "expert_embed", "expert_mlp"))) + wo_s = NamedSharding(mesh_exp, sharding("wo", ("experts", "expert_mlp", "expert_embed"))) + + x_d = jax.device_put(x, tok_s) + w_d = jax.device_put(nat_w, tok_s) + idx_d = jax.device_put(nat_idx, tok_s) + probs_d = jax.device_put(nat_probs, tok_s) + cot_d = jax.device_put(cotangent, tok_s) + wi0_d = jax.device_put(wi_0, w0_s) + wi1_d = jax.device_put(wi_1, w0_s) + wo_d = jax.device_put(wo, wo_s) + + def nat_exp_fwd(x_in, p_in, fi_in, w0, w1, w_out): + w_cur, idx_cur = m3_moe.apply_forced_routed_experts(p_in, fi_in, norm_topk_prob=True) + return exec_exp(x_in, w_cur, idx_cur, w0, w1, w_out) + + def rep_exp_fwd(x_in, p_in, fi_in, w0, w1, w_out): + return exec_exp( + x_in, + w_d, + fi_in, + w0, + w1, + w_out, + forced_routed_experts=fi_in, + router_probs=p_in, + norm_topk_prob=True, + ) + + out_nat = jax.jit(nat_exp_fwd)(x_d, probs_d, idx_d, wi0_d, wi1_d, wo_d) + out_rep = jax.jit(rep_exp_fwd)(x_d, probs_d, idx_d, wi0_d, wi1_d, wo_d) + grads_nat = jax.jit( + jax.grad( + lambda x_in, p_in, fi_in, w0, w1, w_out: jnp.sum(nat_exp_fwd(x_in, p_in, fi_in, w0, w1, w_out) * cot_d), + argnums=(0, 1, 3, 4, 5), + ) + )(x_d, probs_d, idx_d, wi0_d, wi1_d, wo_d) + grads_rep = jax.jit( + jax.grad( + lambda x_in, p_in, fi_in, w0, w1, w_out: jnp.sum(rep_exp_fwd(x_in, p_in, fi_in, w0, w1, w_out) * cot_d), + argnums=(0, 1, 3, 4, 5), + ) + )(x_d, probs_d, idx_d, wi0_d, wi1_d, wo_d) + + self.assertEqual(float(jnp.max(jnp.abs(np.asarray(out_rep) - np.asarray(out_nat)))), 0.0) + for g_r, g_n in zip(grads_rep, grads_nat): + # Explicit mesh sharding parallel reductions introduce up to 1 ULP (~6e-8) float32 roundoff. + self.assertLessEqual(float(jnp.max(jnp.abs(np.asarray(g_r) - np.asarray(g_n)))), 1e-6) + + +class ForcedExpertOverrideAndNegativeControlTest(parameterized.TestCase): + """Verifies non-natural forced expert override matches manual expert math and shifts away from natural top-k.""" + + @parameterized.named_parameters( + ("softmax_norm", "softmax", True, 1.4), + ("topk_softmax", "topk_softmax", False, 1.0), + ("sigmoid_norm", "sigmoid", True, 2.5), + ) + def test_non_natural_forced_override_matches_manual_and_differs_from_natural(self, scoring, norm_topk_prob, scale): + if jax.device_count() < 4: + self.skipTest(f"Requires >= 4 CPU devices; got {jax.device_count()}.") + num_experts = 4 + top_k = 2 + batch, length, embed, mlp = 4, 3, 16, 12 + k0, k1, k2 = jax.random.split(jax.random.key(303), 3) + x = jax.random.normal(k0, (batch, length, embed), dtype=jnp.float32) + logits = jax.random.normal(k1, (batch, length, num_experts), dtype=jnp.float32) + cotangent = jax.random.normal(k2, x.shape, dtype=jnp.float32) + wi_0, wi_1, wo = _make_expert_weights(num_experts=num_experts, embed=embed, mlp=mlp, seed=31) + w_gu = jnp.concatenate([wi_0, wi_1], axis=-1) + + nat_w, nat_idx, _ = m3_moe.route_top_k( + logits, top_k, scoring=scoring, norm_topk_prob=norm_topk_prob, routed_scaling_factor=scale + ) + # Force every token to select the bottom-k experts ((nat_idx + 2) % num_experts), disjoint from natural top-2. + forced_idx = (nat_idx + 2) % num_experts + + def manual_reference(x_in, l_in, w0, w1, w_out): + l_f32 = l_in.astype(jnp.float32) + if scoring == "softmax": + probs = jax.nn.softmax(l_f32, axis=-1) + w = jnp.take_along_axis(probs, forced_idx, axis=-1) + elif scoring == "topk_softmax": + top_l = jnp.take_along_axis(l_f32, forced_idx, axis=-1) + w = jax.nn.softmax(top_l, axis=-1) + else: + probs = jax.nn.sigmoid(l_f32) + w = jnp.take_along_axis(probs, forced_idx, axis=-1) + if norm_topk_prob: + w = w / jnp.sum(w, axis=-1, keepdims=True) + w = w * scale + # Manual per-slot expert MLP evaluation. + gate = jnp.einsum("bsd,edm->bsem", x_in, w0) + up = jnp.einsum("bsd,edm->bsem", x_in, w1) + hid = jax.nn.silu(gate) * up + exp_out = jnp.einsum("bsem,emd->bsed", hid, w_out) + chosen_out = jnp.take_along_axis(exp_out, forced_idx[..., None], axis=2) + return jnp.sum(chosen_out * w[..., None], axis=2) + + ref_out, ref_grads = jax.jit( + jax.value_and_grad( + lambda x_in, l_in, w0, w1, w_out: ( + jnp.sum(manual_reference(x_in, l_in, w0, w1, w_out) * cotangent), + manual_reference(x_in, l_in, w0, w1, w_out), + ), + argnums=(0, 1, 2, 3, 4), + has_aux=True, + ) + )(x, logits, wi_0, wi_1, wo) + ref_y = ref_out[1] + + mesh = create_mesh(ici={"fsdp": 1, "expert": 2, "tensor": 2}, devices=jax.devices()[:4]) + exec_ep = m3_moe.MoEExecutor( + kind="tokamax", + batch=("data", "fsdp"), + experts="expert", + mlp="tensor", + dropless_buffer_factor=1.25, + bwd_wgrad_overlap=True, + ) + + with jax.set_mesh(mesh): + nat_y = exec_ep(x, nat_w, nat_idx, wi_0, wi_1, wo) + # Negative control: forced non-natural routing MUST differ from natural top-k output. + self.assertGreater(float(jnp.max(jnp.abs(ref_y - nat_y))), 1e-2) + + for exec_cur in (m3_moe.DENSE, m3_moe.RAGGED_DOT, exec_ep): + (_, got_y), got_grads = jax.jit( + jax.value_and_grad( + lambda x_in, l_in, w0, w1, w_out, ex=exec_cur: ( + jnp.sum( + ex( + x_in, + *m3_moe.route_top_k( + l_in, + top_k, + scoring=scoring, + norm_topk_prob=norm_topk_prob, + routed_scaling_factor=scale, + forced_routed_experts=forced_idx, + )[:2], + w0, + w1, + w_out, + ) + * cotangent + ), + ex( + x_in, + *m3_moe.route_top_k( + l_in, + top_k, + scoring=scoring, + norm_topk_prob=norm_topk_prob, + routed_scaling_factor=scale, + forced_routed_experts=forced_idx, + )[:2], + w0, + w1, + w_out, + ), + ), + argnums=(0, 1, 2, 3, 4), + has_aux=True, + ) + )(x, logits, wi_0, wi_1, wo) + + np.testing.assert_allclose(np.asarray(got_y), np.asarray(ref_y), rtol=1e-5, atol=1e-5) + for g_got, g_ref in zip(got_grads, ref_grads): + np.testing.assert_allclose(np.asarray(g_got), np.asarray(g_ref), rtol=1e-5, atol=1e-5) + + # Also verify unsharded fused fallback with forced_routed_experts. + w_forced, _ = m3_moe.route_top_k( + logits, + top_k, + scoring=scoring, + norm_topk_prob=norm_topk_prob, + routed_scaling_factor=scale, + forced_routed_experts=forced_idx, + )[:2] + unf_y = m3_moe.grouped_unsharded_fused(x, w_forced, nat_idx, w_gu, wo, forced_routed_experts=forced_idx) + np.testing.assert_allclose(np.asarray(unf_y), np.asarray(ref_y), rtol=1e-5, atol=1e-5) + + +class SentinelPaddingAndZeroGradLeakageTest(parameterized.TestCase): + """Verifies `-1` partial-slot and all-`-1` full-row padding produce 0.0 output/grads and zero expert-0 leakage.""" + + @parameterized.named_parameters( + ("softmax_norm", "softmax", True, 1.5, 0.0), + ("softmax_unnorm", "softmax", False, 1.0, 0.0), + ("topk_softmax_unnorm", "topk_softmax", False, 1.0, 0.0), + ("topk_softmax_norm", "topk_softmax", True, 1.2, 0.0), + ("sigmoid_norm_eps", "sigmoid", True, 2.5, 1e-20), + ("sigmoid_unnorm", "sigmoid", False, 1.0, 0.0), + ) + def test_partial_and_all_minus_one_padding_zero_output_zero_nan_and_zero_expert0_leakage( + self, scoring, norm_topk_prob, scale, eps + ): + if jax.device_count() < 4: + self.skipTest(f"Requires >= 4 CPU devices; got {jax.device_count()}.") + num_experts = 4 + top_k = 2 + batch, length, embed, mlp = 4, 3, 16, 12 + k0, k1, k2 = jax.random.split(jax.random.key(505), 3) + x = jax.random.normal(k0, (batch, length, embed), dtype=jnp.float32) + logits = jax.random.normal(k1, (batch, length, num_experts), dtype=jnp.float32) + cotangent = jax.random.normal(k2, x.shape, dtype=jnp.float32) + wi_0, wi_1, wo = _make_expert_weights(num_experts=num_experts, embed=embed, mlp=mlp, seed=41) + w_gu = jnp.stack([wi_0, wi_1], axis=2) + + # Construct forced_routed_experts where: + # - Expert 0 is NEVER selected by any active (>= 0) slot (active IDs are only in {1, 2, 3}). + # - Token (0, 0) and full batch row 2 have all slots = -1 (all-`-1` full-row padding). + # - Token (0, 1) and token (1, 2) have partial-slot padding `[2, -1]` and `[-1, 3]`. + forced_idx = jnp.array( + [ + [[-1, -1], [2, -1], [1, 3]], + [[3, 1], [2, 3], [-1, 3]], + [[-1, -1], [-1, -1], [-1, -1]], + [[1, 2], [-1, -1], [3, -1]], + ], + dtype=jnp.int32, + ) + self.assertFalse(bool(jnp.any(forced_idx == 0))) + + # Verify compute_ep_overflow_stats does not count -1 slots into expert 0 / EP rank 0. + stats = m3_moe.compute_ep_overflow_stats(forced_idx, num_experts=4, ep_size=2, dropless_buffer_factor=1.25) + # Active slots: expert 1 appears 3 times, expert 2 appears 3 times, expert 3 appears 5 times, expert 0 appears 0 times. + # Rank 0 owns experts {0, 1} -> count = 0 + 3 = 3; Rank 1 owns experts {2, 3} -> count = 3 + 5 = 8. + np.testing.assert_array_equal(np.asarray(stats["dst_counts"]), np.array([3, 8], dtype=np.int32)) + + # Negative control: if -1 were naively clamped to 0, Rank 0 count would be 3 + 13 = 16 instead of 3! + naive_clamped = jnp.maximum(forced_idx, 0) + naive_stats = m3_moe.compute_ep_overflow_stats(naive_clamped, num_experts=4, ep_size=2) + self.assertEqual(int(naive_stats["dst_counts"][0]), 16) + + # Check partial-slot weight normalization on token (0, 1) where forced_idx = [2, -1]: + w_routed, idx_routed, probs_routed = m3_moe.route_top_k( + logits, + top_k, + scoring=scoring, + norm_topk_prob=norm_topk_prob, + routed_scaling_factor=scale, + eps=eps, + forced_routed_experts=forced_idx, + ) + self.assertTrue(bool(jnp.all(jnp.isfinite(w_routed)))) + self.assertTrue(bool(jnp.array_equal(idx_routed, forced_idx))) + # All -1 slots must have exact 0.0 weight. + self.assertEqual(float(jnp.max(jnp.abs(jnp.where(forced_idx < 0, w_routed, 0.0)))), 0.0) + # On partial-padded token (0, 1) with [2, -1], when norm_topk_prob=True or topk_softmax, active slot weight == scale. + if norm_topk_prob or scoring == "topk_softmax": + np.testing.assert_allclose(float(w_routed[0, 1, 0]), scale, rtol=1e-6, atol=1e-6) + np.testing.assert_allclose(float(w_routed[1, 2, 1]), scale, rtol=1e-6, atol=1e-6) + self.assertEqual(float(w_routed[0, 1, 1]), 0.0) + self.assertEqual(float(w_routed[1, 2, 0]), 0.0) + + mesh_ep1 = create_mesh(devices=jax.devices()[:1]) + mesh_ep2 = create_mesh(ici={"fsdp": 1, "expert": 2, "tensor": 2}, devices=jax.devices()[:4]) + exec_ep1 = m3_moe.MoEExecutor(kind="tokamax", bwd_wgrad_overlap=True) + exec_ep2 = m3_moe.MoEExecutor( + kind="tokamax", + batch=("data", "fsdp"), + experts="expert", + mlp="tensor", + dropless_buffer_factor=1.25, + bwd_wgrad_overlap=True, + ) + + calls = {"gmm_group0": []} + + def _record_group0(g0): + calls["gmm_group0"].append(int(np.asarray(g0))) + + def fake_gmm_v2( + *, lhs, rhs, group_sizes, rhs_scale, tile_info, preferred_element_type, maybe_quantize_lhs, transpose_rhs=False + ): + del rhs_scale, tile_info, maybe_quantize_lhs + jax.debug.callback(_record_group0, group_sizes[0]) + rhs_eff = rhs.swapaxes(1, 2) if transpose_rhs else rhs + return jax.lax.ragged_dot(lhs, rhs_eff, group_sizes, preferred_element_type=jnp.float32).astype( + preferred_element_type + ) + + def fake_tgmm_v2( + *, lhs, rhs, group_sizes, num_actual_groups, rhs_scale, precision, preferred_element_type, tile_info + ): + del rhs_scale, precision, tile_info + jax.debug.callback(_record_group0, group_sizes[0]) + _, vjp_fn = jax.vjp( + lambda r_w: jax.lax.ragged_dot(lhs, r_w, group_sizes, preferred_element_type=jnp.float32), + jnp.zeros((num_actual_groups, lhs.shape[1], rhs.shape[1]), dtype=lhs.dtype), + ) + (drhs,) = vjp_fn(rhs.astype(jnp.float32)) + return drhs.astype(preferred_element_type) + + # Check EP=1 mocked TPU gmm_v2/tgmm_v2 group_sizes[0] == 0 and exact 0.0 expert-0 grads. + with ( + mock.patch.object(m3_kernels, "is_tpu_mesh", return_value=True), + mock.patch.object(gmm_v2, "gmm_v2", side_effect=fake_gmm_v2), + mock.patch.object(tgmm_v2, "tgmm_v2", side_effect=fake_tgmm_v2), + jax.set_mesh(mesh_ep1), + ): + (_, out_ep1), grads_ep1 = jax.jit( + jax.value_and_grad( + lambda x_in, l_in, w0, w1, w_out: ( + jnp.sum( + exec_ep1( + x_in, + *m3_moe.route_top_k( + l_in, + top_k, + scoring=scoring, + norm_topk_prob=norm_topk_prob, + routed_scaling_factor=scale, + eps=eps, + forced_routed_experts=forced_idx, + )[:2], + w0, + w1, + w_out, + ) + * cotangent + ), + exec_ep1( + x_in, + *m3_moe.route_top_k( + l_in, + top_k, + scoring=scoring, + norm_topk_prob=norm_topk_prob, + routed_scaling_factor=scale, + eps=eps, + forced_routed_experts=forced_idx, + )[:2], + w0, + w1, + w_out, + ), + ), + argnums=(0, 1, 2, 3, 4), + has_aux=True, + ) + )(x, logits, wi_0, wi_1, wo) + + # Every single gmm_v2 / tgmm_v2 call on EP=1 must see group_sizes[0] == 0! + self.assertNotEmpty(calls["gmm_group0"]) + self.assertEqual(max(calls["gmm_group0"]), 0) + + # Verify across DENSE, RAGGED_DOT, EP=1, and 4-device EP=2 (with overflow rounds) for both separate and fused: + with jax.set_mesh(mesh_ep2): + (_, out_dense), grads_dense = jax.jit( + jax.value_and_grad( + lambda x_in, l_in, w0, w1, w_out: ( + jnp.sum( + m3_moe.DENSE( + x_in, + *m3_moe.route_top_k( + l_in, + top_k, + scoring=scoring, + norm_topk_prob=norm_topk_prob, + routed_scaling_factor=scale, + eps=eps, + forced_routed_experts=forced_idx, + )[:2], + w0, + w1, + w_out, + ) + * cotangent + ), + m3_moe.DENSE( + x_in, + *m3_moe.route_top_k( + l_in, + top_k, + scoring=scoring, + norm_topk_prob=norm_topk_prob, + routed_scaling_factor=scale, + eps=eps, + forced_routed_experts=forced_idx, + )[:2], + w0, + w1, + w_out, + ), + ), + argnums=(0, 1, 2, 3, 4), + has_aux=True, + ) + )(x, logits, wi_0, wi_1, wo) + + (_, out_ep2), grads_ep2 = jax.jit( + jax.value_and_grad( + lambda x_in, p_in, w0, w1, w_out: ( + jnp.sum( + exec_ep2( + x_in, + p_in, + None, + w0, + w1, + w_out, + forced_routed_experts=forced_idx, + norm_topk_prob=norm_topk_prob, + routed_scaling_factor=scale, + scores_are_logits=(scoring == "topk_softmax"), + eps=eps, + ) + * cotangent + ), + exec_ep2( + x_in, + p_in, + None, + w0, + w1, + w_out, + forced_routed_experts=forced_idx, + norm_topk_prob=norm_topk_prob, + routed_scaling_factor=scale, + scores_are_logits=(scoring == "topk_softmax"), + eps=eps, + ), + ), + argnums=(0, 1, 2, 3, 4), + has_aux=True, + ) + )(x, logits if scoring == "topk_softmax" else probs_routed, wi_0, wi_1, wo) + + (_, out_ep2_f), grads_ep2_f = jax.jit( + jax.value_and_grad( + lambda x_in, p_in, gu, dn: ( + jnp.sum( + exec_ep2.fused( + x_in, + p_in, + None, + gu, + dn, + forced_routed_experts=forced_idx, + norm_topk_prob=norm_topk_prob, + routed_scaling_factor=scale, + scores_are_logits=(scoring == "topk_softmax"), + eps=eps, + ) + * cotangent + ), + exec_ep2.fused( + x_in, + p_in, + None, + gu, + dn, + forced_routed_experts=forced_idx, + norm_topk_prob=norm_topk_prob, + routed_scaling_factor=scale, + scores_are_logits=(scoring == "topk_softmax"), + eps=eps, + ), + ), + argnums=(0, 1, 2, 3), + has_aux=True, + ) + )(x, logits if scoring == "topk_softmax" else probs_routed, w_gu, wo) + + for out_cur in (out_ep1, out_dense, out_ep2, out_ep2_f): + self.assertTrue(bool(jnp.all(jnp.isfinite(out_cur)))) + # All-`-1` padded tokens ((0, 0), all of batch 2, and (3, 1)) must have exact 0.0 output! + self.assertEqual(float(jnp.max(jnp.abs(out_cur[0, 0]))), 0.0) + self.assertEqual(float(jnp.max(jnp.abs(out_cur[2]))), 0.0) + self.assertEqual(float(jnp.max(jnp.abs(out_cur[3, 1]))), 0.0) + np.testing.assert_allclose(np.asarray(out_cur), np.asarray(out_dense), rtol=1e-5, atol=1e-5) + + for grads_cur in (grads_ep1, grads_dense, grads_ep2): + dx, dlogits_or_probs, dw0, dw1, dwo = grads_cur + for g_leaf in grads_cur: + self.assertTrue(bool(jnp.all(jnp.isfinite(g_leaf)))) + # All-`-1` padded tokens must have exact 0.0 dx and 0.0 router gradient! + self.assertEqual(float(jnp.max(jnp.abs(dx[0, 0]))), 0.0) + self.assertEqual(float(jnp.max(jnp.abs(dx[2]))), 0.0) + self.assertEqual(float(jnp.max(jnp.abs(dx[3, 1]))), 0.0) + self.assertEqual(float(jnp.max(jnp.abs(dlogits_or_probs[0, 0]))), 0.0) + self.assertEqual(float(jnp.max(jnp.abs(dlogits_or_probs[2]))), 0.0) + self.assertEqual(float(jnp.max(jnp.abs(dlogits_or_probs[3, 1]))), 0.0) + # Expert 0 was never selected by any valid (>= 0) slot, so its weight gradients must be exact 0.0! + self.assertEqual(float(jnp.max(jnp.abs(dw0[0]))), 0.0) + self.assertEqual(float(jnp.max(jnp.abs(dw1[0]))), 0.0) + self.assertEqual(float(jnp.max(jnp.abs(dwo[0]))), 0.0) + + # Also check fused expert 0 weight gradients and router_probs[:, :, 0] gradient on EP=2: + dx_f, dp_f, dgu_f, ddn_f = grads_ep2_f + self.assertEqual(float(jnp.max(jnp.abs(dx_f[2]))), 0.0) + self.assertEqual(float(jnp.max(jnp.abs(dp_f[:, :, 0]))), 0.0) + self.assertEqual(float(jnp.max(jnp.abs(dgu_f[0]))), 0.0) + self.assertEqual(float(jnp.max(jnp.abs(ddn_f[0]))), 0.0) + + np.testing.assert_allclose(np.asarray(grads_ep1[0]), np.asarray(grads_dense[0]), rtol=1e-5, atol=1e-5) + np.testing.assert_allclose(np.asarray(grads_ep1[1]), np.asarray(grads_dense[1]), rtol=1e-5, atol=1e-5) + np.testing.assert_allclose(np.asarray(grads_ep2[0]), np.asarray(grads_dense[0]), rtol=1e-5, atol=1e-5) + + +class ValidationFailClosedTest(parameterized.TestCase): + """Verifies fail-closed validation on invalid shape, rank, dtype, and out-of-range expert IDs.""" + + def test_fail_closed_on_invalid_shape_rank_dtype_and_out_of_range_ids(self): + x = jnp.ones((2, 3, 8), dtype=jnp.float32) + probs = jnp.full((2, 3, 4), 0.25, dtype=jnp.float32) + weights = jnp.full((2, 3, 2), 0.5, dtype=jnp.float32) + indices = jnp.zeros((2, 3, 2), dtype=jnp.int32) + wi_0, wi_1, wo = _make_expert_weights(num_experts=4, embed=8, mlp=6) + w_gu = jnp.concatenate([wi_0, wi_1], axis=-1) + + # Wrong rank (2D instead of 3D). + with self.assertRaisesRegex(ValueError, "rank|Expected 3D forced_routed_experts"): + m3_moe.apply_forced_routed_experts(probs, jnp.zeros((6, 2), dtype=jnp.int32)) + with self.assertRaisesRegex(ValueError, "Expected 3D forced_routed_experts"): + m3_moe.routed_experts(x, weights, indices, wi_0, wi_1, wo, forced_routed_experts=jnp.zeros((6, 2), dtype=jnp.int32)) + + # Mismatched token prefix shape. + with self.assertRaisesRegex(ValueError, "forced_routed_experts shape"): + m3_moe.route_top_k(probs, 2, forced_routed_experts=jnp.zeros((2, 4, 2), dtype=jnp.int32)) + with self.assertRaisesRegex(ValueError, "forced_routed_experts shape"): + m3_moe.routed_experts_fused( + x, weights, indices, w_gu, wo, forced_routed_experts=jnp.zeros((2, 4, 2), dtype=jnp.int32) + ) + + # Mismatched top_k in route_top_k or indices vs forced_routed_experts. + with self.assertRaisesRegex(ValueError, "forced_routed_experts shape"): + m3_moe.route_top_k(probs, 2, forced_routed_experts=jnp.zeros((2, 3, 3), dtype=jnp.int32)) + with self.assertRaisesRegex(ValueError, "indices shape .* must match forced_routed_experts shape"): + m3_moe.routed_experts( + x, weights, indices, wi_0, wi_1, wo, forced_routed_experts=jnp.zeros((2, 3, 3), dtype=jnp.int32) + ) + + # Float or bool dtype rejected. + with self.assertRaisesRegex(ValueError, "integer dtype"): + m3_moe.apply_forced_routed_experts(probs, jnp.zeros((2, 3, 2), dtype=jnp.float32)) + with self.assertRaisesRegex(ValueError, "integer dtype"): + m3_moe.apply_forced_routed_experts(probs, jnp.zeros((2, 3, 2), dtype=jnp.bool_)) + + # Out-of-range ID < -1 (e.g. -2) rejected. + bad_neg = jnp.zeros((2, 3, 2), dtype=jnp.int32).at[0, 1, 0].set(-2) + with self.assertRaisesRegex(ValueError, r"must be in \[-1, 4\)"): + m3_moe.route_top_k(probs, 2, forced_routed_experts=bad_neg) + with self.assertRaisesRegex(ValueError, r"must be in \[-1, 4\)"): + m3_moe.routed_experts(x, weights, indices, wi_0, wi_1, wo, forced_routed_experts=bad_neg) + + # Out-of-range ID >= num_experts (e.g. 4 when num_experts == 4) rejected. + bad_hi = jnp.zeros((2, 3, 2), dtype=jnp.int32).at[1, 0, 1].set(4) + with self.assertRaisesRegex(ValueError, r"must be in \[-1, 4\)"): + m3_moe.apply_forced_routed_experts(probs, bad_hi) + with self.assertRaisesRegex(ValueError, r"must be in \[-1, 4\)"): + m3_moe.routed_experts_fused(x, weights, indices, w_gu, wo, forced_routed_experts=bad_hi) + + # Missing indices when forced_routed_experts is None. + with self.assertRaisesRegex(ValueError, "indices must be provided when forced_routed_experts is None"): + m3_moe.routed_experts(x, weights, None, wi_0, wi_1, wo) + + +if __name__ == "__main__": + absltest.main()