Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 12 additions & 5 deletions src/maxtext/m3/configs/glm5_3_flash_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ def validate_no_packing(decoder_segment_ids: Any) -> None:
" linear_conv_kernel_dim kda_implementation"
" dsa_attention_implementation rope_parameters index_kpool"
" index_kpool_always_select_tail linear_lower_bound mhc_eps"
" enable_debug_stats router_param_dtype"
" enable_debug_stats router_param_dtype sparse_matmul"
).split()
)

Expand Down Expand Up @@ -136,6 +136,7 @@ class Glm5Config:
remat_policy: str = "full"
enable_debug_stats: bool = False
num_vocab_tiling: int = 1
sparse_matmul: bool = True

def __post_init__(self) -> None:
if isinstance(self.dtype, (np.dtype, type)):
Expand Down Expand Up @@ -218,6 +219,8 @@ def __post_init__(self) -> None:
kd, vd = self.linear_key_head_dim, self.linear_value_head_dim
if kd != vd:
raise ValueError(f"linear_key_head_dim ({kd}) must equal linear_value_head_dim ({vd}).")
if not isinstance(self.sparse_matmul, bool):
raise ValueError(f"sparse_matmul must be a bool; got {self.sparse_matmul!r}.")

@property
def qk_head_dim(self) -> int:
Expand Down Expand Up @@ -339,7 +342,7 @@ def from_mesh(cls, mesh: jax.sharding.Mesh) -> "Glm5Sharding":
)
use_tp = mesh.shape["tensor"] > 1
ep_size = int(mesh.shape["expert"])
return cls(use_tp=use_tp, ep_size=ep_size)
return cls(use_tp=use_tp, ep_in_batch=ep_size > 1, ep_size=ep_size)

def map_axis(
self,
Expand All @@ -351,13 +354,17 @@ def map_axis(
act = tensor_type == m3_sharding.TensorType.ACTIVATION
if axis == "batch":
if act:
return (self.dp_axis, self.fsdp_axis, self.ep_axis) if self.ep_in_batch else (self.dp_axis, self.fsdp_axis)
if self.ep_in_batch or self.ep_size > 1 or tensor_name in ("tokens", "moe_tokens"):
return (self.dp_axis, self.fsdp_axis, self.ep_axis)
return (self.dp_axis, self.fsdp_axis)
return None
if axis == "expert":
if axis in {"expert", "experts"}:

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why we have expert and experts axis names

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Glm5Experts originally annotated its stacked 3D expert weights (gate_up_proj, down_proj) with singular logical axis "expert" (matching the physical mesh axis name), whereas shared DefaultSharding in src/maxtext/m3/core/sharding.py (and Qwen3_8Experts / GraniteMoeSharding) uses plural "experts" for the logical expert-count dimension mapped onto the physical "expert" mesh axis. Accepting {"expert", "experts"} here keeps backward compatibility with existing Glm5Experts parameter annotations while supporting DefaultSharding's "experts" convention.

return None if act else self.ep_axis
if axis == "expert_embed":
return None
if axis in {"embed", "q_lora", "kv_lora"}:
return None if act else self.fsdp_axis
if axis in {"vocab", "heads", "kv", "mlp"}:
if axis in {"vocab", "heads", "kv", "mlp", "expert_mlp"}:
return self.tp_axis if self.use_tp else None
if axis == "norm" or (axis == "length" and act):
return None
Expand Down
65 changes: 64 additions & 1 deletion src/maxtext/m3/core/moe.py
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,8 @@ def _pallas_grouped_matmul_bwd(

if backend_kind in ("gmm_v2", "tokamax"):
if not is_tpu:
lhs_safe = jnp.where(valid_rows, lhs, jnp.zeros((), dtype=lhs.dtype))

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔴 lhs_safe in the backward pass is only defined within if not is_tpu:. On TPU (line 252), lhs_pad = jnp.pad(lhs, ...) passes the unmasked lhs to tgmm_v2.tgmm_v2(lhs=lhs_pad, rhs=grad_pad). If unassigned tail rows in lhs contain NaNs, the contraction with grad_pad will propagate NaN into weight gradients (drhs). Define lhs_safe outside if not is_tpu: and pass lhs_safe to lhs_pad.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in 752e2bf4af (and parent #6164 53c481bad3): hoisted lhs_safe = jnp.where(valid_rows, lhs, 0) above if not is_tpu: in _pallas_grouped_matmul_bwd and passed lhs_safe into lhs_pad = jnp.pad(lhs_safe, ...) before tgmm_v2.tgmm_v2 on TPU.

grad_safe = jnp.where(valid_rows, grad.astype(lhs.dtype), jnp.zeros((), dtype=lhs.dtype))
_, vjp_fn = jax.vjp(
lambda l, r: jax.lax.ragged_dot(
l,
Expand Down Expand Up @@ -827,6 +829,8 @@ def _grouped_unsharded(
activation: Callable[[jax.Array], jax.Array],
swiglu_limit: float | None,
executor: MoEExecutor,
backend: str = "ragged_dot",
is_tpu: bool = False,
) -> jax.Array:
"""Portable `ragged_dot` grouped execution without `shard_map`."""
if executor.bwd_wgrad_overlap:
Expand Down Expand Up @@ -866,7 +870,7 @@ def _grouped_unsharded(
sorted_x = _permute_rows(slot_rows, perm)
sorted_x = jnp.where((jnp.arange(num_slots, dtype=jnp.int32) < valid_count)[:, None], sorted_x, 0)

matmul_fn = functools.partial(grouped_matmul, backend="ragged_dot", is_tpu=False)
matmul_fn = functools.partial(grouped_matmul, backend=backend, is_tpu=is_tpu)
expert_out = _run_local_expert_mlp(
sorted_x,
projections,
Expand Down Expand Up @@ -1235,6 +1239,65 @@ def routed_experts(
return checkpoint_name(out.astype(x.dtype), "moe_routed_output")


def grouped_unsharded_fused(
x: jax.Array,
weights: jax.Array,
indices: jax.Array,
gate_up_proj: jax.Array,
down_proj: jax.Array,
*,
executor: MoEExecutor = RAGGED_DOT,
activation: Callable[[jax.Array], jax.Array] = jax.nn.silu,
swiglu_limit: float | None = None,
is_tpu: bool = False,
) -> jax.Array:
"""Executes fused routed experts on a single device when no `Mesh` is in scope."""
if gate_up_proj.ndim not in (3, 4) or down_proj.ndim != 3:
raise ValueError(
f"Expected 3D or 4D gate_up_proj and 3D down_proj; got gate_up_proj={gate_up_proj.shape}, "
f"down_proj={down_proj.shape}."
)
num_experts = gate_up_proj.shape[0]
_validate_routed_inputs(x, weights, indices, num_experts)
projections = (gate_up_proj.astype(x.dtype), down_proj.astype(x.dtype))
backend = executor.kind if executor.kind in SHARDED_EXECUTOR_KINDS else "ragged_dot"
if is_tpu and backend in SHARDED_EXECUTOR_KINDS:
mesh_1d = jax.sharding.Mesh(jax.local_devices()[:1], ("_moe_1d",))
return jax.shard_map(
lambda x_l, w_l, idx_l, gu_l, dn_l: _grouped_unsharded(
x_l,
w_l,
idx_l,
(gu_l, dn_l),
fused=True,
num_experts=num_experts,
activation=activation,
swiglu_limit=swiglu_limit,
executor=executor,
backend=backend,
is_tpu=True,
).astype(x_l.dtype),
mesh=mesh_1d,
in_specs=(P(), P(), P(), P(), P()),
out_specs=P(),
check_vma=False,
)(x, weights, indices, projections[0], projections[1])
out = _grouped_unsharded(
x,
weights,
indices,
projections,
fused=True,
num_experts=num_experts,
activation=activation,
swiglu_limit=swiglu_limit,
executor=executor,
backend=backend,
is_tpu=is_tpu,
)
return out.astype(x.dtype)


def routed_experts_fused(
x: jax.Array,
weights: jax.Array,
Expand Down
Loading
Loading