Repository navigation
feat(m3): adopt shared MoEExecutor for GraniteMoE and multi-rank EP GLM-5.3 #6282
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: yaoyuchen/m3-shared-moe-ep-kernels
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -229,6 +229,8 @@ def _pallas_grouped_matmul_bwd( | |
|
|
||
| if backend_kind in ("gmm_v2", "tokamax"): | ||
| if not is_tpu: | ||
| lhs_safe = jnp.where(valid_rows, lhs, jnp.zeros((), dtype=lhs.dtype)) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🔴
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Fixed in |
||
| grad_safe = jnp.where(valid_rows, grad.astype(lhs.dtype), jnp.zeros((), dtype=lhs.dtype)) | ||
| _, vjp_fn = jax.vjp( | ||
| lambda l, r: jax.lax.ragged_dot( | ||
| l, | ||
|
|
@@ -827,6 +829,8 @@ def _grouped_unsharded( | |
| activation: Callable[[jax.Array], jax.Array], | ||
| swiglu_limit: float | None, | ||
| executor: MoEExecutor, | ||
| backend: str = "ragged_dot", | ||
| is_tpu: bool = False, | ||
| ) -> jax.Array: | ||
| """Portable `ragged_dot` grouped execution without `shard_map`.""" | ||
| if executor.bwd_wgrad_overlap: | ||
|
|
@@ -866,7 +870,7 @@ def _grouped_unsharded( | |
| sorted_x = _permute_rows(slot_rows, perm) | ||
| sorted_x = jnp.where((jnp.arange(num_slots, dtype=jnp.int32) < valid_count)[:, None], sorted_x, 0) | ||
|
|
||
| matmul_fn = functools.partial(grouped_matmul, backend="ragged_dot", is_tpu=False) | ||
| matmul_fn = functools.partial(grouped_matmul, backend=backend, is_tpu=is_tpu) | ||
| expert_out = _run_local_expert_mlp( | ||
| sorted_x, | ||
| projections, | ||
|
|
@@ -1235,6 +1239,65 @@ def routed_experts( | |
| return checkpoint_name(out.astype(x.dtype), "moe_routed_output") | ||
|
|
||
|
|
||
| def grouped_unsharded_fused( | ||
| x: jax.Array, | ||
| weights: jax.Array, | ||
| indices: jax.Array, | ||
| gate_up_proj: jax.Array, | ||
| down_proj: jax.Array, | ||
| *, | ||
| executor: MoEExecutor = RAGGED_DOT, | ||
| activation: Callable[[jax.Array], jax.Array] = jax.nn.silu, | ||
| swiglu_limit: float | None = None, | ||
| is_tpu: bool = False, | ||
| ) -> jax.Array: | ||
| """Executes fused routed experts on a single device when no `Mesh` is in scope.""" | ||
| if gate_up_proj.ndim not in (3, 4) or down_proj.ndim != 3: | ||
| raise ValueError( | ||
| f"Expected 3D or 4D gate_up_proj and 3D down_proj; got gate_up_proj={gate_up_proj.shape}, " | ||
| f"down_proj={down_proj.shape}." | ||
| ) | ||
| num_experts = gate_up_proj.shape[0] | ||
| _validate_routed_inputs(x, weights, indices, num_experts) | ||
| projections = (gate_up_proj.astype(x.dtype), down_proj.astype(x.dtype)) | ||
| backend = executor.kind if executor.kind in SHARDED_EXECUTOR_KINDS else "ragged_dot" | ||
| if is_tpu and backend in SHARDED_EXECUTOR_KINDS: | ||
| mesh_1d = jax.sharding.Mesh(jax.local_devices()[:1], ("_moe_1d",)) | ||
| return jax.shard_map( | ||
| lambda x_l, w_l, idx_l, gu_l, dn_l: _grouped_unsharded( | ||
| x_l, | ||
| w_l, | ||
| idx_l, | ||
| (gu_l, dn_l), | ||
| fused=True, | ||
| num_experts=num_experts, | ||
| activation=activation, | ||
| swiglu_limit=swiglu_limit, | ||
| executor=executor, | ||
| backend=backend, | ||
| is_tpu=True, | ||
| ).astype(x_l.dtype), | ||
| mesh=mesh_1d, | ||
| in_specs=(P(), P(), P(), P(), P()), | ||
| out_specs=P(), | ||
| check_vma=False, | ||
| )(x, weights, indices, projections[0], projections[1]) | ||
| out = _grouped_unsharded( | ||
| x, | ||
| weights, | ||
| indices, | ||
| projections, | ||
| fused=True, | ||
| num_experts=num_experts, | ||
| activation=activation, | ||
| swiglu_limit=swiglu_limit, | ||
| executor=executor, | ||
| backend=backend, | ||
| is_tpu=is_tpu, | ||
| ) | ||
| return out.astype(x.dtype) | ||
|
|
||
|
|
||
| def routed_experts_fused( | ||
| x: jax.Array, | ||
| weights: jax.Array, | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
why we have expert and experts axis names
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Glm5Expertsoriginally annotated its stacked 3D expert weights (gate_up_proj,down_proj) with singular logical axis"expert"(matching the physical mesh axis name), whereas sharedDefaultShardinginsrc/maxtext/m3/core/sharding.py(andQwen3_8Experts/GraniteMoeSharding) uses plural"experts"for the logical expert-count dimension mapped onto the physical"expert"mesh axis. Accepting{"expert", "experts"}here keeps backward compatibility with existingGlm5Expertsparameter annotations while supportingDefaultSharding's"experts"convention.