Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/maxtext/layers/moe.py
Original file line number Diff line number Diff line change
Expand Up @@ -5385,6 +5385,9 @@ def _maybe_native_fused_weight(kernel, scale):
wi_scale = self.wi_scale[...] if self.wi_scale is not None else None
fused_kernel, wi_native_scale = _maybe_native_fused_weight(self.wi[...], wi_scale)
elif cfg.prefuse_moe_weights:
# Split into two whole halves: `model_creation_utils._fuse_moe_weights` prefuses [gate | up] for every
# consumer except tpu-inference's TP fused kernel (the fused path above without expert parallelism), which
# gets one [gate | up] chunk per shard instead.
wi_scale = self.wi_scale[...] if self.wi_scale is not None else None
wi = quantizations.dequantize_weight(self.wi[...], wi_scale, self.dtype)
n = wi.shape[-1] // 2
Expand Down
71 changes: 50 additions & 21 deletions src/maxtext/utils/model_creation_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -328,14 +328,25 @@ def _intersect_with_metadata(target, meta_tree):
_VLLM_EXPERT_PARALLELISM_AXES = ("attn_dp_expert", "expert", "dcp")


def _fused_moe_kernel_uses_ep(config, mesh) -> bool:
"""Whether tpu-inference's fused MoE kernel runs expert-parallel for this config and mesh.
def _fused_moe_kernel_uses_tp(config, mesh) -> bool:
"""Whether MoE layers run through tpu-inference's tensor-parallel fused MoE kernel for this config and mesh.

Mirrors `use_ep = self.get_expert_parallelism_size() > 1` in `RoutedMoE.fused_moe_matmul`.
That kernel is the only consumer of a prefused `wi` that splits each shard of the fused axis into gate and up.
The expert-parallel fused kernel and `RoutedMoE`'s non-fused paths (non-vLLM attention, hash-routed layers)
split the whole axis in half. Mirrors `use_ep = self.get_expert_parallelism_size() > 1` being False in
`RoutedMoE.fused_moe_matmul`.
"""
if config is None or mesh is None or getattr(config, "attention", None) not in _VLLM_FUSED_MOE_ATTENTION:
return False
return math.prod(mesh.shape.get(axis, 1) for axis in _VLLM_EXPERT_PARALLELISM_AXES) > 1
return math.prod(mesh.shape.get(axis, 1) for axis in _VLLM_EXPERT_PARALLELISM_AXES) == 1


def _is_hash_routed_moe(model_node) -> bool:
"""Whether `model_node`, the arrays of a `RoutedMoE`, belongs to a hash-routed layer (it holds `tid2eid`).

Hash-routed layers never run the fused MoE kernel (see `is_fused_moe_path` in `RoutedMoE.__call__`).
"""
return "tid2eid" in model_node


# `_fuse_moe_halves_concatenated` builds the fused wi in this many slices of the embed axis.
Expand All @@ -350,14 +361,24 @@ def _sharded_zeros(shape, dtype, out_sharding):
@partial(jax.jit, donate_argnums=0)
def _write_embed_slice(fused, part, start):
"""Writes `part` into `fused` at `start` along the embed (second-to-last) axis, reusing `fused`'s buffer."""
return jax.lax.dynamic_update_slice_in_dim(fused, part, start, axis=fused.ndim - 2)
return jax.lax.dynamic_update_slice_in_dim(fused, part.astype(fused.dtype), start, axis=fused.ndim - 2)


def _check_half_splits_into_shards(half_size, half_dim, n_shards):
"""Raises unless a MoE half of `half_size` can be padded to `half_dim` shard by shard."""
if half_size % n_shards or half_dim % n_shards:
raise ValueError(
f"Cannot pad a MoE half of size {half_size} to {half_dim} shard by shard: both must be multiples of "
f"the {n_shards} shards of the fused axis."
)


@partial(jax.jit, static_argnames=("n_shards", "half_dim", "out_sharding"))
def _concat_halves_padded_per_shard(wi_0, wi_1, n_shards, half_dim, out_sharding):
"""Returns [wi_0 | wi_1] along the last axis, each half zero-padded to `half_dim` at the tail of each shard."""

def _pad(arr):
_check_half_splits_into_shards(arr.shape[-1], half_dim, n_shards)
chunk = arr.shape[-1] // n_shards
arr = arr.reshape(*arr.shape[:-1], n_shards, chunk)
arr = jnp.pad(arr, [(0, 0)] * (arr.ndim - 1) + [(0, half_dim // n_shards - chunk)])
Expand All @@ -376,9 +397,15 @@ def _fuse_moe_halves_concatenated(wi_0, wi_1, wi_model, n_shards, num_slices=_FU
wi_sharding = wi_model.sharding
embed_axis = wi_model.ndim - 2
embed_dim = wi_model.shape[embed_axis]
if embed_axis < len(wi_sharding.spec) and wi_sharding.spec[embed_axis] is not None:
num_slices = 1 # Slices of a sharded embed axis would not match `wi_sharding`.
fused = _sharded_zeros(wi_model.shape, jnp.dtype(wi_0.dtype), wi_sharding)
if num_slices > 1 and embed_axis < len(wi_sharding.spec) and wi_sharding.spec[embed_axis] is not None:
# Slices of a sharded embed axis would not match `wi_sharding`.
max_logging.info(
f"Fusing MoE wi of shape {wi_model.shape} in one pass instead of {num_slices} embed slices, since its embed "
f"axis is sharded ({wi_sharding.spec}); this needs more temporary memory."
)
num_slices = 1
# In the model's dtype rather than the checkpoint's, e.g. half the size for an f32 checkpoint into a bf16 model.
fused = _sharded_zeros(wi_model.shape, jnp.dtype(wi_model.dtype), wi_sharding)
step = -(-embed_dim // num_slices)
Comment thread
chiajunglien marked this conversation as resolved.
Resolved
for start in range(0, embed_dim, step):
stop = min(start + step, embed_dim)
Expand All @@ -402,18 +429,19 @@ def _fuse_moe_weights(ckpt_tree, model_arrays_tree, config=None):
on a per-shard basis to satisfy kernel constraints.

The per-shard interleave is the layout tpu-inference's TP fused MoE kernel
expects: it shards the fused axis and splits each shard into gate/up. When
that kernel runs expert-parallel instead (vLLM attention with an expert
parallelism size > 1), it does not shard the fused axis and splits the whole
axis in half (`gate = wi[..., :H]`, `up = wi[..., H:]`), so the gate and up
halves are concatenated rather than interleaved. Each half still keeps the
per-shard padding, which matches how `wo` is padded along the MLP dim.
expects: it shards the fused axis and splits each shard into gate/up. It is
the only consumer that does. The expert-parallel fused kernel and
`RoutedMoE`'s non-fused paths (non-vLLM attention, hash-routed layers) split
the whole axis in half (`gate = wi[..., :H]`, `up = wi[..., H:]`), so for
everything else the gate and up halves are concatenated rather than
interleaved. Each half still keeps the per-shard padding, which matches how
`wo` is padded along the MLP dim. Both layouts are built in the model's dtype.

Args:
ckpt_tree: Restored checkpoint tree, possibly holding wi_0/wi_1 pairs.
model_arrays_tree: Model arrays tree; a fused model holds `wi` instead.
config: Model config, used to detect the expert-parallel fused MoE kernel.
When None, the per-shard interleaved layout is used.
config: Model config, used to detect the TP fused MoE kernel. When None,
the halves are concatenated.
"""
is_vllm_rpa = config is None or config.attention in _VLLM_FUSED_MOE_ATTENTION

Expand Down Expand Up @@ -458,13 +486,13 @@ def _maybe_fuse(path, ckpt_node):

# Determine the number of shards (TP degree) along the concatenated axis
n_shards = 1
use_ep = False
concatenate = False
wi_sharding = getattr(wi_model, "sharding", None)
if is_vllm_rpa and isinstance(wi_sharding, jax.sharding.NamedSharding):
spec = wi_sharding.spec
partition = spec[axis] if axis < len(spec) else None
n_shards = _partition_size(partition, wi_sharding.mesh)
use_ep = _fused_moe_kernel_uses_ep(config, wi_sharding.mesh)
concatenate = not _fused_moe_kernel_uses_tp(config, wi_sharding.mesh) or _is_hash_routed_moe(model_node)

# Target size for a single half (wi_0 or wi_1) AFTER padding
target_half_dim = wi_model.shape[-1] // 2
Expand All @@ -473,6 +501,7 @@ def _maybe_fuse(path, ckpt_node):
def _pad_and_chunk(arr, target_total_size):
shape = arr.shape
current_total_size = shape[-1]
_check_half_splits_into_shards(current_total_size, target_total_size, n_shards)

# Calculate per-shard chunk sizes
chunk_size = current_total_size // n_shards
Expand All @@ -490,8 +519,8 @@ def _pad_and_chunk(arr, target_total_size):

return arr_reshaped

# Expert-parallel kernel: [gate | up] instead of the per-shard interleave below.
if use_ep:
# Everything but the TP fused kernel: [gate | up] instead of the per-shard interleave below.
if concatenate:
new_node = {k: v for k, v in ckpt_node.items() if k not in ("wi_0", "wi_1")}
new_node["wi"] = _fuse_moe_halves_concatenated(ckpt_node["wi_0"], ckpt_node["wi_1"], wi_model, n_shards)
return new_node
Expand All @@ -506,7 +535,7 @@ def _pad_and_chunk(arr, target_total_size):

# Flatten the n_shards dimension back out to match the final model shape, drop wi_0/wi_1.
new_node = {k: v for k, v in ckpt_node.items() if k not in ("wi_0", "wi_1")}
new_node["wi"] = wi_interleaved.reshape(*wi_model.shape)
new_node["wi"] = wi_interleaved.reshape(*wi_model.shape).astype(wi_model.dtype)
return new_node

return jax.tree_util.tree_map_with_path(_maybe_fuse, ckpt_tree, is_leaf=_is_fusion_site)
Expand Down
133 changes: 102 additions & 31 deletions tests/unit/model_creation_utils_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,8 @@
_fix_restore_args_for_shape_mismatch,
_fuse_moe_weights,
_fuse_moe_halves_concatenated,
_fused_moe_kernel_uses_ep,
_concat_halves_padded_per_shard,
_fused_moe_kernel_uses_tp,
_logical_partition_specs,
_stored_shape_evenly_shardable,
_zero_pad_axis,
Expand Down Expand Up @@ -381,36 +382,40 @@ def test_recurses_through_nested_dicts(self):
np.testing.assert_array_equal(out["layers"]["0"]["other"], other)


class TestFusedMoeKernelUsesEp(unittest.TestCase):
"""_fused_moe_kernel_uses_ep must match `use_ep` in `RoutedMoE.fused_moe_matmul`."""
class TestFusedMoeKernelUsesTp(unittest.TestCase):
"""_fused_moe_kernel_uses_tp must be True exactly when `RoutedMoE.fused_moe_matmul` runs with `use_ep` False."""

def _uses_ep(self, attention, mesh_shape):
return _fused_moe_kernel_uses_ep(types.SimpleNamespace(attention=attention), types.SimpleNamespace(shape=mesh_shape))
def _uses_tp(self, attention, mesh_shape):
return _fused_moe_kernel_uses_tp(types.SimpleNamespace(attention=attention), types.SimpleNamespace(shape=mesh_shape))

def test_tensor_parallel_only(self):
self.assertFalse(self._uses_ep("vllm_rpa", {"data": 1, "model": 4, "expert": 1}))
self.assertTrue(self._uses_tp("vllm_rpa", {"data": 1, "model": 4, "expert": 1}))
self.assertTrue(self._uses_tp("vllm_batched_rpa", {"model": 4}))

def test_expert_parallel_axes(self):
self.assertTrue(self._uses_ep("vllm_rpa", {"model": 2, "expert": 2}))
self.assertTrue(self._uses_ep("vllm_batched_rpa", {"model": 2, "attn_dp_expert": 2}))
self.assertTrue(self._uses_ep("vllm_rpa", {"model": 2, "dcp": 2}))
self.assertFalse(self._uses_tp("vllm_rpa", {"model": 2, "expert": 2}))
self.assertFalse(self._uses_tp("vllm_batched_rpa", {"model": 2, "attn_dp_expert": 2}))
self.assertFalse(self._uses_tp("vllm_rpa", {"model": 2, "dcp": 2}))

def test_non_vllm_attention(self):
self.assertFalse(self._uses_ep("dot_product", {"model": 2, "expert": 2}))
self.assertFalse(self._uses_tp("dot_product", {"model": 4}))

def test_missing_config_or_mesh(self):
self.assertFalse(_fused_moe_kernel_uses_ep(None, types.SimpleNamespace(shape={"expert": 2})))
self.assertFalse(_fused_moe_kernel_uses_ep(types.SimpleNamespace(attention="vllm_rpa"), None))
self.assertFalse(_fused_moe_kernel_uses_tp(None, types.SimpleNamespace(shape={"model": 4})))
self.assertFalse(_fused_moe_kernel_uses_tp(types.SimpleNamespace(attention="vllm_rpa"), None))


class TestFuseMoeWeightsFusedKernelLayout(unittest.TestCase):
"""The prefused wi layout must match how tpu-inference's fused MoE kernel splits it into gate/up.
"""The prefused wi layout must match how its consumer splits it into gate/up.

The TP kernel splits each shard of the fused axis, so each shard holds [gate_i | up_i]. The
expert-parallel kernel splits the whole fused axis in half, so it needs [gate | up], with each
half keeping the per-shard MLP-dim padding that `wo` gets.
tpu-inference's TP fused kernel splits each shard of the fused axis, so each shard holds
[gate_i | up_i]. Every other consumer (the expert-parallel kernel, `RoutedMoE`'s non-fused
paths) splits the whole fused axis in half, so it needs [gate | up], with each half keeping the
per-shard MLP-dim padding that `wo` gets.
"""

_VLLM_CONFIG = types.SimpleNamespace(attention="vllm_rpa")

# Checkpoint halves are (experts, embed, mlp) = (2, 3, 8); the model pads mlp to 16 per half.
_E, _D, _H, _PADDED_H = 2, 3, 8, 16

Expand All @@ -421,14 +426,15 @@ def _make_mesh(self, expert, model):
devices = np.array(jax.devices()[:num_devices]).reshape(expert, model)
return Mesh(devices, ("expert", "model"))

def _fuse(self, mesh, padded_h):
def _fuse(self, mesh, padded_h, config=_VLLM_CONFIG, model_dtype=jnp.float32, hash_routed=False):
wi_0 = np.arange(1, self._E * self._D * self._H + 1, dtype=np.float32).reshape(self._E, self._D, self._H)
wi_1 = -wi_0
sharding = jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec("expert", None, "model"))
model = {"wi": jax.device_put(jnp.zeros((self._E, self._D, 2 * padded_h), dtype=jnp.float32), sharding)}
config = types.SimpleNamespace(attention="vllm_rpa")
model = {"wi": jax.device_put(jnp.zeros((self._E, self._D, 2 * padded_h), dtype=model_dtype), sharding)}
if hash_routed:
model["tid2eid"] = jnp.zeros((8, 2), dtype=jnp.float32)
out = _fuse_moe_weights({"wi_0": wi_0, "wi_1": wi_1}, model, config)
return wi_0, wi_1, np.asarray(out["wi"])
return wi_0, wi_1, out["wi"]

@staticmethod
def _pad_each_shard(arr, n_shards, padded_size):
Expand All @@ -437,28 +443,93 @@ def _pad_each_shard(arr, n_shards, padded_size):
pad_width = [(0, 0)] * (arr.ndim - 1) + [(0, pad)]
return [np.pad(chunk, pad_width) for chunk in np.split(arr, n_shards, axis=-1)]

def _assert_interleaved(self, wi_0, wi_1, out, n_shards):
gate = self._pad_each_shard(wi_0, n_shards, self._PADDED_H)
up = self._pad_each_shard(wi_1, n_shards, self._PADDED_H)
expected = np.concatenate([chunk for pair in zip(gate, up) for chunk in pair], axis=-1)
np.testing.assert_array_equal(np.asarray(out, np.float32), expected)

def test_tensor_parallel_interleaves_gate_and_up_per_shard(self):
mesh = self._make_mesh(expert=1, model=4)
wi_0, wi_1, out = self._fuse(mesh, self._PADDED_H)
gate = self._pad_each_shard(wi_0, 4, self._PADDED_H)
up = self._pad_each_shard(wi_1, 4, self._PADDED_H)
expected = np.concatenate([chunk for pair in zip(gate, up) for chunk in pair], axis=-1)
np.testing.assert_array_equal(out, expected)
self._assert_interleaved(*self._fuse(mesh, self._PADDED_H), n_shards=4)

def test_interleaved_layout_is_built_in_the_model_dtype(self):
mesh = self._make_mesh(expert=1, model=4)
wi_0, wi_1, out = self._fuse(mesh, self._PADDED_H, model_dtype=jnp.bfloat16)
self.assertEqual(out.dtype, jnp.bfloat16)
self._assert_interleaved(wi_0, wi_1, out, n_shards=4) # The test values are exact in bf16.

def _assert_concatenated(self, wi_0, wi_1, out, n_shards):
gate = self._pad_each_shard(wi_0, n_shards, self._PADDED_H)
up = self._pad_each_shard(wi_1, n_shards, self._PADDED_H)
np.testing.assert_array_equal(np.asarray(out, np.float32), np.concatenate(gate + up, axis=-1))

def test_expert_parallel_concatenates_gate_and_up(self):
mesh = self._make_mesh(expert=2, model=2)
wi_0, wi_1, out = self._fuse(mesh, self._PADDED_H)
gate = self._pad_each_shard(wi_0, 2, self._PADDED_H)
up = self._pad_each_shard(wi_1, 2, self._PADDED_H)
np.testing.assert_array_equal(out, np.concatenate(gate + up, axis=-1))
self._assert_concatenated(*self._fuse(mesh, self._PADDED_H), n_shards=2)

def test_expert_parallel_with_unsharded_fused_axis_builds_in_embed_slices(self):
"""With EP alone the fused axis is unsharded, and the fused wi is still built in memory-bounded slices."""
mesh = self._make_mesh(expert=2, model=1)
with patch(
"maxtext.utils.model_creation_utils._fuse_moe_halves_concatenated", wraps=_fuse_moe_halves_concatenated
) as builder:
wi_0, wi_1, out = self._fuse(mesh, self._PADDED_H)
builder.assert_called_once()
self._assert_concatenated(wi_0, wi_1, out, n_shards=1)

def test_without_config_concatenates_gate_and_up(self):
"""Without a config the TP fused kernel cannot be detected, so the halves are not interleaved."""
mesh = self._make_mesh(expert=1, model=4)
self._assert_concatenated(*self._fuse(mesh, self._PADDED_H, config=None), n_shards=4)

def test_hash_routed_layer_concatenates_gate_and_up(self):
"""Hash-routed layers take the non-fused path, which splits wi in half, even under the TP fused kernel."""
mesh = self._make_mesh(expert=1, model=4)
self._assert_concatenated(*self._fuse(mesh, self._PADDED_H, hash_routed=True), n_shards=4)

def test_concatenated_layout_is_built_in_the_model_dtype(self):
mesh = self._make_mesh(expert=2, model=2)
wi_0, wi_1, out = self._fuse(mesh, self._PADDED_H, model_dtype=jnp.bfloat16)
self.assertEqual(out.dtype, jnp.bfloat16)
self._assert_concatenated(wi_0, wi_1, out, n_shards=2) # The test values are exact in bf16.

def test_expert_parallel_without_padding_is_plain_concat(self):
mesh = self._make_mesh(expert=2, model=2)
wi_0, wi_1, out = self._fuse(mesh, self._H)
np.testing.assert_array_equal(out, np.concatenate([wi_0, wi_1], axis=-1))
np.testing.assert_array_equal(np.asarray(out), np.concatenate([wi_0, wi_1], axis=-1))

def test_concatenation_rejects_halves_the_shards_do_not_divide(self):
mesh = self._make_mesh(expert=1, model=1)
sharding = jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec(None, None, "model"))
half = np.zeros((self._E, self._D, 6), dtype=np.float32)
with self.assertRaisesRegex(ValueError, "multiples of the 4 shards"):
_concat_halves_padded_per_shard(half, half, n_shards=4, half_dim=8, out_sharding=sharding)

def test_interleave_rejects_halves_the_shards_do_not_divide(self):
mesh = self._make_mesh(expert=1, model=4)
sharding = jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec("expert", None, "model"))
model = {"wi": jax.device_put(jnp.zeros((self._E, self._D, 2 * self._PADDED_H), dtype=jnp.float32), sharding)}
half = np.zeros((self._E, self._D, 6), dtype=np.float32)
with self.assertRaisesRegex(ValueError, "multiples of the 4 shards"):
_fuse_moe_weights({"wi_0": half, "wi_1": half}, model, self._VLLM_CONFIG)

def test_sharded_embed_axis_is_fused_in_one_logged_pass(self):
mesh = self._make_mesh(expert=1, model=2)
sharding = jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec(None, "model", None))
wi_0 = np.arange(2 * 4 * 8, dtype=np.float32).reshape(2, 4, 8)
wi_model = jax.device_put(jnp.zeros((2, 4, 16), dtype=jnp.float32), sharding)
with patch("maxtext.utils.model_creation_utils.max_logging.info") as info:
out = _fuse_moe_halves_concatenated(wi_0, -wi_0, wi_model, n_shards=1, num_slices=2)
info.assert_called_once()
np.testing.assert_array_equal(np.asarray(out), np.concatenate([wi_0, -wi_0], axis=-1))

def test_expert_parallel_embed_slices_match_single_pass(self):
"""Building [gate | up] in uneven embed slices gives the same array and sharding as one pass."""
"""Building [gate | up] in uneven embed slices gives the same array and sharding as one pass.

Donation is a no-op on CPU, so this does not check that the slices are written into the result buffer in
place; the load-time measurements on TPU in #5988 cover that.
"""
mesh = self._make_mesh(expert=2, model=2)
sharding = jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec("expert", None, "model"))
wi_0 = np.arange(2 * 5 * 8, dtype=np.float32).reshape(2, 5, 8)
Expand Down
Loading