diff --git a/src/maxtext/layers/moe.py b/src/maxtext/layers/moe.py index 3727373f58..40d1bc7da6 100644 --- a/src/maxtext/layers/moe.py +++ b/src/maxtext/layers/moe.py @@ -5385,6 +5385,9 @@ def _maybe_native_fused_weight(kernel, scale): wi_scale = self.wi_scale[...] if self.wi_scale is not None else None fused_kernel, wi_native_scale = _maybe_native_fused_weight(self.wi[...], wi_scale) elif cfg.prefuse_moe_weights: + # Split into two whole halves: `model_creation_utils._fuse_moe_weights` prefuses [gate | up] for every + # consumer except tpu-inference's TP fused kernel (the fused path above without expert parallelism), which + # gets one [gate | up] chunk per shard instead. wi_scale = self.wi_scale[...] if self.wi_scale is not None else None wi = quantizations.dequantize_weight(self.wi[...], wi_scale, self.dtype) n = wi.shape[-1] // 2 diff --git a/src/maxtext/utils/model_creation_utils.py b/src/maxtext/utils/model_creation_utils.py index be387e2682..dcd6b8dfb0 100644 --- a/src/maxtext/utils/model_creation_utils.py +++ b/src/maxtext/utils/model_creation_utils.py @@ -328,14 +328,25 @@ def _intersect_with_metadata(target, meta_tree): _VLLM_EXPERT_PARALLELISM_AXES = ("attn_dp_expert", "expert", "dcp") -def _fused_moe_kernel_uses_ep(config, mesh) -> bool: - """Whether tpu-inference's fused MoE kernel runs expert-parallel for this config and mesh. +def _fused_moe_kernel_uses_tp(config, mesh) -> bool: + """Whether MoE layers run through tpu-inference's tensor-parallel fused MoE kernel for this config and mesh. - Mirrors `use_ep = self.get_expert_parallelism_size() > 1` in `RoutedMoE.fused_moe_matmul`. + That kernel is the only consumer of a prefused `wi` that splits each shard of the fused axis into gate and up. + The expert-parallel fused kernel and `RoutedMoE`'s non-fused paths (non-vLLM attention, hash-routed layers) + split the whole axis in half. Mirrors `use_ep = self.get_expert_parallelism_size() > 1` being False in + `RoutedMoE.fused_moe_matmul`. """ if config is None or mesh is None or getattr(config, "attention", None) not in _VLLM_FUSED_MOE_ATTENTION: return False - return math.prod(mesh.shape.get(axis, 1) for axis in _VLLM_EXPERT_PARALLELISM_AXES) > 1 + return math.prod(mesh.shape.get(axis, 1) for axis in _VLLM_EXPERT_PARALLELISM_AXES) == 1 + + +def _is_hash_routed_moe(model_node) -> bool: + """Whether `model_node`, the arrays of a `RoutedMoE`, belongs to a hash-routed layer (it holds `tid2eid`). + + Hash-routed layers never run the fused MoE kernel (see `is_fused_moe_path` in `RoutedMoE.__call__`). + """ + return "tid2eid" in model_node # `_fuse_moe_halves_concatenated` builds the fused wi in this many slices of the embed axis. @@ -350,7 +361,16 @@ def _sharded_zeros(shape, dtype, out_sharding): @partial(jax.jit, donate_argnums=0) def _write_embed_slice(fused, part, start): """Writes `part` into `fused` at `start` along the embed (second-to-last) axis, reusing `fused`'s buffer.""" - return jax.lax.dynamic_update_slice_in_dim(fused, part, start, axis=fused.ndim - 2) + return jax.lax.dynamic_update_slice_in_dim(fused, part.astype(fused.dtype), start, axis=fused.ndim - 2) + + +def _check_half_splits_into_shards(half_size, half_dim, n_shards): + """Raises unless a MoE half of `half_size` can be padded to `half_dim` shard by shard.""" + if half_size % n_shards or half_dim % n_shards: + raise ValueError( + f"Cannot pad a MoE half of size {half_size} to {half_dim} shard by shard: both must be multiples of " + f"the {n_shards} shards of the fused axis." + ) @partial(jax.jit, static_argnames=("n_shards", "half_dim", "out_sharding")) @@ -358,6 +378,7 @@ def _concat_halves_padded_per_shard(wi_0, wi_1, n_shards, half_dim, out_sharding """Returns [wi_0 | wi_1] along the last axis, each half zero-padded to `half_dim` at the tail of each shard.""" def _pad(arr): + _check_half_splits_into_shards(arr.shape[-1], half_dim, n_shards) chunk = arr.shape[-1] // n_shards arr = arr.reshape(*arr.shape[:-1], n_shards, chunk) arr = jnp.pad(arr, [(0, 0)] * (arr.ndim - 1) + [(0, half_dim // n_shards - chunk)]) @@ -376,9 +397,15 @@ def _fuse_moe_halves_concatenated(wi_0, wi_1, wi_model, n_shards, num_slices=_FU wi_sharding = wi_model.sharding embed_axis = wi_model.ndim - 2 embed_dim = wi_model.shape[embed_axis] - if embed_axis < len(wi_sharding.spec) and wi_sharding.spec[embed_axis] is not None: - num_slices = 1 # Slices of a sharded embed axis would not match `wi_sharding`. - fused = _sharded_zeros(wi_model.shape, jnp.dtype(wi_0.dtype), wi_sharding) + if num_slices > 1 and embed_axis < len(wi_sharding.spec) and wi_sharding.spec[embed_axis] is not None: + # Slices of a sharded embed axis would not match `wi_sharding`. + max_logging.info( + f"Fusing MoE wi of shape {wi_model.shape} in one pass instead of {num_slices} embed slices, since its embed " + f"axis is sharded ({wi_sharding.spec}); this needs more temporary memory." + ) + num_slices = 1 + # In the model's dtype rather than the checkpoint's, e.g. half the size for an f32 checkpoint into a bf16 model. + fused = _sharded_zeros(wi_model.shape, jnp.dtype(wi_model.dtype), wi_sharding) step = -(-embed_dim // num_slices) for start in range(0, embed_dim, step): stop = min(start + step, embed_dim) @@ -402,18 +429,19 @@ def _fuse_moe_weights(ckpt_tree, model_arrays_tree, config=None): on a per-shard basis to satisfy kernel constraints. The per-shard interleave is the layout tpu-inference's TP fused MoE kernel - expects: it shards the fused axis and splits each shard into gate/up. When - that kernel runs expert-parallel instead (vLLM attention with an expert - parallelism size > 1), it does not shard the fused axis and splits the whole - axis in half (`gate = wi[..., :H]`, `up = wi[..., H:]`), so the gate and up - halves are concatenated rather than interleaved. Each half still keeps the - per-shard padding, which matches how `wo` is padded along the MLP dim. + expects: it shards the fused axis and splits each shard into gate/up. It is + the only consumer that does. The expert-parallel fused kernel and + `RoutedMoE`'s non-fused paths (non-vLLM attention, hash-routed layers) split + the whole axis in half (`gate = wi[..., :H]`, `up = wi[..., H:]`), so for + everything else the gate and up halves are concatenated rather than + interleaved. Each half still keeps the per-shard padding, which matches how + `wo` is padded along the MLP dim. Both layouts are built in the model's dtype. Args: ckpt_tree: Restored checkpoint tree, possibly holding wi_0/wi_1 pairs. model_arrays_tree: Model arrays tree; a fused model holds `wi` instead. - config: Model config, used to detect the expert-parallel fused MoE kernel. - When None, the per-shard interleaved layout is used. + config: Model config, used to detect the TP fused MoE kernel. When None, + the halves are concatenated. """ is_vllm_rpa = config is None or config.attention in _VLLM_FUSED_MOE_ATTENTION @@ -458,13 +486,13 @@ def _maybe_fuse(path, ckpt_node): # Determine the number of shards (TP degree) along the concatenated axis n_shards = 1 - use_ep = False + concatenate = False wi_sharding = getattr(wi_model, "sharding", None) if is_vllm_rpa and isinstance(wi_sharding, jax.sharding.NamedSharding): spec = wi_sharding.spec partition = spec[axis] if axis < len(spec) else None n_shards = _partition_size(partition, wi_sharding.mesh) - use_ep = _fused_moe_kernel_uses_ep(config, wi_sharding.mesh) + concatenate = not _fused_moe_kernel_uses_tp(config, wi_sharding.mesh) or _is_hash_routed_moe(model_node) # Target size for a single half (wi_0 or wi_1) AFTER padding target_half_dim = wi_model.shape[-1] // 2 @@ -473,6 +501,7 @@ def _maybe_fuse(path, ckpt_node): def _pad_and_chunk(arr, target_total_size): shape = arr.shape current_total_size = shape[-1] + _check_half_splits_into_shards(current_total_size, target_total_size, n_shards) # Calculate per-shard chunk sizes chunk_size = current_total_size // n_shards @@ -490,8 +519,8 @@ def _pad_and_chunk(arr, target_total_size): return arr_reshaped - # Expert-parallel kernel: [gate | up] instead of the per-shard interleave below. - if use_ep: + # Everything but the TP fused kernel: [gate | up] instead of the per-shard interleave below. + if concatenate: new_node = {k: v for k, v in ckpt_node.items() if k not in ("wi_0", "wi_1")} new_node["wi"] = _fuse_moe_halves_concatenated(ckpt_node["wi_0"], ckpt_node["wi_1"], wi_model, n_shards) return new_node @@ -506,7 +535,7 @@ def _pad_and_chunk(arr, target_total_size): # Flatten the n_shards dimension back out to match the final model shape, drop wi_0/wi_1. new_node = {k: v for k, v in ckpt_node.items() if k not in ("wi_0", "wi_1")} - new_node["wi"] = wi_interleaved.reshape(*wi_model.shape) + new_node["wi"] = wi_interleaved.reshape(*wi_model.shape).astype(wi_model.dtype) return new_node return jax.tree_util.tree_map_with_path(_maybe_fuse, ckpt_tree, is_leaf=_is_fusion_site) diff --git a/tests/unit/model_creation_utils_test.py b/tests/unit/model_creation_utils_test.py index ffd05aa06b..60d64d7d75 100644 --- a/tests/unit/model_creation_utils_test.py +++ b/tests/unit/model_creation_utils_test.py @@ -37,7 +37,8 @@ _fix_restore_args_for_shape_mismatch, _fuse_moe_weights, _fuse_moe_halves_concatenated, - _fused_moe_kernel_uses_ep, + _concat_halves_padded_per_shard, + _fused_moe_kernel_uses_tp, _logical_partition_specs, _stored_shape_evenly_shardable, _zero_pad_axis, @@ -381,36 +382,40 @@ def test_recurses_through_nested_dicts(self): np.testing.assert_array_equal(out["layers"]["0"]["other"], other) -class TestFusedMoeKernelUsesEp(unittest.TestCase): - """_fused_moe_kernel_uses_ep must match `use_ep` in `RoutedMoE.fused_moe_matmul`.""" +class TestFusedMoeKernelUsesTp(unittest.TestCase): + """_fused_moe_kernel_uses_tp must be True exactly when `RoutedMoE.fused_moe_matmul` runs with `use_ep` False.""" - def _uses_ep(self, attention, mesh_shape): - return _fused_moe_kernel_uses_ep(types.SimpleNamespace(attention=attention), types.SimpleNamespace(shape=mesh_shape)) + def _uses_tp(self, attention, mesh_shape): + return _fused_moe_kernel_uses_tp(types.SimpleNamespace(attention=attention), types.SimpleNamespace(shape=mesh_shape)) def test_tensor_parallel_only(self): - self.assertFalse(self._uses_ep("vllm_rpa", {"data": 1, "model": 4, "expert": 1})) + self.assertTrue(self._uses_tp("vllm_rpa", {"data": 1, "model": 4, "expert": 1})) + self.assertTrue(self._uses_tp("vllm_batched_rpa", {"model": 4})) def test_expert_parallel_axes(self): - self.assertTrue(self._uses_ep("vllm_rpa", {"model": 2, "expert": 2})) - self.assertTrue(self._uses_ep("vllm_batched_rpa", {"model": 2, "attn_dp_expert": 2})) - self.assertTrue(self._uses_ep("vllm_rpa", {"model": 2, "dcp": 2})) + self.assertFalse(self._uses_tp("vllm_rpa", {"model": 2, "expert": 2})) + self.assertFalse(self._uses_tp("vllm_batched_rpa", {"model": 2, "attn_dp_expert": 2})) + self.assertFalse(self._uses_tp("vllm_rpa", {"model": 2, "dcp": 2})) def test_non_vllm_attention(self): - self.assertFalse(self._uses_ep("dot_product", {"model": 2, "expert": 2})) + self.assertFalse(self._uses_tp("dot_product", {"model": 4})) def test_missing_config_or_mesh(self): - self.assertFalse(_fused_moe_kernel_uses_ep(None, types.SimpleNamespace(shape={"expert": 2}))) - self.assertFalse(_fused_moe_kernel_uses_ep(types.SimpleNamespace(attention="vllm_rpa"), None)) + self.assertFalse(_fused_moe_kernel_uses_tp(None, types.SimpleNamespace(shape={"model": 4}))) + self.assertFalse(_fused_moe_kernel_uses_tp(types.SimpleNamespace(attention="vllm_rpa"), None)) class TestFuseMoeWeightsFusedKernelLayout(unittest.TestCase): - """The prefused wi layout must match how tpu-inference's fused MoE kernel splits it into gate/up. + """The prefused wi layout must match how its consumer splits it into gate/up. - The TP kernel splits each shard of the fused axis, so each shard holds [gate_i | up_i]. The - expert-parallel kernel splits the whole fused axis in half, so it needs [gate | up], with each - half keeping the per-shard MLP-dim padding that `wo` gets. + tpu-inference's TP fused kernel splits each shard of the fused axis, so each shard holds + [gate_i | up_i]. Every other consumer (the expert-parallel kernel, `RoutedMoE`'s non-fused + paths) splits the whole fused axis in half, so it needs [gate | up], with each half keeping the + per-shard MLP-dim padding that `wo` gets. """ + _VLLM_CONFIG = types.SimpleNamespace(attention="vllm_rpa") + # Checkpoint halves are (experts, embed, mlp) = (2, 3, 8); the model pads mlp to 16 per half. _E, _D, _H, _PADDED_H = 2, 3, 8, 16 @@ -421,14 +426,15 @@ def _make_mesh(self, expert, model): devices = np.array(jax.devices()[:num_devices]).reshape(expert, model) return Mesh(devices, ("expert", "model")) - def _fuse(self, mesh, padded_h): + def _fuse(self, mesh, padded_h, config=_VLLM_CONFIG, model_dtype=jnp.float32, hash_routed=False): wi_0 = np.arange(1, self._E * self._D * self._H + 1, dtype=np.float32).reshape(self._E, self._D, self._H) wi_1 = -wi_0 sharding = jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec("expert", None, "model")) - model = {"wi": jax.device_put(jnp.zeros((self._E, self._D, 2 * padded_h), dtype=jnp.float32), sharding)} - config = types.SimpleNamespace(attention="vllm_rpa") + model = {"wi": jax.device_put(jnp.zeros((self._E, self._D, 2 * padded_h), dtype=model_dtype), sharding)} + if hash_routed: + model["tid2eid"] = jnp.zeros((8, 2), dtype=jnp.float32) out = _fuse_moe_weights({"wi_0": wi_0, "wi_1": wi_1}, model, config) - return wi_0, wi_1, np.asarray(out["wi"]) + return wi_0, wi_1, out["wi"] @staticmethod def _pad_each_shard(arr, n_shards, padded_size): @@ -437,28 +443,93 @@ def _pad_each_shard(arr, n_shards, padded_size): pad_width = [(0, 0)] * (arr.ndim - 1) + [(0, pad)] return [np.pad(chunk, pad_width) for chunk in np.split(arr, n_shards, axis=-1)] + def _assert_interleaved(self, wi_0, wi_1, out, n_shards): + gate = self._pad_each_shard(wi_0, n_shards, self._PADDED_H) + up = self._pad_each_shard(wi_1, n_shards, self._PADDED_H) + expected = np.concatenate([chunk for pair in zip(gate, up) for chunk in pair], axis=-1) + np.testing.assert_array_equal(np.asarray(out, np.float32), expected) + def test_tensor_parallel_interleaves_gate_and_up_per_shard(self): mesh = self._make_mesh(expert=1, model=4) - wi_0, wi_1, out = self._fuse(mesh, self._PADDED_H) - gate = self._pad_each_shard(wi_0, 4, self._PADDED_H) - up = self._pad_each_shard(wi_1, 4, self._PADDED_H) - expected = np.concatenate([chunk for pair in zip(gate, up) for chunk in pair], axis=-1) - np.testing.assert_array_equal(out, expected) + self._assert_interleaved(*self._fuse(mesh, self._PADDED_H), n_shards=4) + + def test_interleaved_layout_is_built_in_the_model_dtype(self): + mesh = self._make_mesh(expert=1, model=4) + wi_0, wi_1, out = self._fuse(mesh, self._PADDED_H, model_dtype=jnp.bfloat16) + self.assertEqual(out.dtype, jnp.bfloat16) + self._assert_interleaved(wi_0, wi_1, out, n_shards=4) # The test values are exact in bf16. + + def _assert_concatenated(self, wi_0, wi_1, out, n_shards): + gate = self._pad_each_shard(wi_0, n_shards, self._PADDED_H) + up = self._pad_each_shard(wi_1, n_shards, self._PADDED_H) + np.testing.assert_array_equal(np.asarray(out, np.float32), np.concatenate(gate + up, axis=-1)) def test_expert_parallel_concatenates_gate_and_up(self): mesh = self._make_mesh(expert=2, model=2) - wi_0, wi_1, out = self._fuse(mesh, self._PADDED_H) - gate = self._pad_each_shard(wi_0, 2, self._PADDED_H) - up = self._pad_each_shard(wi_1, 2, self._PADDED_H) - np.testing.assert_array_equal(out, np.concatenate(gate + up, axis=-1)) + self._assert_concatenated(*self._fuse(mesh, self._PADDED_H), n_shards=2) + + def test_expert_parallel_with_unsharded_fused_axis_builds_in_embed_slices(self): + """With EP alone the fused axis is unsharded, and the fused wi is still built in memory-bounded slices.""" + mesh = self._make_mesh(expert=2, model=1) + with patch( + "maxtext.utils.model_creation_utils._fuse_moe_halves_concatenated", wraps=_fuse_moe_halves_concatenated + ) as builder: + wi_0, wi_1, out = self._fuse(mesh, self._PADDED_H) + builder.assert_called_once() + self._assert_concatenated(wi_0, wi_1, out, n_shards=1) + + def test_without_config_concatenates_gate_and_up(self): + """Without a config the TP fused kernel cannot be detected, so the halves are not interleaved.""" + mesh = self._make_mesh(expert=1, model=4) + self._assert_concatenated(*self._fuse(mesh, self._PADDED_H, config=None), n_shards=4) + + def test_hash_routed_layer_concatenates_gate_and_up(self): + """Hash-routed layers take the non-fused path, which splits wi in half, even under the TP fused kernel.""" + mesh = self._make_mesh(expert=1, model=4) + self._assert_concatenated(*self._fuse(mesh, self._PADDED_H, hash_routed=True), n_shards=4) + + def test_concatenated_layout_is_built_in_the_model_dtype(self): + mesh = self._make_mesh(expert=2, model=2) + wi_0, wi_1, out = self._fuse(mesh, self._PADDED_H, model_dtype=jnp.bfloat16) + self.assertEqual(out.dtype, jnp.bfloat16) + self._assert_concatenated(wi_0, wi_1, out, n_shards=2) # The test values are exact in bf16. def test_expert_parallel_without_padding_is_plain_concat(self): mesh = self._make_mesh(expert=2, model=2) wi_0, wi_1, out = self._fuse(mesh, self._H) - np.testing.assert_array_equal(out, np.concatenate([wi_0, wi_1], axis=-1)) + np.testing.assert_array_equal(np.asarray(out), np.concatenate([wi_0, wi_1], axis=-1)) + + def test_concatenation_rejects_halves_the_shards_do_not_divide(self): + mesh = self._make_mesh(expert=1, model=1) + sharding = jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec(None, None, "model")) + half = np.zeros((self._E, self._D, 6), dtype=np.float32) + with self.assertRaisesRegex(ValueError, "multiples of the 4 shards"): + _concat_halves_padded_per_shard(half, half, n_shards=4, half_dim=8, out_sharding=sharding) + + def test_interleave_rejects_halves_the_shards_do_not_divide(self): + mesh = self._make_mesh(expert=1, model=4) + sharding = jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec("expert", None, "model")) + model = {"wi": jax.device_put(jnp.zeros((self._E, self._D, 2 * self._PADDED_H), dtype=jnp.float32), sharding)} + half = np.zeros((self._E, self._D, 6), dtype=np.float32) + with self.assertRaisesRegex(ValueError, "multiples of the 4 shards"): + _fuse_moe_weights({"wi_0": half, "wi_1": half}, model, self._VLLM_CONFIG) + + def test_sharded_embed_axis_is_fused_in_one_logged_pass(self): + mesh = self._make_mesh(expert=1, model=2) + sharding = jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec(None, "model", None)) + wi_0 = np.arange(2 * 4 * 8, dtype=np.float32).reshape(2, 4, 8) + wi_model = jax.device_put(jnp.zeros((2, 4, 16), dtype=jnp.float32), sharding) + with patch("maxtext.utils.model_creation_utils.max_logging.info") as info: + out = _fuse_moe_halves_concatenated(wi_0, -wi_0, wi_model, n_shards=1, num_slices=2) + info.assert_called_once() + np.testing.assert_array_equal(np.asarray(out), np.concatenate([wi_0, -wi_0], axis=-1)) def test_expert_parallel_embed_slices_match_single_pass(self): - """Building [gate | up] in uneven embed slices gives the same array and sharding as one pass.""" + """Building [gate | up] in uneven embed slices gives the same array and sharding as one pass. + + Donation is a no-op on CPU, so this does not check that the slices are written into the result buffer in + place; the load-time measurements on TPU in #5988 cover that. + """ mesh = self._make_mesh(expert=2, model=2) sharding = jax.sharding.NamedSharding(mesh, jax.sharding.PartitionSpec("expert", None, "model")) wi_0 = np.arange(2 * 5 * 8, dtype=np.float32).reshape(2, 5, 8)