diff --git a/src/maxtext/inference/vllm_decode.py b/src/maxtext/inference/vllm_decode.py index e6fbf20958..e86d3432c8 100644 --- a/src/maxtext/inference/vllm_decode.py +++ b/src/maxtext/inference/vllm_decode.py @@ -41,6 +41,7 @@ from maxtext.configs import pyconfig from maxtext.integration.tunix.tunix_adapter import TunixMaxTextAdapter import maxtext.integration.vllm.maxtext_vllm_adapter as adapter +from maxtext.integration.vllm.convert_utils import uses_maxtext_vllm_adapter from maxtext.multimodal import utils as mm_utils from maxtext.utils import lora_utils from maxtext.utils import max_logging @@ -125,6 +126,14 @@ def create_vllm_llm(config: Config, **overrides: Any) -> LLM: } if config.load_parameters_path: + if not uses_maxtext_vllm_adapter(config): + # Only MaxTextForCausalLM reads load_parameters_path. vLLM's own model implementation loads the full + # Hugging Face checkpoint of `model` through host memory instead (e.g. ~40 GB for gpt-oss-20b in BF16). + max_logging.warning( + f"load_parameters_path={config.load_parameters_path} is ignored and vLLM will load the Hugging Face " + f"checkpoint of {config.tokenizer_path} instead, which can run out of host memory. Set " + "vllm_hf_overrides='{architectures: [\"MaxTextForCausalLM\"]}' to decode the MaxText checkpoint." + ) vllm_args["additional_config"]["maxtext_config"]["load_parameters_path"] = config.load_parameters_path else: vllm_args["load_format"] = "dummy" @@ -230,17 +239,9 @@ def decode_with_tunix( mesh: The JAX mesh for parallelism. """ # Wrap the model for Tunix - use_no_op_mappings = False - if hasattr(config, "vllm_hf_overrides") and config.vllm_hf_overrides: - overrides = config.vllm_hf_overrides - if isinstance(overrides, str) and "MaxTextForCausalLM" in overrides: - use_no_op_mappings = True - elif isinstance(overrides, dict) and "MaxTextForCausalLM" in overrides.get("architectures", []): - use_no_op_mappings = True - tunix_model = TunixMaxTextAdapter( base_model=model, - use_no_op_mappings=use_no_op_mappings, + use_no_op_mappings=uses_maxtext_vllm_adapter(config), ) # Load the tokenizer diff --git a/src/maxtext/integration/vllm/convert_utils.py b/src/maxtext/integration/vllm/convert_utils.py index 71e6a7fa2f..de87b84a52 100644 --- a/src/maxtext/integration/vllm/convert_utils.py +++ b/src/maxtext/integration/vllm/convert_utils.py @@ -68,6 +68,34 @@ def pad_to_tpu_lanes(dim: int, lane_size: int = DEFAULT_TPU_NUM_LANES) -> int: return ((dim + lane_size - 1) // lane_size) * lane_size +def get_moe_intermediate_size(hf_config: Any) -> Optional[int]: + """Returns the per-expert MLP dimension of a Hugging Face config, or None for a dense model. + + Most MoE configs store it as `moe_intermediate_size`. Models whose MLPs are all MoE (e.g. GPT-OSS, + Mixtral) have no such field; their per-expert MLP dimension is `intermediate_size`. + """ + moe_intermediate_size = getattr(hf_config, "moe_intermediate_size", None) + if moe_intermediate_size is not None: + return moe_intermediate_size + num_experts = getattr(hf_config, "num_local_experts", None) or getattr(hf_config, "num_experts", None) + if num_experts is not None and num_experts > 1: + return getattr(hf_config, "intermediate_size", None) + return None + + +def uses_maxtext_vllm_adapter(config: Any) -> bool: + """Returns whether vLLM is configured to instantiate MaxTextForCausalLM.""" + overrides = getattr(config, "vllm_hf_overrides", None) + if isinstance(overrides, str): + return "MaxTextForCausalLM" in overrides + if isinstance(overrides, dict): + architectures = overrides.get("architectures", ()) + if isinstance(architectures, str): + architectures = (architectures,) + return "MaxTextForCausalLM" in architectures + return False + + def compute_padded_moe_mlp_dim( hidden_size: Optional[int], moe_mlp_tp_size: int, @@ -80,7 +108,7 @@ def compute_padded_moe_mlp_dim( `2 * num_lanes * moe_mlp_tp_size`. Args: - hidden_size: Unpadded MoE intermediate size (`moe_intermediate_size`). + hidden_size: Unpadded MoE intermediate size (see `get_moe_intermediate_size`). moe_mlp_tp_size: TP degree across the MLP dimension (`tp * attn_dp`). num_lanes: TPU vector-register lane count. Pass the runtime-probed `pltpu.get_tpu_info().num_lanes` when a TPU handle is available; the default diff --git a/src/maxtext/integration/vllm/maxtext_vllm_adapter/adapter.py b/src/maxtext/integration/vllm/maxtext_vllm_adapter/adapter.py index de55a792fd..fa034c47e0 100644 --- a/src/maxtext/integration/vllm/maxtext_vllm_adapter/adapter.py +++ b/src/maxtext/integration/vllm/maxtext_vllm_adapter/adapter.py @@ -24,7 +24,11 @@ import numpy as np from maxtext.common.common_types import MODEL_MODE_AUTOREGRESSIVE from maxtext.configs import pyconfig -from maxtext.integration.vllm.convert_utils import DEFAULT_TPU_NUM_LANES, compute_padded_moe_mlp_dim +from maxtext.integration.vllm.convert_utils import ( + DEFAULT_TPU_NUM_LANES, + compute_padded_moe_mlp_dim, + get_moe_intermediate_size, +) from maxtext.integration.vllm.hybrid_cache_utils import ( build_qwen_gdn_cache_layout, call_with_supported_kwargs, @@ -126,7 +130,7 @@ def generate_maxtext_config(vllm_config: VllmConfig) -> pyconfig.HyperParameters if hasattr(vllm_config.model_config.hf_config, "text_config") else vllm_config.model_config.hf_config ) - hidden_size = getattr(hf_config, "moe_intermediate_size", None) + hidden_size = get_moe_intermediate_size(hf_config) try: num_lanes = pltpu.get_tpu_info().num_lanes except Exception: # pylint: disable=broad-exception-caught diff --git a/src/maxtext/integration/vllm/maxtext_vllm_rollout.py b/src/maxtext/integration/vllm/maxtext_vllm_rollout.py index 2ef8ff8038..f9d46cc357 100644 --- a/src/maxtext/integration/vllm/maxtext_vllm_rollout.py +++ b/src/maxtext/integration/vllm/maxtext_vllm_rollout.py @@ -37,7 +37,7 @@ from flax.traverse_util import flatten_dict, unflatten_dict import jax import jax.numpy as jnp -from maxtext.integration.vllm.convert_utils import MOE_MLP_WEIGHTS, _sharding_summary +from maxtext.integration.vllm.convert_utils import MOE_MLP_WEIGHTS, _sharding_summary, uses_maxtext_vllm_adapter from maxtext.integration.vllm.torchax_converter.gemma4_moe import Gemma4MaxTextToVLLMConverter from maxtext.integration.vllm.torchax_converter.qwen35_moe import Qwen35MaxTextToVLLMConverter from maxtext.integration.vllm.torchax_converter.qwen3_moe import Qwen3MaxTextToVLLMConverter @@ -177,19 +177,6 @@ def _as_dict(value: Any) -> dict: return {} -def uses_maxtext_vllm_adapter(config: Any) -> bool: - """Returns whether vLLM is configured to instantiate MaxTextForCausalLM.""" - overrides = getattr(config, "vllm_hf_overrides", None) - if isinstance(overrides, str): - return "MaxTextForCausalLM" in overrides - if isinstance(overrides, dict): - architectures = overrides.get("architectures", ()) - if isinstance(architectures, str): - architectures = (architectures,) - return "MaxTextForCausalLM" in architectures - return False - - def requires_maxtext_scanned_weight_unroll(config: Any) -> bool: """Returns whether direct MaxText-to-MaxText sync needs a custom unroll.""" return bool(getattr(config, "scan_layers", False) and uses_maxtext_vllm_adapter(config)) diff --git a/src/maxtext/layers/moe.py b/src/maxtext/layers/moe.py index 3727373f58..a9fb8225ac 100644 --- a/src/maxtext/layers/moe.py +++ b/src/maxtext/layers/moe.py @@ -607,6 +607,68 @@ def fused_moe_lhs_scale_kwargs(quant, fused_moe_fn) -> dict[str, jax.Array]: return {"w1_lhs_scale": scale, "w2_lhs_scale": scale} +# GPT-OSS's SwiGLU clamp limit. tpu-inference's fused "swigluoai" activation hardcodes it, along with +# alpha=1.702 (see swigluoai in tpu_inference/kernels/megablox/gmm_v2.py). +_FUSED_SWIGLUOAI_LIMIT = 7.0 + + +def fused_moe_activation(config) -> str: + """Name of the tpu-inference fused MoE activation that matches `RoutedMoE.apply_ffn_activation`.""" + if config.decoder_block == ctypes.DecoderBlockType.GPT_OSS: + # GPT-OSS's clamped SwiGLU is the kernel's "swigluoai". Its mlp_activations[0] ("sigmoid") is not a + # kernel activation at all. + if config.mlp_activations[0] != "sigmoid" or config.mlp_activations_limit != _FUSED_SWIGLUOAI_LIMIT: + raise NotImplementedError( + "The fused MoE kernel only implements GPT-OSS's SwiGLU with mlp_activations[0]='sigmoid' and " + f"mlp_activations_limit={_FUSED_SWIGLUOAI_LIMIT}, got {config.mlp_activations[0]!r} and " + f"{config.mlp_activations_limit}." + ) + return "swigluoai" + return config.mlp_activations[0] + + +def fuse_moe_biases_for_kernel(w0_bias, w1_bias, fused_size, num_shards, per_shard_layout) -> jax.Array: + """Lays out the gate/up expert biases as the `w1_bias` of tpu-inference's `fused_moe_func`. + + The kernel adds `w1_bias` to `x @ w1` before the activation, so the bias has to follow the layout + the kernel reads `w1` in. The vLLM weight loader zero-pads the tail of every MLP-dim shard of the + expert kernels (GMM_v2 alignment), so the bias is padded the same way. Then gate and up are either + interleaved per shard, for the tensor-parallel kernel, which splits each shard of the fused axis + into gate and up (the layout of a prefused `wi`, see `model_creation_utils._fuse_moe_weights`), or + concatenated whole, for the expert-parallel kernel, which splits the whole fused axis. + + Args: + w0_bias: Gate bias, [num_experts, mlp_dim]. + w1_bias: Up bias, [num_experts, mlp_dim]. + fused_size: Last dim of the fused kernel, i.e. twice the (padded) MoE MLP dim. + num_shards: Number of shards along the expert kernels' MLP dim. + per_shard_layout: Whether each shard holds its own [gate | up] chunk (tensor-parallel kernel) + instead of the kernel holding [all of gate | all of up] (expert-parallel kernel). + + Returns: + The float32 bias, [num_experts, 1, fused_size]. + """ + num_experts, mlp_dim = w0_bias.shape + padded_mlp_dim = fused_size // 2 + if mlp_dim % num_shards or padded_mlp_dim % num_shards or padded_mlp_dim < mlp_dim: + raise ValueError( + f"Cannot lay out MoE biases of MLP dim {mlp_dim} for a fused kernel of MLP dim {padded_mlp_dim} " + f"over {num_shards} shards." + ) + + def pad_shards(bias): + """[E, mlp_dim] -> [E, num_shards, padded_mlp_dim // num_shards], zero-padding each shard's tail.""" + bias = bias.astype(jnp.float32).reshape(num_experts, num_shards, mlp_dim // num_shards) + return jnp.pad(bias, ((0, 0), (0, 0), (0, (padded_mlp_dim - mlp_dim) // num_shards))) + + gate, up = pad_shards(w0_bias), pad_shards(w1_bias) + if per_shard_layout: + fused = jnp.concatenate([gate, up], axis=-1) + else: + fused = jnp.concatenate([gate.reshape(num_experts, -1), up.reshape(num_experts, -1)], axis=-1) + return fused.reshape(num_experts, 1, fused_size) + + class Tid2EidVar(nnx.Variable): """Custom variable to hold tid2eid without trainable param overhead.""" @@ -1194,7 +1256,10 @@ def _init_expert_kernel(shape, dtype): ) if self.config.mlp_bias: - wi_bias_axes = ("exp", "activation_mlp") + # vLLM's rules map "activation_mlp" onto mesh axes that "exp" already takes, which makes no valid + # NamedSharding; there "activation_mlp_moe" shards the bias like the expert kernels' MLP dim. + is_vllm = self.config.attention in ("vllm_rpa", "vllm_batched_rpa") + wi_bias_axes = ("exp", "activation_mlp_moe" if is_vllm else "activation_mlp") wo_bias_axes = ("exp", "activation_embed") wi_bias_shape = (self.num_experts, self.intermediate_dim) wo_bias_shape = (self.num_experts, self.moe_expert_input_dim) @@ -1389,6 +1454,21 @@ def get_tensor_parallelism_size(self): def get_context_autoregressive_parallelism_size(self): return self.mesh.shape.get("context_autoregressive", 1) + def _moe_mlp_num_shards(self): + """Number of shards along the expert kernels' MLP dim. + + These are the chunks the vLLM weight loader pads and fuses the expert kernels by: it reads them off + the sharding of the same logical axis (see `model_creation_utils._fuse_moe_weights`). + """ + if self.mesh is None: + return 1 + mlp_axes = self._logical_to_mesh_axes(self.wi_kernel_axes)[-1] + if mlp_axes is None: + return 1 + if isinstance(mlp_axes, str): + mlp_axes = (mlp_axes,) + return math.prod(self.mesh.shape.get(axis, 1) for axis in mlp_axes) + def should_update_load_balance(self): """Determines if loss-free load balancing updates should be applied. @@ -4972,11 +5052,15 @@ def fused_moe_matmul( forced_routed_experts=None, native_w1_scale=None, native_w2_scale=None, + w0_bias=None, + w1_bias=None, + wo_bias=None, ) -> tuple[jax.Array, None, None]: """Fused MoE via tpu_inference fused_moe_func (vllm_rpa path only). fused_moe_func handles routing, GMM, and weighted combination internally. - It does not compute lb_loss or bias_updates (inference-only). + It does not compute lb_loss or bias_updates (inference-only). Expert biases + (`w0_bias`, `w1_bias`, `wo_bias`, e.g. GPT-OSS) are applied by the kernel. """ if forced_routed_experts is not None: raise NotImplementedError("Forced routing via forced_routed_experts is not supported with" " fused_moe_matmul.") @@ -5005,8 +5089,16 @@ def fused_moe_matmul( # Use expert parallelism if the expert axis has size > 1 use_ep = self.get_expert_parallelism_size() > 1 + # The kernel takes the biases as [E, 1, N], the gate/up one in the layout it reads the fused kernel in. + fused_w1_bias = None + if w0_bias is not None and w1_bias is not None: + fused_w1_bias = fuse_moe_biases_for_kernel( + w0_bias, w1_bias, fused_kernel.shape[-1], self._moe_mlp_num_shards(), per_shard_layout=not use_ep + ) + fused_w2_bias = None if wo_bias is None else wo_bias.astype(jnp.float32)[:, None, :] + # Map MaxText config fields to fused_moe_func args - activation = self.config.mlp_activations[0] # e.g. "silu" + activation = fused_moe_activation(self.config) # e.g. "silu" scoring_fn = self.config.routed_score_func if self.config.routed_score_func else "softmax" # Check if the model architecture intrinsically renormalizes weights @@ -5040,8 +5132,8 @@ def fused_moe_matmul( w2=quantized_w2, w1_scale=w1_scale, w2_scale=w2_scale, - w1_bias=None, - w2_bias=None, + w1_bias=fused_w1_bias, + w2_bias=fused_w2_bias, gating_output=gating_output, topk=self.num_experts_per_tok, renormalize=renormalize, @@ -5448,6 +5540,9 @@ def _maybe_native_fused_weight(kernel, scale): forced_routed_experts=forced_routed_experts, native_w1_scale=wi_native_scale, native_w2_scale=wo_native_scale, + w0_bias=w0_bias, + w1_bias=w1_bias, + wo_bias=wo_bias, ) elif cfg.sparse_matmul: if quantizations.in_serve_mode(self.quant): diff --git a/tests/end_to_end/tpu/gpt_oss/20b/test_gpt_oss_rl.sh b/tests/end_to_end/tpu/gpt_oss/20b/test_gpt_oss_rl.sh index 1c903624a2..a5e69e7897 100644 --- a/tests/end_to_end/tpu/gpt_oss/20b/test_gpt_oss_rl.sh +++ b/tests/end_to_end/tpu/gpt_oss/20b/test_gpt_oss_rl.sh @@ -3,7 +3,9 @@ # Validates the GPT-OSS-20b RL pipeline using a pre-converted MaxText checkpoint. # The flow of this script is as follows: -# 1. Run RL starting from the pre-converted checkpoint. +# 1. Run inference on the pre-converted checkpoint. +# 2. Run RL starting from the pre-converted checkpoint. +# 3. Run inference on the checkpoint produced by the RL run. # Usage: # export HF_TOKEN= @@ -28,12 +30,36 @@ fi run_id=$1 use_pathways=${2:-false} export MODEL_NAME='gpt-oss-20b' +TOKENIZER_PATH='unsloth/gpt-oss-20b-BF16' # Non-Googlers please remember to point BASE_OUTPUT_DIRECTORY to the GCS paths where you have the scanned and unscanned checkpoints stored BASE_OUTPUT_DIRECTORY=gs://runner-maxtext-logs/${MODEL_NAME} +UNSCANNED_CKPT_PATH=${BASE_OUTPUT_DIRECTORY}/to_maxtext/unscanned/${run_id}/0/items SCANNED_CKPT_PATH=${BASE_OUTPUT_DIRECTORY}/to_maxtext/scanned/${run_id}/0/items -# Step 1: Run RL on the converted checkpoint +# Inference runs the MaxText implementation of GPT-OSS in vLLM. Its MoE layers then use tpu-inference's fused MoE kernel, +# tensor-parallel here, which needs prefuse_moe_weights=True. Tensor x data parallelism spans the 32 chips of a v6e-32. + +# Step 1: Run inference on the pre-converted checkpoint +python3 -m maxtext.inference.vllm_decode \ + model_name=${MODEL_NAME} \ + tokenizer_path=${TOKENIZER_PATH} \ + load_parameters_path=${UNSCANNED_CKPT_PATH} \ + vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \ + hbm_utilization_vllm=0.5 \ + prompt='Suggest some famous landmarks in London.' \ + decode_sampling_temperature=0.0 \ + max_target_length=256 \ + max_num_batched_tokens=256 \ + use_chat_template=True \ + scan_layers=false \ + prefuse_moe_weights=True \ + ici_tensor_parallelism=4 \ + ici_data_parallelism=8 \ + weight_dtype=bfloat16 \ + enable_single_controller=${use_pathways} + +# Step 2: Run RL on the converted checkpoint python3 -m maxtext.trainers.post_train.rl.train_rl \ base_output_directory=${BASE_OUTPUT_DIRECTORY}/rl \ load_parameters_path=${SCANNED_CKPT_PATH} \ @@ -61,4 +87,23 @@ python3 -m maxtext.trainers.post_train.rl.train_rl \ checkpoint_storage_use_ocdbt=False \ rollout_data_parallelism=-1 \ rollout_tensor_parallelism=4 \ - hbm_utilization_vllm=0.8 \ No newline at end of file + hbm_utilization_vllm=0.8 + +# Step 3: Run inference on the checkpoint produced by the RL run +python3 -m maxtext.inference.vllm_decode \ + model_name=${MODEL_NAME} \ + tokenizer_path=${TOKENIZER_PATH} \ + load_parameters_path=${BASE_OUTPUT_DIRECTORY}/rl/${run_id}/checkpoints/actor/2/model_params \ + vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \ + hbm_utilization_vllm=0.5 \ + prompt='Suggest some famous landmarks in London.' \ + decode_sampling_temperature=0.0 \ + max_target_length=256 \ + max_num_batched_tokens=256 \ + use_chat_template=True \ + scan_layers=true \ + prefuse_moe_weights=True \ + ici_tensor_parallelism=4 \ + ici_data_parallelism=8 \ + weight_dtype=bfloat16 \ + enable_single_controller=${use_pathways} \ No newline at end of file diff --git a/tests/end_to_end/tpu/gpt_oss/20b/test_gpt_oss_sft.sh b/tests/end_to_end/tpu/gpt_oss/20b/test_gpt_oss_sft.sh index 33d347c680..72ea04e3d7 100644 --- a/tests/end_to_end/tpu/gpt_oss/20b/test_gpt_oss_sft.sh +++ b/tests/end_to_end/tpu/gpt_oss/20b/test_gpt_oss_sft.sh @@ -3,7 +3,9 @@ # Validates the GPT-OSS-20b SFT pipeline using a pre-converted MaxText checkpoint. # The flow of this script is as follows: -# 1. Run SFT starting from the pre-converted checkpoint. +# 1. Run inference on the pre-converted checkpoint. +# 2. Run SFT starting from the pre-converted checkpoint. +# 3. Run inference on the checkpoint produced by the SFT run. # Usage: # export HF_TOKEN= @@ -21,12 +23,36 @@ fi run_id=$1 use_pathways=${2:-false} export MODEL_NAME='gpt-oss-20b' +TOKENIZER_PATH='unsloth/gpt-oss-20b-BF16' # Non-Googlers please remember to point BASE_OUTPUT_DIRECTORY to the GCS paths where you have the scanned and unscanned checkpoints stored BASE_OUTPUT_DIRECTORY=gs://runner-maxtext-logs/${MODEL_NAME} +UNSCANNED_CKPT_PATH=${BASE_OUTPUT_DIRECTORY}/to_maxtext/unscanned/${run_id}/0/items SCANNED_CKPT_PATH=${BASE_OUTPUT_DIRECTORY}/to_maxtext/scanned/${run_id}/0/items -# Step 1: Run SFT on the converted checkpoint +# Inference runs the MaxText implementation of GPT-OSS in vLLM. Its MoE layers then use tpu-inference's fused MoE kernel, +# tensor-parallel here, which needs prefuse_moe_weights=True. Tensor x data parallelism spans the 32 chips of a v6e-32. + +# Step 1: Run inference on the pre-converted checkpoint +python3 -m maxtext.inference.vllm_decode \ + model_name=${MODEL_NAME} \ + tokenizer_path=${TOKENIZER_PATH} \ + load_parameters_path=${UNSCANNED_CKPT_PATH} \ + vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \ + hbm_utilization_vllm=0.5 \ + prompt='Suggest some famous landmarks in London.' \ + decode_sampling_temperature=0.0 \ + max_target_length=256 \ + max_num_batched_tokens=256 \ + use_chat_template=True \ + scan_layers=false \ + prefuse_moe_weights=True \ + ici_tensor_parallelism=4 \ + ici_data_parallelism=8 \ + weight_dtype=bfloat16 \ + enable_single_controller=${use_pathways} + +# Step 2: Run SFT on the converted checkpoint python3 -m maxtext.trainers.post_train.sft.train_sft \ base_output_directory=${BASE_OUTPUT_DIRECTORY}/sft \ load_parameters_path=${SCANNED_CKPT_PATH} \ @@ -38,4 +64,23 @@ python3 -m maxtext.trainers.post_train.sft.train_sft \ tokenizer_path='unsloth/gpt-oss-20b-BF16' \ enable_single_controller=${use_pathways} \ checkpoint_storage_use_zarr3=False \ - checkpoint_storage_use_ocdbt=False \ No newline at end of file + checkpoint_storage_use_ocdbt=False + +# Step 3: Run inference on the checkpoint produced by the SFT run +python3 -m maxtext.inference.vllm_decode \ + model_name=${MODEL_NAME} \ + tokenizer_path=${TOKENIZER_PATH} \ + load_parameters_path=${BASE_OUTPUT_DIRECTORY}/sft/${run_id}/checkpoints/2/model_params \ + vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \ + hbm_utilization_vllm=0.5 \ + prompt='Suggest some famous landmarks in London.' \ + decode_sampling_temperature=0.0 \ + max_target_length=256 \ + max_num_batched_tokens=256 \ + use_chat_template=True \ + scan_layers=true \ + prefuse_moe_weights=True \ + ici_tensor_parallelism=4 \ + ici_data_parallelism=8 \ + weight_dtype=bfloat16 \ + enable_single_controller=${use_pathways} \ No newline at end of file diff --git a/tests/post_training/unit/convert_utils_test.py b/tests/post_training/unit/convert_utils_test.py index 69f059f439..c22b6a7704 100644 --- a/tests/post_training/unit/convert_utils_test.py +++ b/tests/post_training/unit/convert_utils_test.py @@ -15,6 +15,7 @@ """Tests for MaxText convert_utils functions.""" import os +from types import SimpleNamespace import unittest import jax.numpy as jnp @@ -25,6 +26,7 @@ _interleave_moe_weights, compute_padded_moe_mlp_dim, DEFAULT_TPU_NUM_LANES, + get_moe_intermediate_size, is_verify_weights_enabled, pad_to_tpu_lanes, resolve_prefuse_moe_weights, @@ -164,6 +166,24 @@ def test_compute_padded_moe_mlp_dim_leaves_aligned_dims_alone(self): self.assertEqual(compute_padded_moe_mlp_dim(2560, 2, lanes), 2560) self.assertIsNone(compute_padded_moe_mlp_dim(None, 4, lanes)) + def test_get_moe_intermediate_size(self): + # Qwen3-30B-A3B: the expert MLP dim has its own field, distinct from the dense one. + qwen3_moe = SimpleNamespace(moe_intermediate_size=768, intermediate_size=6144, num_experts=128) + self.assertEqual(get_moe_intermediate_size(qwen3_moe), 768) + # GPT-OSS-20B: every MLP is an MoE, so the expert MLP dim is `intermediate_size`. + gpt_oss = SimpleNamespace(intermediate_size=2880, num_local_experts=32) + self.assertEqual(get_moe_intermediate_size(gpt_oss), 2880) + # Dense models have no expert MLP dim. + self.assertIsNone(get_moe_intermediate_size(SimpleNamespace(intermediate_size=14336))) + self.assertIsNone(get_moe_intermediate_size(SimpleNamespace(intermediate_size=14336, num_local_experts=1))) + + def test_gpt_oss_moe_mlp_dim_is_padded_for_gmm_v2(self): + # 2880 / tp is not a multiple of 2 * lanes for any tp, so GPT-OSS always needs padding. + lanes = DEFAULT_TPU_NUM_LANES + self.assertEqual(compute_padded_moe_mlp_dim(2880, 1, lanes), 3072) + self.assertEqual(compute_padded_moe_mlp_dim(2880, 4, lanes), 3072) + self.assertEqual(compute_padded_moe_mlp_dim(2880, 8, lanes), 4096) + def test_is_verify_weights_enabled(self): orig = os.environ.get("VERIFY_WEIGHTS") try: diff --git a/tests/unit/fused_moe_qwix_test.py b/tests/unit/fused_moe_qwix_test.py index 4a241c0b0c..b53175115b 100644 --- a/tests/unit/fused_moe_qwix_test.py +++ b/tests/unit/fused_moe_qwix_test.py @@ -14,22 +14,39 @@ # pylint: disable=missing-class-docstring,missing-function-docstring,unbalanced-tuple-unpacking -"""Tests for the qwix boundary around tpu-inference's fused MoE kernel.""" +"""Tests for RoutedMoE.fused_moe_matmul (tpu-inference's fused MoE kernel) and the qwix boundary around it.""" import sys import types import unittest from unittest import mock +from absl.testing import parameterized from flax import nnx +from flax.linen import partitioning as nn_partitioning import jax import jax.numpy as jnp +from jax.sharding import Mesh, NamedSharding, PartitionSpec as P import numpy as np +import pytest import qwix from maxtext.common import common_types as ctypes +from maxtext.configs import pyconfig +from maxtext.integration.vllm.convert_utils import compute_padded_moe_mlp_dim +from maxtext.kernels.megablox.pallas_mosaic_tpu_v2_gmm_kernel import swigluoai from maxtext.layers import moe from maxtext.layers import quantizations +from maxtext.layers.initializers import nd_dense_init +from maxtext.utils import model_creation_utils +from tests.utils.test_helpers import get_test_config_path + +# A GPT-OSS MoE small enough for CPU. Its MLP dim, like GPT-OSS's 2880, needs padding for GMM_v2. +_GPT_OSS_EMB_DIM, _GPT_OSS_MLP_DIM, _GPT_OSS_NUM_EXPERTS, _GPT_OSS_TOP_K, _GPT_OSS_TOKENS = 64, 96, 4, 2, 8 + +# Mesh axes tpu-inference shards the fused kernel's MLP dim over in tensor-parallel mode +# (ShardingAxisNameBase.MLP_TENSOR in tpu_inference/layers/common/sharding.py). +_TPU_INFERENCE_MLP_TENSOR_AXES = ("attn_dp", "attn_dp_expert", "expert", "model", "dcp", "pcp") def _assert_error_within(actual, ref, bound): @@ -154,34 +171,168 @@ def opaque(y): self.assertNotIn("float8", jaxpr) -class FusedMoeMatmulTest(unittest.TestCase): +def _fake_tpu_inference(fused_moe_func): + """Stand-in for the tpu-inference modules fused_moe_matmul imports, with `fused_moe_func` as the kernel.""" + envs = types.SimpleNamespace( + ENABLE_RS_KERNEL=False, USE_GMM_FUSED_RS_KERNEL=False, ONEHOT_MOE_PERMUTE_THRESHOLD=0, VLLM_MOE_CHUNK_SIZE=0 + ) + pkg = types.ModuleType("tpu_inference") + pkg.envs = envs + layers = types.ModuleType("tpu_inference.layers") + common = types.ModuleType("tpu_inference.layers.common") + gmm = types.ModuleType("tpu_inference.layers.common.fused_moe_gmm") + gmm.fused_moe_func = fused_moe_func + return { + "tpu_inference": pkg, + "tpu_inference.envs": envs, + "tpu_inference.layers": layers, + "tpu_inference.layers.common": common, + "tpu_inference.layers.common.fused_moe_gmm": gmm, + } + + +def _emulated_fused_moe_func( + *, + hidden_states, + w1, + w2, + w1_scale, + w2_scale, + w1_bias, + w2_bias, + gating_output, + topk, + renormalize, + mesh, + use_ep, + activation, + scoring_fn, + **unused_kwargs, +): + """Pure-JAX stand-in for tpu-inference's unquantized `fused_moe_func`. + + It routes like the kernel (score function over all experts, top-k, renormalization) and, in + tensor-parallel mode, gives every MLP shard its contiguous chunk of `w1` and `w1_bias`, whose first + half is the gate and second half the up projection: the layout gmm_v2 reads. + """ + del unused_kwargs + assert w1_scale is None and w2_scale is None + if activation != "swigluoai" or scoring_fn != "softmax": + raise NotImplementedError(f"Unsupported {activation=}, {scoring_fn=}.") + weights, experts = jax.lax.top_k(jax.nn.softmax(gating_output.astype(jnp.float32), axis=-1), topk) + if renormalize: + weights /= weights.sum(axis=-1, keepdims=True) + + num_shards = 1 if use_ep else int(np.prod([mesh.shape.get(axis, 1) for axis in _TPU_INFERENCE_MLP_TENSOR_AXES])) + chunk = w1.shape[-1] // num_shards + x = hidden_states.astype(jnp.float32) + expert_out = 0.0 + for shard in range(num_shards): + cols = slice(shard * chunk, (shard + 1) * chunk) + h = jnp.einsum("td,edn->ten", x, w1[..., cols].astype(jnp.float32)) + if w1_bias is not None: + h += w1_bias[:, 0, cols] + act = swigluoai(h[..., : chunk // 2], h[..., chunk // 2 :]) + rows = slice(shard * chunk // 2, (shard + 1) * chunk // 2) + expert_out += jnp.einsum("ten,end->ted", act, w2[:, rows].astype(jnp.float32)) + if w2_bias is not None: + expert_out += w2_bias[:, 0] + selected = jnp.take_along_axis(expert_out, experts[..., None], axis=1) + return jnp.einsum("tkd,tk->td", selected, weights).astype(hidden_states.dtype) + + +def _make_moe(cfg, mesh): + """A RoutedMoE built like GPT-OSS's decoder layer builds it.""" + with jax.set_mesh(mesh), nn_partitioning.axis_rules(cfg.logical_axis_rules): + return moe.RoutedMoE( + config=cfg, + num_experts=cfg.num_experts, + num_experts_per_tok=cfg.num_experts_per_tok, + mesh=mesh, + kernel_init=nd_dense_init(1.0, "fan_in", "truncated_normal"), + kernel_axes=("embed", None), + intermediate_dim=cfg.mlp_dim, + weight_dtype=cfg.weight_dtype, + dtype=cfg.dtype, + rngs=nnx.Rngs(params=0), + ) + + +def _mesh(cfg, **axis_sizes): + """A mesh over the first devices with `axis_sizes` (e.g. model=2) and every other axis of size 1.""" + shape = [axis_sizes.get(name, 1) for name in cfg.mesh_axes] + return Mesh(np.asarray(jax.devices()[: int(np.prod(shape))]).reshape(shape), cfg.mesh_axes) + + +def _randomize_biases(layer, seed): + """The default bias init is zeros, which would not test where the biases end up.""" + keys = iter(jax.random.split(jax.random.key(seed), 4)) + for bias in (layer.wi_0_bias, layer.wi_1_bias, layer.wo_bias, layer.gate.bias): + value = bias[...] + bias.set_value(jax.device_put(jax.random.normal(next(keys), value.shape, value.dtype), value.sharding)) + + +def _load_like_vllm(src, dst, dst_cfg, dst_mesh): + """Loads the unpadded expert weights of `src` into the padded vLLM layer `dst` like the vLLM loader. + + That is `model_creation_utils.from_pretrained`: it restores the checkpoint sharded like the model, + then `_fuse_moe_weights` builds a prefused `wi`, one [gate | up] chunk per MLP shard, and + `_align_checkpoint_to_model_shapes` zero-pads every shard of `wi_0` / `wi_1` / `wo` along the MLP dim. + The biases keep their unpadded shape. + """ + with jax.set_mesh(dst_mesh), nn_partitioning.axis_rules(dst_cfg.logical_axis_rules): + + def sharding(logical_axes): + return NamedSharding(dst_mesh, dst._logical_to_mesh_axes(logical_axes)) # pylint: disable=protected-access + + def target(name, logical_axes): + value = getattr(dst, name)[...] + return jax.ShapeDtypeStruct(value.shape, value.dtype, sharding=sharding(logical_axes)) + + def restore(name, logical_axes): + # Like a checkpoint saved in the serving weight dtype. + return jax.device_put(getattr(src, name)[...].astype(dst.wo[...].dtype), sharding(logical_axes)) + + def align(name, logical_axes): + return model_creation_utils._align_checkpoint_to_model_shapes( # pylint: disable=protected-access + restore(name, logical_axes), target(name, logical_axes), logical_axes + ) + + if dst_cfg.prefuse_moe_weights: + checkpoint = {"wi_0": restore("wi_0", dst.wi_kernel_axes), "wi_1": restore("wi_1", dst.wi_kernel_axes)} + dst.wi.set_value( + model_creation_utils._fuse_moe_weights( # pylint: disable=protected-access + checkpoint, {"wi": target("wi", dst.wi_kernel_axes)}, dst_cfg + )["wi"] + ) + else: + dst.wi_0.set_value(align("wi_0", dst.wi_kernel_axes)) + dst.wi_1.set_value(align("wi_1", dst.wi_kernel_axes)) + dst.wo.set_value(align("wo", dst.wo_kernel_axes)) + + def copy(src_variable, dst_variable): + value = src_variable[...].astype(dst_variable[...].dtype) + dst_variable.set_value(jax.device_put(value, NamedSharding(dst_mesh, P()))) + + for name in ("wi_0_bias", "wi_1_bias", "wo_bias"): + copy(getattr(src, name), getattr(dst, name)) + copy(src.gate.kernel, dst.gate.kernel) + copy(src.gate.bias, dst.gate.bias) + + +class FusedMoeMatmulTest(parameterized.TestCase): """RoutedMoE.fused_moe_matmul hands the kernel pre-quantized weights, outside qwix.""" num_experts, top_k, emb_dim, mlp_dim, tokens = 4, 2, 64, 128, 16 - def _fake_tpu_inference(self, calls): - """Stand-in for the tpu-inference modules fused_moe_matmul imports.""" + def _recording_tpu_inference(self, calls): + """tpu-inference stand-in whose kernel records its arguments (and the qwix rule it sees) in `calls`.""" def fused_moe_func(**kwargs): calls.append(dict(kwargs, rule_inside=quantizations.get_fused_moe_rule())) return jnp.zeros((kwargs["hidden_states"].shape[0], self.emb_dim), jnp.bfloat16) - envs = types.SimpleNamespace( - ENABLE_RS_KERNEL=False, USE_GMM_FUSED_RS_KERNEL=False, ONEHOT_MOE_PERMUTE_THRESHOLD=0, VLLM_MOE_CHUNK_SIZE=0 - ) - pkg = types.ModuleType("tpu_inference") - pkg.envs = envs - layers = types.ModuleType("tpu_inference.layers") - common = types.ModuleType("tpu_inference.layers.common") - gmm = types.ModuleType("tpu_inference.layers.common.fused_moe_gmm") - gmm.fused_moe_func = fused_moe_func - return { - "tpu_inference": pkg, - "tpu_inference.envs": envs, - "tpu_inference.layers": layers, - "tpu_inference.layers.common": common, - "tpu_inference.layers.common.fused_moe_gmm": gmm, - } + return _fake_tpu_inference(fused_moe_func) def _make_layer(self, calls, return_routed_experts=False, mesh=None): test = self @@ -210,7 +361,7 @@ def get_expert_parallelism_size(self): return 1 def __call__(self, inputs, gate_logits): - with mock.patch.dict(sys.modules, test._fake_tpu_inference(calls)): + with mock.patch.dict(sys.modules, test._recording_tpu_inference(calls)): out, _, _ = moe.RoutedMoE.fused_moe_matmul(self, inputs, gate_logits, self.wo[...], self.w0[...], self.w1[...]) return out @@ -299,6 +450,242 @@ def test_dot_general_only_rule_keeps_experts_unquantized(self): self.assertIsNone(call["w1_scale"]) self.assertIsNone(call["rule_inside"]) + # GPT-OSS needs four things from this path: its clamped SwiGLU ("swigluoai"), its expert biases, its MLP dim + # (2880) padded so every tensor-parallel shard satisfies the GMM_v2 kernel's alignment, and valid shardings for + # those biases under vLLM's logical axis rules. + + def _model_config(self, model_name, config_file="base.yml", **overrides): + return pyconfig.initialize( + [None, get_test_config_path(config_file)], + run_name="fused_moe_matmul_test", + model_name=model_name, + enable_checkpointing=False, + log_config=False, + **overrides, + ) + + def _small_gpt_oss_config(self, config_file, **overrides): + return self._model_config( + "gpt-oss-20b", + config_file, + override_model_config=True, + base_emb_dim=_GPT_OSS_EMB_DIM, + base_mlp_dim=_GPT_OSS_MLP_DIM, + base_moe_mlp_dim=_GPT_OSS_MLP_DIM, + num_experts=_GPT_OSS_NUM_EXPERTS, + num_experts_per_tok=_GPT_OSS_TOP_K, + dtype="float32", + weight_dtype="float32", + matmul_precision="highest", + max_target_length=_GPT_OSS_TOKENS, + per_device_batch_size=1, + **overrides, + ) + + def test_fused_moe_activation_gpt_oss_uses_swigluoai(self): + # mlp_activations[0] is "sigmoid", which the kernel rejects. + self.assertEqual(moe.fused_moe_activation(self._model_config("gpt-oss-20b")), "swigluoai") + + def test_fused_moe_activation_other_models_keep_their_activation(self): + self.assertEqual(moe.fused_moe_activation(self._model_config("mixtral-8x7b")), "silu") + + def test_fused_moe_activation_rejects_another_gpt_oss_clamp_limit(self): + # The kernel hardcodes GPT-OSS's limit, so a different one cannot silently run with 7.0. + cfg = self._model_config("gpt-oss-20b", override_model_config=True, mlp_activations_limit=10.0) + with self.assertRaisesRegex(NotImplementedError, "mlp_activations_limit"): + moe.fused_moe_activation(cfg) + + def _gate_and_up_biases(self, num_experts=2, mlp_dim=12): + gate = jnp.arange(num_experts * mlp_dim, dtype=jnp.float32).reshape(num_experts, mlp_dim) + return gate + 1.0, -(gate + 1.0) + + @parameterized.parameters((1, 12), (1, 16), (2, 16), (4, 16), (4, 24)) + def test_fuse_moe_biases_per_shard_layout_matches_fused_kernel(self, num_shards, padded_mlp_dim): + """A bias is a kernel with one input row, so `_fuse_moe_weights` must produce the same layout.""" + gate, up = self._gate_and_up_biases() + mesh = jax.sharding.AbstractMesh((num_shards,), ("model",)) + target = jax.ShapeDtypeStruct( + (gate.shape[0], 1, 2 * padded_mlp_dim), jnp.float32, sharding=NamedSharding(mesh, P(None, None, "model")) + ) + expected = model_creation_utils._fuse_moe_weights( # pylint: disable=protected-access + {"wi_0": gate[:, None, :], "wi_1": up[:, None, :]}, {"wi": target}, types.SimpleNamespace(attention="vllm_rpa") + )["wi"] + + actual = moe.fuse_moe_biases_for_kernel(gate, up, 2 * padded_mlp_dim, num_shards, per_shard_layout=True) + + self.assertEqual(actual.dtype, jnp.float32) + np.testing.assert_array_equal(actual, expected) + + def test_fuse_moe_biases_concatenated_layout(self): + """Unfused `wi_0` / `wi_1` (each padded per shard) are concatenated whole by `fused_moe_matmul`.""" + gate, up = self._gate_and_up_biases() + actual = moe.fuse_moe_biases_for_kernel(gate, up, 32, num_shards=2, per_shard_layout=False) + + def pad_shards(bias): + return np.concatenate([np.pad(shard, ((0, 0), (0, 2))) for shard in np.split(np.asarray(bias), 2, axis=-1)], -1) + + expected = np.concatenate([pad_shards(gate), pad_shards(up)], axis=-1)[:, None, :] + np.testing.assert_array_equal(actual, expected) + + def test_fuse_moe_biases_rejects_unshardable_dims(self): + gate, up = self._gate_and_up_biases() + with self.assertRaises(ValueError): + moe.fuse_moe_biases_for_kernel(gate, up, 32, num_shards=5, per_shard_layout=True) + + @parameterized.named_parameters( + ("unfused", 1, 1, False), + ("prefused", 1, 1, True), + ("prefused_tp2", 2, 1, True), + ("prefused_tp4", 4, 1, True), + # The expert-parallel kernel reads gate/up as two whole halves, which is how unfused wi_0/wi_1 arrive. + ("unfused_tp2_ep2", 2, 2, False), + ) + def test_gpt_oss_matches_dense(self, tp, ep, prefuse): + """GPT-OSS through `fused_moe_matmul` matches the dense MaxText implementation (the kernel is emulated).""" + if jax.device_count() < tp * ep: + self.skipTest(f"Needs {tp * ep} devices (e.g. XLA_FLAGS=--xla_force_host_platform_device_count={tp * ep}).") + dense_cfg = self._small_gpt_oss_config("base.yml", sparse_matmul=False, megablox=False) + dense = _make_moe(dense_cfg, _mesh(dense_cfg, fsdp=1)) + _randomize_biases(dense, seed=1) + + # What the vLLM adapter sets up for GPT-OSS: the MLP dim is padded for GMM_v2 on every shard. + padded_mlp_dim = compute_padded_moe_mlp_dim(_GPT_OSS_MLP_DIM, tp) + self.assertGreater(padded_mlp_dim, _GPT_OSS_MLP_DIM) + fused_cfg = self._small_gpt_oss_config( + "inference/vllm.yml", + ici_tensor_parallelism=tp, + ici_expert_parallelism=ep, + padded_base_moe_mlp_dim=padded_mlp_dim, + prefuse_moe_weights=prefuse, + ) + fused_mesh = _mesh(fused_cfg, model=tp, expert=ep) + fused = _make_moe(fused_cfg, fused_mesh) + _load_like_vllm(dense, fused, fused_cfg, fused_mesh) + + inputs = jax.random.normal(jax.random.key(2), (1, _GPT_OSS_TOKENS, _GPT_OSS_EMB_DIM), jnp.float32) + with nn_partitioning.axis_rules(dense_cfg.logical_axis_rules): + expected, _, _ = dense(inputs) + calls = [] + + def fused_moe_func(**kwargs): + calls.append(kwargs) + return _emulated_fused_moe_func(**kwargs) + + with ( + mock.patch.dict(sys.modules, _fake_tpu_inference(fused_moe_func)), + jax.set_mesh(fused_mesh), + nn_partitioning.axis_rules(fused_cfg.logical_axis_rules), + ): + actual, lb_loss, bias_updates = fused(jax.device_put(inputs, NamedSharding(fused_mesh, P()))) + + self.assertLen(calls, 1) + call = calls[0] + self.assertEqual(call["activation"], "swigluoai") + self.assertEqual(call["w1_bias"].shape, (_GPT_OSS_NUM_EXPERTS, 1, 2 * padded_mlp_dim)) + self.assertEqual(call["w2_bias"].shape, (_GPT_OSS_NUM_EXPERTS, 1, _GPT_OSS_EMB_DIM)) + self.assertIsNone(lb_loss) + self.assertIsNone(bias_updates) + np.testing.assert_allclose(np.asarray(actual), np.asarray(expected), rtol=1e-5, atol=1e-5) + + def test_gpt_oss_vllm_model_shards_biases_like_kernels(self): + """The GPT-OSS model builds under vLLM's logical axis rules, as `MaxTextForCausalLM` builds it.""" + cfg = self._model_config( + "gpt-oss-20b", + "inference/vllm.yml", + override_model_config=True, + base_emb_dim=_GPT_OSS_EMB_DIM, + base_mlp_dim=_GPT_OSS_MLP_DIM, + base_moe_mlp_dim=_GPT_OSS_MLP_DIM, + base_num_query_heads=8, + base_num_kv_heads=2, + head_dim=16, + base_num_decoder_layers=2, + vocab_size=512, + num_experts=_GPT_OSS_NUM_EXPERTS, + num_experts_per_tok=_GPT_OSS_TOP_K, + padded_base_moe_mlp_dim=256, + prefuse_moe_weights=True, + max_target_length=_GPT_OSS_TOKENS, + per_device_batch_size=1, + ) + mesh = _mesh(cfg) + # tpu-inference creates the model under its mesh, so every Variable's sharding becomes a NamedSharding. + # That rejects a spec that maps one mesh axis to two dimensions, which vLLM's rules produced for the + # expert biases' ("exp", "activation_mlp"). + with jax.set_mesh(mesh): + _, abstract_model = model_creation_utils.create_nnx_abstract_model( + cfg, mesh, model_mode=ctypes.MODEL_MODE_AUTOREGRESSIVE + ) + + moe_block = abstract_model.decoder.layers_0.GptOssMlp + wi_spec = moe_block.wi.get_value().sharding.spec + for bias in (moe_block.wi_0_bias, moe_block.wi_1_bias): + # Sharded like the expert kernels: experts, then the MLP dim. + self.assertEqual(tuple(bias.get_value().sharding.spec), (wi_spec[0], wi_spec[-1])) + + +@pytest.mark.tpu_only +@pytest.mark.post_training +class GptOssFusedMoeTPUTest(parameterized.TestCase): + """GPT-OSS through the real tpu-inference kernel matches the dense MaxText implementation.""" + + def setUp(self): + super().setUp() + try: + import tpu_inference # pylint: disable=import-outside-toplevel,unused-import + except ImportError: + self.skipTest("Fused MoE tests require tpu-inference package to be installed.") + + def _config(self, config_file, **overrides): + # A smaller GPT-OSS MoE. Its MLP dim (360, an eighth of 2880) still needs padding on every shard. + return pyconfig.initialize( + [None, get_test_config_path(config_file)], + run_name="gpt_oss_fused_moe_tpu", + model_name="gpt-oss-20b", + override_model_config=True, + base_emb_dim=256, + base_mlp_dim=360, + base_moe_mlp_dim=360, + num_experts=16, + dtype="bfloat16", + weight_dtype="bfloat16", + enable_checkpointing=False, + log_config=False, + max_target_length=16, + per_device_batch_size=1, + **overrides, + ) + + @parameterized.named_parameters(("expert_parallel", "expert", False), ("tensor_parallel", "model", True)) + def test_matches_dense(self, axis, prefuse): + num_devices = jax.device_count() + dense_cfg = self._config("base.yml", sparse_matmul=False, megablox=False) + dense = _make_moe(dense_cfg, _mesh(dense_cfg, fsdp=1)) + _randomize_biases(dense, seed=1) + + tp = num_devices if axis == "model" else 1 + fused_cfg = self._config( + "inference/vllm.yml", + ici_tensor_parallelism=tp, + ici_expert_parallelism=num_devices // tp, + padded_base_moe_mlp_dim=compute_padded_moe_mlp_dim(dense_cfg.mlp_dim, tp), + prefuse_moe_weights=prefuse, + ) + fused_mesh = _mesh(fused_cfg, **{axis: num_devices}) + fused = _make_moe(fused_cfg, fused_mesh) + _load_like_vllm(dense, fused, fused_cfg, fused_mesh) + + inputs = jax.random.normal(jax.random.key(2), (1, 16, dense_cfg.emb_dim), jnp.bfloat16) + with nn_partitioning.axis_rules(dense_cfg.logical_axis_rules): + expected, _, _ = dense(inputs) + with jax.set_mesh(fused_mesh), nn_partitioning.axis_rules(fused_cfg.logical_axis_rules): + actual, _, _ = fused(jax.device_put(inputs, NamedSharding(fused_mesh, P()))) + + # bf16 everywhere: compare the whole output rather than element by element. A missing bias or the + # wrong activation is off by O(1) relative error. + actual, expected = np.asarray(actual, np.float32), np.asarray(expected, np.float32) + self.assertLess(np.linalg.norm(actual - expected) / np.linalg.norm(expected), 2e-2) + if __name__ == "__main__": unittest.main()