Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 10 additions & 9 deletions src/maxtext/inference/vllm_decode.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
from maxtext.configs import pyconfig
from maxtext.integration.tunix.tunix_adapter import TunixMaxTextAdapter
import maxtext.integration.vllm.maxtext_vllm_adapter as adapter
from maxtext.integration.vllm.convert_utils import uses_maxtext_vllm_adapter
from maxtext.multimodal import utils as mm_utils
from maxtext.utils import lora_utils
from maxtext.utils import max_logging
Expand Down Expand Up @@ -125,6 +126,14 @@ def create_vllm_llm(config: Config, **overrides: Any) -> LLM:
}

if config.load_parameters_path:
if not uses_maxtext_vllm_adapter(config):
# Only MaxTextForCausalLM reads load_parameters_path. vLLM's own model implementation loads the full
# Hugging Face checkpoint of `model` through host memory instead (e.g. ~40 GB for gpt-oss-20b in BF16).
max_logging.warning(
f"load_parameters_path={config.load_parameters_path} is ignored and vLLM will load the Hugging Face "
f"checkpoint of {config.tokenizer_path} instead, which can run out of host memory. Set "
"vllm_hf_overrides='{architectures: [\"MaxTextForCausalLM\"]}' to decode the MaxText checkpoint."
)
vllm_args["additional_config"]["maxtext_config"]["load_parameters_path"] = config.load_parameters_path
else:
vllm_args["load_format"] = "dummy"
Expand Down Expand Up @@ -230,17 +239,9 @@ def decode_with_tunix(
mesh: The JAX mesh for parallelism.
"""
# Wrap the model for Tunix
use_no_op_mappings = False
if hasattr(config, "vllm_hf_overrides") and config.vllm_hf_overrides:
overrides = config.vllm_hf_overrides
if isinstance(overrides, str) and "MaxTextForCausalLM" in overrides:
use_no_op_mappings = True
elif isinstance(overrides, dict) and "MaxTextForCausalLM" in overrides.get("architectures", []):
use_no_op_mappings = True

tunix_model = TunixMaxTextAdapter(
base_model=model,
use_no_op_mappings=use_no_op_mappings,
use_no_op_mappings=uses_maxtext_vllm_adapter(config),
)

# Load the tokenizer
Expand Down
30 changes: 29 additions & 1 deletion src/maxtext/integration/vllm/convert_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,34 @@ def pad_to_tpu_lanes(dim: int, lane_size: int = DEFAULT_TPU_NUM_LANES) -> int:
return ((dim + lane_size - 1) // lane_size) * lane_size


def get_moe_intermediate_size(hf_config: Any) -> Optional[int]:
"""Returns the per-expert MLP dimension of a Hugging Face config, or None for a dense model.

Most MoE configs store it as `moe_intermediate_size`. Models whose MLPs are all MoE (e.g. GPT-OSS,
Mixtral) have no such field; their per-expert MLP dimension is `intermediate_size`.
"""
moe_intermediate_size = getattr(hf_config, "moe_intermediate_size", None)
if moe_intermediate_size is not None:
return moe_intermediate_size
num_experts = getattr(hf_config, "num_local_experts", None) or getattr(hf_config, "num_experts", None)
if num_experts is not None and num_experts > 1:
return getattr(hf_config, "intermediate_size", None)
return None


def uses_maxtext_vllm_adapter(config: Any) -> bool:
"""Returns whether vLLM is configured to instantiate MaxTextForCausalLM."""
overrides = getattr(config, "vllm_hf_overrides", None)
if isinstance(overrides, str):
return "MaxTextForCausalLM" in overrides
if isinstance(overrides, dict):
architectures = overrides.get("architectures", ())
if isinstance(architectures, str):
architectures = (architectures,)
return "MaxTextForCausalLM" in architectures
return False


def compute_padded_moe_mlp_dim(
hidden_size: Optional[int],
moe_mlp_tp_size: int,
Expand All @@ -80,7 +108,7 @@ def compute_padded_moe_mlp_dim(
`2 * num_lanes * moe_mlp_tp_size`.

Args:
hidden_size: Unpadded MoE intermediate size (`moe_intermediate_size`).
hidden_size: Unpadded MoE intermediate size (see `get_moe_intermediate_size`).
moe_mlp_tp_size: TP degree across the MLP dimension (`tp * attn_dp`).
num_lanes: TPU vector-register lane count. Pass the runtime-probed
`pltpu.get_tpu_info().num_lanes` when a TPU handle is available; the default
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,11 @@
import numpy as np
from maxtext.common.common_types import MODEL_MODE_AUTOREGRESSIVE
from maxtext.configs import pyconfig
from maxtext.integration.vllm.convert_utils import DEFAULT_TPU_NUM_LANES, compute_padded_moe_mlp_dim
from maxtext.integration.vllm.convert_utils import (
DEFAULT_TPU_NUM_LANES,
compute_padded_moe_mlp_dim,
get_moe_intermediate_size,
)
from maxtext.integration.vllm.hybrid_cache_utils import (
build_qwen_gdn_cache_layout,
call_with_supported_kwargs,
Expand Down Expand Up @@ -126,7 +130,7 @@ def generate_maxtext_config(vllm_config: VllmConfig) -> pyconfig.HyperParameters
if hasattr(vllm_config.model_config.hf_config, "text_config")
else vllm_config.model_config.hf_config
)
hidden_size = getattr(hf_config, "moe_intermediate_size", None)
hidden_size = get_moe_intermediate_size(hf_config)
try:
num_lanes = pltpu.get_tpu_info().num_lanes
except Exception: # pylint: disable=broad-exception-caught
Expand Down
15 changes: 1 addition & 14 deletions src/maxtext/integration/vllm/maxtext_vllm_rollout.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
from flax.traverse_util import flatten_dict, unflatten_dict
import jax
import jax.numpy as jnp
from maxtext.integration.vllm.convert_utils import MOE_MLP_WEIGHTS, _sharding_summary
from maxtext.integration.vllm.convert_utils import MOE_MLP_WEIGHTS, _sharding_summary, uses_maxtext_vllm_adapter
from maxtext.integration.vllm.torchax_converter.gemma4_moe import Gemma4MaxTextToVLLMConverter
from maxtext.integration.vllm.torchax_converter.qwen35_moe import Qwen35MaxTextToVLLMConverter
from maxtext.integration.vllm.torchax_converter.qwen3_moe import Qwen3MaxTextToVLLMConverter
Expand Down Expand Up @@ -177,19 +177,6 @@ def _as_dict(value: Any) -> dict:
return {}


def uses_maxtext_vllm_adapter(config: Any) -> bool:
"""Returns whether vLLM is configured to instantiate MaxTextForCausalLM."""
overrides = getattr(config, "vllm_hf_overrides", None)
if isinstance(overrides, str):
return "MaxTextForCausalLM" in overrides
if isinstance(overrides, dict):
architectures = overrides.get("architectures", ())
if isinstance(architectures, str):
architectures = (architectures,)
return "MaxTextForCausalLM" in architectures
return False


def requires_maxtext_scanned_weight_unroll(config: Any) -> bool:
"""Returns whether direct MaxText-to-MaxText sync needs a custom unroll."""
return bool(getattr(config, "scan_layers", False) and uses_maxtext_vllm_adapter(config))
Expand Down
105 changes: 100 additions & 5 deletions src/maxtext/layers/moe.py
Original file line number Diff line number Diff line change
Expand Up @@ -607,6 +607,68 @@ def fused_moe_lhs_scale_kwargs(quant, fused_moe_fn) -> dict[str, jax.Array]:
return {"w1_lhs_scale": scale, "w2_lhs_scale": scale}


# GPT-OSS's SwiGLU clamp limit. tpu-inference's fused "swigluoai" activation hardcodes it, along with
# alpha=1.702 (see swigluoai in tpu_inference/kernels/megablox/gmm_v2.py).
_FUSED_SWIGLUOAI_LIMIT = 7.0


def fused_moe_activation(config) -> str:
"""Name of the tpu-inference fused MoE activation that matches `RoutedMoE.apply_ffn_activation`."""
if config.decoder_block == ctypes.DecoderBlockType.GPT_OSS:
# GPT-OSS's clamped SwiGLU is the kernel's "swigluoai". Its mlp_activations[0] ("sigmoid") is not a
# kernel activation at all.
if config.mlp_activations[0] != "sigmoid" or config.mlp_activations_limit != _FUSED_SWIGLUOAI_LIMIT:
raise NotImplementedError(
"The fused MoE kernel only implements GPT-OSS's SwiGLU with mlp_activations[0]='sigmoid' and "
f"mlp_activations_limit={_FUSED_SWIGLUOAI_LIMIT}, got {config.mlp_activations[0]!r} and "
f"{config.mlp_activations_limit}."
)
return "swigluoai"
return config.mlp_activations[0]


def fuse_moe_biases_for_kernel(w0_bias, w1_bias, fused_size, num_shards, per_shard_layout) -> jax.Array:
"""Lays out the gate/up expert biases as the `w1_bias` of tpu-inference's `fused_moe_func`.

The kernel adds `w1_bias` to `x @ w1` before the activation, so the bias has to follow the layout
the kernel reads `w1` in. The vLLM weight loader zero-pads the tail of every MLP-dim shard of the
expert kernels (GMM_v2 alignment), so the bias is padded the same way. Then gate and up are either
interleaved per shard, for the tensor-parallel kernel, which splits each shard of the fused axis
into gate and up (the layout of a prefused `wi`, see `model_creation_utils._fuse_moe_weights`), or
concatenated whole, for the expert-parallel kernel, which splits the whole fused axis.

Args:
w0_bias: Gate bias, [num_experts, mlp_dim].
w1_bias: Up bias, [num_experts, mlp_dim].
fused_size: Last dim of the fused kernel, i.e. twice the (padded) MoE MLP dim.
num_shards: Number of shards along the expert kernels' MLP dim.
per_shard_layout: Whether each shard holds its own [gate | up] chunk (tensor-parallel kernel)
instead of the kernel holding [all of gate | all of up] (expert-parallel kernel).

Returns:
The float32 bias, [num_experts, 1, fused_size].
"""
num_experts, mlp_dim = w0_bias.shape
padded_mlp_dim = fused_size // 2
if mlp_dim % num_shards or padded_mlp_dim % num_shards or padded_mlp_dim < mlp_dim:
raise ValueError(
f"Cannot lay out MoE biases of MLP dim {mlp_dim} for a fused kernel of MLP dim {padded_mlp_dim} "
f"over {num_shards} shards."
)

def pad_shards(bias):
"""[E, mlp_dim] -> [E, num_shards, padded_mlp_dim // num_shards], zero-padding each shard's tail."""
bias = bias.astype(jnp.float32).reshape(num_experts, num_shards, mlp_dim // num_shards)
return jnp.pad(bias, ((0, 0), (0, 0), (0, (padded_mlp_dim - mlp_dim) // num_shards)))

gate, up = pad_shards(w0_bias), pad_shards(w1_bias)
if per_shard_layout:
fused = jnp.concatenate([gate, up], axis=-1)
else:
fused = jnp.concatenate([gate.reshape(num_experts, -1), up.reshape(num_experts, -1)], axis=-1)
return fused.reshape(num_experts, 1, fused_size)


class Tid2EidVar(nnx.Variable):
"""Custom variable to hold tid2eid without trainable param overhead."""

Expand Down Expand Up @@ -1194,7 +1256,10 @@ def _init_expert_kernel(shape, dtype):
)

if self.config.mlp_bias:
wi_bias_axes = ("exp", "activation_mlp")
# vLLM's rules map "activation_mlp" onto mesh axes that "exp" already takes, which makes no valid
# NamedSharding; there "activation_mlp_moe" shards the bias like the expert kernels' MLP dim.
is_vllm = self.config.attention in ("vllm_rpa", "vllm_batched_rpa")
wi_bias_axes = ("exp", "activation_mlp_moe" if is_vllm else "activation_mlp")
wo_bias_axes = ("exp", "activation_embed")
wi_bias_shape = (self.num_experts, self.intermediate_dim)
wo_bias_shape = (self.num_experts, self.moe_expert_input_dim)
Expand Down Expand Up @@ -1389,6 +1454,21 @@ def get_tensor_parallelism_size(self):
def get_context_autoregressive_parallelism_size(self):
return self.mesh.shape.get("context_autoregressive", 1)

def _moe_mlp_num_shards(self):
"""Number of shards along the expert kernels' MLP dim.

These are the chunks the vLLM weight loader pads and fuses the expert kernels by: it reads them off
the sharding of the same logical axis (see `model_creation_utils._fuse_moe_weights`).
"""
if self.mesh is None:
return 1
mlp_axes = self._logical_to_mesh_axes(self.wi_kernel_axes)[-1]
if mlp_axes is None:
return 1
if isinstance(mlp_axes, str):
mlp_axes = (mlp_axes,)
return math.prod(self.mesh.shape.get(axis, 1) for axis in mlp_axes)

def should_update_load_balance(self):
"""Determines if loss-free load balancing updates should be applied.

Expand Down Expand Up @@ -4972,11 +5052,15 @@ def fused_moe_matmul(
forced_routed_experts=None,
native_w1_scale=None,
native_w2_scale=None,
w0_bias=None,
w1_bias=None,
wo_bias=None,
) -> tuple[jax.Array, None, None]:
"""Fused MoE via tpu_inference fused_moe_func (vllm_rpa path only).

fused_moe_func handles routing, GMM, and weighted combination internally.
It does not compute lb_loss or bias_updates (inference-only).
It does not compute lb_loss or bias_updates (inference-only). Expert biases
(`w0_bias`, `w1_bias`, `wo_bias`, e.g. GPT-OSS) are applied by the kernel.
"""
if forced_routed_experts is not None:
raise NotImplementedError("Forced routing via forced_routed_experts is not supported with" " fused_moe_matmul.")
Expand Down Expand Up @@ -5005,8 +5089,16 @@ def fused_moe_matmul(
# Use expert parallelism if the expert axis has size > 1
use_ep = self.get_expert_parallelism_size() > 1

# The kernel takes the biases as [E, 1, N], the gate/up one in the layout it reads the fused kernel in.
fused_w1_bias = None
if w0_bias is not None and w1_bias is not None:
fused_w1_bias = fuse_moe_biases_for_kernel(
w0_bias, w1_bias, fused_kernel.shape[-1], self._moe_mlp_num_shards(), per_shard_layout=not use_ep
)
fused_w2_bias = None if wo_bias is None else wo_bias.astype(jnp.float32)[:, None, :]

# Map MaxText config fields to fused_moe_func args
activation = self.config.mlp_activations[0] # e.g. "silu"
activation = fused_moe_activation(self.config) # e.g. "silu"
scoring_fn = self.config.routed_score_func if self.config.routed_score_func else "softmax"

# Check if the model architecture intrinsically renormalizes weights
Expand Down Expand Up @@ -5040,8 +5132,8 @@ def fused_moe_matmul(
w2=quantized_w2,
w1_scale=w1_scale,
w2_scale=w2_scale,
w1_bias=None,
w2_bias=None,
w1_bias=fused_w1_bias,
w2_bias=fused_w2_bias,
gating_output=gating_output,
topk=self.num_experts_per_tok,
renormalize=renormalize,
Expand Down Expand Up @@ -5448,6 +5540,9 @@ def _maybe_native_fused_weight(kernel, scale):
forced_routed_experts=forced_routed_experts,
native_w1_scale=wi_native_scale,
native_w2_scale=wo_native_scale,
w0_bias=w0_bias,
w1_bias=w1_bias,
wo_bias=wo_bias,
)
elif cfg.sparse_matmul:
if quantizations.in_serve_mode(self.quant):
Expand Down
51 changes: 48 additions & 3 deletions tests/end_to_end/tpu/gpt_oss/20b/test_gpt_oss_rl.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@
# Validates the GPT-OSS-20b RL pipeline using a pre-converted MaxText checkpoint.

# The flow of this script is as follows:
# 1. Run RL starting from the pre-converted checkpoint.
# 1. Run inference on the pre-converted checkpoint.
# 2. Run RL starting from the pre-converted checkpoint.
# 3. Run inference on the checkpoint produced by the RL run.

# Usage:
# export HF_TOKEN=<your Hugging Face access token>
Expand All @@ -28,12 +30,36 @@ fi
run_id=$1
use_pathways=${2:-false}
export MODEL_NAME='gpt-oss-20b'
TOKENIZER_PATH='unsloth/gpt-oss-20b-BF16'

# Non-Googlers please remember to point BASE_OUTPUT_DIRECTORY to the GCS paths where you have the scanned and unscanned checkpoints stored
BASE_OUTPUT_DIRECTORY=gs://runner-maxtext-logs/${MODEL_NAME}
UNSCANNED_CKPT_PATH=${BASE_OUTPUT_DIRECTORY}/to_maxtext/unscanned/${run_id}/0/items
SCANNED_CKPT_PATH=${BASE_OUTPUT_DIRECTORY}/to_maxtext/scanned/${run_id}/0/items

# Step 1: Run RL on the converted checkpoint
# Inference runs the MaxText implementation of GPT-OSS in vLLM. Its MoE layers then use tpu-inference's fused MoE kernel,
# tensor-parallel here, which needs prefuse_moe_weights=True. Tensor x data parallelism spans the 32 chips of a v6e-32.

# Step 1: Run inference on the pre-converted checkpoint
python3 -m maxtext.inference.vllm_decode \
model_name=${MODEL_NAME} \
tokenizer_path=${TOKENIZER_PATH} \
load_parameters_path=${UNSCANNED_CKPT_PATH} \
vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \
hbm_utilization_vllm=0.5 \
prompt='Suggest some famous landmarks in London.' \
decode_sampling_temperature=0.0 \
max_target_length=256 \
max_num_batched_tokens=256 \
use_chat_template=True \
scan_layers=false \
prefuse_moe_weights=True \
ici_tensor_parallelism=4 \
ici_data_parallelism=8 \
weight_dtype=bfloat16 \
enable_single_controller=${use_pathways}

# Step 2: Run RL on the converted checkpoint
python3 -m maxtext.trainers.post_train.rl.train_rl \
base_output_directory=${BASE_OUTPUT_DIRECTORY}/rl \
load_parameters_path=${SCANNED_CKPT_PATH} \
Expand Down Expand Up @@ -61,4 +87,23 @@ python3 -m maxtext.trainers.post_train.rl.train_rl \
checkpoint_storage_use_ocdbt=False \
rollout_data_parallelism=-1 \
rollout_tensor_parallelism=4 \
hbm_utilization_vllm=0.8
hbm_utilization_vllm=0.8

# Step 3: Run inference on the checkpoint produced by the RL run
python3 -m maxtext.inference.vllm_decode \
model_name=${MODEL_NAME} \
tokenizer_path=${TOKENIZER_PATH} \
load_parameters_path=${BASE_OUTPUT_DIRECTORY}/rl/${run_id}/checkpoints/actor/2/model_params \
vllm_hf_overrides='{architectures: ["MaxTextForCausalLM"]}' \
hbm_utilization_vllm=0.5 \
prompt='Suggest some famous landmarks in London.' \
decode_sampling_temperature=0.0 \
max_target_length=256 \
max_num_batched_tokens=256 \
use_chat_template=True \
scan_layers=true \
prefuse_moe_weights=True \
ici_tensor_parallelism=4 \
ici_data_parallelism=8 \
weight_dtype=bfloat16 \
enable_single_controller=${use_pathways}
Loading
Loading