Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/maxtext/configs/base.yml
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,7 @@ quantize_kvcache: false # set to true to quantize kv cache values, defaults to f
# - "dkv" is expected with better accuracy but degraded computation
kv_quant_axis: "heads_and_dkv"
kv_quant_dtype: "int8"
quantize_mtp: false # set to true to quantize mtp layers when quantization is enabled. requires mtp_num_layers > 0 and quantization=fp8_full.
checkpoint_is_quantized: false # set to true if reading from a saved aqt quantized checkpoint
# saves params quantized on fly at following path
save_quantized_params_path: ""
Expand Down
12 changes: 12 additions & 0 deletions src/maxtext/configs/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -494,6 +494,13 @@ class Quantization(BaseModel):
)
quant_cfg_path: PathStr = Field("", description="Path to the configuration file for 'intmp' quantization.")
quantize_kvcache: bool = Field(False, description="If True, quantizes the Key-Value cache.")
quantize_mtp: bool = Field(
False,
description=(
"If True, quantizes the Multi-Token Prediction (MTP) block. Only supported with"
" `mtp_num_layers > 0` and `quantization=fp8_full`."
),
)
kv_quant_axis: KvQuantAxis = Field(KvQuantAxis.HEADS_AND_DKV, description="Axes to quantize over for the KV cache.")
kv_quant_dtype: Literal["int8", "int4"] = Field("int8", description="Data type for KV cache quantization.")
quantization_local_shard_count: int = Field(-1, description="Shards the range finding operation for quantization.")
Expand Down Expand Up @@ -4214,6 +4221,11 @@ def calculate_global_batch_sizes(per_device_batch_size, expansion_factor, num_de
raise ValueError("`block_diffusion_canvas_policy='seed_and_mask'` requires `causal_block_size >= 2`.")
if self.quantize_kvcache and not self.kv_quant_axis:
raise ValueError("`kv_quant_axis` cannot be empty when quantize_kvcache is True.")
if self.quantize_mtp:
if self.mtp_num_layers <= 0:
raise ValueError("`quantize_mtp` can only be enabled when `mtp_num_layers > 0`.")
if self.quantization != "fp8_full":
raise ValueError("`quantize_mtp` can only be enabled when `quantization='fp8_full'`.")
if (
self.quantization in ("fp8", "nanoo_fp8", "fp8_gpu", "te_fp8_delayedscaling")
and self.gradient_accumulation_steps > 1
Expand Down
8 changes: 6 additions & 2 deletions src/maxtext/kernels/megablox/ops.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,8 +91,12 @@ def gmm(
# get_current_rule has to be called outside of the _gmm_fwd function.
# 2. for batchsplit, explicitly pass the rule
quantization_rule = qwix_rule if qwix_rule else qpl.get_current_rule("gmm")
if not quantization_rule or not isinstance(quantization_rule, qwix.QtRule):
raise ValueError(f"Expect a QtRule for quantized training. But get quantization_rule={quantization_rule} for gmm.")
# Only "fp8_full" defines a Qwix rule for GMM.
# quantization_rule is None when:
# 1. Quantization scheme does not target GMM (e.g., "fp8", "int8").
# 2. Module is excluded from quantization (e.g., MTP when quantize_mtp=False).
if quantization_rule and not isinstance(quantization_rule, qwix.QtRule):
raise ValueError("Expect a QtRule for quantized training.")
else:
# Handcraft a rule that matches the AQT's behavior.
if lhs_quantize_dtype or rhs_quantize_dtype:
Expand Down
4 changes: 1 addition & 3 deletions src/maxtext/layers/moe.py
Original file line number Diff line number Diff line change
Expand Up @@ -1698,9 +1698,7 @@ def extract_vma(tensor):
group_offset=group_offset,
lhs_quantize_dtype=lhs_quantize_dtype,
rhs_quantize_dtype=rhs_quantize_dtype,
# Only "fp8_full" quantizes GMM; other schemes (e.g. "fp8", "int8")
# do not define a GMM quantization rule.
use_qwix_quantization=bool(self.config.quantization == "fp8_full") and self.config.use_qwix_quantization,
use_qwix_quantization=self.config.use_qwix_quantization,
use_tokamax_backend=self.config.use_tokamax_gmm,
weight_gather_axes=weight_gather_axes,
lhs_vma_axes=lhs_vma_axes,
Expand Down
9 changes: 8 additions & 1 deletion src/maxtext/layers/quantizations.py
Original file line number Diff line number Diff line change
Expand Up @@ -875,6 +875,7 @@ def dot_general(self, *args, **kwargs):


def get_fp8_full_qwix_rule_w_sparsity(config: Config):
"""Returns Qwix quantization rules for fp8_full with optional weight sparsity."""
sparsity_rule = None
if config.weight_sparsity_n and config.weight_sparsity_m:
sparsity_rule = sparsity.SparsityRule(
Expand All @@ -883,9 +884,15 @@ def get_fp8_full_qwix_rule_w_sparsity(config: Config):
weight_sparsity_update_step=config.weight_sparsity_update_step,
weight_sparsity_start_step=config.weight_sparsity_start_step,
)

if config.quantize_mtp:
module_path = "(decoder/.*layers.*|mtp_block/.*)"
else:
module_path = "decoder/.*layers.*"

return [
qwix.QtRule(
module_path="decoder/.*layers.*",
module_path=module_path,
weight_qtype=jnp.float8_e4m3fn,
act_qtype=jnp.float8_e4m3fn,
bwd_qtype=jnp.float8_e5m2,
Expand Down
38 changes: 38 additions & 0 deletions tests/unit/multi_token_prediction_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import unittest
import functools
from types import SimpleNamespace
from absl import logging as absl_logging


import jax
Expand All @@ -35,6 +36,7 @@
from maxtext.utils import max_logging
from maxtext.utils import max_utils
from maxtext.utils import maxtext_utils
from maxtext.utils import model_creation_utils

from tests.utils.test_helpers import get_test_config_path

Expand Down Expand Up @@ -679,6 +681,42 @@ def test_quantize_without_mtp_still_works(self):
self.assertIsNotNone(quantized)


class MTPQwixInterceptionTest(unittest.TestCase):

def _assert_mtp_interception(self, quantize_mtp: bool, expected_rule: str):
"""Verifies Qwix interception behavior for MTP layers."""
cfg = pyconfig.initialize(
[
"",
get_test_config_path(),
"model_name=deepseek3-671b",
"quantization=fp8_full",
"use_qwix_quantization=true",
"per_device_batch_size=1",
"max_target_length=16",
"mtp_num_layers=1",
f"quantize_mtp={quantize_mtp}",
],
run_name="deepseek3_mtp_quantize_test",
skip_jax_distributed_system=True,
)
with self.assertLogs(absl_logging.get_absl_logger(), level="DEBUG") as cm:
# Create abstract model using nnx.eval_shape (0 FLOPs, 0 device allocation)
_, _ = model_creation_utils.create_nnx_abstract_model(cfg)
mtp_logs = [log for log in cm.output if "module='mtp_block" in log and "op=dot_general" in log]
self.assertTrue(mtp_logs, "Expected MTP dot_general operations to be traced by Qwix")
for log in mtp_logs:
self.assertIn(expected_rule, log)

def test_deepseek3_quantize_mtp_true_intercepts_mtp_ops(self):
"""DeepSeek3 with quantize_mtp=True must intercept mtp_block dot_general operations."""
self._assert_mtp_interception(quantize_mtp=True, expected_rule="rule=0")

def test_deepseek3_quantize_mtp_false_skips_mtp_ops(self):
"""DeepSeek3 with quantize_mtp=False must leave mtp_block dot_general unquantized (rule=None)."""
self._assert_mtp_interception(quantize_mtp=False, expected_rule="rule=None")


try:
from maxtext.input_pipeline.synthetic_data_processing import _make_packed_segment_ids as _fn_make_packed_segs
except ImportError:
Expand Down
Loading