diff --git a/src/maxtext/configs/base.yml b/src/maxtext/configs/base.yml index 5669e012ae..69d8dc0f60 100644 --- a/src/maxtext/configs/base.yml +++ b/src/maxtext/configs/base.yml @@ -150,6 +150,7 @@ quantize_kvcache: false # set to true to quantize kv cache values, defaults to f # - "dkv" is expected with better accuracy but degraded computation kv_quant_axis: "heads_and_dkv" kv_quant_dtype: "int8" +quantize_mtp: false # set to true to quantize mtp layers when quantization is enabled. requires mtp_num_layers > 0 and quantization=fp8_full. checkpoint_is_quantized: false # set to true if reading from a saved aqt quantized checkpoint # saves params quantized on fly at following path save_quantized_params_path: "" diff --git a/src/maxtext/configs/types.py b/src/maxtext/configs/types.py index 353656c665..c3b7874140 100644 --- a/src/maxtext/configs/types.py +++ b/src/maxtext/configs/types.py @@ -494,6 +494,13 @@ class Quantization(BaseModel): ) quant_cfg_path: PathStr = Field("", description="Path to the configuration file for 'intmp' quantization.") quantize_kvcache: bool = Field(False, description="If True, quantizes the Key-Value cache.") + quantize_mtp: bool = Field( + False, + description=( + "If True, quantizes the Multi-Token Prediction (MTP) block. Only supported with" + " `mtp_num_layers > 0` and `quantization=fp8_full`." + ), + ) kv_quant_axis: KvQuantAxis = Field(KvQuantAxis.HEADS_AND_DKV, description="Axes to quantize over for the KV cache.") kv_quant_dtype: Literal["int8", "int4"] = Field("int8", description="Data type for KV cache quantization.") quantization_local_shard_count: int = Field(-1, description="Shards the range finding operation for quantization.") @@ -4214,6 +4221,11 @@ def calculate_global_batch_sizes(per_device_batch_size, expansion_factor, num_de raise ValueError("`block_diffusion_canvas_policy='seed_and_mask'` requires `causal_block_size >= 2`.") if self.quantize_kvcache and not self.kv_quant_axis: raise ValueError("`kv_quant_axis` cannot be empty when quantize_kvcache is True.") + if self.quantize_mtp: + if self.mtp_num_layers <= 0: + raise ValueError("`quantize_mtp` can only be enabled when `mtp_num_layers > 0`.") + if self.quantization != "fp8_full": + raise ValueError("`quantize_mtp` can only be enabled when `quantization='fp8_full'`.") if ( self.quantization in ("fp8", "nanoo_fp8", "fp8_gpu", "te_fp8_delayedscaling") and self.gradient_accumulation_steps > 1 diff --git a/src/maxtext/kernels/megablox/ops.py b/src/maxtext/kernels/megablox/ops.py index f88e0aa0ab..2810516531 100644 --- a/src/maxtext/kernels/megablox/ops.py +++ b/src/maxtext/kernels/megablox/ops.py @@ -91,8 +91,12 @@ def gmm( # get_current_rule has to be called outside of the _gmm_fwd function. # 2. for batchsplit, explicitly pass the rule quantization_rule = qwix_rule if qwix_rule else qpl.get_current_rule("gmm") - if not quantization_rule or not isinstance(quantization_rule, qwix.QtRule): - raise ValueError(f"Expect a QtRule for quantized training. But get quantization_rule={quantization_rule} for gmm.") + # Only "fp8_full" defines a Qwix rule for GMM. + # quantization_rule is None when: + # 1. Quantization scheme does not target GMM (e.g., "fp8", "int8"). + # 2. Module is excluded from quantization (e.g., MTP when quantize_mtp=False). + if quantization_rule and not isinstance(quantization_rule, qwix.QtRule): + raise ValueError("Expect a QtRule for quantized training.") else: # Handcraft a rule that matches the AQT's behavior. if lhs_quantize_dtype or rhs_quantize_dtype: diff --git a/src/maxtext/layers/moe.py b/src/maxtext/layers/moe.py index f3c02ff069..c74fc5f8bc 100644 --- a/src/maxtext/layers/moe.py +++ b/src/maxtext/layers/moe.py @@ -1698,9 +1698,7 @@ def extract_vma(tensor): group_offset=group_offset, lhs_quantize_dtype=lhs_quantize_dtype, rhs_quantize_dtype=rhs_quantize_dtype, - # Only "fp8_full" quantizes GMM; other schemes (e.g. "fp8", "int8") - # do not define a GMM quantization rule. - use_qwix_quantization=bool(self.config.quantization == "fp8_full") and self.config.use_qwix_quantization, + use_qwix_quantization=self.config.use_qwix_quantization, use_tokamax_backend=self.config.use_tokamax_gmm, weight_gather_axes=weight_gather_axes, lhs_vma_axes=lhs_vma_axes, diff --git a/src/maxtext/layers/quantizations.py b/src/maxtext/layers/quantizations.py index f6df6c52ef..74aee0541c 100644 --- a/src/maxtext/layers/quantizations.py +++ b/src/maxtext/layers/quantizations.py @@ -875,6 +875,7 @@ def dot_general(self, *args, **kwargs): def get_fp8_full_qwix_rule_w_sparsity(config: Config): + """Returns Qwix quantization rules for fp8_full with optional weight sparsity.""" sparsity_rule = None if config.weight_sparsity_n and config.weight_sparsity_m: sparsity_rule = sparsity.SparsityRule( @@ -883,9 +884,15 @@ def get_fp8_full_qwix_rule_w_sparsity(config: Config): weight_sparsity_update_step=config.weight_sparsity_update_step, weight_sparsity_start_step=config.weight_sparsity_start_step, ) + + if config.quantize_mtp: + module_path = "(decoder/.*layers.*|mtp_block/.*)" + else: + module_path = "decoder/.*layers.*" + return [ qwix.QtRule( - module_path="decoder/.*layers.*", + module_path=module_path, weight_qtype=jnp.float8_e4m3fn, act_qtype=jnp.float8_e4m3fn, bwd_qtype=jnp.float8_e5m2, diff --git a/tests/unit/multi_token_prediction_test.py b/tests/unit/multi_token_prediction_test.py index 93c8ef6f81..93bcb3b531 100644 --- a/tests/unit/multi_token_prediction_test.py +++ b/tests/unit/multi_token_prediction_test.py @@ -16,6 +16,7 @@ import unittest import functools from types import SimpleNamespace +from absl import logging as absl_logging import jax @@ -35,6 +36,7 @@ from maxtext.utils import max_logging from maxtext.utils import max_utils from maxtext.utils import maxtext_utils +from maxtext.utils import model_creation_utils from tests.utils.test_helpers import get_test_config_path @@ -679,6 +681,42 @@ def test_quantize_without_mtp_still_works(self): self.assertIsNotNone(quantized) +class MTPQwixInterceptionTest(unittest.TestCase): + + def _assert_mtp_interception(self, quantize_mtp: bool, expected_rule: str): + """Verifies Qwix interception behavior for MTP layers.""" + cfg = pyconfig.initialize( + [ + "", + get_test_config_path(), + "model_name=deepseek3-671b", + "quantization=fp8_full", + "use_qwix_quantization=true", + "per_device_batch_size=1", + "max_target_length=16", + "mtp_num_layers=1", + f"quantize_mtp={quantize_mtp}", + ], + run_name="deepseek3_mtp_quantize_test", + skip_jax_distributed_system=True, + ) + with self.assertLogs(absl_logging.get_absl_logger(), level="DEBUG") as cm: + # Create abstract model using nnx.eval_shape (0 FLOPs, 0 device allocation) + _, _ = model_creation_utils.create_nnx_abstract_model(cfg) + mtp_logs = [log for log in cm.output if "module='mtp_block" in log and "op=dot_general" in log] + self.assertTrue(mtp_logs, "Expected MTP dot_general operations to be traced by Qwix") + for log in mtp_logs: + self.assertIn(expected_rule, log) + + def test_deepseek3_quantize_mtp_true_intercepts_mtp_ops(self): + """DeepSeek3 with quantize_mtp=True must intercept mtp_block dot_general operations.""" + self._assert_mtp_interception(quantize_mtp=True, expected_rule="rule=0") + + def test_deepseek3_quantize_mtp_false_skips_mtp_ops(self): + """DeepSeek3 with quantize_mtp=False must leave mtp_block dot_general unquantized (rule=None).""" + self._assert_mtp_interception(quantize_mtp=False, expected_rule="rule=None") + + try: from maxtext.input_pipeline.synthetic_data_processing import _make_packed_segment_ids as _fn_make_packed_segs except ImportError: