diff --git a/tests/equiv_chunk_test.py b/tests/equiv_chunk_test.py
index 96bfd7b27c..8728d3788c 100644
--- a/tests/equiv_chunk_test.py
+++ b/tests/equiv_chunk_test.py
@@ -44,8 +44,10 @@ def build_cfg(n_chunks):
[None, test_helpers.get_test_config_path()],
run_name=f"equiv_chunk_{n_chunks}",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
override_model_config=True,
+ num_experts=8,
+ num_experts_per_tok=2,
base_emb_dim=2048,
base_mlp_dim=256,
base_moe_mlp_dim=256,
diff --git a/tests/inference/benchmark_offline_engine.py b/tests/inference/benchmark_offline_engine.py
index 9fcb942f84..e7a7da8904 100644
--- a/tests/inference/benchmark_offline_engine.py
+++ b/tests/inference/benchmark_offline_engine.py
@@ -56,13 +56,13 @@ def init_pyconfig(**kwargs):
"max_target_length": 256,
"return_log_prob": True, # <=== set this to True
# Model
- "model_name": "gemma2-2b",
+ "model_name": "gemma3-4b",
"attention": "dot_product",
"allow_split_physical_axes": True,
"scan_layers": False,
# Checkpoints
"tokenizer_type": "huggingface",
- "tokenizer_path": "google/gemma-2-2b-it",
+ "tokenizer_path": "google/gemma-3-4b-it",
"hf_path": "trl-lib/tldr",
# "load_parameters_path": "gs://inference-benchmarks/models/llama2-70b-chat/quant/int8_",
"enable_single_controller": True, # for pathways
diff --git a/tests/integration/diloco_test.py b/tests/integration/diloco_test.py
index 7a48f9dafd..592e0acfc4 100644
--- a/tests/integration/diloco_test.py
+++ b/tests/integration/diloco_test.py
@@ -332,7 +332,7 @@ def test_diloco_two_slices(self):
"ici_fsdp_parallelism=-1",
"dcn_diloco_parallelism=2",
"enable_diloco=true",
- "model_name=gemma2-2b",
+ "model_name=qwen3-0.6b",
"override_model_config=True",
"base_emb_dim=32",
"base_num_decoder_layers=1",
@@ -360,7 +360,7 @@ def test_streaming_diloco_two_slices(self):
"enable_diloco=true",
"enable_streaming_diloco=true",
"num_diloco_fragments=2",
- "model_name=gemma2-2b",
+ "model_name=qwen3-0.6b",
"override_model_config=True",
"base_emb_dim=32",
"base_num_decoder_layers=2",
@@ -389,7 +389,7 @@ def test_streaming_diloco_two_slices_bucketized(self):
"enable_streaming_diloco=true",
"num_diloco_fragments=3",
"diloco_bucketize_non_scanned=true",
- "model_name=gemma2-2b",
+ "model_name=qwen3-0.6b",
"override_model_config=True",
"base_emb_dim=32",
"base_num_decoder_layers=2",
diff --git a/tests/post_training/integration/grpo_trainer_correctness_test.py b/tests/post_training/integration/grpo_trainer_correctness_test.py
index 725e19a455..15edbca3dc 100644
--- a/tests/post_training/integration/grpo_trainer_correctness_test.py
+++ b/tests/post_training/integration/grpo_trainer_correctness_test.py
@@ -264,9 +264,9 @@ def init_pyconfig(self, config_file, **kwargs):
"dataset_type": "hf",
"hf_path": "trl-lib/tldr",
"tokenizer_type": "huggingface",
- "tokenizer_path": "google/gemma-2-2b-it",
+ "tokenizer_path": "google/gemma-3-4b-it",
"attention": "dot_product",
- "model_name": "gemma2-2b",
+ "model_name": "gemma3-4b",
} | kwargs
config = pyconfig.initialize(
[
diff --git a/tests/post_training/unit/lora_utils_test.py b/tests/post_training/unit/lora_utils_test.py
index a84945acaf..3fa4011205 100644
--- a/tests/post_training/unit/lora_utils_test.py
+++ b/tests/post_training/unit/lora_utils_test.py
@@ -55,7 +55,7 @@
"base_num_kv_heads": 2,
"base_mlp_dim": 256,
"max_prefill_predict_length": 4,
- "model_name": "llama2-7b",
+ "model_name": "llama3.1-8b",
"override_model_config": True,
"weight_dtype": "bfloat16",
}
diff --git a/tests/post_training/unit/rl_lr_schedule_test.py b/tests/post_training/unit/rl_lr_schedule_test.py
index 950518cde3..da3ec0c08f 100644
--- a/tests/post_training/unit/rl_lr_schedule_test.py
+++ b/tests/post_training/unit/rl_lr_schedule_test.py
@@ -65,7 +65,7 @@
"base_num_kv_heads": 2,
"base_mlp_dim": 256,
"max_prefill_predict_length": 4,
- "model_name": "llama2-7b",
+ "model_name": "llama3.1-8b",
"override_model_config": True,
"weight_dtype": "bfloat16",
}
diff --git a/tests/unit/aqt_serve_roundtrip_nnx_test.py b/tests/unit/aqt_serve_roundtrip_nnx_test.py
index 006a4580d2..da4597f006 100644
--- a/tests/unit/aqt_serve_roundtrip_nnx_test.py
+++ b/tests/unit/aqt_serve_roundtrip_nnx_test.py
@@ -66,14 +66,23 @@ class ServeModeRoundTripTest(unittest.TestCase):
def _init_cfg(self, ckpt_path, *, checkpoint_is_quantized):
"""Build a pyconfig for save or reload."""
- # Use base.yml + gpt3-52k. The decoupled test config strips
+ # Use base.yml + qwen3-0.6b with small dimensions. The decoupled test config strips
# logical_axis_rules (e.g. "norm"), which the AQT serve-mode model
# construction needs.
base_yml = os.path.join(MAXTEXT_PKG_DIR, "configs", "base.yml")
args = [
sys.argv[0],
base_yml,
- "model_name=gpt3-52k",
+ "model_name=qwen3-0.6b",
+ "override_model_config=true",
+ "base_emb_dim=16",
+ "base_num_query_heads=2",
+ "base_num_kv_heads=2",
+ "base_mlp_dim=64",
+ "base_num_decoder_layers=1",
+ "head_dim=8",
+ "vocab_size=1024",
+ "attention=dot_product",
"max_target_length=64",
"max_prefill_predict_length=16",
"per_device_batch_size=1",
diff --git a/tests/unit/configs_value_test.py b/tests/unit/configs_value_test.py
index b53e63ce2a..fa0a455c68 100644
--- a/tests/unit/configs_value_test.py
+++ b/tests/unit/configs_value_test.py
@@ -60,11 +60,11 @@ def test_type_conversion(self):
def test_model_override(self):
"""Tests that model-specific configs override base.yml."""
- argv = ["", _BASE_CONFIG_PATH, "model_name=llama2-7b", "run_name=test"]
+ argv = ["", _BASE_CONFIG_PATH, "model_name=llama3.1-8b", "run_name=test"]
config = pyconfig.initialize(argv)
- self.assertEqual(config.base_emb_dim, 4096) # From llama2-7b.yml
- self.assertEqual(config.base_num_decoder_layers, 32) # From llama2-7b.yml
- self.assertEqual(config.decoder_block, types.DecoderBlockType.LLAMA2) # from llama2-7b.yml
+ self.assertEqual(config.base_emb_dim, 4096) # From llama3.1-8b.yml
+ self.assertEqual(config.base_num_decoder_layers, 32) # From llama3.1-8b.yml
+ self.assertEqual(config.decoder_block, types.DecoderBlockType.LLAMA2) # from llama3.1-8b.yml
self.assertEqual(config.steps, 150001) # From base.yml, not overridden
def test_derived_values(self):
@@ -1251,7 +1251,7 @@ def test_llama3_tokenizer_correction(self):
argv = [
"",
_BASE_CONFIG_PATH,
- "model_name=llama3-8b",
+ "model_name=llama3.1-8b",
"tokenizer_path=assets/tokenizer_llama3.tiktoken",
"run_name=test",
]
@@ -1841,10 +1841,10 @@ def test_rl_config_sa_context_split_requires_custom_remat(self):
"base_num_kv_heads": 2,
"base_mlp_dim": 256,
"max_prefill_predict_length": 4,
- "model_name": "llama2-7b",
+ "model_name": "llama3.1-8b",
"override_model_config": True,
"weight_dtype": "bfloat16",
- "tokenizer_path": "meta-llama/Llama-2-7b",
+ "tokenizer_path": "meta-llama/Llama-3.1-8B",
}
def rl_config(*cli_args):
diff --git a/tests/unit/decoder_layer_model_mode_test.py b/tests/unit/decoder_layer_model_mode_test.py
index c48fccf30e..e16815e96e 100644
--- a/tests/unit/decoder_layer_model_mode_test.py
+++ b/tests/unit/decoder_layer_model_mode_test.py
@@ -84,7 +84,7 @@
}
_DEEPSEEK = {
- "model_name": "deepseek2-16b",
+ "model_name": "deepseek3-test",
"override_model_config": True,
"mla_naive_kvcache": False,
"first_num_dense_layers": 1,
diff --git a/tests/unit/deepseek_decode_consistency_test.py b/tests/unit/deepseek_decode_consistency_test.py
index 89f86e9f12..f20cf54780 100644
--- a/tests/unit/deepseek_decode_consistency_test.py
+++ b/tests/unit/deepseek_decode_consistency_test.py
@@ -71,7 +71,7 @@
}
_DEEPSEEK = {
- "model_name": "deepseek2-16b",
+ "model_name": "deepseek3-test",
"override_model_config": True,
"num_experts": 4,
"num_experts_per_tok": 2,
diff --git a/tests/unit/defer_small_all_reduces_test.py b/tests/unit/defer_small_all_reduces_test.py
index 3d7e2dc7ce..d5b2119ac5 100644
--- a/tests/unit/defer_small_all_reduces_test.py
+++ b/tests/unit/defer_small_all_reduces_test.py
@@ -295,8 +295,10 @@ def run(defer, chunks, rbf):
[None, get_test_config_path()],
run_name="defer_small_ar_layer",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
override_model_config=True,
+ num_experts=8,
+ num_experts_per_tok=2,
base_emb_dim=256,
base_mlp_dim=128,
base_moe_mlp_dim=128,
diff --git a/tests/unit/flop_calculation_test.py b/tests/unit/flop_calculation_test.py
index 3979474011..74e4fd912b 100644
--- a/tests/unit/flop_calculation_test.py
+++ b/tests/unit/flop_calculation_test.py
@@ -313,10 +313,10 @@ def test_qwen3_5_9b_flops(self):
self.assertFlopsAlmostEqual(calculated_tflops, golden_tflops)
- def test_llama2_7b_flops(self):
- """Test Llama2 7b Flops calculation with default parameters"""
+ def test_llama3_3_70b_flops(self):
+ """Test Llama3.3 70b Flops calculation with default parameters"""
cfg = self._initialize_model_config(
- "llama2-7b",
+ "llama3.3-70b",
max_target_length=2048,
per_device_batch_size=12,
)
@@ -324,17 +324,16 @@ def test_llama2_7b_flops(self):
B = cfg.per_device_batch_size
S = cfg.max_target_length
attention_flops = self.compute_regular_attention_flops_per_device(kwargs)
- # Llama2-7b has ~6.74B parameters
- # https://adithyask.medium.com/from-7b-to-8b-parameters-understanding-weight-matrix-changes-in-llama-transformer-models-31ea7ed5fd88
- golden_param_size = 6.74e9
+ # Llama3.3-70b has ~69.5B non-embedding + unembedding active parameters
+ golden_param_size = 69.5e9
golden_tflops = 6 * B * S * golden_param_size / 1e12 + attention_flops
calculated_tflops, _, _ = calculate_tflops_training_per_device(cfg)
self.assertFlopsAlmostEqual(calculated_tflops, golden_tflops)
- def test_llama3_8b_flops(self):
- """Test Llama3 8b Flops calculation with default parameters"""
+ def test_llama3_1_8b_flops(self):
+ """Test Llama3.1 8b Flops calculation with default parameters"""
cfg = self._initialize_model_config(
- "llama3-8b",
+ "llama3.1-8b",
max_target_length=2048,
per_device_batch_size=4,
)
@@ -342,19 +341,16 @@ def test_llama3_8b_flops(self):
B = cfg.per_device_batch_size
S = cfg.max_target_length
attention_flops = self.compute_regular_attention_flops_per_device(kwargs)
- # LLaMA3-8b has ~8.03B parameters
- # https://adithyask.medium.com/from-7b-to-8b-parameters-understanding-weight-matrix-changes-in-llama-transformer-models-31ea7ed5fd88
- # Note: The commonly cited 8.03B parameter count for Llama 3 8B corresponds to a version with UNTIED embeddings.
- # Here we consider TIED embedding table, which reduces param count to 7.50B
+ # LLaMA3.1-8b has ~8.03B parameters (7.50B excluding input embedding lookup)
golden_param_size = 7.50e9
golden_tflops = 6 * B * S * golden_param_size / 1e12 + attention_flops
calculated_tflops, _, _ = calculate_tflops_training_per_device(cfg)
self.assertFlopsAlmostEqual(calculated_tflops, golden_tflops)
- def test_mixtral_8x7b_flops(self):
- """Test Mixtral 8x7b Flops calculation"""
+ def test_qwen3_30b_a3b_flops(self):
+ """Test Qwen3 30B-A3B Flops calculation"""
cfg = self._initialize_model_config(
- "mixtral-8x7b",
+ "qwen3-30b-a3b",
max_target_length=8192,
per_device_batch_size=4,
)
@@ -362,17 +358,16 @@ def test_mixtral_8x7b_flops(self):
B = cfg.per_device_batch_size
S = cfg.max_target_length
attention_flops = self.compute_regular_attention_flops_per_device(kwargs)
- # mixtral-8x7b has ~12.9B active parameters
- # https://mistral.ai/news/mixtral-of-experts
- golden_param_size = 12.9e9
+ # qwen3-30b-a3b has ~3.05B active parameters
+ golden_param_size = 3.05e9
golden_tflops = 6 * B * S * golden_param_size / 1e12 + attention_flops
calculated_tflops, _, _ = calculate_tflops_training_per_device(cfg)
self.assertFlopsAlmostEqual(calculated_tflops, golden_tflops)
- def test_deepseek2_16b_flops(self):
- """Test DeepSeek2-16b FLops calculation"""
+ def test_deepseek3_test_flops(self):
+ """Test DeepSeek3-test FLops calculation"""
cfg = self._initialize_model_config(
- "deepseek2-16b",
+ "deepseek3-test",
max_target_length=8192,
per_device_batch_size=4,
)
@@ -380,9 +375,27 @@ def test_deepseek2_16b_flops(self):
B = cfg.per_device_batch_size
S = cfg.max_target_length
attention_flops = self.compute_deepseek_attention_flops_per_device(kwargs)
- # deepseek2-16b has ~2.4B active parameters
- # https://arxiv.org/pdf/2405.04434
- golden_param_size = 2.4e9
+ # deepseek3-test has ~4.47B active parameters
+ golden_param_size = 4.47e9
+ golden_tflops = 6 * B * S * golden_param_size / 1e12 + attention_flops
+ calculated_tflops, _, _ = calculate_tflops_training_per_device(cfg)
+ self.assertFlopsAlmostEqual(calculated_tflops, golden_tflops)
+
+ def test_deepseek_mla_no_q_lora_flops(self):
+ """Test MLA FLOPs calculation when q_lora_rank == 0 (direct Q projection)."""
+ cfg = self._initialize_model_config(
+ "deepseek3-test",
+ max_target_length=8192,
+ per_device_batch_size=4,
+ override_model_config=True,
+ q_lora_rank=0,
+ )
+ kwargs = cfg.get_keys()
+ B = cfg.per_device_batch_size
+ S = cfg.max_target_length
+ attention_flops = self.compute_deepseek_attention_flops_per_device(kwargs)
+ # deepseek3-test with q_lora_rank=0 has ~2.89B active parameters
+ golden_param_size = 2.89e9
golden_tflops = 6 * B * S * golden_param_size / 1e12 + attention_flops
calculated_tflops, _, _ = calculate_tflops_training_per_device(cfg)
self.assertFlopsAlmostEqual(calculated_tflops, golden_tflops)
@@ -540,7 +553,7 @@ def test_deepseek4_1600b_flops(self):
def test_custom_engram_flops(self):
"""Test model with Engram Flops calculation"""
cfg = self._initialize_model_config(
- "deepseek2-16b",
+ "deepseek3-test",
max_target_length=8192,
per_device_batch_size=4,
)
@@ -556,9 +569,8 @@ def test_custom_engram_flops(self):
N = cfg.engram_max_ngram_size
attention_flops = self.compute_deepseek_attention_flops_per_device(kwargs)
- # deepseek2-16b has ~2.4B active parameters
- # https://arxiv.org/pdf/2405.04434
- golden_param_size = 2.4e9
+ # deepseek3-test has ~4.47B active parameters
+ golden_param_size = 4.47e9
# calculate Engram active params
engram_dim = H * H_D * (N - 1)
@@ -1000,10 +1012,10 @@ def test_calculate_routed_and_shared_ffn_tflops_per_device(self):
self.assertAlmostEqual(ffn_tflops, expected_total, places=5)
def test_mtp_dense_flops(self):
- """Test model with MTP modules on a Dense architecture (Llama3)."""
+ """Test model with MTP modules on a Dense architecture (Llama3.1)."""
# Inject MTP modules during initialization
cfg = self._initialize_model_config(
- "llama3-8b",
+ "llama3.1-8b",
max_target_length=2048,
per_device_batch_size=4,
mtp_num_layers=2,
@@ -1013,9 +1025,9 @@ def test_mtp_dense_flops(self):
B = cfg.per_device_batch_size
S = cfg.max_target_length
- # 1. Base Llama3-8b FLOPs
+ # 1. Base Llama3.1-8b FLOPs
base_attention_flops = self.compute_regular_attention_flops_per_device(kwargs)
- # LLaMA3-8b has ~7.50B active parameters with tied embeddings
+ # LLaMA3.1-8b has ~7.50B active parameters (excluding input embedding lookup)
golden_base_param_size = 7.50e9
golden_base_tflops = 6 * B * S * golden_base_param_size / 1e12 + base_attention_flops
@@ -1161,72 +1173,72 @@ def test_hy4_preview_flops(self):
self.assertAlmostEqual(short_attention, expected_short_attention, delta=expected_short_attention * 1e-9)
def test_mtp_moe_flops(self):
- """Test model with MTP modules on an MoE architecture (DeepSeek)."""
- # Inject MTP modules during initialization
- cfg = self._initialize_model_config(
- "deepseek2-16b",
- max_target_length=8192,
- per_device_batch_size=4,
- mtp_num_layers=2,
- )
-
- kwargs = cfg.get_keys()
- B = cfg.per_device_batch_size
- S = cfg.max_target_length
-
- # 1. Base DeepSeek FLOPs
- base_attention_flops = self.compute_deepseek_attention_flops_per_device(kwargs)
- # deepseek2-16b has ~2.4B active parameters
- golden_base_param_size = 2.4e9
- golden_base_tflops = 6 * B * S * golden_base_param_size / 1e12 + base_attention_flops
-
- # 2. MTP Module Active Params for MoE (DeepSeek style)
- emb_dim = kwargs["base_emb_dim"]
- num_experts = kwargs["num_experts"]
- moe_mlp_dim = kwargs["base_moe_mlp_dim"]
- shared_experts = kwargs.get("shared_experts", 1)
- num_experts_per_tok = kwargs["num_experts_per_tok"]
-
- # FFN Params (Gate + Shared + Routed)
- # SwiGLU requires 3 matrices (Gate, Up, Down) per expert
- gate_params = emb_dim * num_experts
- shared_params = 3 * emb_dim * moe_mlp_dim * shared_experts
- routed_params = 3 * emb_dim * moe_mlp_dim * num_experts_per_tok
- ffn_params = gate_params + shared_params + routed_params
-
- # MLA Attention Params (Q, KV, and Out Projections)
- qk_nope_hd = kwargs["qk_nope_head_dim"]
- qk_rope_hd = kwargs["qk_rope_head_dim"]
- v_hd = kwargs["v_head_dim"]
- H_q = kwargs["base_num_query_heads"]
- q_lora_rank = kwargs.get("q_lora_rank", 0)
- kv_lora_rank = kwargs.get("kv_lora_rank", 0)
-
- qk_head_dim_sum = qk_nope_hd + qk_rope_hd
- if q_lora_rank == 0:
- q_params = emb_dim * H_q * qk_head_dim_sum
- else:
- q_params = emb_dim * q_lora_rank + q_lora_rank * H_q * qk_head_dim_sum
-
- kv_params = emb_dim * (kv_lora_rank + qk_rope_hd) + kv_lora_rank * H_q * (qk_nope_hd + v_hd)
- proj_params = emb_dim * H_q * v_hd
- attn_params = q_params + kv_params + proj_params
-
- # Total MTP Active Params
- mtp_active_params = (attn_params + ffn_params) * cfg.mtp_num_layers
- mtp_weight_tflops = 6 * B * S * mtp_active_params / 1e12
-
- # 3. MTP Attention FLOPs
- # Causal attention flops for MLA (3 for fwd + bwd pass)
- mtp_attention_tflops = 3 * cfg.mtp_num_layers * B * (S**2) * H_q * (qk_nope_hd + qk_rope_hd + v_hd) / 1e12
-
- # Final Expected TFLOPs
- golden_total_tflops = golden_base_tflops + mtp_weight_tflops + mtp_attention_tflops
-
- # Run Calculation
- calculated_tflops, _, _ = calculate_tflops_training_per_device(cfg)
+ """Test model with MTP modules on an MoE architecture (DeepSeek, with and without Q-LoRA)."""
+ for q_lora_rank, golden_base_param_size in ((1536, 4.47e9), (0, 2.89e9)):
+ with self.subTest(q_lora_rank=q_lora_rank):
+ cfg = self._initialize_model_config(
+ "deepseek3-test",
+ max_target_length=8192,
+ per_device_batch_size=4,
+ mtp_num_layers=2,
+ override_model_config=True,
+ q_lora_rank=q_lora_rank,
+ )
- self.assertFlopsAlmostEqual(calculated_tflops, golden_total_tflops)
+ kwargs = cfg.get_keys()
+ B = cfg.per_device_batch_size
+ S = cfg.max_target_length
+
+ # 1. Base DeepSeek FLOPs
+ base_attention_flops = self.compute_deepseek_attention_flops_per_device(kwargs)
+ golden_base_tflops = 6 * B * S * golden_base_param_size / 1e12 + base_attention_flops
+
+ # 2. MTP Module Active Params for MoE (DeepSeek style)
+ emb_dim = kwargs["base_emb_dim"]
+ num_experts = kwargs["num_experts"]
+ moe_mlp_dim = kwargs["base_moe_mlp_dim"]
+ shared_experts = kwargs.get("shared_experts", 1)
+ num_experts_per_tok = kwargs["num_experts_per_tok"]
+
+ # FFN Params (Gate + Shared + Routed)
+ # SwiGLU requires 3 matrices (Gate, Up, Down) per expert
+ gate_params = emb_dim * num_experts
+ shared_params = 3 * emb_dim * moe_mlp_dim * shared_experts
+ routed_params = 3 * emb_dim * moe_mlp_dim * num_experts_per_tok
+ ffn_params = gate_params + shared_params + routed_params
+
+ # MLA Attention Params (Q, KV, and Out Projections)
+ qk_nope_hd = kwargs["qk_nope_head_dim"]
+ qk_rope_hd = kwargs["qk_rope_head_dim"]
+ v_hd = kwargs["v_head_dim"]
+ H_q = kwargs["base_num_query_heads"]
+ kv_lora_rank = kwargs.get("kv_lora_rank", 0)
+
+ qk_head_dim_sum = qk_nope_hd + qk_rope_hd
+ if q_lora_rank == 0:
+ q_params = emb_dim * H_q * qk_head_dim_sum
+ else:
+ q_params = emb_dim * q_lora_rank + q_lora_rank * H_q * qk_head_dim_sum
+
+ kv_params = emb_dim * (kv_lora_rank + qk_rope_hd) + kv_lora_rank * H_q * (qk_nope_hd + v_hd)
+ proj_params = emb_dim * H_q * v_hd
+ attn_params = q_params + kv_params + proj_params
+
+ # Total MTP Active Params
+ mtp_active_params = (attn_params + ffn_params) * cfg.mtp_num_layers
+ mtp_weight_tflops = 6 * B * S * mtp_active_params / 1e12
+
+ # 3. MTP Attention FLOPs
+ # Causal attention flops for MLA (3 for fwd + bwd pass)
+ mtp_attention_tflops = 3 * cfg.mtp_num_layers * B * (S**2) * H_q * (qk_nope_hd + qk_rope_hd + v_hd) / 1e12
+
+ # Final Expected TFLOPs
+ golden_total_tflops = golden_base_tflops + mtp_weight_tflops + mtp_attention_tflops
+
+ # Run Calculation
+ calculated_tflops, _, _ = calculate_tflops_training_per_device(cfg)
+
+ self.assertFlopsAlmostEqual(calculated_tflops, golden_total_tflops)
# ========== Parameterized Tests for Multiple Standard Models ==========
@@ -1403,7 +1415,8 @@ def _verify_long_sequence_flops(self, model_name):
self._verify_flops(model_name, max_target_length=8192)
@parameterized.parameters(
- ("llama3-8b",),
+ ("llama3.1-8b",),
+ ("llama3.3-70b",),
("llama4-17b-16e",),
("gemma3-4b",),
("gemma3-12b",),
@@ -1424,7 +1437,8 @@ def test_short_sequence_flops(self, model_name):
self._verify_short_sequence_flops(model_name)
@parameterized.parameters(
- ("llama3-8b",),
+ ("llama3.1-8b",),
+ ("llama3.3-70b",),
("llama4-17b-16e",),
("gemma3-4b",),
("gemma3-12b",),
diff --git a/tests/unit/managed_mldiagnostics_test.py b/tests/unit/managed_mldiagnostics_test.py
index 55be291d43..ffe1566f30 100644
--- a/tests/unit/managed_mldiagnostics_test.py
+++ b/tests/unit/managed_mldiagnostics_test.py
@@ -290,8 +290,8 @@ def get_keys(self):
def test_pydantic_rl_config_without_get_keys(self):
"""The RL trainer passes pydantic `RLConfig` objects (initialize_pydantic), which have no get_keys()."""
trainer = types.RLConfig(
- model_name="llama2-7b",
- tokenizer_path="meta-llama/Llama-2-7b",
+ model_name="llama3.1-8b",
+ tokenizer_path="meta-llama/Llama-3.1-8B",
run_name="rl_run",
base_output_directory="gs://bucket/out",
managed_mldiagnostics=True,
@@ -306,7 +306,7 @@ def test_pydantic_rl_config_without_get_keys(self):
self.assertEqual(kwargs["gcs_path"], "gs://bucket/out/rl_run/managed-mldiagnostics")
self.assertEqual(kwargs["region"], "us-central1")
self.assertFalse(kwargs["on_demand_xprof"])
- self.assertEqual(kwargs["configs"]["model_name"], "llama2-7b")
+ self.assertEqual(kwargs["configs"]["model_name"], "llama3.1-8b")
self.assertEqual(kwargs["configs"]["sampler.debug"], sampler.debug)
self.assertNotIn("sampler.model_name", kwargs["configs"])
self.assertFalse(kwargs["metric_only_run"])
diff --git a/tests/unit/moe_test.py b/tests/unit/moe_test.py
index 4477b11484..ffd65f19ef 100644
--- a/tests/unit/moe_test.py
+++ b/tests/unit/moe_test.py
@@ -370,7 +370,11 @@ def setUp(self):
[None, get_test_config_path()],
run_name="token_dropping_test",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ override_model_config=True,
+ num_experts=8,
+ num_experts_per_tok=2,
+ norm_topk_prob=False,
dtype="bfloat16",
megablox=False,
sparse_matmul=False,
@@ -494,7 +498,7 @@ def setUp(self):
[None, get_test_config_path()],
run_name="mlp_block_init_test",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
dtype="bfloat16",
megablox=False,
sparse_matmul=False,
@@ -1252,7 +1256,9 @@ def test_megablox(self):
[None, get_test_config_path()],
run_name="moe_block_megablox_test",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ num_experts=8,
+ num_experts_per_tok=2,
override_model_config=True,
base_emb_dim=512,
base_mlp_dim=256,
@@ -1286,7 +1292,9 @@ def test_ragged_dot(self):
[None, get_test_config_path()],
run_name="moe_block_ragged_dot_test",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ num_experts=8,
+ num_experts_per_tok=2,
override_model_config=True,
base_emb_dim=512,
base_mlp_dim=256,
@@ -1318,9 +1326,9 @@ def test_ragged_dot(self):
def test_te_grouped_gemm(self):
"""TransformerEngine grouped_dense matches dense MoE and ragged_dot.
- Mixtral-style routing (8 experts, top-2, SwiGLU, no expert bias) rather than
- GPT-OSS. TE v1 copies group_sizes D2H inside its FFI, so this requires one
- local device; shrink Mixtral dims so the case stays a unit test on GPU.
+ 8-expert top-2 SwiGLU routing (no expert bias) rather than GPT-OSS. TE v1
+ copies group_sizes D2H inside its FFI, so this requires one local device;
+ shrink Qwen3-MoE dims so the case stays a unit test on GPU.
"""
_skip_unless_te_single_local_device(self)
@@ -1329,7 +1337,9 @@ def _build_cfg(use_te_grouped_gemm, run_name):
[None, get_test_config_path()],
run_name=run_name,
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ num_experts=8,
+ num_experts_per_tok=2,
hardware="gpu",
override_model_config=True,
dtype="bfloat16",
@@ -1444,7 +1454,9 @@ def test_dense(self):
[None, get_test_config_path()],
run_name="moe_block_dense_test",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ num_experts=8,
+ num_experts_per_tok=2,
override_model_config=True,
base_emb_dim=512,
base_mlp_dim=256,
@@ -1478,7 +1490,9 @@ def test_moe_emb_chunking_random_routing(self):
[None, get_test_config_path()],
run_name="moe_block_chunking_rr_test",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ num_experts=8,
+ num_experts_per_tok=2,
override_model_config=True,
base_emb_dim=1024,
base_mlp_dim=256,
@@ -1504,7 +1518,9 @@ def test_moe_emb_chunking_random_routing(self):
[None, get_test_config_path()],
run_name="moe_block_non_chunking_rr_test",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ num_experts=8,
+ num_experts_per_tok=2,
override_model_config=True,
base_emb_dim=1024,
base_mlp_dim=256,
@@ -1579,7 +1595,7 @@ def test_moe_emb_chunking_gmm_v2(self):
[None, get_test_config_path()],
run_name="moe_block_chunking_gmm_v2_test",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
dtype="bfloat16",
weight_dtype="bfloat16",
megablox=False,
@@ -1617,7 +1633,9 @@ def test_megablox_expert_parallelism(self):
[None, get_test_config_path()],
run_name="moe_block_megablox_ep_test",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ num_experts=8,
+ num_experts_per_tok=2,
override_model_config=True,
base_emb_dim=512,
base_mlp_dim=256,
@@ -1653,7 +1671,9 @@ def test_ring_of_expert_and_tensor_parallelism(self):
[None, get_test_config_path()],
run_name="moe_block_ring_ep_tp_test",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ num_experts=8,
+ num_experts_per_tok=2,
override_model_config=True,
base_emb_dim=512,
base_mlp_dim=256,
@@ -1708,7 +1728,9 @@ def _build_cfg(use_ragged_sort: bool):
[None, get_test_config_path()],
run_name=(f"moe_block_use_ragged_sort_{use_ragged_sort}" f"_ring_{use_ring_of_experts}_test"),
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ num_experts=8,
+ num_experts_per_tok=2,
override_model_config=True,
# emb dim must be >= 2048 (multiple of 1024) for ragged_gather_reduce_v2 on TPU v6e SparseCore.
base_emb_dim=2048,
@@ -1848,7 +1870,9 @@ def _run_tc_ragged_sort_loss_and_grad(
def _build_cfg(moe_tc_ragged_sort: bool):
kwargs = {
"enable_checkpointing": False,
- "model_name": "mixtral-8x7b",
+ "model_name": "qwen3-30b-a3b",
+ "num_experts": 8,
+ "num_experts_per_tok": 2,
"override_model_config": True,
"base_emb_dim": 2048,
"base_mlp_dim": 256,
@@ -2018,7 +2042,9 @@ def _run_topk_before_ep_all_gather_loss_and_grad(self, **overrides):
def _build_cfg(topk_before: bool):
kwargs = {
"enable_checkpointing": False,
- "model_name": "mixtral-8x7b",
+ "model_name": "qwen3-30b-a3b",
+ "num_experts": 8,
+ "num_experts_per_tok": 2,
"override_model_config": True,
"base_emb_dim": 512,
"base_mlp_dim": 256,
@@ -2120,7 +2146,9 @@ def _moe_routing_maps_cfg(self, moe_routing_maps: str, **overrides):
"""Tiny ring-of-experts ragged-sort MoE config (EP2) for the moe_routing_maps test."""
kwargs = {
"enable_checkpointing": False,
- "model_name": "mixtral-8x7b",
+ "model_name": "qwen3-30b-a3b",
+ "num_experts": 8,
+ "num_experts_per_tok": 2,
"override_model_config": True,
"base_emb_dim": 512,
"base_mlp_dim": 256,
@@ -2531,7 +2559,9 @@ def test_ragged_sort_overflow_detection(self, ragged_buffer_factor, expect_overf
[None, get_test_config_path()],
run_name=f"moe_overflow_detection_{ragged_buffer_factor}",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ num_experts=8,
+ num_experts_per_tok=2,
override_model_config=True,
base_emb_dim=2048,
base_mlp_dim=256,
@@ -2595,11 +2625,14 @@ def test_ragged_sort_overflow_detection(self, ragged_buffer_factor, expect_overf
self.assertFalse(any_overflow, "Expected has_overflow=False with a dropless (worst-case) buffer.")
def _build_retry_test_mesh(self):
+ """Builds an EP=2 device mesh for retry unit tests."""
cfg = pyconfig.initialize(
[None, get_test_config_path()],
run_name="moe_retry_test_probe",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ num_experts=8,
+ num_experts_per_tok=2,
override_model_config=True,
ici_expert_parallelism=2,
)
@@ -2613,14 +2646,16 @@ def _build_retry_test_model(
force_dropless: bool = False,
use_ring_of_experts: bool = True,
):
- """Builds a mixtral-8x7b RoutedMoE with the given ragged buffer/retry settings."""
+ """Builds a qwen3-30b-a3b RoutedMoE with the given ragged buffer/retry settings."""
cfg = pyconfig.initialize(
[None, get_test_config_path()],
run_name=(
f"moe_retry_test_{ragged_buffer_factor}_{moe_dropless_fallback}_{force_dropless}_{use_ring_of_experts}"
),
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ num_experts=8,
+ num_experts_per_tok=2,
override_model_config=True,
base_emb_dim=2048,
base_mlp_dim=256,
@@ -2884,7 +2919,9 @@ def test_moe_fsdp_two_stage_parallelism_tpu_only(self):
[None, get_test_config_path()],
run_name="moe_block_megablox_ep_test",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ num_experts=8,
+ num_experts_per_tok=2,
override_model_config=True,
base_emb_dim=512,
base_mlp_dim=256,
@@ -2925,7 +2962,9 @@ def test_shard_embed_moe_on_fsdp(self):
[None, get_test_config_path()],
run_name="moe_block_shard_embed_test",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ num_experts=8,
+ num_experts_per_tok=2,
override_model_config=True,
base_emb_dim=512,
base_mlp_dim=256,
@@ -3004,7 +3043,9 @@ def test_megablox_context_parallelism(self):
[None, get_test_config_path()],
run_name="moe_block_megablox_cp_test",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ num_experts=8,
+ num_experts_per_tok=2,
override_model_config=True,
base_emb_dim=512,
base_mlp_dim=256,
@@ -3040,7 +3081,9 @@ def test_megablox_expert_context_parallelism(self):
[None, get_test_config_path()],
run_name="moe_block_megablox_ep_cp_test",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ num_experts=8,
+ num_experts_per_tok=2,
override_model_config=True,
base_emb_dim=512,
base_mlp_dim=256,
@@ -3078,7 +3121,9 @@ def test_megablox_expert_tensor_parallelism(self):
[None, get_test_config_path()],
run_name="moe_block_megablox_ep_tp_test",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ num_experts=8,
+ num_experts_per_tok=2,
override_model_config=True,
base_emb_dim=512,
base_mlp_dim=256,
@@ -3116,7 +3161,13 @@ def test_tp_as_ep_matches_dense_reference(self):
[None, get_test_config_path()],
run_name="moe_block_tp_as_ep_test",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ num_experts=8,
+ num_experts_per_tok=2,
+ override_model_config=True,
+ base_emb_dim=512,
+ base_mlp_dim=256,
+ base_moe_mlp_dim=256,
dtype="bfloat16",
sparse_matmul=True,
per_device_batch_size=4,
@@ -3144,13 +3195,15 @@ def test_tp_as_ep_matches_dense_reference(self):
assert_moe_close(actual_output, expected_output, cfg.dtype)
def _tp_as_ep_test_config(self, rule, **overrides):
- """Small float32 Mixtral on an expert=2 x tensor=2 mesh, with the default rules (rule="") or tp-as-ep."""
+ """Small float32 Qwen3-MoE (8-expert top-2) on an expert=2 x tensor=2 mesh, with default rules (rule="") or tp-as-ep."""
return pyconfig.initialize(
[None, get_test_config_path()],
**{
"run_name": f"moe_tp_as_ep_{rule or 'default'}",
"enable_checkpointing": False,
- "model_name": "mixtral-8x7b",
+ "model_name": "qwen3-30b-a3b",
+ "num_experts": 8,
+ "num_experts_per_tok": 2,
"override_model_config": True,
"base_emb_dim": 64,
"base_mlp_dim": 64,
@@ -3652,7 +3705,7 @@ def _assert_te_bf16_close(name, actual, expected, relative_norm_threshold=5e-2):
@pytest.mark.gpu_only
class TeGroupedGemmTest(unittest.TestCase):
- """TransformerEngine grouped_dense vs jax.lax.ragged_dot on Mixtral-style MoE.
+ """TransformerEngine grouped_dense vs jax.lax.ragged_dot on 8-expert top-2 SwiGLU MoE.
Covers paths that are not GPT-OSS-specific: mlp_bias, EP>1, ragged-sort, and
mixed-dtype wo activations. EP>1 cases need one process per GPU; see
@@ -3664,7 +3717,7 @@ def _te_cfg(self, use_te_grouped_gemm, expert_parallelism, mlp_bias=False, **ove
[None, get_test_config_path()],
run_name=f"moe_te_gmm_ep{expert_parallelism}_{int(use_te_grouped_gemm)}_{int(mlp_bias)}",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
hardware="gpu",
override_model_config=True,
dtype="bfloat16",
@@ -3679,7 +3732,7 @@ def _te_cfg(self, use_te_grouped_gemm, expert_parallelism, mlp_bias=False, **ove
base_moe_mlp_dim=64,
mlp_bias=mlp_bias,
float32_gate_logits=True,
- **{"megablox": False, **overrides},
+ **{"megablox": False, "num_experts": 8, "num_experts_per_tok": 2, **overrides},
)
def _run_te_vs_ragged_dot(self, expert_parallelism, mlp_bias=False, **overrides):
@@ -4151,7 +4204,9 @@ def _build_cfg(
[None, get_test_config_path()],
run_name="gmm_grad_equivalence_test",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ num_experts=8,
+ num_experts_per_tok=2,
override_model_config=True,
base_emb_dim=1024,
base_mlp_dim=1024,
@@ -4281,7 +4336,9 @@ def _build_cfg(moe_quantize_token_all_gather: bool):
[None, get_test_config_path()],
run_name="moe_quantize_token_all_gather_test",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ num_experts=8,
+ num_experts_per_tok=2,
override_model_config=True,
base_emb_dim=emb_dim,
base_mlp_dim=512,
@@ -4412,7 +4469,9 @@ def _build_cfg(bwd_method_val: str):
[None, get_test_config_path()],
run_name="moe_quantize_combine_bwd_test",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ num_experts=8,
+ num_experts_per_tok=2,
override_model_config=True,
base_emb_dim=2048,
base_mlp_dim=512,
@@ -4949,7 +5008,13 @@ def setUp(self):
[None, get_test_config_path()],
run_name="fused_moe_dense_ref",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ override_model_config=True,
+ num_experts=8,
+ num_experts_per_tok=2,
+ base_mlp_dim=2048,
+ base_moe_mlp_dim=2048,
+ norm_topk_prob=False,
dtype="bfloat16",
sparse_matmul=False,
megablox=False,
@@ -4967,7 +5032,13 @@ def setUp(self):
[None, get_test_config_path("inference/vllm.yml")],
run_name="fused_moe_vllm",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ override_model_config=True,
+ num_experts=8,
+ num_experts_per_tok=2,
+ base_mlp_dim=2048,
+ base_moe_mlp_dim=2048,
+ norm_topk_prob=False,
dtype="bfloat16",
ici_expert_parallelism=jax.device_count(),
log_config=False,
@@ -5004,7 +5075,13 @@ def test_fused_vs_sparse_softmax(self):
[None, get_test_config_path()],
run_name="fused_moe_sparse_ref",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ override_model_config=True,
+ num_experts=8,
+ num_experts_per_tok=2,
+ base_mlp_dim=2048,
+ base_moe_mlp_dim=2048,
+ norm_topk_prob=False,
dtype="bfloat16",
sparse_matmul=True,
megablox=True,
@@ -5049,7 +5126,12 @@ def test_fused_vs_dense_renormalize(self):
[None, get_test_config_path()],
run_name="fused_moe_dense_renorm",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ override_model_config=True,
+ num_experts=8,
+ num_experts_per_tok=2,
+ base_mlp_dim=2048,
+ base_moe_mlp_dim=2048,
dtype="bfloat16",
sparse_matmul=False,
megablox=False,
@@ -5067,7 +5149,12 @@ def test_fused_vs_dense_renormalize(self):
[None, get_test_config_path("inference/vllm.yml")],
run_name="fused_moe_vllm_renorm",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ override_model_config=True,
+ num_experts=8,
+ num_experts_per_tok=2,
+ base_mlp_dim=2048,
+ base_moe_mlp_dim=2048,
dtype="bfloat16",
norm_topk_prob=True,
ici_expert_parallelism=jax.device_count(),
@@ -5099,7 +5186,13 @@ def test_prefused_vs_dense_softmax(self):
[None, get_test_config_path("inference/vllm.yml")],
run_name="fused_moe_vllm_prefused",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ override_model_config=True,
+ num_experts=8,
+ num_experts_per_tok=2,
+ base_mlp_dim=2048,
+ base_moe_mlp_dim=2048,
+ norm_topk_prob=False,
dtype="bfloat16",
ici_expert_parallelism=jax.device_count(),
log_config=False,
@@ -5131,7 +5224,13 @@ def test_prefused_vs_sparse_softmax(self):
[None, get_test_config_path()],
run_name="fused_moe_sparse_ref2",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ override_model_config=True,
+ num_experts=8,
+ num_experts_per_tok=2,
+ base_mlp_dim=2048,
+ base_moe_mlp_dim=2048,
+ norm_topk_prob=False,
dtype="bfloat16",
sparse_matmul=True,
megablox=True,
@@ -5148,7 +5247,13 @@ def test_prefused_vs_sparse_softmax(self):
[None, get_test_config_path("inference/vllm.yml")],
run_name="fused_moe_vllm_prefused2",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ override_model_config=True,
+ num_experts=8,
+ num_experts_per_tok=2,
+ base_mlp_dim=2048,
+ base_moe_mlp_dim=2048,
+ norm_topk_prob=False,
dtype="bfloat16",
ici_expert_parallelism=jax.device_count(),
prefuse_moe_weights=True,
@@ -5199,7 +5304,11 @@ def test_k_axis_only_scale_falls_back_to_dequantize(self):
[None, get_test_config_path()],
run_name="moe_native_gmm_k_axis_scale",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ override_model_config=True,
+ num_experts=8,
+ num_experts_per_tok=2,
+ norm_topk_prob=False,
dtype="bfloat16",
weight_dtype="float8_e4m3fn",
weight_block_size=block_size,
@@ -5269,7 +5378,11 @@ def test_per_tensor_scale_matches_dequantize(self):
[None, get_test_config_path()],
run_name="moe_native_gmm_per_tensor_scale",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ override_model_config=True,
+ num_experts=8,
+ num_experts_per_tok=2,
+ norm_topk_prob=False,
dtype="bfloat16",
weight_dtype="float8_e4m3fn",
weight_block_size=None,
@@ -5403,7 +5516,11 @@ def _make_config(self, run_name, **overrides):
kwargs = dict( # pylint: disable=use-dict-literal
run_name=run_name,
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ override_model_config=True,
+ num_experts=8,
+ num_experts_per_tok=2,
+ norm_topk_prob=False,
dtype="bfloat16",
weight_dtype="float8_e4m3fn",
weight_block_size=block_size,
@@ -5544,9 +5661,9 @@ def test_sparse_matmul_gmm_v2_flags_dont_crash_under_vllm_rpa(self):
@pytest.mark.parametrize(
"model_name,flag",
[
- ("mixtral-8x7b", True), # flag on: expert stays on E, peeled off the batch dim
- ("mixtral-8x22b", True), # flag on
- ("mixtral-8x7b", False), # flag off (default): batch dim keeps 'expert'
+ ("qwen3-30b-a3b", True), # flag on: expert stays on E, peeled off the batch dim
+ ("qwen3-235b-a22b", True), # flag on
+ ("qwen3-30b-a3b", False), # flag off (default): batch dim keeps 'expert'
],
)
def test_moe_dispatch_keeps_expert_on_expert_dim(model_name, flag):
@@ -5605,7 +5722,9 @@ def test_moe_dispatch_no_expert_sharding_dense_forward():
[None, get_test_config_path()],
run_name="moe_dense_no_exp_fwd",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ num_experts=8,
+ num_experts_per_tok=2,
override_model_config=True,
base_emb_dim=256,
base_mlp_dim=256,
@@ -5647,7 +5766,9 @@ def setUp(self):
[None, get_test_config_path()],
run_name="fused_mlp_moe_ref",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ num_experts=8,
+ num_experts_per_tok=2,
override_model_config=True,
base_emb_dim=512,
base_mlp_dim=256,
@@ -5673,7 +5794,9 @@ def test_prefuse_moe_weights_matches_unfused(self):
[None, get_test_config_path()],
run_name="fused_mlp_moe_fused",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ num_experts=8,
+ num_experts_per_tok=2,
override_model_config=True,
base_emb_dim=512,
base_mlp_dim=256,
@@ -5711,7 +5834,7 @@ def test_config_moe_pin_sparse_core_all_gathers(self):
[None, get_test_config_path()],
run_name="test_moe_pin_sc",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
moe_pin_sparse_core_all_gathers=True,
moe_fsdp_all_gather_sparse_core_id=0,
moe_ep_all_gather_sparse_core_id=1,
@@ -5721,12 +5844,12 @@ def test_config_moe_pin_sparse_core_all_gathers(self):
self.assertEqual(cfg.moe_ep_all_gather_sparse_core_id, 1)
def test_moe_pin_sparse_core_with_two_stage_all_gather_raises(self):
- with self.assertRaises(ValueError):
+ with self.assertRaisesRegex(ValueError, "moe_fsdp_use_two_stage_all_gather"):
pyconfig.initialize(
[None, get_test_config_path()],
run_name="test_moe_pin_sc_two_stage_error",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
moe_pin_sparse_core_all_gathers=True,
moe_fsdp_use_two_stage_all_gather=True,
)
@@ -5736,7 +5859,9 @@ def test_moe_model_init_with_pin_sparse_core(self):
[None, get_test_config_path()],
run_name="test_moe_pin_sc_model",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ num_experts=8,
+ num_experts_per_tok=2,
override_model_config=True,
base_emb_dim=256,
base_mlp_dim=256,
@@ -5763,7 +5888,9 @@ def test_moe_pin_sparse_core_with_quantize_token_all_gather_init(self):
[None, get_test_config_path()],
run_name="test_moe_pin_sc_quant_model",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ num_experts=8,
+ num_experts_per_tok=2,
override_model_config=True,
base_emb_dim=256,
base_mlp_dim=256,
@@ -5801,7 +5928,7 @@ def _make_fp8_cfg(self, prefuse_moe_weights=True, weight_block_size=None, **over
[None, get_test_config_path()],
run_name="fp8_moe_test",
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
override_model_config=True,
weight_dtype="float8_e4m3fn",
dtype="bfloat16",
@@ -6298,7 +6425,7 @@ def _make_config(self, run_name, weight_block_size=None, **overrides):
[None, get_test_config_path()],
run_name=run_name,
enable_checkpointing=False,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
override_model_config=True,
base_emb_dim=1024,
base_mlp_dim=256,
@@ -6312,7 +6439,7 @@ def _make_config(self, run_name, weight_block_size=None, **overrides):
max_target_length=128,
float32_gate_logits=True,
log_config=False,
- **overrides,
+ **{"num_experts": 8, "num_experts_per_tok": 2, **overrides},
)
def _build_model(self, cfg, mesh):
diff --git a/tests/unit/multimodal_utils_test.py b/tests/unit/multimodal_utils_test.py
index 71adc98615..3177c9cec3 100644
--- a/tests/unit/multimodal_utils_test.py
+++ b/tests/unit/multimodal_utils_test.py
@@ -418,7 +418,7 @@ def test_reformat_prompt_text_only_fallback(self):
mm_processor.reformat_prompt("Tell me a story", "
", "qwen3-4b", num_images=0), "Tell me a story"
)
self.assertEqual(
- mm_processor.reformat_prompt("Tell me a story", "
", "llama2-7b", num_images=0), "Tell me a story"
+ mm_processor.reformat_prompt("Tell me a story", "
", "llama3.1-8b", num_images=0), "Tell me a story"
)
def test_get_vision_and_decoder_block_routing(self):
@@ -439,9 +439,9 @@ def test_get_vision_and_decoder_block_routing(self):
self.assertIsNone(mm_processor._get_vision_block("qwen3-4b"))
self.assertEqual(mm_processor._get_decoder_block("qwen3-4b"), "qwen3-4b")
- # Text-only Llama 2 should have empty vision block and fallback to model_name
- self.assertIsNone(mm_processor._get_vision_block("llama2-7b"))
- self.assertEqual(mm_processor._get_decoder_block("llama2-7b"), "llama2-7b")
+ # Text-only Llama 3.1 should have empty vision block and fallback to model_name
+ self.assertIsNone(mm_processor._get_vision_block("llama3.1-8b"))
+ self.assertEqual(mm_processor._get_decoder_block("llama3.1-8b"), "llama3.1-8b")
# Ming has an audio decoder but no vision encoder.
self.assertIsNone(mm_processor._get_vision_block("ming-3.0-audio-tiny"))
@@ -506,7 +506,7 @@ def test_get_dummy_image_shape_for_init(self):
# Text-only models should return empty tuple ()
self.assertEqual(mm_processor.get_dummy_image_shape_for_init("qwen3-4b"), ())
- self.assertEqual(mm_processor.get_dummy_image_shape_for_init("llama2-7b"), ())
+ self.assertEqual(mm_processor.get_dummy_image_shape_for_init("llama3.1-8b"), ())
def test_preprocess_image_for_training(self):
dummy_image = np.zeros((224, 224, 3), dtype=np.uint8)
diff --git a/tests/unit/muon_utils_test.py b/tests/unit/muon_utils_test.py
index 51e09003e0..c860922820 100644
--- a/tests/unit/muon_utils_test.py
+++ b/tests/unit/muon_utils_test.py
@@ -740,7 +740,7 @@ class TestScannedLayoutDimensionNumbers(parameterized.TestCase):
"""Muon reduces over the same feature axes of a real model whatever its param_scan_axis."""
@parameterized.product(
- model_name=("llama2-7b", "qwen3-30b-a3b", "qwen3-next-80b-a3b", "deepseek3-test"), sharded=(False, True)
+ model_name=("llama3.1-8b", "qwen3-30b-a3b", "qwen3-next-80b-a3b", "deepseek3-test"), sharded=(False, True)
)
def test_feature_axes_match_across_scan_axes(self, model_name, sharded):
layer_major = _dimension_numbers_by_path(model_name, param_scan_axis=0, sharded=sharded)
diff --git a/tests/unit/nnx_decoders_test.py b/tests/unit/nnx_decoders_test.py
index 9c5918fb53..6099558730 100644
--- a/tests/unit/nnx_decoders_test.py
+++ b/tests/unit/nnx_decoders_test.py
@@ -379,7 +379,7 @@ def test_get_decoder_layers_returns_list(self):
def test_llama2_decoder_block(self):
- decoder = self._make_decoder(model_name="llama2-7b")
+ decoder = self._make_decoder(model_name="llama3.1-8b")
layers = decoder.get_decoder_layers()
self.assertEqual(layers, [LlamaDecoderLayer])
@@ -415,7 +415,7 @@ def test_default_returns_rms_norm(self):
def test_gpt3_returns_gpt3_layer_norm(self):
- cfg = _make_config(model_name="gpt3-52k")
+ cfg = _make_config(decoder_block=DecoderBlockType.GPT3)
mesh = _make_mesh(cfg)
decoder = NNXDecoder(config=cfg, mesh=mesh, rngs=nnx.Rngs(params=0, dropout=1))
self.assertIsInstance(decoder.decoder_norm, Gpt3LayerNorm)
diff --git a/tests/unit/optimizers_test.py b/tests/unit/optimizers_test.py
index 78815f92d8..0626f156eb 100644
--- a/tests/unit/optimizers_test.py
+++ b/tests/unit/optimizers_test.py
@@ -26,15 +26,14 @@
from maxtext.configs import pyconfig
from maxtext.optimizers import optimizers
from maxtext.optimizers.muon import ShardedMuonDimensionNumbers as smdn
-from maxtext.utils import maxtext_utils, muon_utils
+from maxtext.utils import maxtext_utils, model_creation_utils, muon_utils
from tests.utils.test_helpers import get_test_config_path
import numpy as np
import optax
from optax.contrib._muon import MuonDimensionNumbers as mdn
import pytest
-# deepseek2, specific: q_lora_rank=0
-# applicable: deepseek2-16, but not deepseek2-236b (q_lora_rank=1536)
+# MLA without Q-LoRA (q_lora_rank=0, e.g. deepseek3-test with q_lora_rank=0 override)
_DEEPSEEK2_ATTENTION = {
"self_attention": {
"kv_norm": {"scale": None},
@@ -589,13 +588,11 @@ class MuonDimensionTest(parameterized.TestCase):
"""
@parameterized.named_parameters(
- ("deepseek2-16b", "deepseek2-16b", DEEPSEEK2_DIMENSION_NUMBER),
+ ("deepseek3-test", "deepseek3-test", DEEPSEEK3_DIMENSION_NUMBER),
("deepseek3-671b", "deepseek3-671b", DEEPSEEK3_DIMENSION_NUMBER),
("deepseek4-284b", "deepseek4-284b", DEEPSEEK4_DIMENSION_NUMBER),
("deepseek4-1600b", "deepseek4-1600b", DEEPSEEK4_1600B_DIMENSION_NUMBER),
("kimi-k2-1t", "kimi-k2-1t", DEEPSEEK3_DIMENSION_NUMBER),
- ("llama2-7b", "llama2-7b", LLAMA2_DIMENSION_NUMBER),
- ("llama3-8b", "llama3-8b", LLAMA2_DIMENSION_NUMBER),
("llama3.1-8b", "llama3.1-8b", LLAMA2_DIMENSION_NUMBER),
("llama3.3-70b", "llama3.3-70b", LLAMA2_DIMENSION_NUMBER),
("gemma3-4b", "gemma3-4b", GEMMA3_DIMENSION_NUMBER),
@@ -614,6 +611,27 @@ def test_model_integration(self, model_name, expected_output):
actual_output = muon_utils.get_model_mdn(model_name, scan_layers=True, param_scan_axis=1)
self.assertEqual(actual_output, expected_output)
+ def test_deepseek_no_q_lora_mdn(self):
+ """Verifies Muon dimension numbers on the MLA q_lora_rank=0 (direct query projection) path."""
+ cfg = pyconfig.initialize(
+ [
+ None,
+ get_test_config_path(),
+ "model_name=deepseek3-test",
+ "override_model_config=True",
+ "q_lora_rank=0",
+ "scan_layers=True",
+ "param_scan_axis=1",
+ "attention=dot_product",
+ "skip_jax_distributed_system=True",
+ ]
+ )
+ devices_array = maxtext_utils.create_device_mesh(cfg)
+ mesh = jax.sharding.Mesh(devices_array, cfg.mesh_axes)
+ _, model = model_creation_utils.create_nnx_abstract_model(cfg, mesh)
+ mdn_tree = muon_utils.get_muon_weight_dimension_numbers(model, cfg)
+ self.assertEqual({"params": nnx.to_pure_dict(mdn_tree)}, DEEPSEEK2_DIMENSION_NUMBER)
+
class AdamWMaskTest(parameterized.TestCase):
"""Tests for the AdamW mask functionality"""
@@ -1172,7 +1190,7 @@ def test_muon_newton_schulz_config(self):
"run_name=test",
"opt_type=muon",
"muon_type=maxtext_muon",
- "model_name=llama2-7b",
+ "model_name=llama3.1-8b",
"muon_use_all_to_all=False",
]
config_no_a2a = pyconfig.initialize(argv_no_a2a)
@@ -1197,7 +1215,7 @@ def test_muon_newton_schulz_config(self):
get_test_config_path(),
"run_name=test",
"opt_type=muon",
- "model_name=llama2-7b",
+ "model_name=llama3.1-8b",
]
config_default = pyconfig.initialize(argv_default)
@@ -1222,7 +1240,7 @@ def test_muon_newton_schulz_config(self):
"run_name=test",
"opt_type=muon",
"muon_type=optax_muon",
- "model_name=llama2-7b",
+ "model_name=llama3.1-8b",
]
config_optax = pyconfig.initialize(argv_optax)
@@ -1247,7 +1265,7 @@ def test_muon_newton_schulz_config(self):
"run_name=test",
"opt_type=muon",
"muon_type=optax_muon",
- "model_name=llama2-7b",
+ "model_name=llama3.1-8b",
"muon_use_all_to_all=True",
]
config_optax_a2a = pyconfig.initialize(argv_optax_a2a)
@@ -1261,7 +1279,7 @@ def test_muon_newton_schulz_config(self):
"run_name=test",
"opt_type=muon",
"muon_type=invalid_type",
- "model_name=llama2-7b",
+ "model_name=llama3.1-8b",
]
config_invalid = pyconfig.initialize(argv_invalid)
with self.assertRaises(ValueError):
@@ -1290,7 +1308,7 @@ def test_maxtext_muon_wires_named_sharding_and_mesh(self):
"run_name=test",
"opt_type=muon",
"muon_type=maxtext_muon",
- "model_name=llama2-7b",
+ "model_name=llama3.1-8b",
]
config = pyconfig.initialize(argv)
diff --git a/tests/unit/pyconfig_deprecated_test.py b/tests/unit/pyconfig_deprecated_test.py
index eb2ddcff60..1f0d825bd2 100644
--- a/tests/unit/pyconfig_deprecated_test.py
+++ b/tests/unit/pyconfig_deprecated_test.py
@@ -130,13 +130,13 @@ def test_overriding_model(self):
config = pyconfig.initialize(
[os.path.join(MAXTEXT_PKG_DIR, "train.py"), get_test_config_path()],
skip_jax_distributed_system=True,
- model_name="gemma-7b",
+ model_name="gemma3-4b",
override_model_config=True,
- base_emb_dim=1024, # Defined as 3072 in gemma-7b
+ base_emb_dim=1024, # Defined as 2560 in gemma3-4b
)
self.assertEqual(config.base_emb_dim, 1024)
- self.assertEqual(config.base_mlp_dim, 24576)
+ self.assertEqual(config.base_mlp_dim, 10240)
def test_resolve_config_path(self):
self.assertEqual(resolve_config_path("foo"), os.path.join("src", "foo"))
diff --git a/tests/unit/pyconfig_test.py b/tests/unit/pyconfig_test.py
index 5f13d4b4bd..513830c323 100644
--- a/tests/unit/pyconfig_test.py
+++ b/tests/unit/pyconfig_test.py
@@ -63,7 +63,7 @@ def initialize(**kwargs):
return pyconfig.initialize(
[os.path.join(MAXTEXT_PKG_DIR, "train.py"), get_test_config_path()],
skip_jax_distributed_system=True,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
megablox=True,
sparse_matmul=True,
use_ring_of_experts=True,
@@ -354,13 +354,13 @@ def test_overriding_model(self):
config = pyconfig.initialize(
[os.path.join(MAXTEXT_PKG_DIR, "train.py"), get_test_config_path()],
skip_jax_distributed_system=True,
- model_name="gemma-7b",
+ model_name="gemma3-4b",
override_model_config=True,
- base_emb_dim=1024, # Defined as 3072 in gemma-7b
+ base_emb_dim=1024, # Defined as 2560 in gemma3-4b
)
self.assertEqual(config.base_emb_dim, 1024) # override
- self.assertEqual(config.base_mlp_dim, 24576) # unchanged
+ self.assertEqual(config.base_mlp_dim, 10240) # unchanged
def test_overriding_model_raises_error(self):
"""Test that overriding a model config with override_model_config=False raises an error."""
@@ -368,9 +368,9 @@ def test_overriding_model_raises_error(self):
pyconfig.initialize(
[os.path.join(MAXTEXT_PKG_DIR, "train.py"), get_test_config_path()],
skip_jax_distributed_system=True,
- model_name="gemma-7b",
+ model_name="gemma3-4b",
override_model_config=False,
- base_emb_dim=1024, # Defined as 3072 in gemma-7b
+ base_emb_dim=1024, # Defined as 2560 in gemma3-4b
)
def test_overriding_model_in_sft(self):
diff --git a/tests/unit/rl_profiling_config_test.py b/tests/unit/rl_profiling_config_test.py
index 2aa92f07f6..61b384c6c5 100644
--- a/tests/unit/rl_profiling_config_test.py
+++ b/tests/unit/rl_profiling_config_test.py
@@ -39,10 +39,10 @@
"base_num_kv_heads": 2,
"base_mlp_dim": 256,
"max_prefill_predict_length": 4,
- "model_name": "llama2-7b",
+ "model_name": "llama3.1-8b",
"override_model_config": True,
"weight_dtype": "bfloat16",
- "tokenizer_path": "meta-llama/Llama-2-7b",
+ "tokenizer_path": "meta-llama/Llama-3.1-8B",
}
# Step-window profiling flags from the `Profiling` mixin that `RLConfig`
@@ -125,7 +125,7 @@ def test_profiler_max_num_hosts_allows_managed_mldiagnostics(self):
)
def test_invocation_window_bounds(self, **bad):
with self.assertRaises(pydantic.ValidationError):
- types.RLConfig(model_name="llama2-7b", tokenizer_path="meta-llama/Llama-2-7b", **bad)
+ types.RLConfig(model_name="llama3.1-8b", tokenizer_path="meta-llama/Llama-3.1-8B", **bad)
@parameterized.parameters(*_LEGACY_PROFILING_FLAGS)
def test_legacy_step_profiling_flags_are_accepted_by_rl_config(self, flag):
@@ -265,8 +265,8 @@ def test_remat_policy_none_rejects_prefetch_fsdp_weights_and_pipeline_parallelis
with self.subTest(bad_kwargs=bad_kwargs):
with self.assertRaisesRegex(pydantic.ValidationError, error_regex):
types.RLConfig(
- model_name="llama2-7b",
- tokenizer_path="meta-llama/Llama-2-7b",
+ model_name="llama3.1-8b",
+ tokenizer_path="meta-llama/Llama-3.1-8B",
remat_policy="none",
**bad_kwargs,
)
diff --git a/tests/unit/router_replay_test.py b/tests/unit/router_replay_test.py
index 6b1878e00a..407035db7e 100644
--- a/tests/unit/router_replay_test.py
+++ b/tests/unit/router_replay_test.py
@@ -848,7 +848,9 @@ def test_loss_fn_with_forced_routed_experts_scanned_mixtral(self):
override_model_config=True,
base_num_decoder_layers=num_layers,
num_decoder_layers=num_layers,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ decoder_block="mixtral",
+ norm_topk_prob=False,
num_experts=num_experts,
num_experts_per_tok=top_k,
base_emb_dim=256,
@@ -1039,7 +1041,7 @@ def _weights(
decoder_block=ctypes.DecoderBlockType.MIXTRAL,
):
"""Computes top-k routing weights with forced routed experts."""
- config = DummyConfig(model_name="mixtral-8x7b", decoder_block=decoder_block)
+ config = DummyConfig(model_name="qwen3-30b-a3b", decoder_block=decoder_block)
config.norm_topk_prob = norm_topk_prob
config.routed_scaling_factor = 1.0
# Expert 2 has a large logit; it is the one -1 wraps onto.
@@ -1081,7 +1083,7 @@ class ForcedRoutingGradientTest(unittest.TestCase):
"""
def test_gate_logits_receive_gradient_under_forced_routing(self):
- config = DummyConfig(model_name="mixtral-8x7b", decoder_block=ctypes.DecoderBlockType.MIXTRAL)
+ config = DummyConfig(model_name="qwen3-30b-a3b", decoder_block=ctypes.DecoderBlockType.MIXTRAL)
config.routed_scaling_factor = 1.0
model = DummyRoutedMoE(config)
forced = jnp.array([[[1, -1], [0, 2]]])
@@ -1095,7 +1097,7 @@ def loss(gate_logits):
self.assertGreater(float(jnp.max(jnp.abs(grads))), 0.0)
def test_fully_padded_token_has_zero_gradient(self):
- config = DummyConfig(model_name="mixtral-8x7b", decoder_block=ctypes.DecoderBlockType.MIXTRAL)
+ config = DummyConfig(model_name="qwen3-30b-a3b", decoder_block=ctypes.DecoderBlockType.MIXTRAL)
config.routed_scaling_factor = 1.0
model = DummyRoutedMoE(config)
@@ -1121,7 +1123,7 @@ class OutOfRangeExpertIdTest(unittest.TestCase):
"""
def _model(self):
- config = DummyConfig(model_name="mixtral-8x7b", decoder_block=ctypes.DecoderBlockType.MIXTRAL)
+ config = DummyConfig(model_name="qwen3-30b-a3b", decoder_block=ctypes.DecoderBlockType.MIXTRAL)
config.routed_scaling_factor = 1.0
model = DummyRoutedMoE(config)
model.num_experts = 4
@@ -1312,7 +1314,10 @@ def _build_cfg(self, use_ragged_sort):
enable_checkpointing=False,
log_config=False,
skip_jax_distributed_system=True,
- model_name="mixtral-8x7b",
+ model_name="qwen3-30b-a3b",
+ num_experts=8,
+ num_experts_per_tok=2,
+ norm_topk_prob=False,
override_model_config=True,
base_emb_dim=2048, # >= 2048 so SparseCore ragged_gather_reduce satisfies num_row_partitions <= 8
base_mlp_dim=256,