diff --git a/tests/equiv_chunk_test.py b/tests/equiv_chunk_test.py index 96bfd7b27c..8728d3788c 100644 --- a/tests/equiv_chunk_test.py +++ b/tests/equiv_chunk_test.py @@ -44,8 +44,10 @@ def build_cfg(n_chunks): [None, test_helpers.get_test_config_path()], run_name=f"equiv_chunk_{n_chunks}", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", override_model_config=True, + num_experts=8, + num_experts_per_tok=2, base_emb_dim=2048, base_mlp_dim=256, base_moe_mlp_dim=256, diff --git a/tests/inference/benchmark_offline_engine.py b/tests/inference/benchmark_offline_engine.py index 9fcb942f84..e7a7da8904 100644 --- a/tests/inference/benchmark_offline_engine.py +++ b/tests/inference/benchmark_offline_engine.py @@ -56,13 +56,13 @@ def init_pyconfig(**kwargs): "max_target_length": 256, "return_log_prob": True, # <=== set this to True # Model - "model_name": "gemma2-2b", + "model_name": "gemma3-4b", "attention": "dot_product", "allow_split_physical_axes": True, "scan_layers": False, # Checkpoints "tokenizer_type": "huggingface", - "tokenizer_path": "google/gemma-2-2b-it", + "tokenizer_path": "google/gemma-3-4b-it", "hf_path": "trl-lib/tldr", # "load_parameters_path": "gs://inference-benchmarks/models/llama2-70b-chat/quant/int8_", "enable_single_controller": True, # for pathways diff --git a/tests/integration/diloco_test.py b/tests/integration/diloco_test.py index 7a48f9dafd..592e0acfc4 100644 --- a/tests/integration/diloco_test.py +++ b/tests/integration/diloco_test.py @@ -332,7 +332,7 @@ def test_diloco_two_slices(self): "ici_fsdp_parallelism=-1", "dcn_diloco_parallelism=2", "enable_diloco=true", - "model_name=gemma2-2b", + "model_name=qwen3-0.6b", "override_model_config=True", "base_emb_dim=32", "base_num_decoder_layers=1", @@ -360,7 +360,7 @@ def test_streaming_diloco_two_slices(self): "enable_diloco=true", "enable_streaming_diloco=true", "num_diloco_fragments=2", - "model_name=gemma2-2b", + "model_name=qwen3-0.6b", "override_model_config=True", "base_emb_dim=32", "base_num_decoder_layers=2", @@ -389,7 +389,7 @@ def test_streaming_diloco_two_slices_bucketized(self): "enable_streaming_diloco=true", "num_diloco_fragments=3", "diloco_bucketize_non_scanned=true", - "model_name=gemma2-2b", + "model_name=qwen3-0.6b", "override_model_config=True", "base_emb_dim=32", "base_num_decoder_layers=2", diff --git a/tests/post_training/integration/grpo_trainer_correctness_test.py b/tests/post_training/integration/grpo_trainer_correctness_test.py index 725e19a455..15edbca3dc 100644 --- a/tests/post_training/integration/grpo_trainer_correctness_test.py +++ b/tests/post_training/integration/grpo_trainer_correctness_test.py @@ -264,9 +264,9 @@ def init_pyconfig(self, config_file, **kwargs): "dataset_type": "hf", "hf_path": "trl-lib/tldr", "tokenizer_type": "huggingface", - "tokenizer_path": "google/gemma-2-2b-it", + "tokenizer_path": "google/gemma-3-4b-it", "attention": "dot_product", - "model_name": "gemma2-2b", + "model_name": "gemma3-4b", } | kwargs config = pyconfig.initialize( [ diff --git a/tests/post_training/unit/lora_utils_test.py b/tests/post_training/unit/lora_utils_test.py index a84945acaf..3fa4011205 100644 --- a/tests/post_training/unit/lora_utils_test.py +++ b/tests/post_training/unit/lora_utils_test.py @@ -55,7 +55,7 @@ "base_num_kv_heads": 2, "base_mlp_dim": 256, "max_prefill_predict_length": 4, - "model_name": "llama2-7b", + "model_name": "llama3.1-8b", "override_model_config": True, "weight_dtype": "bfloat16", } diff --git a/tests/post_training/unit/rl_lr_schedule_test.py b/tests/post_training/unit/rl_lr_schedule_test.py index 950518cde3..da3ec0c08f 100644 --- a/tests/post_training/unit/rl_lr_schedule_test.py +++ b/tests/post_training/unit/rl_lr_schedule_test.py @@ -65,7 +65,7 @@ "base_num_kv_heads": 2, "base_mlp_dim": 256, "max_prefill_predict_length": 4, - "model_name": "llama2-7b", + "model_name": "llama3.1-8b", "override_model_config": True, "weight_dtype": "bfloat16", } diff --git a/tests/unit/aqt_serve_roundtrip_nnx_test.py b/tests/unit/aqt_serve_roundtrip_nnx_test.py index 006a4580d2..da4597f006 100644 --- a/tests/unit/aqt_serve_roundtrip_nnx_test.py +++ b/tests/unit/aqt_serve_roundtrip_nnx_test.py @@ -66,14 +66,23 @@ class ServeModeRoundTripTest(unittest.TestCase): def _init_cfg(self, ckpt_path, *, checkpoint_is_quantized): """Build a pyconfig for save or reload.""" - # Use base.yml + gpt3-52k. The decoupled test config strips + # Use base.yml + qwen3-0.6b with small dimensions. The decoupled test config strips # logical_axis_rules (e.g. "norm"), which the AQT serve-mode model # construction needs. base_yml = os.path.join(MAXTEXT_PKG_DIR, "configs", "base.yml") args = [ sys.argv[0], base_yml, - "model_name=gpt3-52k", + "model_name=qwen3-0.6b", + "override_model_config=true", + "base_emb_dim=16", + "base_num_query_heads=2", + "base_num_kv_heads=2", + "base_mlp_dim=64", + "base_num_decoder_layers=1", + "head_dim=8", + "vocab_size=1024", + "attention=dot_product", "max_target_length=64", "max_prefill_predict_length=16", "per_device_batch_size=1", diff --git a/tests/unit/configs_value_test.py b/tests/unit/configs_value_test.py index b53e63ce2a..fa0a455c68 100644 --- a/tests/unit/configs_value_test.py +++ b/tests/unit/configs_value_test.py @@ -60,11 +60,11 @@ def test_type_conversion(self): def test_model_override(self): """Tests that model-specific configs override base.yml.""" - argv = ["", _BASE_CONFIG_PATH, "model_name=llama2-7b", "run_name=test"] + argv = ["", _BASE_CONFIG_PATH, "model_name=llama3.1-8b", "run_name=test"] config = pyconfig.initialize(argv) - self.assertEqual(config.base_emb_dim, 4096) # From llama2-7b.yml - self.assertEqual(config.base_num_decoder_layers, 32) # From llama2-7b.yml - self.assertEqual(config.decoder_block, types.DecoderBlockType.LLAMA2) # from llama2-7b.yml + self.assertEqual(config.base_emb_dim, 4096) # From llama3.1-8b.yml + self.assertEqual(config.base_num_decoder_layers, 32) # From llama3.1-8b.yml + self.assertEqual(config.decoder_block, types.DecoderBlockType.LLAMA2) # from llama3.1-8b.yml self.assertEqual(config.steps, 150001) # From base.yml, not overridden def test_derived_values(self): @@ -1251,7 +1251,7 @@ def test_llama3_tokenizer_correction(self): argv = [ "", _BASE_CONFIG_PATH, - "model_name=llama3-8b", + "model_name=llama3.1-8b", "tokenizer_path=assets/tokenizer_llama3.tiktoken", "run_name=test", ] @@ -1841,10 +1841,10 @@ def test_rl_config_sa_context_split_requires_custom_remat(self): "base_num_kv_heads": 2, "base_mlp_dim": 256, "max_prefill_predict_length": 4, - "model_name": "llama2-7b", + "model_name": "llama3.1-8b", "override_model_config": True, "weight_dtype": "bfloat16", - "tokenizer_path": "meta-llama/Llama-2-7b", + "tokenizer_path": "meta-llama/Llama-3.1-8B", } def rl_config(*cli_args): diff --git a/tests/unit/decoder_layer_model_mode_test.py b/tests/unit/decoder_layer_model_mode_test.py index c48fccf30e..e16815e96e 100644 --- a/tests/unit/decoder_layer_model_mode_test.py +++ b/tests/unit/decoder_layer_model_mode_test.py @@ -84,7 +84,7 @@ } _DEEPSEEK = { - "model_name": "deepseek2-16b", + "model_name": "deepseek3-test", "override_model_config": True, "mla_naive_kvcache": False, "first_num_dense_layers": 1, diff --git a/tests/unit/deepseek_decode_consistency_test.py b/tests/unit/deepseek_decode_consistency_test.py index 89f86e9f12..f20cf54780 100644 --- a/tests/unit/deepseek_decode_consistency_test.py +++ b/tests/unit/deepseek_decode_consistency_test.py @@ -71,7 +71,7 @@ } _DEEPSEEK = { - "model_name": "deepseek2-16b", + "model_name": "deepseek3-test", "override_model_config": True, "num_experts": 4, "num_experts_per_tok": 2, diff --git a/tests/unit/defer_small_all_reduces_test.py b/tests/unit/defer_small_all_reduces_test.py index 3d7e2dc7ce..d5b2119ac5 100644 --- a/tests/unit/defer_small_all_reduces_test.py +++ b/tests/unit/defer_small_all_reduces_test.py @@ -295,8 +295,10 @@ def run(defer, chunks, rbf): [None, get_test_config_path()], run_name="defer_small_ar_layer", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", override_model_config=True, + num_experts=8, + num_experts_per_tok=2, base_emb_dim=256, base_mlp_dim=128, base_moe_mlp_dim=128, diff --git a/tests/unit/flop_calculation_test.py b/tests/unit/flop_calculation_test.py index 3979474011..74e4fd912b 100644 --- a/tests/unit/flop_calculation_test.py +++ b/tests/unit/flop_calculation_test.py @@ -313,10 +313,10 @@ def test_qwen3_5_9b_flops(self): self.assertFlopsAlmostEqual(calculated_tflops, golden_tflops) - def test_llama2_7b_flops(self): - """Test Llama2 7b Flops calculation with default parameters""" + def test_llama3_3_70b_flops(self): + """Test Llama3.3 70b Flops calculation with default parameters""" cfg = self._initialize_model_config( - "llama2-7b", + "llama3.3-70b", max_target_length=2048, per_device_batch_size=12, ) @@ -324,17 +324,16 @@ def test_llama2_7b_flops(self): B = cfg.per_device_batch_size S = cfg.max_target_length attention_flops = self.compute_regular_attention_flops_per_device(kwargs) - # Llama2-7b has ~6.74B parameters - # https://adithyask.medium.com/from-7b-to-8b-parameters-understanding-weight-matrix-changes-in-llama-transformer-models-31ea7ed5fd88 - golden_param_size = 6.74e9 + # Llama3.3-70b has ~69.5B non-embedding + unembedding active parameters + golden_param_size = 69.5e9 golden_tflops = 6 * B * S * golden_param_size / 1e12 + attention_flops calculated_tflops, _, _ = calculate_tflops_training_per_device(cfg) self.assertFlopsAlmostEqual(calculated_tflops, golden_tflops) - def test_llama3_8b_flops(self): - """Test Llama3 8b Flops calculation with default parameters""" + def test_llama3_1_8b_flops(self): + """Test Llama3.1 8b Flops calculation with default parameters""" cfg = self._initialize_model_config( - "llama3-8b", + "llama3.1-8b", max_target_length=2048, per_device_batch_size=4, ) @@ -342,19 +341,16 @@ def test_llama3_8b_flops(self): B = cfg.per_device_batch_size S = cfg.max_target_length attention_flops = self.compute_regular_attention_flops_per_device(kwargs) - # LLaMA3-8b has ~8.03B parameters - # https://adithyask.medium.com/from-7b-to-8b-parameters-understanding-weight-matrix-changes-in-llama-transformer-models-31ea7ed5fd88 - # Note: The commonly cited 8.03B parameter count for Llama 3 8B corresponds to a version with UNTIED embeddings. - # Here we consider TIED embedding table, which reduces param count to 7.50B + # LLaMA3.1-8b has ~8.03B parameters (7.50B excluding input embedding lookup) golden_param_size = 7.50e9 golden_tflops = 6 * B * S * golden_param_size / 1e12 + attention_flops calculated_tflops, _, _ = calculate_tflops_training_per_device(cfg) self.assertFlopsAlmostEqual(calculated_tflops, golden_tflops) - def test_mixtral_8x7b_flops(self): - """Test Mixtral 8x7b Flops calculation""" + def test_qwen3_30b_a3b_flops(self): + """Test Qwen3 30B-A3B Flops calculation""" cfg = self._initialize_model_config( - "mixtral-8x7b", + "qwen3-30b-a3b", max_target_length=8192, per_device_batch_size=4, ) @@ -362,17 +358,16 @@ def test_mixtral_8x7b_flops(self): B = cfg.per_device_batch_size S = cfg.max_target_length attention_flops = self.compute_regular_attention_flops_per_device(kwargs) - # mixtral-8x7b has ~12.9B active parameters - # https://mistral.ai/news/mixtral-of-experts - golden_param_size = 12.9e9 + # qwen3-30b-a3b has ~3.05B active parameters + golden_param_size = 3.05e9 golden_tflops = 6 * B * S * golden_param_size / 1e12 + attention_flops calculated_tflops, _, _ = calculate_tflops_training_per_device(cfg) self.assertFlopsAlmostEqual(calculated_tflops, golden_tflops) - def test_deepseek2_16b_flops(self): - """Test DeepSeek2-16b FLops calculation""" + def test_deepseek3_test_flops(self): + """Test DeepSeek3-test FLops calculation""" cfg = self._initialize_model_config( - "deepseek2-16b", + "deepseek3-test", max_target_length=8192, per_device_batch_size=4, ) @@ -380,9 +375,27 @@ def test_deepseek2_16b_flops(self): B = cfg.per_device_batch_size S = cfg.max_target_length attention_flops = self.compute_deepseek_attention_flops_per_device(kwargs) - # deepseek2-16b has ~2.4B active parameters - # https://arxiv.org/pdf/2405.04434 - golden_param_size = 2.4e9 + # deepseek3-test has ~4.47B active parameters + golden_param_size = 4.47e9 + golden_tflops = 6 * B * S * golden_param_size / 1e12 + attention_flops + calculated_tflops, _, _ = calculate_tflops_training_per_device(cfg) + self.assertFlopsAlmostEqual(calculated_tflops, golden_tflops) + + def test_deepseek_mla_no_q_lora_flops(self): + """Test MLA FLOPs calculation when q_lora_rank == 0 (direct Q projection).""" + cfg = self._initialize_model_config( + "deepseek3-test", + max_target_length=8192, + per_device_batch_size=4, + override_model_config=True, + q_lora_rank=0, + ) + kwargs = cfg.get_keys() + B = cfg.per_device_batch_size + S = cfg.max_target_length + attention_flops = self.compute_deepseek_attention_flops_per_device(kwargs) + # deepseek3-test with q_lora_rank=0 has ~2.89B active parameters + golden_param_size = 2.89e9 golden_tflops = 6 * B * S * golden_param_size / 1e12 + attention_flops calculated_tflops, _, _ = calculate_tflops_training_per_device(cfg) self.assertFlopsAlmostEqual(calculated_tflops, golden_tflops) @@ -540,7 +553,7 @@ def test_deepseek4_1600b_flops(self): def test_custom_engram_flops(self): """Test model with Engram Flops calculation""" cfg = self._initialize_model_config( - "deepseek2-16b", + "deepseek3-test", max_target_length=8192, per_device_batch_size=4, ) @@ -556,9 +569,8 @@ def test_custom_engram_flops(self): N = cfg.engram_max_ngram_size attention_flops = self.compute_deepseek_attention_flops_per_device(kwargs) - # deepseek2-16b has ~2.4B active parameters - # https://arxiv.org/pdf/2405.04434 - golden_param_size = 2.4e9 + # deepseek3-test has ~4.47B active parameters + golden_param_size = 4.47e9 # calculate Engram active params engram_dim = H * H_D * (N - 1) @@ -1000,10 +1012,10 @@ def test_calculate_routed_and_shared_ffn_tflops_per_device(self): self.assertAlmostEqual(ffn_tflops, expected_total, places=5) def test_mtp_dense_flops(self): - """Test model with MTP modules on a Dense architecture (Llama3).""" + """Test model with MTP modules on a Dense architecture (Llama3.1).""" # Inject MTP modules during initialization cfg = self._initialize_model_config( - "llama3-8b", + "llama3.1-8b", max_target_length=2048, per_device_batch_size=4, mtp_num_layers=2, @@ -1013,9 +1025,9 @@ def test_mtp_dense_flops(self): B = cfg.per_device_batch_size S = cfg.max_target_length - # 1. Base Llama3-8b FLOPs + # 1. Base Llama3.1-8b FLOPs base_attention_flops = self.compute_regular_attention_flops_per_device(kwargs) - # LLaMA3-8b has ~7.50B active parameters with tied embeddings + # LLaMA3.1-8b has ~7.50B active parameters (excluding input embedding lookup) golden_base_param_size = 7.50e9 golden_base_tflops = 6 * B * S * golden_base_param_size / 1e12 + base_attention_flops @@ -1161,72 +1173,72 @@ def test_hy4_preview_flops(self): self.assertAlmostEqual(short_attention, expected_short_attention, delta=expected_short_attention * 1e-9) def test_mtp_moe_flops(self): - """Test model with MTP modules on an MoE architecture (DeepSeek).""" - # Inject MTP modules during initialization - cfg = self._initialize_model_config( - "deepseek2-16b", - max_target_length=8192, - per_device_batch_size=4, - mtp_num_layers=2, - ) - - kwargs = cfg.get_keys() - B = cfg.per_device_batch_size - S = cfg.max_target_length - - # 1. Base DeepSeek FLOPs - base_attention_flops = self.compute_deepseek_attention_flops_per_device(kwargs) - # deepseek2-16b has ~2.4B active parameters - golden_base_param_size = 2.4e9 - golden_base_tflops = 6 * B * S * golden_base_param_size / 1e12 + base_attention_flops - - # 2. MTP Module Active Params for MoE (DeepSeek style) - emb_dim = kwargs["base_emb_dim"] - num_experts = kwargs["num_experts"] - moe_mlp_dim = kwargs["base_moe_mlp_dim"] - shared_experts = kwargs.get("shared_experts", 1) - num_experts_per_tok = kwargs["num_experts_per_tok"] - - # FFN Params (Gate + Shared + Routed) - # SwiGLU requires 3 matrices (Gate, Up, Down) per expert - gate_params = emb_dim * num_experts - shared_params = 3 * emb_dim * moe_mlp_dim * shared_experts - routed_params = 3 * emb_dim * moe_mlp_dim * num_experts_per_tok - ffn_params = gate_params + shared_params + routed_params - - # MLA Attention Params (Q, KV, and Out Projections) - qk_nope_hd = kwargs["qk_nope_head_dim"] - qk_rope_hd = kwargs["qk_rope_head_dim"] - v_hd = kwargs["v_head_dim"] - H_q = kwargs["base_num_query_heads"] - q_lora_rank = kwargs.get("q_lora_rank", 0) - kv_lora_rank = kwargs.get("kv_lora_rank", 0) - - qk_head_dim_sum = qk_nope_hd + qk_rope_hd - if q_lora_rank == 0: - q_params = emb_dim * H_q * qk_head_dim_sum - else: - q_params = emb_dim * q_lora_rank + q_lora_rank * H_q * qk_head_dim_sum - - kv_params = emb_dim * (kv_lora_rank + qk_rope_hd) + kv_lora_rank * H_q * (qk_nope_hd + v_hd) - proj_params = emb_dim * H_q * v_hd - attn_params = q_params + kv_params + proj_params - - # Total MTP Active Params - mtp_active_params = (attn_params + ffn_params) * cfg.mtp_num_layers - mtp_weight_tflops = 6 * B * S * mtp_active_params / 1e12 - - # 3. MTP Attention FLOPs - # Causal attention flops for MLA (3 for fwd + bwd pass) - mtp_attention_tflops = 3 * cfg.mtp_num_layers * B * (S**2) * H_q * (qk_nope_hd + qk_rope_hd + v_hd) / 1e12 - - # Final Expected TFLOPs - golden_total_tflops = golden_base_tflops + mtp_weight_tflops + mtp_attention_tflops - - # Run Calculation - calculated_tflops, _, _ = calculate_tflops_training_per_device(cfg) + """Test model with MTP modules on an MoE architecture (DeepSeek, with and without Q-LoRA).""" + for q_lora_rank, golden_base_param_size in ((1536, 4.47e9), (0, 2.89e9)): + with self.subTest(q_lora_rank=q_lora_rank): + cfg = self._initialize_model_config( + "deepseek3-test", + max_target_length=8192, + per_device_batch_size=4, + mtp_num_layers=2, + override_model_config=True, + q_lora_rank=q_lora_rank, + ) - self.assertFlopsAlmostEqual(calculated_tflops, golden_total_tflops) + kwargs = cfg.get_keys() + B = cfg.per_device_batch_size + S = cfg.max_target_length + + # 1. Base DeepSeek FLOPs + base_attention_flops = self.compute_deepseek_attention_flops_per_device(kwargs) + golden_base_tflops = 6 * B * S * golden_base_param_size / 1e12 + base_attention_flops + + # 2. MTP Module Active Params for MoE (DeepSeek style) + emb_dim = kwargs["base_emb_dim"] + num_experts = kwargs["num_experts"] + moe_mlp_dim = kwargs["base_moe_mlp_dim"] + shared_experts = kwargs.get("shared_experts", 1) + num_experts_per_tok = kwargs["num_experts_per_tok"] + + # FFN Params (Gate + Shared + Routed) + # SwiGLU requires 3 matrices (Gate, Up, Down) per expert + gate_params = emb_dim * num_experts + shared_params = 3 * emb_dim * moe_mlp_dim * shared_experts + routed_params = 3 * emb_dim * moe_mlp_dim * num_experts_per_tok + ffn_params = gate_params + shared_params + routed_params + + # MLA Attention Params (Q, KV, and Out Projections) + qk_nope_hd = kwargs["qk_nope_head_dim"] + qk_rope_hd = kwargs["qk_rope_head_dim"] + v_hd = kwargs["v_head_dim"] + H_q = kwargs["base_num_query_heads"] + kv_lora_rank = kwargs.get("kv_lora_rank", 0) + + qk_head_dim_sum = qk_nope_hd + qk_rope_hd + if q_lora_rank == 0: + q_params = emb_dim * H_q * qk_head_dim_sum + else: + q_params = emb_dim * q_lora_rank + q_lora_rank * H_q * qk_head_dim_sum + + kv_params = emb_dim * (kv_lora_rank + qk_rope_hd) + kv_lora_rank * H_q * (qk_nope_hd + v_hd) + proj_params = emb_dim * H_q * v_hd + attn_params = q_params + kv_params + proj_params + + # Total MTP Active Params + mtp_active_params = (attn_params + ffn_params) * cfg.mtp_num_layers + mtp_weight_tflops = 6 * B * S * mtp_active_params / 1e12 + + # 3. MTP Attention FLOPs + # Causal attention flops for MLA (3 for fwd + bwd pass) + mtp_attention_tflops = 3 * cfg.mtp_num_layers * B * (S**2) * H_q * (qk_nope_hd + qk_rope_hd + v_hd) / 1e12 + + # Final Expected TFLOPs + golden_total_tflops = golden_base_tflops + mtp_weight_tflops + mtp_attention_tflops + + # Run Calculation + calculated_tflops, _, _ = calculate_tflops_training_per_device(cfg) + + self.assertFlopsAlmostEqual(calculated_tflops, golden_total_tflops) # ========== Parameterized Tests for Multiple Standard Models ========== @@ -1403,7 +1415,8 @@ def _verify_long_sequence_flops(self, model_name): self._verify_flops(model_name, max_target_length=8192) @parameterized.parameters( - ("llama3-8b",), + ("llama3.1-8b",), + ("llama3.3-70b",), ("llama4-17b-16e",), ("gemma3-4b",), ("gemma3-12b",), @@ -1424,7 +1437,8 @@ def test_short_sequence_flops(self, model_name): self._verify_short_sequence_flops(model_name) @parameterized.parameters( - ("llama3-8b",), + ("llama3.1-8b",), + ("llama3.3-70b",), ("llama4-17b-16e",), ("gemma3-4b",), ("gemma3-12b",), diff --git a/tests/unit/managed_mldiagnostics_test.py b/tests/unit/managed_mldiagnostics_test.py index 55be291d43..ffe1566f30 100644 --- a/tests/unit/managed_mldiagnostics_test.py +++ b/tests/unit/managed_mldiagnostics_test.py @@ -290,8 +290,8 @@ def get_keys(self): def test_pydantic_rl_config_without_get_keys(self): """The RL trainer passes pydantic `RLConfig` objects (initialize_pydantic), which have no get_keys().""" trainer = types.RLConfig( - model_name="llama2-7b", - tokenizer_path="meta-llama/Llama-2-7b", + model_name="llama3.1-8b", + tokenizer_path="meta-llama/Llama-3.1-8B", run_name="rl_run", base_output_directory="gs://bucket/out", managed_mldiagnostics=True, @@ -306,7 +306,7 @@ def test_pydantic_rl_config_without_get_keys(self): self.assertEqual(kwargs["gcs_path"], "gs://bucket/out/rl_run/managed-mldiagnostics") self.assertEqual(kwargs["region"], "us-central1") self.assertFalse(kwargs["on_demand_xprof"]) - self.assertEqual(kwargs["configs"]["model_name"], "llama2-7b") + self.assertEqual(kwargs["configs"]["model_name"], "llama3.1-8b") self.assertEqual(kwargs["configs"]["sampler.debug"], sampler.debug) self.assertNotIn("sampler.model_name", kwargs["configs"]) self.assertFalse(kwargs["metric_only_run"]) diff --git a/tests/unit/moe_test.py b/tests/unit/moe_test.py index 4477b11484..ffd65f19ef 100644 --- a/tests/unit/moe_test.py +++ b/tests/unit/moe_test.py @@ -370,7 +370,11 @@ def setUp(self): [None, get_test_config_path()], run_name="token_dropping_test", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + override_model_config=True, + num_experts=8, + num_experts_per_tok=2, + norm_topk_prob=False, dtype="bfloat16", megablox=False, sparse_matmul=False, @@ -494,7 +498,7 @@ def setUp(self): [None, get_test_config_path()], run_name="mlp_block_init_test", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", dtype="bfloat16", megablox=False, sparse_matmul=False, @@ -1252,7 +1256,9 @@ def test_megablox(self): [None, get_test_config_path()], run_name="moe_block_megablox_test", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + num_experts=8, + num_experts_per_tok=2, override_model_config=True, base_emb_dim=512, base_mlp_dim=256, @@ -1286,7 +1292,9 @@ def test_ragged_dot(self): [None, get_test_config_path()], run_name="moe_block_ragged_dot_test", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + num_experts=8, + num_experts_per_tok=2, override_model_config=True, base_emb_dim=512, base_mlp_dim=256, @@ -1318,9 +1326,9 @@ def test_ragged_dot(self): def test_te_grouped_gemm(self): """TransformerEngine grouped_dense matches dense MoE and ragged_dot. - Mixtral-style routing (8 experts, top-2, SwiGLU, no expert bias) rather than - GPT-OSS. TE v1 copies group_sizes D2H inside its FFI, so this requires one - local device; shrink Mixtral dims so the case stays a unit test on GPU. + 8-expert top-2 SwiGLU routing (no expert bias) rather than GPT-OSS. TE v1 + copies group_sizes D2H inside its FFI, so this requires one local device; + shrink Qwen3-MoE dims so the case stays a unit test on GPU. """ _skip_unless_te_single_local_device(self) @@ -1329,7 +1337,9 @@ def _build_cfg(use_te_grouped_gemm, run_name): [None, get_test_config_path()], run_name=run_name, enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + num_experts=8, + num_experts_per_tok=2, hardware="gpu", override_model_config=True, dtype="bfloat16", @@ -1444,7 +1454,9 @@ def test_dense(self): [None, get_test_config_path()], run_name="moe_block_dense_test", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + num_experts=8, + num_experts_per_tok=2, override_model_config=True, base_emb_dim=512, base_mlp_dim=256, @@ -1478,7 +1490,9 @@ def test_moe_emb_chunking_random_routing(self): [None, get_test_config_path()], run_name="moe_block_chunking_rr_test", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + num_experts=8, + num_experts_per_tok=2, override_model_config=True, base_emb_dim=1024, base_mlp_dim=256, @@ -1504,7 +1518,9 @@ def test_moe_emb_chunking_random_routing(self): [None, get_test_config_path()], run_name="moe_block_non_chunking_rr_test", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + num_experts=8, + num_experts_per_tok=2, override_model_config=True, base_emb_dim=1024, base_mlp_dim=256, @@ -1579,7 +1595,7 @@ def test_moe_emb_chunking_gmm_v2(self): [None, get_test_config_path()], run_name="moe_block_chunking_gmm_v2_test", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", dtype="bfloat16", weight_dtype="bfloat16", megablox=False, @@ -1617,7 +1633,9 @@ def test_megablox_expert_parallelism(self): [None, get_test_config_path()], run_name="moe_block_megablox_ep_test", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + num_experts=8, + num_experts_per_tok=2, override_model_config=True, base_emb_dim=512, base_mlp_dim=256, @@ -1653,7 +1671,9 @@ def test_ring_of_expert_and_tensor_parallelism(self): [None, get_test_config_path()], run_name="moe_block_ring_ep_tp_test", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + num_experts=8, + num_experts_per_tok=2, override_model_config=True, base_emb_dim=512, base_mlp_dim=256, @@ -1708,7 +1728,9 @@ def _build_cfg(use_ragged_sort: bool): [None, get_test_config_path()], run_name=(f"moe_block_use_ragged_sort_{use_ragged_sort}" f"_ring_{use_ring_of_experts}_test"), enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + num_experts=8, + num_experts_per_tok=2, override_model_config=True, # emb dim must be >= 2048 (multiple of 1024) for ragged_gather_reduce_v2 on TPU v6e SparseCore. base_emb_dim=2048, @@ -1848,7 +1870,9 @@ def _run_tc_ragged_sort_loss_and_grad( def _build_cfg(moe_tc_ragged_sort: bool): kwargs = { "enable_checkpointing": False, - "model_name": "mixtral-8x7b", + "model_name": "qwen3-30b-a3b", + "num_experts": 8, + "num_experts_per_tok": 2, "override_model_config": True, "base_emb_dim": 2048, "base_mlp_dim": 256, @@ -2018,7 +2042,9 @@ def _run_topk_before_ep_all_gather_loss_and_grad(self, **overrides): def _build_cfg(topk_before: bool): kwargs = { "enable_checkpointing": False, - "model_name": "mixtral-8x7b", + "model_name": "qwen3-30b-a3b", + "num_experts": 8, + "num_experts_per_tok": 2, "override_model_config": True, "base_emb_dim": 512, "base_mlp_dim": 256, @@ -2120,7 +2146,9 @@ def _moe_routing_maps_cfg(self, moe_routing_maps: str, **overrides): """Tiny ring-of-experts ragged-sort MoE config (EP2) for the moe_routing_maps test.""" kwargs = { "enable_checkpointing": False, - "model_name": "mixtral-8x7b", + "model_name": "qwen3-30b-a3b", + "num_experts": 8, + "num_experts_per_tok": 2, "override_model_config": True, "base_emb_dim": 512, "base_mlp_dim": 256, @@ -2531,7 +2559,9 @@ def test_ragged_sort_overflow_detection(self, ragged_buffer_factor, expect_overf [None, get_test_config_path()], run_name=f"moe_overflow_detection_{ragged_buffer_factor}", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + num_experts=8, + num_experts_per_tok=2, override_model_config=True, base_emb_dim=2048, base_mlp_dim=256, @@ -2595,11 +2625,14 @@ def test_ragged_sort_overflow_detection(self, ragged_buffer_factor, expect_overf self.assertFalse(any_overflow, "Expected has_overflow=False with a dropless (worst-case) buffer.") def _build_retry_test_mesh(self): + """Builds an EP=2 device mesh for retry unit tests.""" cfg = pyconfig.initialize( [None, get_test_config_path()], run_name="moe_retry_test_probe", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + num_experts=8, + num_experts_per_tok=2, override_model_config=True, ici_expert_parallelism=2, ) @@ -2613,14 +2646,16 @@ def _build_retry_test_model( force_dropless: bool = False, use_ring_of_experts: bool = True, ): - """Builds a mixtral-8x7b RoutedMoE with the given ragged buffer/retry settings.""" + """Builds a qwen3-30b-a3b RoutedMoE with the given ragged buffer/retry settings.""" cfg = pyconfig.initialize( [None, get_test_config_path()], run_name=( f"moe_retry_test_{ragged_buffer_factor}_{moe_dropless_fallback}_{force_dropless}_{use_ring_of_experts}" ), enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + num_experts=8, + num_experts_per_tok=2, override_model_config=True, base_emb_dim=2048, base_mlp_dim=256, @@ -2884,7 +2919,9 @@ def test_moe_fsdp_two_stage_parallelism_tpu_only(self): [None, get_test_config_path()], run_name="moe_block_megablox_ep_test", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + num_experts=8, + num_experts_per_tok=2, override_model_config=True, base_emb_dim=512, base_mlp_dim=256, @@ -2925,7 +2962,9 @@ def test_shard_embed_moe_on_fsdp(self): [None, get_test_config_path()], run_name="moe_block_shard_embed_test", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + num_experts=8, + num_experts_per_tok=2, override_model_config=True, base_emb_dim=512, base_mlp_dim=256, @@ -3004,7 +3043,9 @@ def test_megablox_context_parallelism(self): [None, get_test_config_path()], run_name="moe_block_megablox_cp_test", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + num_experts=8, + num_experts_per_tok=2, override_model_config=True, base_emb_dim=512, base_mlp_dim=256, @@ -3040,7 +3081,9 @@ def test_megablox_expert_context_parallelism(self): [None, get_test_config_path()], run_name="moe_block_megablox_ep_cp_test", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + num_experts=8, + num_experts_per_tok=2, override_model_config=True, base_emb_dim=512, base_mlp_dim=256, @@ -3078,7 +3121,9 @@ def test_megablox_expert_tensor_parallelism(self): [None, get_test_config_path()], run_name="moe_block_megablox_ep_tp_test", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + num_experts=8, + num_experts_per_tok=2, override_model_config=True, base_emb_dim=512, base_mlp_dim=256, @@ -3116,7 +3161,13 @@ def test_tp_as_ep_matches_dense_reference(self): [None, get_test_config_path()], run_name="moe_block_tp_as_ep_test", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + num_experts=8, + num_experts_per_tok=2, + override_model_config=True, + base_emb_dim=512, + base_mlp_dim=256, + base_moe_mlp_dim=256, dtype="bfloat16", sparse_matmul=True, per_device_batch_size=4, @@ -3144,13 +3195,15 @@ def test_tp_as_ep_matches_dense_reference(self): assert_moe_close(actual_output, expected_output, cfg.dtype) def _tp_as_ep_test_config(self, rule, **overrides): - """Small float32 Mixtral on an expert=2 x tensor=2 mesh, with the default rules (rule="") or tp-as-ep.""" + """Small float32 Qwen3-MoE (8-expert top-2) on an expert=2 x tensor=2 mesh, with default rules (rule="") or tp-as-ep.""" return pyconfig.initialize( [None, get_test_config_path()], **{ "run_name": f"moe_tp_as_ep_{rule or 'default'}", "enable_checkpointing": False, - "model_name": "mixtral-8x7b", + "model_name": "qwen3-30b-a3b", + "num_experts": 8, + "num_experts_per_tok": 2, "override_model_config": True, "base_emb_dim": 64, "base_mlp_dim": 64, @@ -3652,7 +3705,7 @@ def _assert_te_bf16_close(name, actual, expected, relative_norm_threshold=5e-2): @pytest.mark.gpu_only class TeGroupedGemmTest(unittest.TestCase): - """TransformerEngine grouped_dense vs jax.lax.ragged_dot on Mixtral-style MoE. + """TransformerEngine grouped_dense vs jax.lax.ragged_dot on 8-expert top-2 SwiGLU MoE. Covers paths that are not GPT-OSS-specific: mlp_bias, EP>1, ragged-sort, and mixed-dtype wo activations. EP>1 cases need one process per GPU; see @@ -3664,7 +3717,7 @@ def _te_cfg(self, use_te_grouped_gemm, expert_parallelism, mlp_bias=False, **ove [None, get_test_config_path()], run_name=f"moe_te_gmm_ep{expert_parallelism}_{int(use_te_grouped_gemm)}_{int(mlp_bias)}", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", hardware="gpu", override_model_config=True, dtype="bfloat16", @@ -3679,7 +3732,7 @@ def _te_cfg(self, use_te_grouped_gemm, expert_parallelism, mlp_bias=False, **ove base_moe_mlp_dim=64, mlp_bias=mlp_bias, float32_gate_logits=True, - **{"megablox": False, **overrides}, + **{"megablox": False, "num_experts": 8, "num_experts_per_tok": 2, **overrides}, ) def _run_te_vs_ragged_dot(self, expert_parallelism, mlp_bias=False, **overrides): @@ -4151,7 +4204,9 @@ def _build_cfg( [None, get_test_config_path()], run_name="gmm_grad_equivalence_test", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + num_experts=8, + num_experts_per_tok=2, override_model_config=True, base_emb_dim=1024, base_mlp_dim=1024, @@ -4281,7 +4336,9 @@ def _build_cfg(moe_quantize_token_all_gather: bool): [None, get_test_config_path()], run_name="moe_quantize_token_all_gather_test", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + num_experts=8, + num_experts_per_tok=2, override_model_config=True, base_emb_dim=emb_dim, base_mlp_dim=512, @@ -4412,7 +4469,9 @@ def _build_cfg(bwd_method_val: str): [None, get_test_config_path()], run_name="moe_quantize_combine_bwd_test", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + num_experts=8, + num_experts_per_tok=2, override_model_config=True, base_emb_dim=2048, base_mlp_dim=512, @@ -4949,7 +5008,13 @@ def setUp(self): [None, get_test_config_path()], run_name="fused_moe_dense_ref", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + override_model_config=True, + num_experts=8, + num_experts_per_tok=2, + base_mlp_dim=2048, + base_moe_mlp_dim=2048, + norm_topk_prob=False, dtype="bfloat16", sparse_matmul=False, megablox=False, @@ -4967,7 +5032,13 @@ def setUp(self): [None, get_test_config_path("inference/vllm.yml")], run_name="fused_moe_vllm", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + override_model_config=True, + num_experts=8, + num_experts_per_tok=2, + base_mlp_dim=2048, + base_moe_mlp_dim=2048, + norm_topk_prob=False, dtype="bfloat16", ici_expert_parallelism=jax.device_count(), log_config=False, @@ -5004,7 +5075,13 @@ def test_fused_vs_sparse_softmax(self): [None, get_test_config_path()], run_name="fused_moe_sparse_ref", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + override_model_config=True, + num_experts=8, + num_experts_per_tok=2, + base_mlp_dim=2048, + base_moe_mlp_dim=2048, + norm_topk_prob=False, dtype="bfloat16", sparse_matmul=True, megablox=True, @@ -5049,7 +5126,12 @@ def test_fused_vs_dense_renormalize(self): [None, get_test_config_path()], run_name="fused_moe_dense_renorm", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + override_model_config=True, + num_experts=8, + num_experts_per_tok=2, + base_mlp_dim=2048, + base_moe_mlp_dim=2048, dtype="bfloat16", sparse_matmul=False, megablox=False, @@ -5067,7 +5149,12 @@ def test_fused_vs_dense_renormalize(self): [None, get_test_config_path("inference/vllm.yml")], run_name="fused_moe_vllm_renorm", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + override_model_config=True, + num_experts=8, + num_experts_per_tok=2, + base_mlp_dim=2048, + base_moe_mlp_dim=2048, dtype="bfloat16", norm_topk_prob=True, ici_expert_parallelism=jax.device_count(), @@ -5099,7 +5186,13 @@ def test_prefused_vs_dense_softmax(self): [None, get_test_config_path("inference/vllm.yml")], run_name="fused_moe_vllm_prefused", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + override_model_config=True, + num_experts=8, + num_experts_per_tok=2, + base_mlp_dim=2048, + base_moe_mlp_dim=2048, + norm_topk_prob=False, dtype="bfloat16", ici_expert_parallelism=jax.device_count(), log_config=False, @@ -5131,7 +5224,13 @@ def test_prefused_vs_sparse_softmax(self): [None, get_test_config_path()], run_name="fused_moe_sparse_ref2", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + override_model_config=True, + num_experts=8, + num_experts_per_tok=2, + base_mlp_dim=2048, + base_moe_mlp_dim=2048, + norm_topk_prob=False, dtype="bfloat16", sparse_matmul=True, megablox=True, @@ -5148,7 +5247,13 @@ def test_prefused_vs_sparse_softmax(self): [None, get_test_config_path("inference/vllm.yml")], run_name="fused_moe_vllm_prefused2", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + override_model_config=True, + num_experts=8, + num_experts_per_tok=2, + base_mlp_dim=2048, + base_moe_mlp_dim=2048, + norm_topk_prob=False, dtype="bfloat16", ici_expert_parallelism=jax.device_count(), prefuse_moe_weights=True, @@ -5199,7 +5304,11 @@ def test_k_axis_only_scale_falls_back_to_dequantize(self): [None, get_test_config_path()], run_name="moe_native_gmm_k_axis_scale", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + override_model_config=True, + num_experts=8, + num_experts_per_tok=2, + norm_topk_prob=False, dtype="bfloat16", weight_dtype="float8_e4m3fn", weight_block_size=block_size, @@ -5269,7 +5378,11 @@ def test_per_tensor_scale_matches_dequantize(self): [None, get_test_config_path()], run_name="moe_native_gmm_per_tensor_scale", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + override_model_config=True, + num_experts=8, + num_experts_per_tok=2, + norm_topk_prob=False, dtype="bfloat16", weight_dtype="float8_e4m3fn", weight_block_size=None, @@ -5403,7 +5516,11 @@ def _make_config(self, run_name, **overrides): kwargs = dict( # pylint: disable=use-dict-literal run_name=run_name, enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + override_model_config=True, + num_experts=8, + num_experts_per_tok=2, + norm_topk_prob=False, dtype="bfloat16", weight_dtype="float8_e4m3fn", weight_block_size=block_size, @@ -5544,9 +5661,9 @@ def test_sparse_matmul_gmm_v2_flags_dont_crash_under_vllm_rpa(self): @pytest.mark.parametrize( "model_name,flag", [ - ("mixtral-8x7b", True), # flag on: expert stays on E, peeled off the batch dim - ("mixtral-8x22b", True), # flag on - ("mixtral-8x7b", False), # flag off (default): batch dim keeps 'expert' + ("qwen3-30b-a3b", True), # flag on: expert stays on E, peeled off the batch dim + ("qwen3-235b-a22b", True), # flag on + ("qwen3-30b-a3b", False), # flag off (default): batch dim keeps 'expert' ], ) def test_moe_dispatch_keeps_expert_on_expert_dim(model_name, flag): @@ -5605,7 +5722,9 @@ def test_moe_dispatch_no_expert_sharding_dense_forward(): [None, get_test_config_path()], run_name="moe_dense_no_exp_fwd", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + num_experts=8, + num_experts_per_tok=2, override_model_config=True, base_emb_dim=256, base_mlp_dim=256, @@ -5647,7 +5766,9 @@ def setUp(self): [None, get_test_config_path()], run_name="fused_mlp_moe_ref", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + num_experts=8, + num_experts_per_tok=2, override_model_config=True, base_emb_dim=512, base_mlp_dim=256, @@ -5673,7 +5794,9 @@ def test_prefuse_moe_weights_matches_unfused(self): [None, get_test_config_path()], run_name="fused_mlp_moe_fused", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + num_experts=8, + num_experts_per_tok=2, override_model_config=True, base_emb_dim=512, base_mlp_dim=256, @@ -5711,7 +5834,7 @@ def test_config_moe_pin_sparse_core_all_gathers(self): [None, get_test_config_path()], run_name="test_moe_pin_sc", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", moe_pin_sparse_core_all_gathers=True, moe_fsdp_all_gather_sparse_core_id=0, moe_ep_all_gather_sparse_core_id=1, @@ -5721,12 +5844,12 @@ def test_config_moe_pin_sparse_core_all_gathers(self): self.assertEqual(cfg.moe_ep_all_gather_sparse_core_id, 1) def test_moe_pin_sparse_core_with_two_stage_all_gather_raises(self): - with self.assertRaises(ValueError): + with self.assertRaisesRegex(ValueError, "moe_fsdp_use_two_stage_all_gather"): pyconfig.initialize( [None, get_test_config_path()], run_name="test_moe_pin_sc_two_stage_error", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", moe_pin_sparse_core_all_gathers=True, moe_fsdp_use_two_stage_all_gather=True, ) @@ -5736,7 +5859,9 @@ def test_moe_model_init_with_pin_sparse_core(self): [None, get_test_config_path()], run_name="test_moe_pin_sc_model", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + num_experts=8, + num_experts_per_tok=2, override_model_config=True, base_emb_dim=256, base_mlp_dim=256, @@ -5763,7 +5888,9 @@ def test_moe_pin_sparse_core_with_quantize_token_all_gather_init(self): [None, get_test_config_path()], run_name="test_moe_pin_sc_quant_model", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + num_experts=8, + num_experts_per_tok=2, override_model_config=True, base_emb_dim=256, base_mlp_dim=256, @@ -5801,7 +5928,7 @@ def _make_fp8_cfg(self, prefuse_moe_weights=True, weight_block_size=None, **over [None, get_test_config_path()], run_name="fp8_moe_test", enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", override_model_config=True, weight_dtype="float8_e4m3fn", dtype="bfloat16", @@ -6298,7 +6425,7 @@ def _make_config(self, run_name, weight_block_size=None, **overrides): [None, get_test_config_path()], run_name=run_name, enable_checkpointing=False, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", override_model_config=True, base_emb_dim=1024, base_mlp_dim=256, @@ -6312,7 +6439,7 @@ def _make_config(self, run_name, weight_block_size=None, **overrides): max_target_length=128, float32_gate_logits=True, log_config=False, - **overrides, + **{"num_experts": 8, "num_experts_per_tok": 2, **overrides}, ) def _build_model(self, cfg, mesh): diff --git a/tests/unit/multimodal_utils_test.py b/tests/unit/multimodal_utils_test.py index 71adc98615..3177c9cec3 100644 --- a/tests/unit/multimodal_utils_test.py +++ b/tests/unit/multimodal_utils_test.py @@ -418,7 +418,7 @@ def test_reformat_prompt_text_only_fallback(self): mm_processor.reformat_prompt("Tell me a story", "", "qwen3-4b", num_images=0), "Tell me a story" ) self.assertEqual( - mm_processor.reformat_prompt("Tell me a story", "", "llama2-7b", num_images=0), "Tell me a story" + mm_processor.reformat_prompt("Tell me a story", "", "llama3.1-8b", num_images=0), "Tell me a story" ) def test_get_vision_and_decoder_block_routing(self): @@ -439,9 +439,9 @@ def test_get_vision_and_decoder_block_routing(self): self.assertIsNone(mm_processor._get_vision_block("qwen3-4b")) self.assertEqual(mm_processor._get_decoder_block("qwen3-4b"), "qwen3-4b") - # Text-only Llama 2 should have empty vision block and fallback to model_name - self.assertIsNone(mm_processor._get_vision_block("llama2-7b")) - self.assertEqual(mm_processor._get_decoder_block("llama2-7b"), "llama2-7b") + # Text-only Llama 3.1 should have empty vision block and fallback to model_name + self.assertIsNone(mm_processor._get_vision_block("llama3.1-8b")) + self.assertEqual(mm_processor._get_decoder_block("llama3.1-8b"), "llama3.1-8b") # Ming has an audio decoder but no vision encoder. self.assertIsNone(mm_processor._get_vision_block("ming-3.0-audio-tiny")) @@ -506,7 +506,7 @@ def test_get_dummy_image_shape_for_init(self): # Text-only models should return empty tuple () self.assertEqual(mm_processor.get_dummy_image_shape_for_init("qwen3-4b"), ()) - self.assertEqual(mm_processor.get_dummy_image_shape_for_init("llama2-7b"), ()) + self.assertEqual(mm_processor.get_dummy_image_shape_for_init("llama3.1-8b"), ()) def test_preprocess_image_for_training(self): dummy_image = np.zeros((224, 224, 3), dtype=np.uint8) diff --git a/tests/unit/muon_utils_test.py b/tests/unit/muon_utils_test.py index 51e09003e0..c860922820 100644 --- a/tests/unit/muon_utils_test.py +++ b/tests/unit/muon_utils_test.py @@ -740,7 +740,7 @@ class TestScannedLayoutDimensionNumbers(parameterized.TestCase): """Muon reduces over the same feature axes of a real model whatever its param_scan_axis.""" @parameterized.product( - model_name=("llama2-7b", "qwen3-30b-a3b", "qwen3-next-80b-a3b", "deepseek3-test"), sharded=(False, True) + model_name=("llama3.1-8b", "qwen3-30b-a3b", "qwen3-next-80b-a3b", "deepseek3-test"), sharded=(False, True) ) def test_feature_axes_match_across_scan_axes(self, model_name, sharded): layer_major = _dimension_numbers_by_path(model_name, param_scan_axis=0, sharded=sharded) diff --git a/tests/unit/nnx_decoders_test.py b/tests/unit/nnx_decoders_test.py index 9c5918fb53..6099558730 100644 --- a/tests/unit/nnx_decoders_test.py +++ b/tests/unit/nnx_decoders_test.py @@ -379,7 +379,7 @@ def test_get_decoder_layers_returns_list(self): def test_llama2_decoder_block(self): - decoder = self._make_decoder(model_name="llama2-7b") + decoder = self._make_decoder(model_name="llama3.1-8b") layers = decoder.get_decoder_layers() self.assertEqual(layers, [LlamaDecoderLayer]) @@ -415,7 +415,7 @@ def test_default_returns_rms_norm(self): def test_gpt3_returns_gpt3_layer_norm(self): - cfg = _make_config(model_name="gpt3-52k") + cfg = _make_config(decoder_block=DecoderBlockType.GPT3) mesh = _make_mesh(cfg) decoder = NNXDecoder(config=cfg, mesh=mesh, rngs=nnx.Rngs(params=0, dropout=1)) self.assertIsInstance(decoder.decoder_norm, Gpt3LayerNorm) diff --git a/tests/unit/optimizers_test.py b/tests/unit/optimizers_test.py index 78815f92d8..0626f156eb 100644 --- a/tests/unit/optimizers_test.py +++ b/tests/unit/optimizers_test.py @@ -26,15 +26,14 @@ from maxtext.configs import pyconfig from maxtext.optimizers import optimizers from maxtext.optimizers.muon import ShardedMuonDimensionNumbers as smdn -from maxtext.utils import maxtext_utils, muon_utils +from maxtext.utils import maxtext_utils, model_creation_utils, muon_utils from tests.utils.test_helpers import get_test_config_path import numpy as np import optax from optax.contrib._muon import MuonDimensionNumbers as mdn import pytest -# deepseek2, specific: q_lora_rank=0 -# applicable: deepseek2-16, but not deepseek2-236b (q_lora_rank=1536) +# MLA without Q-LoRA (q_lora_rank=0, e.g. deepseek3-test with q_lora_rank=0 override) _DEEPSEEK2_ATTENTION = { "self_attention": { "kv_norm": {"scale": None}, @@ -589,13 +588,11 @@ class MuonDimensionTest(parameterized.TestCase): """ @parameterized.named_parameters( - ("deepseek2-16b", "deepseek2-16b", DEEPSEEK2_DIMENSION_NUMBER), + ("deepseek3-test", "deepseek3-test", DEEPSEEK3_DIMENSION_NUMBER), ("deepseek3-671b", "deepseek3-671b", DEEPSEEK3_DIMENSION_NUMBER), ("deepseek4-284b", "deepseek4-284b", DEEPSEEK4_DIMENSION_NUMBER), ("deepseek4-1600b", "deepseek4-1600b", DEEPSEEK4_1600B_DIMENSION_NUMBER), ("kimi-k2-1t", "kimi-k2-1t", DEEPSEEK3_DIMENSION_NUMBER), - ("llama2-7b", "llama2-7b", LLAMA2_DIMENSION_NUMBER), - ("llama3-8b", "llama3-8b", LLAMA2_DIMENSION_NUMBER), ("llama3.1-8b", "llama3.1-8b", LLAMA2_DIMENSION_NUMBER), ("llama3.3-70b", "llama3.3-70b", LLAMA2_DIMENSION_NUMBER), ("gemma3-4b", "gemma3-4b", GEMMA3_DIMENSION_NUMBER), @@ -614,6 +611,27 @@ def test_model_integration(self, model_name, expected_output): actual_output = muon_utils.get_model_mdn(model_name, scan_layers=True, param_scan_axis=1) self.assertEqual(actual_output, expected_output) + def test_deepseek_no_q_lora_mdn(self): + """Verifies Muon dimension numbers on the MLA q_lora_rank=0 (direct query projection) path.""" + cfg = pyconfig.initialize( + [ + None, + get_test_config_path(), + "model_name=deepseek3-test", + "override_model_config=True", + "q_lora_rank=0", + "scan_layers=True", + "param_scan_axis=1", + "attention=dot_product", + "skip_jax_distributed_system=True", + ] + ) + devices_array = maxtext_utils.create_device_mesh(cfg) + mesh = jax.sharding.Mesh(devices_array, cfg.mesh_axes) + _, model = model_creation_utils.create_nnx_abstract_model(cfg, mesh) + mdn_tree = muon_utils.get_muon_weight_dimension_numbers(model, cfg) + self.assertEqual({"params": nnx.to_pure_dict(mdn_tree)}, DEEPSEEK2_DIMENSION_NUMBER) + class AdamWMaskTest(parameterized.TestCase): """Tests for the AdamW mask functionality""" @@ -1172,7 +1190,7 @@ def test_muon_newton_schulz_config(self): "run_name=test", "opt_type=muon", "muon_type=maxtext_muon", - "model_name=llama2-7b", + "model_name=llama3.1-8b", "muon_use_all_to_all=False", ] config_no_a2a = pyconfig.initialize(argv_no_a2a) @@ -1197,7 +1215,7 @@ def test_muon_newton_schulz_config(self): get_test_config_path(), "run_name=test", "opt_type=muon", - "model_name=llama2-7b", + "model_name=llama3.1-8b", ] config_default = pyconfig.initialize(argv_default) @@ -1222,7 +1240,7 @@ def test_muon_newton_schulz_config(self): "run_name=test", "opt_type=muon", "muon_type=optax_muon", - "model_name=llama2-7b", + "model_name=llama3.1-8b", ] config_optax = pyconfig.initialize(argv_optax) @@ -1247,7 +1265,7 @@ def test_muon_newton_schulz_config(self): "run_name=test", "opt_type=muon", "muon_type=optax_muon", - "model_name=llama2-7b", + "model_name=llama3.1-8b", "muon_use_all_to_all=True", ] config_optax_a2a = pyconfig.initialize(argv_optax_a2a) @@ -1261,7 +1279,7 @@ def test_muon_newton_schulz_config(self): "run_name=test", "opt_type=muon", "muon_type=invalid_type", - "model_name=llama2-7b", + "model_name=llama3.1-8b", ] config_invalid = pyconfig.initialize(argv_invalid) with self.assertRaises(ValueError): @@ -1290,7 +1308,7 @@ def test_maxtext_muon_wires_named_sharding_and_mesh(self): "run_name=test", "opt_type=muon", "muon_type=maxtext_muon", - "model_name=llama2-7b", + "model_name=llama3.1-8b", ] config = pyconfig.initialize(argv) diff --git a/tests/unit/pyconfig_deprecated_test.py b/tests/unit/pyconfig_deprecated_test.py index eb2ddcff60..1f0d825bd2 100644 --- a/tests/unit/pyconfig_deprecated_test.py +++ b/tests/unit/pyconfig_deprecated_test.py @@ -130,13 +130,13 @@ def test_overriding_model(self): config = pyconfig.initialize( [os.path.join(MAXTEXT_PKG_DIR, "train.py"), get_test_config_path()], skip_jax_distributed_system=True, - model_name="gemma-7b", + model_name="gemma3-4b", override_model_config=True, - base_emb_dim=1024, # Defined as 3072 in gemma-7b + base_emb_dim=1024, # Defined as 2560 in gemma3-4b ) self.assertEqual(config.base_emb_dim, 1024) - self.assertEqual(config.base_mlp_dim, 24576) + self.assertEqual(config.base_mlp_dim, 10240) def test_resolve_config_path(self): self.assertEqual(resolve_config_path("foo"), os.path.join("src", "foo")) diff --git a/tests/unit/pyconfig_test.py b/tests/unit/pyconfig_test.py index 5f13d4b4bd..513830c323 100644 --- a/tests/unit/pyconfig_test.py +++ b/tests/unit/pyconfig_test.py @@ -63,7 +63,7 @@ def initialize(**kwargs): return pyconfig.initialize( [os.path.join(MAXTEXT_PKG_DIR, "train.py"), get_test_config_path()], skip_jax_distributed_system=True, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", megablox=True, sparse_matmul=True, use_ring_of_experts=True, @@ -354,13 +354,13 @@ def test_overriding_model(self): config = pyconfig.initialize( [os.path.join(MAXTEXT_PKG_DIR, "train.py"), get_test_config_path()], skip_jax_distributed_system=True, - model_name="gemma-7b", + model_name="gemma3-4b", override_model_config=True, - base_emb_dim=1024, # Defined as 3072 in gemma-7b + base_emb_dim=1024, # Defined as 2560 in gemma3-4b ) self.assertEqual(config.base_emb_dim, 1024) # override - self.assertEqual(config.base_mlp_dim, 24576) # unchanged + self.assertEqual(config.base_mlp_dim, 10240) # unchanged def test_overriding_model_raises_error(self): """Test that overriding a model config with override_model_config=False raises an error.""" @@ -368,9 +368,9 @@ def test_overriding_model_raises_error(self): pyconfig.initialize( [os.path.join(MAXTEXT_PKG_DIR, "train.py"), get_test_config_path()], skip_jax_distributed_system=True, - model_name="gemma-7b", + model_name="gemma3-4b", override_model_config=False, - base_emb_dim=1024, # Defined as 3072 in gemma-7b + base_emb_dim=1024, # Defined as 2560 in gemma3-4b ) def test_overriding_model_in_sft(self): diff --git a/tests/unit/rl_profiling_config_test.py b/tests/unit/rl_profiling_config_test.py index 2aa92f07f6..61b384c6c5 100644 --- a/tests/unit/rl_profiling_config_test.py +++ b/tests/unit/rl_profiling_config_test.py @@ -39,10 +39,10 @@ "base_num_kv_heads": 2, "base_mlp_dim": 256, "max_prefill_predict_length": 4, - "model_name": "llama2-7b", + "model_name": "llama3.1-8b", "override_model_config": True, "weight_dtype": "bfloat16", - "tokenizer_path": "meta-llama/Llama-2-7b", + "tokenizer_path": "meta-llama/Llama-3.1-8B", } # Step-window profiling flags from the `Profiling` mixin that `RLConfig` @@ -125,7 +125,7 @@ def test_profiler_max_num_hosts_allows_managed_mldiagnostics(self): ) def test_invocation_window_bounds(self, **bad): with self.assertRaises(pydantic.ValidationError): - types.RLConfig(model_name="llama2-7b", tokenizer_path="meta-llama/Llama-2-7b", **bad) + types.RLConfig(model_name="llama3.1-8b", tokenizer_path="meta-llama/Llama-3.1-8B", **bad) @parameterized.parameters(*_LEGACY_PROFILING_FLAGS) def test_legacy_step_profiling_flags_are_accepted_by_rl_config(self, flag): @@ -265,8 +265,8 @@ def test_remat_policy_none_rejects_prefetch_fsdp_weights_and_pipeline_parallelis with self.subTest(bad_kwargs=bad_kwargs): with self.assertRaisesRegex(pydantic.ValidationError, error_regex): types.RLConfig( - model_name="llama2-7b", - tokenizer_path="meta-llama/Llama-2-7b", + model_name="llama3.1-8b", + tokenizer_path="meta-llama/Llama-3.1-8B", remat_policy="none", **bad_kwargs, ) diff --git a/tests/unit/router_replay_test.py b/tests/unit/router_replay_test.py index 6b1878e00a..407035db7e 100644 --- a/tests/unit/router_replay_test.py +++ b/tests/unit/router_replay_test.py @@ -848,7 +848,9 @@ def test_loss_fn_with_forced_routed_experts_scanned_mixtral(self): override_model_config=True, base_num_decoder_layers=num_layers, num_decoder_layers=num_layers, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + decoder_block="mixtral", + norm_topk_prob=False, num_experts=num_experts, num_experts_per_tok=top_k, base_emb_dim=256, @@ -1039,7 +1041,7 @@ def _weights( decoder_block=ctypes.DecoderBlockType.MIXTRAL, ): """Computes top-k routing weights with forced routed experts.""" - config = DummyConfig(model_name="mixtral-8x7b", decoder_block=decoder_block) + config = DummyConfig(model_name="qwen3-30b-a3b", decoder_block=decoder_block) config.norm_topk_prob = norm_topk_prob config.routed_scaling_factor = 1.0 # Expert 2 has a large logit; it is the one -1 wraps onto. @@ -1081,7 +1083,7 @@ class ForcedRoutingGradientTest(unittest.TestCase): """ def test_gate_logits_receive_gradient_under_forced_routing(self): - config = DummyConfig(model_name="mixtral-8x7b", decoder_block=ctypes.DecoderBlockType.MIXTRAL) + config = DummyConfig(model_name="qwen3-30b-a3b", decoder_block=ctypes.DecoderBlockType.MIXTRAL) config.routed_scaling_factor = 1.0 model = DummyRoutedMoE(config) forced = jnp.array([[[1, -1], [0, 2]]]) @@ -1095,7 +1097,7 @@ def loss(gate_logits): self.assertGreater(float(jnp.max(jnp.abs(grads))), 0.0) def test_fully_padded_token_has_zero_gradient(self): - config = DummyConfig(model_name="mixtral-8x7b", decoder_block=ctypes.DecoderBlockType.MIXTRAL) + config = DummyConfig(model_name="qwen3-30b-a3b", decoder_block=ctypes.DecoderBlockType.MIXTRAL) config.routed_scaling_factor = 1.0 model = DummyRoutedMoE(config) @@ -1121,7 +1123,7 @@ class OutOfRangeExpertIdTest(unittest.TestCase): """ def _model(self): - config = DummyConfig(model_name="mixtral-8x7b", decoder_block=ctypes.DecoderBlockType.MIXTRAL) + config = DummyConfig(model_name="qwen3-30b-a3b", decoder_block=ctypes.DecoderBlockType.MIXTRAL) config.routed_scaling_factor = 1.0 model = DummyRoutedMoE(config) model.num_experts = 4 @@ -1312,7 +1314,10 @@ def _build_cfg(self, use_ragged_sort): enable_checkpointing=False, log_config=False, skip_jax_distributed_system=True, - model_name="mixtral-8x7b", + model_name="qwen3-30b-a3b", + num_experts=8, + num_experts_per_tok=2, + norm_topk_prob=False, override_model_config=True, base_emb_dim=2048, # >= 2048 so SparseCore ragged_gather_reduce satisfies num_row_partitions <= 8 base_mlp_dim=256,