diff --git a/tests/utils/run_sharding_dump.py b/tests/utils/run_sharding_dump.py index 7d3156fe00..48184a8354 100644 --- a/tests/utils/run_sharding_dump.py +++ b/tests/utils/run_sharding_dump.py @@ -39,7 +39,7 @@ python3 -m tests.utils.run_sharding_dump --model_name --topology --num_slice Example: - python3 -m tests.utils.run_sharding_dump --model_name gemma-7b --topology v5p-256 --num_slice 1 + python3 -m tests.utils.run_sharding_dump --model_name gemma3-4b --topology v5p-256 --num_slice 1 """ @@ -84,6 +84,7 @@ def run_single_dump(model_name: str, topology: str, num_slice: str, custom_mesh_ def main(argv: Sequence[str]) -> None: """Generate json files for every combination of model, topology, slices and rule.""" + del argv if FLAGS.model_name and FLAGS.topology and FLAGS.num_slice: cmr = FLAGS.custom_mesh_and_rule if FLAGS.custom_mesh_and_rule is not None else "" # We do not natively support overrides via CLI FLAGS. To test explicit cases, diff --git a/tests/utils/sharding_dump.py b/tests/utils/sharding_dump.py index 6989114456..2e6d54903e 100644 --- a/tests/utils/sharding_dump.py +++ b/tests/utils/sharding_dump.py @@ -38,46 +38,46 @@ TEST_CASES = [ # (model_name, topology, num_slice, custom_mesh_and_rule, sharding_strategy_overrides) - ("deepseek2-16b", "tpu7x-16", 1, "", ()), - ("deepseek2-16b", "tpu7x-16", 1, "pure-fsdp", ()), - ("deepseek2-16b", "v6e-16", 1, "", ("ici_fsdp_parallelism=-1", "ici_expert_parallelism=4")), + ("deepseek3-test", "tpu7x-16", 1, "", ()), + ("deepseek3-test", "tpu7x-16", 1, "pure-fsdp", ()), + ("deepseek3-test", "v6e-16", 1, "", ("ici_fsdp_parallelism=-1", "ici_expert_parallelism=4")), ( - "deepseek2-16b", + "deepseek3-test", "v6e-16", 1, "pipeline-large-moe", ("ici_fsdp_parallelism=-1", "ici_expert_parallelism=4", "use_ring_of_experts=true"), ), ( - "deepseek2-16b", + "deepseek3-test", "tpu7x-8", 1, "ep-as-cp", ("ici_fsdp_parallelism=-1", "ici_expert_parallelism=2"), ), ( - "deepseek2-16b", + "deepseek3-test", "tpu7x-8", 1, "cp-as-ep", ("ici_fsdp_parallelism=-1", "ici_context_parallelism=2", "ici_expert_parallelism=2"), ), ( - "deepseek2-16b", + "deepseek3-test", "v6e-16", 1, "tp-as-ep", ("ici_fsdp_parallelism=-1", "ici_expert_parallelism=4", "ici_tensor_parallelism=4"), ), ( - "deepseek2-16b", + "deepseek3-test", "tpu7x-16", 1, "ep-as-dp", ("ici_fsdp_parallelism=-1", "ici_expert_parallelism=2", "use_ring_of_experts=true"), ), ( - "deepseek2-16b", + "deepseek3-test", "tpu7x-8", 1, "shard-exp-on-fsdp", diff --git a/tests/utils/sharding_info/deepseek2-16b/tpu7x-16/slice_1/rule_default/logical_shardings.json b/tests/utils/sharding_info/deepseek2-16b/tpu7x-16/slice_1/rule_default/logical_shardings.json deleted file mode 100644 index d00fd35e3e..0000000000 --- a/tests/utils/sharding_info/deepseek2-16b/tpu7x-16/slice_1/rule_default/logical_shardings.json +++ /dev/null @@ -1,980 +0,0 @@ -{ - ".step": { - "partition_spec": [], - "shape": [] - }, - ".params/['params']/['decoder']/['decoder_norm']/['scale']": { - "partition_spec": [ - "norm" - ], - "shape": [ - 2048 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "dense_layers", - "embed" - ], - "shape": [ - 10944, - 1, - 2048 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 512, - 1 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "dense_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 1, - 128, - 2048 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 1, - 16, - 192 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 1, - 576 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "dense_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 1, - 16, - 256 - ] - }, - ".params/['params']/['decoder']/['logits_dense']/['kernel']": { - "partition_spec": [ - "embed_vocab", - "vocab" - ], - "shape": [ - 2048, - 102400 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { - "partition_spec": [ - "embed_moe", - "moe_layers", - null - ], - "shape": [ - 2048, - 26, - 64 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { - "partition_spec": [ - "exp", - "moe_layers", - "mlp_moe", - "embed_moe" - ], - "shape": [ - 64, - 26, - 1408, - 2048 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "moe_layers", - "embed" - ], - "shape": [ - 2816, - 26, - 2048 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 512, - 26 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "moe_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 26, - 128, - 2048 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 26, - 16, - 192 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 26, - 576 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "moe_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 26, - 16, - 256 - ] - }, - ".params/['params']/['token_embedder']/['embedding']": { - "partition_spec": [ - "vocab", - "embed_vocab" - ], - "shape": [ - 102400, - 2048 - ] - }, - ".opt_state/[0]/.count": { - "partition_spec": [], - "shape": [] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['decoder_norm']/['scale']": { - "partition_spec": [ - "norm" - ], - "shape": [ - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "dense_layers", - "embed" - ], - "shape": [ - 10944, - 1, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 512, - 1 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "dense_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 1, - 128, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 1, - 16, - 192 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 1, - 576 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "dense_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 1, - 16, - 256 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['logits_dense']/['kernel']": { - "partition_spec": [ - "embed_vocab", - "vocab" - ], - "shape": [ - 2048, - 102400 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { - "partition_spec": [ - "embed_moe", - "moe_layers", - null - ], - "shape": [ - 2048, - 26, - 64 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { - "partition_spec": [ - "exp", - "moe_layers", - "mlp_moe", - "embed_moe" - ], - "shape": [ - 64, - 26, - 1408, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "moe_layers", - "embed" - ], - "shape": [ - 2816, - 26, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 512, - 26 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "moe_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 26, - 128, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 26, - 16, - 192 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 26, - 576 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "moe_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 26, - 16, - 256 - ] - }, - ".opt_state/[0]/.mu/['params']/['token_embedder']/['embedding']": { - "partition_spec": [ - "vocab", - "embed_vocab" - ], - "shape": [ - 102400, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['decoder_norm']/['scale']": { - "partition_spec": [ - "norm" - ], - "shape": [ - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "dense_layers", - "embed" - ], - "shape": [ - 10944, - 1, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 512, - 1 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "dense_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 1, - 128, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 1, - 16, - 192 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 1, - 576 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "dense_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 1, - 16, - 256 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['logits_dense']/['kernel']": { - "partition_spec": [ - "embed_vocab", - "vocab" - ], - "shape": [ - 2048, - 102400 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { - "partition_spec": [ - "embed_moe", - "moe_layers", - null - ], - "shape": [ - 2048, - 26, - 64 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { - "partition_spec": [ - "exp", - "moe_layers", - "mlp_moe", - "embed_moe" - ], - "shape": [ - 64, - 26, - 1408, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "moe_layers", - "embed" - ], - "shape": [ - 2816, - 26, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 512, - 26 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "moe_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 26, - 128, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 26, - 16, - 192 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 26, - 576 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "moe_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 26, - 16, - 256 - ] - }, - ".opt_state/[0]/.nu/['params']/['token_embedder']/['embedding']": { - "partition_spec": [ - "vocab", - "embed_vocab" - ], - "shape": [ - 102400, - 2048 - ] - }, - ".opt_state/[2]/.count": { - "partition_spec": [], - "shape": [] - } -} \ No newline at end of file diff --git a/tests/utils/sharding_info/deepseek2-16b/tpu7x-16/slice_1/rule_ep-as-dp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2_use_ring_of_experts=true/logical_shardings.json b/tests/utils/sharding_info/deepseek2-16b/tpu7x-16/slice_1/rule_ep-as-dp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2_use_ring_of_experts=true/logical_shardings.json deleted file mode 100644 index d00fd35e3e..0000000000 --- a/tests/utils/sharding_info/deepseek2-16b/tpu7x-16/slice_1/rule_ep-as-dp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2_use_ring_of_experts=true/logical_shardings.json +++ /dev/null @@ -1,980 +0,0 @@ -{ - ".step": { - "partition_spec": [], - "shape": [] - }, - ".params/['params']/['decoder']/['decoder_norm']/['scale']": { - "partition_spec": [ - "norm" - ], - "shape": [ - 2048 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "dense_layers", - "embed" - ], - "shape": [ - 10944, - 1, - 2048 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 512, - 1 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "dense_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 1, - 128, - 2048 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 1, - 16, - 192 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 1, - 576 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "dense_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 1, - 16, - 256 - ] - }, - ".params/['params']/['decoder']/['logits_dense']/['kernel']": { - "partition_spec": [ - "embed_vocab", - "vocab" - ], - "shape": [ - 2048, - 102400 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { - "partition_spec": [ - "embed_moe", - "moe_layers", - null - ], - "shape": [ - 2048, - 26, - 64 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { - "partition_spec": [ - "exp", - "moe_layers", - "mlp_moe", - "embed_moe" - ], - "shape": [ - 64, - 26, - 1408, - 2048 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "moe_layers", - "embed" - ], - "shape": [ - 2816, - 26, - 2048 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 512, - 26 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "moe_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 26, - 128, - 2048 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 26, - 16, - 192 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 26, - 576 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "moe_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 26, - 16, - 256 - ] - }, - ".params/['params']/['token_embedder']/['embedding']": { - "partition_spec": [ - "vocab", - "embed_vocab" - ], - "shape": [ - 102400, - 2048 - ] - }, - ".opt_state/[0]/.count": { - "partition_spec": [], - "shape": [] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['decoder_norm']/['scale']": { - "partition_spec": [ - "norm" - ], - "shape": [ - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "dense_layers", - "embed" - ], - "shape": [ - 10944, - 1, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 512, - 1 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "dense_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 1, - 128, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 1, - 16, - 192 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 1, - 576 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "dense_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 1, - 16, - 256 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['logits_dense']/['kernel']": { - "partition_spec": [ - "embed_vocab", - "vocab" - ], - "shape": [ - 2048, - 102400 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { - "partition_spec": [ - "embed_moe", - "moe_layers", - null - ], - "shape": [ - 2048, - 26, - 64 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { - "partition_spec": [ - "exp", - "moe_layers", - "mlp_moe", - "embed_moe" - ], - "shape": [ - 64, - 26, - 1408, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "moe_layers", - "embed" - ], - "shape": [ - 2816, - 26, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 512, - 26 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "moe_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 26, - 128, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 26, - 16, - 192 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 26, - 576 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "moe_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 26, - 16, - 256 - ] - }, - ".opt_state/[0]/.mu/['params']/['token_embedder']/['embedding']": { - "partition_spec": [ - "vocab", - "embed_vocab" - ], - "shape": [ - 102400, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['decoder_norm']/['scale']": { - "partition_spec": [ - "norm" - ], - "shape": [ - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "dense_layers", - "embed" - ], - "shape": [ - 10944, - 1, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 512, - 1 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "dense_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 1, - 128, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 1, - 16, - 192 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 1, - 576 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "dense_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 1, - 16, - 256 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['logits_dense']/['kernel']": { - "partition_spec": [ - "embed_vocab", - "vocab" - ], - "shape": [ - 2048, - 102400 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { - "partition_spec": [ - "embed_moe", - "moe_layers", - null - ], - "shape": [ - 2048, - 26, - 64 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { - "partition_spec": [ - "exp", - "moe_layers", - "mlp_moe", - "embed_moe" - ], - "shape": [ - 64, - 26, - 1408, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "moe_layers", - "embed" - ], - "shape": [ - 2816, - 26, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 512, - 26 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "moe_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 26, - 128, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 26, - 16, - 192 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 26, - 576 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "moe_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 26, - 16, - 256 - ] - }, - ".opt_state/[0]/.nu/['params']/['token_embedder']/['embedding']": { - "partition_spec": [ - "vocab", - "embed_vocab" - ], - "shape": [ - 102400, - 2048 - ] - }, - ".opt_state/[2]/.count": { - "partition_spec": [], - "shape": [] - } -} \ No newline at end of file diff --git a/tests/utils/sharding_info/deepseek2-16b/tpu7x-16/slice_1/rule_pure-fsdp/logical_shardings.json b/tests/utils/sharding_info/deepseek2-16b/tpu7x-16/slice_1/rule_pure-fsdp/logical_shardings.json deleted file mode 100644 index d00fd35e3e..0000000000 --- a/tests/utils/sharding_info/deepseek2-16b/tpu7x-16/slice_1/rule_pure-fsdp/logical_shardings.json +++ /dev/null @@ -1,980 +0,0 @@ -{ - ".step": { - "partition_spec": [], - "shape": [] - }, - ".params/['params']/['decoder']/['decoder_norm']/['scale']": { - "partition_spec": [ - "norm" - ], - "shape": [ - 2048 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "dense_layers", - "embed" - ], - "shape": [ - 10944, - 1, - 2048 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 512, - 1 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "dense_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 1, - 128, - 2048 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 1, - 16, - 192 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 1, - 576 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "dense_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 1, - 16, - 256 - ] - }, - ".params/['params']/['decoder']/['logits_dense']/['kernel']": { - "partition_spec": [ - "embed_vocab", - "vocab" - ], - "shape": [ - 2048, - 102400 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { - "partition_spec": [ - "embed_moe", - "moe_layers", - null - ], - "shape": [ - 2048, - 26, - 64 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { - "partition_spec": [ - "exp", - "moe_layers", - "mlp_moe", - "embed_moe" - ], - "shape": [ - 64, - 26, - 1408, - 2048 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "moe_layers", - "embed" - ], - "shape": [ - 2816, - 26, - 2048 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 512, - 26 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "moe_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 26, - 128, - 2048 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 26, - 16, - 192 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 26, - 576 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "moe_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 26, - 16, - 256 - ] - }, - ".params/['params']/['token_embedder']/['embedding']": { - "partition_spec": [ - "vocab", - "embed_vocab" - ], - "shape": [ - 102400, - 2048 - ] - }, - ".opt_state/[0]/.count": { - "partition_spec": [], - "shape": [] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['decoder_norm']/['scale']": { - "partition_spec": [ - "norm" - ], - "shape": [ - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "dense_layers", - "embed" - ], - "shape": [ - 10944, - 1, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 512, - 1 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "dense_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 1, - 128, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 1, - 16, - 192 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 1, - 576 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "dense_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 1, - 16, - 256 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['logits_dense']/['kernel']": { - "partition_spec": [ - "embed_vocab", - "vocab" - ], - "shape": [ - 2048, - 102400 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { - "partition_spec": [ - "embed_moe", - "moe_layers", - null - ], - "shape": [ - 2048, - 26, - 64 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { - "partition_spec": [ - "exp", - "moe_layers", - "mlp_moe", - "embed_moe" - ], - "shape": [ - 64, - 26, - 1408, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "moe_layers", - "embed" - ], - "shape": [ - 2816, - 26, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 512, - 26 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "moe_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 26, - 128, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 26, - 16, - 192 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 26, - 576 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "moe_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 26, - 16, - 256 - ] - }, - ".opt_state/[0]/.mu/['params']/['token_embedder']/['embedding']": { - "partition_spec": [ - "vocab", - "embed_vocab" - ], - "shape": [ - 102400, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['decoder_norm']/['scale']": { - "partition_spec": [ - "norm" - ], - "shape": [ - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "dense_layers", - "embed" - ], - "shape": [ - 10944, - 1, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 512, - 1 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "dense_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 1, - 128, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 1, - 16, - 192 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 1, - 576 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "dense_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 1, - 16, - 256 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['logits_dense']/['kernel']": { - "partition_spec": [ - "embed_vocab", - "vocab" - ], - "shape": [ - 2048, - 102400 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { - "partition_spec": [ - "embed_moe", - "moe_layers", - null - ], - "shape": [ - 2048, - 26, - 64 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { - "partition_spec": [ - "exp", - "moe_layers", - "mlp_moe", - "embed_moe" - ], - "shape": [ - 64, - 26, - 1408, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "moe_layers", - "embed" - ], - "shape": [ - 2816, - 26, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 512, - 26 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "moe_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 26, - 128, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 26, - 16, - 192 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 26, - 576 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "moe_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 26, - 16, - 256 - ] - }, - ".opt_state/[0]/.nu/['params']/['token_embedder']/['embedding']": { - "partition_spec": [ - "vocab", - "embed_vocab" - ], - "shape": [ - 102400, - 2048 - ] - }, - ".opt_state/[2]/.count": { - "partition_spec": [], - "shape": [] - } -} \ No newline at end of file diff --git a/tests/utils/sharding_info/deepseek2-16b/tpu7x-16/slice_1/rule_pure-fsdp/named_shardings.json b/tests/utils/sharding_info/deepseek2-16b/tpu7x-16/slice_1/rule_pure-fsdp/named_shardings.json deleted file mode 100644 index c4b18d9379..0000000000 --- a/tests/utils/sharding_info/deepseek2-16b/tpu7x-16/slice_1/rule_pure-fsdp/named_shardings.json +++ /dev/null @@ -1,1652 +0,0 @@ -{ - ".step": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [], - "shape": [] - }, - ".params/['params']/['decoder']/['decoder_norm']/['scale']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null - ], - "shape": [ - 2048 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null, - null - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null, - null - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null, - "fsdp" - ], - "shape": [ - 10944, - 1, - 2048 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null - ], - "shape": [ - 2048, - 1 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null - ], - "shape": [ - 2048, - 1 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null - ], - "shape": [ - 512, - 1 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null, - null, - "fsdp" - ], - "shape": [ - 16, - 1, - 128, - 2048 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null, - null, - null - ], - "shape": [ - 2048, - 1, - 16, - 192 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null, - null - ], - "shape": [ - 2048, - 1, - 576 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null, - null, - null - ], - "shape": [ - 512, - 1, - 16, - 256 - ] - }, - ".params/['params']/['decoder']/['logits_dense']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null - ], - "shape": [ - 2048, - 102400 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null, - null - ], - "shape": [ - 2048, - 26, - 64 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null, - "fsdp", - null - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null, - "fsdp", - null - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null, - null, - "fsdp" - ], - "shape": [ - 64, - 26, - 1408, - 2048 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null, - null - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null, - null - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null, - "fsdp" - ], - "shape": [ - 2816, - 26, - 2048 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null - ], - "shape": [ - 2048, - 26 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null - ], - "shape": [ - 2048, - 26 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null - ], - "shape": [ - 512, - 26 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null, - null, - "fsdp" - ], - "shape": [ - 16, - 26, - 128, - 2048 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null, - null, - null - ], - "shape": [ - 2048, - 26, - 16, - 192 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null, - null - ], - "shape": [ - 2048, - 26, - 576 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null, - null, - null - ], - "shape": [ - 512, - 26, - 16, - 256 - ] - }, - ".params/['params']/['token_embedder']/['embedding']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - "fsdp" - ], - "shape": [ - 102400, - 2048 - ] - }, - ".opt_state/[0]/.count": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [], - "shape": [] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['decoder_norm']/['scale']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null - ], - "shape": [ - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null, - null - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null, - null - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null, - "fsdp" - ], - "shape": [ - 10944, - 1, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null - ], - "shape": [ - 512, - 1 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null, - null, - "fsdp" - ], - "shape": [ - 16, - 1, - 128, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null, - null, - null - ], - "shape": [ - 2048, - 1, - 16, - 192 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null, - null - ], - "shape": [ - 2048, - 1, - 576 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null, - null, - null - ], - "shape": [ - 512, - 1, - 16, - 256 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['logits_dense']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null - ], - "shape": [ - 2048, - 102400 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null, - null - ], - "shape": [ - 2048, - 26, - 64 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null, - "fsdp", - null - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null, - "fsdp", - null - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null, - null, - "fsdp" - ], - "shape": [ - 64, - 26, - 1408, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null, - null - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null, - null - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null, - "fsdp" - ], - "shape": [ - 2816, - 26, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null - ], - "shape": [ - 512, - 26 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null, - null, - "fsdp" - ], - "shape": [ - 16, - 26, - 128, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null, - null, - null - ], - "shape": [ - 2048, - 26, - 16, - 192 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null, - null - ], - "shape": [ - 2048, - 26, - 576 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null, - null, - null - ], - "shape": [ - 512, - 26, - 16, - 256 - ] - }, - ".opt_state/[0]/.mu/['params']/['token_embedder']/['embedding']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - "fsdp" - ], - "shape": [ - 102400, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['decoder_norm']/['scale']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null - ], - "shape": [ - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null, - null - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null, - null - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null, - "fsdp" - ], - "shape": [ - 10944, - 1, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null - ], - "shape": [ - 512, - 1 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null, - null, - "fsdp" - ], - "shape": [ - 16, - 1, - 128, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null, - null, - null - ], - "shape": [ - 2048, - 1, - 16, - 192 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null, - null - ], - "shape": [ - 2048, - 1, - 576 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null, - null, - null - ], - "shape": [ - 512, - 1, - 16, - 256 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['logits_dense']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null - ], - "shape": [ - 2048, - 102400 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null, - null - ], - "shape": [ - 2048, - 26, - 64 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null, - "fsdp", - null - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null, - "fsdp", - null - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null, - null, - "fsdp" - ], - "shape": [ - 64, - 26, - 1408, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null, - null - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null, - null - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null, - "fsdp" - ], - "shape": [ - 2816, - 26, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null - ], - "shape": [ - 512, - 26 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - null, - null, - "fsdp" - ], - "shape": [ - 16, - 26, - 128, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null, - null, - null - ], - "shape": [ - 2048, - 26, - 16, - 192 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null, - null - ], - "shape": [ - 2048, - 26, - 576 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - "fsdp", - null, - null, - null - ], - "shape": [ - 512, - 26, - 16, - 256 - ] - }, - ".opt_state/[0]/.nu/['params']/['token_embedder']/['embedding']": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [ - null, - "fsdp" - ], - "shape": [ - 102400, - 2048 - ] - }, - ".opt_state/[2]/.count": { - "mesh": { - "axis_names": [ - "fsdp" - ], - "shape": { - "fsdp": 16 - } - }, - "partition_spec": [], - "shape": [] - } -} \ No newline at end of file diff --git a/tests/utils/sharding_info/deepseek2-16b/tpu7x-8/slice_1/rule_cp-as-ep_ici_fsdp_parallelism=-1_ici_context_parallelism=2_ici_expert_parallelism=2/logical_shardings.json b/tests/utils/sharding_info/deepseek2-16b/tpu7x-8/slice_1/rule_cp-as-ep_ici_fsdp_parallelism=-1_ici_context_parallelism=2_ici_expert_parallelism=2/logical_shardings.json deleted file mode 100644 index d00fd35e3e..0000000000 --- a/tests/utils/sharding_info/deepseek2-16b/tpu7x-8/slice_1/rule_cp-as-ep_ici_fsdp_parallelism=-1_ici_context_parallelism=2_ici_expert_parallelism=2/logical_shardings.json +++ /dev/null @@ -1,980 +0,0 @@ -{ - ".step": { - "partition_spec": [], - "shape": [] - }, - ".params/['params']/['decoder']/['decoder_norm']/['scale']": { - "partition_spec": [ - "norm" - ], - "shape": [ - 2048 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "dense_layers", - "embed" - ], - "shape": [ - 10944, - 1, - 2048 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 512, - 1 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "dense_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 1, - 128, - 2048 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 1, - 16, - 192 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 1, - 576 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "dense_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 1, - 16, - 256 - ] - }, - ".params/['params']/['decoder']/['logits_dense']/['kernel']": { - "partition_spec": [ - "embed_vocab", - "vocab" - ], - "shape": [ - 2048, - 102400 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { - "partition_spec": [ - "embed_moe", - "moe_layers", - null - ], - "shape": [ - 2048, - 26, - 64 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { - "partition_spec": [ - "exp", - "moe_layers", - "mlp_moe", - "embed_moe" - ], - "shape": [ - 64, - 26, - 1408, - 2048 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "moe_layers", - "embed" - ], - "shape": [ - 2816, - 26, - 2048 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 512, - 26 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "moe_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 26, - 128, - 2048 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 26, - 16, - 192 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 26, - 576 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "moe_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 26, - 16, - 256 - ] - }, - ".params/['params']/['token_embedder']/['embedding']": { - "partition_spec": [ - "vocab", - "embed_vocab" - ], - "shape": [ - 102400, - 2048 - ] - }, - ".opt_state/[0]/.count": { - "partition_spec": [], - "shape": [] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['decoder_norm']/['scale']": { - "partition_spec": [ - "norm" - ], - "shape": [ - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "dense_layers", - "embed" - ], - "shape": [ - 10944, - 1, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 512, - 1 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "dense_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 1, - 128, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 1, - 16, - 192 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 1, - 576 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "dense_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 1, - 16, - 256 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['logits_dense']/['kernel']": { - "partition_spec": [ - "embed_vocab", - "vocab" - ], - "shape": [ - 2048, - 102400 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { - "partition_spec": [ - "embed_moe", - "moe_layers", - null - ], - "shape": [ - 2048, - 26, - 64 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { - "partition_spec": [ - "exp", - "moe_layers", - "mlp_moe", - "embed_moe" - ], - "shape": [ - 64, - 26, - 1408, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "moe_layers", - "embed" - ], - "shape": [ - 2816, - 26, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 512, - 26 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "moe_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 26, - 128, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 26, - 16, - 192 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 26, - 576 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "moe_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 26, - 16, - 256 - ] - }, - ".opt_state/[0]/.mu/['params']/['token_embedder']/['embedding']": { - "partition_spec": [ - "vocab", - "embed_vocab" - ], - "shape": [ - 102400, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['decoder_norm']/['scale']": { - "partition_spec": [ - "norm" - ], - "shape": [ - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "dense_layers", - "embed" - ], - "shape": [ - 10944, - 1, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 512, - 1 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "dense_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 1, - 128, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 1, - 16, - 192 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 1, - 576 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "dense_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 1, - 16, - 256 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['logits_dense']/['kernel']": { - "partition_spec": [ - "embed_vocab", - "vocab" - ], - "shape": [ - 2048, - 102400 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { - "partition_spec": [ - "embed_moe", - "moe_layers", - null - ], - "shape": [ - 2048, - 26, - 64 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { - "partition_spec": [ - "exp", - "moe_layers", - "mlp_moe", - "embed_moe" - ], - "shape": [ - 64, - 26, - 1408, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "moe_layers", - "embed" - ], - "shape": [ - 2816, - 26, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 512, - 26 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "moe_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 26, - 128, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 26, - 16, - 192 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 26, - 576 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "moe_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 26, - 16, - 256 - ] - }, - ".opt_state/[0]/.nu/['params']/['token_embedder']/['embedding']": { - "partition_spec": [ - "vocab", - "embed_vocab" - ], - "shape": [ - 102400, - 2048 - ] - }, - ".opt_state/[2]/.count": { - "partition_spec": [], - "shape": [] - } -} \ No newline at end of file diff --git a/tests/utils/sharding_info/deepseek2-16b/tpu7x-8/slice_1/rule_ep-as-cp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/logical_shardings.json b/tests/utils/sharding_info/deepseek2-16b/tpu7x-8/slice_1/rule_ep-as-cp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/logical_shardings.json deleted file mode 100644 index d00fd35e3e..0000000000 --- a/tests/utils/sharding_info/deepseek2-16b/tpu7x-8/slice_1/rule_ep-as-cp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/logical_shardings.json +++ /dev/null @@ -1,980 +0,0 @@ -{ - ".step": { - "partition_spec": [], - "shape": [] - }, - ".params/['params']/['decoder']/['decoder_norm']/['scale']": { - "partition_spec": [ - "norm" - ], - "shape": [ - 2048 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "dense_layers", - "embed" - ], - "shape": [ - 10944, - 1, - 2048 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 512, - 1 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "dense_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 1, - 128, - 2048 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 1, - 16, - 192 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 1, - 576 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "dense_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 1, - 16, - 256 - ] - }, - ".params/['params']/['decoder']/['logits_dense']/['kernel']": { - "partition_spec": [ - "embed_vocab", - "vocab" - ], - "shape": [ - 2048, - 102400 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { - "partition_spec": [ - "embed_moe", - "moe_layers", - null - ], - "shape": [ - 2048, - 26, - 64 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { - "partition_spec": [ - "exp", - "moe_layers", - "mlp_moe", - "embed_moe" - ], - "shape": [ - 64, - 26, - 1408, - 2048 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "moe_layers", - "embed" - ], - "shape": [ - 2816, - 26, - 2048 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 512, - 26 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "moe_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 26, - 128, - 2048 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 26, - 16, - 192 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 26, - 576 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "moe_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 26, - 16, - 256 - ] - }, - ".params/['params']/['token_embedder']/['embedding']": { - "partition_spec": [ - "vocab", - "embed_vocab" - ], - "shape": [ - 102400, - 2048 - ] - }, - ".opt_state/[0]/.count": { - "partition_spec": [], - "shape": [] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['decoder_norm']/['scale']": { - "partition_spec": [ - "norm" - ], - "shape": [ - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "dense_layers", - "embed" - ], - "shape": [ - 10944, - 1, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 512, - 1 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "dense_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 1, - 128, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 1, - 16, - 192 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 1, - 576 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "dense_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 1, - 16, - 256 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['logits_dense']/['kernel']": { - "partition_spec": [ - "embed_vocab", - "vocab" - ], - "shape": [ - 2048, - 102400 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { - "partition_spec": [ - "embed_moe", - "moe_layers", - null - ], - "shape": [ - 2048, - 26, - 64 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { - "partition_spec": [ - "exp", - "moe_layers", - "mlp_moe", - "embed_moe" - ], - "shape": [ - 64, - 26, - 1408, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "moe_layers", - "embed" - ], - "shape": [ - 2816, - 26, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 512, - 26 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "moe_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 26, - 128, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 26, - 16, - 192 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 26, - 576 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "moe_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 26, - 16, - 256 - ] - }, - ".opt_state/[0]/.mu/['params']/['token_embedder']/['embedding']": { - "partition_spec": [ - "vocab", - "embed_vocab" - ], - "shape": [ - 102400, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['decoder_norm']/['scale']": { - "partition_spec": [ - "norm" - ], - "shape": [ - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "dense_layers", - "embed" - ], - "shape": [ - 10944, - 1, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 512, - 1 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "dense_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 1, - 128, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 1, - 16, - 192 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 1, - 576 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "dense_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 1, - 16, - 256 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['logits_dense']/['kernel']": { - "partition_spec": [ - "embed_vocab", - "vocab" - ], - "shape": [ - 2048, - 102400 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { - "partition_spec": [ - "embed_moe", - "moe_layers", - null - ], - "shape": [ - 2048, - 26, - 64 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { - "partition_spec": [ - "exp", - "moe_layers", - "mlp_moe", - "embed_moe" - ], - "shape": [ - 64, - 26, - 1408, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "moe_layers", - "embed" - ], - "shape": [ - 2816, - 26, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 512, - 26 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "moe_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 26, - 128, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 26, - 16, - 192 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 26, - 576 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "moe_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 26, - 16, - 256 - ] - }, - ".opt_state/[0]/.nu/['params']/['token_embedder']/['embedding']": { - "partition_spec": [ - "vocab", - "embed_vocab" - ], - "shape": [ - 102400, - 2048 - ] - }, - ".opt_state/[2]/.count": { - "partition_spec": [], - "shape": [] - } -} \ No newline at end of file diff --git a/tests/utils/sharding_info/deepseek2-16b/tpu7x-8/slice_1/rule_shard-exp-on-fsdp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/logical_shardings.json b/tests/utils/sharding_info/deepseek2-16b/tpu7x-8/slice_1/rule_shard-exp-on-fsdp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/logical_shardings.json deleted file mode 100644 index d00fd35e3e..0000000000 --- a/tests/utils/sharding_info/deepseek2-16b/tpu7x-8/slice_1/rule_shard-exp-on-fsdp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/logical_shardings.json +++ /dev/null @@ -1,980 +0,0 @@ -{ - ".step": { - "partition_spec": [], - "shape": [] - }, - ".params/['params']/['decoder']/['decoder_norm']/['scale']": { - "partition_spec": [ - "norm" - ], - "shape": [ - 2048 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "dense_layers", - "embed" - ], - "shape": [ - 10944, - 1, - 2048 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 512, - 1 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "dense_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 1, - 128, - 2048 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 1, - 16, - 192 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 1, - 576 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "dense_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 1, - 16, - 256 - ] - }, - ".params/['params']/['decoder']/['logits_dense']/['kernel']": { - "partition_spec": [ - "embed_vocab", - "vocab" - ], - "shape": [ - 2048, - 102400 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { - "partition_spec": [ - "embed_moe", - "moe_layers", - null - ], - "shape": [ - 2048, - 26, - 64 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { - "partition_spec": [ - "exp", - "moe_layers", - "mlp_moe", - "embed_moe" - ], - "shape": [ - 64, - 26, - 1408, - 2048 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "moe_layers", - "embed" - ], - "shape": [ - 2816, - 26, - 2048 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 512, - 26 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "moe_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 26, - 128, - 2048 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 26, - 16, - 192 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 26, - 576 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "moe_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 26, - 16, - 256 - ] - }, - ".params/['params']/['token_embedder']/['embedding']": { - "partition_spec": [ - "vocab", - "embed_vocab" - ], - "shape": [ - 102400, - 2048 - ] - }, - ".opt_state/[0]/.count": { - "partition_spec": [], - "shape": [] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['decoder_norm']/['scale']": { - "partition_spec": [ - "norm" - ], - "shape": [ - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "dense_layers", - "embed" - ], - "shape": [ - 10944, - 1, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 512, - 1 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "dense_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 1, - 128, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 1, - 16, - 192 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 1, - 576 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "dense_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 1, - 16, - 256 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['logits_dense']/['kernel']": { - "partition_spec": [ - "embed_vocab", - "vocab" - ], - "shape": [ - 2048, - 102400 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { - "partition_spec": [ - "embed_moe", - "moe_layers", - null - ], - "shape": [ - 2048, - 26, - 64 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { - "partition_spec": [ - "exp", - "moe_layers", - "mlp_moe", - "embed_moe" - ], - "shape": [ - 64, - 26, - 1408, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "moe_layers", - "embed" - ], - "shape": [ - 2816, - 26, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 512, - 26 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "moe_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 26, - 128, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 26, - 16, - 192 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 26, - 576 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "moe_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 26, - 16, - 256 - ] - }, - ".opt_state/[0]/.mu/['params']/['token_embedder']/['embedding']": { - "partition_spec": [ - "vocab", - "embed_vocab" - ], - "shape": [ - 102400, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['decoder_norm']/['scale']": { - "partition_spec": [ - "norm" - ], - "shape": [ - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "dense_layers", - "embed" - ], - "shape": [ - 10944, - 1, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 512, - 1 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "dense_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 1, - 128, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 1, - 16, - 192 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 1, - 576 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "dense_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 1, - 16, - 256 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['logits_dense']/['kernel']": { - "partition_spec": [ - "embed_vocab", - "vocab" - ], - "shape": [ - 2048, - 102400 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { - "partition_spec": [ - "embed_moe", - "moe_layers", - null - ], - "shape": [ - 2048, - 26, - 64 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { - "partition_spec": [ - "exp", - "moe_layers", - "mlp_moe", - "embed_moe" - ], - "shape": [ - 64, - 26, - 1408, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "moe_layers", - "embed" - ], - "shape": [ - 2816, - 26, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 512, - 26 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "moe_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 26, - 128, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 26, - 16, - 192 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 26, - 576 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "moe_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 26, - 16, - 256 - ] - }, - ".opt_state/[0]/.nu/['params']/['token_embedder']/['embedding']": { - "partition_spec": [ - "vocab", - "embed_vocab" - ], - "shape": [ - 102400, - 2048 - ] - }, - ".opt_state/[2]/.count": { - "partition_spec": [], - "shape": [] - } -} \ No newline at end of file diff --git a/tests/utils/sharding_info/deepseek2-16b/v6e-16/slice_1/rule_default_ici_fsdp_parallelism=-1_ici_expert_parallelism=4/logical_shardings.json b/tests/utils/sharding_info/deepseek2-16b/v6e-16/slice_1/rule_default_ici_fsdp_parallelism=-1_ici_expert_parallelism=4/logical_shardings.json deleted file mode 100644 index d00fd35e3e..0000000000 --- a/tests/utils/sharding_info/deepseek2-16b/v6e-16/slice_1/rule_default_ici_fsdp_parallelism=-1_ici_expert_parallelism=4/logical_shardings.json +++ /dev/null @@ -1,980 +0,0 @@ -{ - ".step": { - "partition_spec": [], - "shape": [] - }, - ".params/['params']/['decoder']/['decoder_norm']/['scale']": { - "partition_spec": [ - "norm" - ], - "shape": [ - 2048 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "dense_layers", - "embed" - ], - "shape": [ - 10944, - 1, - 2048 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 512, - 1 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "dense_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 1, - 128, - 2048 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 1, - 16, - 192 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 1, - 576 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "dense_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 1, - 16, - 256 - ] - }, - ".params/['params']/['decoder']/['logits_dense']/['kernel']": { - "partition_spec": [ - "embed_vocab", - "vocab" - ], - "shape": [ - 2048, - 102400 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { - "partition_spec": [ - "embed_moe", - "moe_layers", - null - ], - "shape": [ - 2048, - 26, - 64 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { - "partition_spec": [ - "exp", - "moe_layers", - "mlp_moe", - "embed_moe" - ], - "shape": [ - 64, - 26, - 1408, - 2048 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "moe_layers", - "embed" - ], - "shape": [ - 2816, - 26, - 2048 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 512, - 26 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "moe_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 26, - 128, - 2048 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 26, - 16, - 192 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 26, - 576 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "moe_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 26, - 16, - 256 - ] - }, - ".params/['params']/['token_embedder']/['embedding']": { - "partition_spec": [ - "vocab", - "embed_vocab" - ], - "shape": [ - 102400, - 2048 - ] - }, - ".opt_state/[0]/.count": { - "partition_spec": [], - "shape": [] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['decoder_norm']/['scale']": { - "partition_spec": [ - "norm" - ], - "shape": [ - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "dense_layers", - "embed" - ], - "shape": [ - 10944, - 1, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 512, - 1 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "dense_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 1, - 128, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 1, - 16, - 192 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 1, - 576 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "dense_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 1, - 16, - 256 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['logits_dense']/['kernel']": { - "partition_spec": [ - "embed_vocab", - "vocab" - ], - "shape": [ - 2048, - 102400 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { - "partition_spec": [ - "embed_moe", - "moe_layers", - null - ], - "shape": [ - 2048, - 26, - 64 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { - "partition_spec": [ - "exp", - "moe_layers", - "mlp_moe", - "embed_moe" - ], - "shape": [ - 64, - 26, - 1408, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "moe_layers", - "embed" - ], - "shape": [ - 2816, - 26, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 512, - 26 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "moe_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 26, - 128, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 26, - 16, - 192 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 26, - 576 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "moe_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 26, - 16, - 256 - ] - }, - ".opt_state/[0]/.mu/['params']/['token_embedder']/['embedding']": { - "partition_spec": [ - "vocab", - "embed_vocab" - ], - "shape": [ - 102400, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['decoder_norm']/['scale']": { - "partition_spec": [ - "norm" - ], - "shape": [ - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "dense_layers", - "embed" - ], - "shape": [ - 10944, - 1, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 512, - 1 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "dense_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 1, - 128, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 1, - 16, - 192 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 1, - 576 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "dense_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 1, - 16, - 256 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['logits_dense']/['kernel']": { - "partition_spec": [ - "embed_vocab", - "vocab" - ], - "shape": [ - 2048, - 102400 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { - "partition_spec": [ - "embed_moe", - "moe_layers", - null - ], - "shape": [ - 2048, - 26, - 64 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { - "partition_spec": [ - "exp", - "moe_layers", - "mlp_moe", - "embed_moe" - ], - "shape": [ - 64, - 26, - 1408, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "moe_layers", - "embed" - ], - "shape": [ - 2816, - 26, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 512, - 26 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "moe_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 26, - 128, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 26, - 16, - 192 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 26, - 576 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "moe_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 26, - 16, - 256 - ] - }, - ".opt_state/[0]/.nu/['params']/['token_embedder']/['embedding']": { - "partition_spec": [ - "vocab", - "embed_vocab" - ], - "shape": [ - 102400, - 2048 - ] - }, - ".opt_state/[2]/.count": { - "partition_spec": [], - "shape": [] - } -} \ No newline at end of file diff --git a/tests/utils/sharding_info/deepseek2-16b/v6e-16/slice_1/rule_pipeline-large-moe_ici_fsdp_parallelism=-1_ici_expert_parallelism=4_use_ring_of_experts=true/logical_shardings.json b/tests/utils/sharding_info/deepseek2-16b/v6e-16/slice_1/rule_pipeline-large-moe_ici_fsdp_parallelism=-1_ici_expert_parallelism=4_use_ring_of_experts=true/logical_shardings.json deleted file mode 100644 index d00fd35e3e..0000000000 --- a/tests/utils/sharding_info/deepseek2-16b/v6e-16/slice_1/rule_pipeline-large-moe_ici_fsdp_parallelism=-1_ici_expert_parallelism=4_use_ring_of_experts=true/logical_shardings.json +++ /dev/null @@ -1,980 +0,0 @@ -{ - ".step": { - "partition_spec": [], - "shape": [] - }, - ".params/['params']/['decoder']/['decoder_norm']/['scale']": { - "partition_spec": [ - "norm" - ], - "shape": [ - 2048 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "dense_layers", - "embed" - ], - "shape": [ - 10944, - 1, - 2048 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 512, - 1 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "dense_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 1, - 128, - 2048 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 1, - 16, - 192 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 1, - 576 - ] - }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "dense_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 1, - 16, - 256 - ] - }, - ".params/['params']/['decoder']/['logits_dense']/['kernel']": { - "partition_spec": [ - "embed_vocab", - "vocab" - ], - "shape": [ - 2048, - 102400 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { - "partition_spec": [ - "embed_moe", - "moe_layers", - null - ], - "shape": [ - 2048, - 26, - 64 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { - "partition_spec": [ - "exp", - "moe_layers", - "mlp_moe", - "embed_moe" - ], - "shape": [ - 64, - 26, - 1408, - 2048 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "moe_layers", - "embed" - ], - "shape": [ - 2816, - 26, - 2048 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 512, - 26 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "moe_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 26, - 128, - 2048 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 26, - 16, - 192 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 26, - 576 - ] - }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "moe_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 26, - 16, - 256 - ] - }, - ".params/['params']/['token_embedder']/['embedding']": { - "partition_spec": [ - "vocab", - "embed_vocab" - ], - "shape": [ - 102400, - 2048 - ] - }, - ".opt_state/[0]/.count": { - "partition_spec": [], - "shape": [] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['decoder_norm']/['scale']": { - "partition_spec": [ - "norm" - ], - "shape": [ - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "dense_layers", - "embed" - ], - "shape": [ - 10944, - 1, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 512, - 1 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "dense_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 1, - 128, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 1, - 16, - 192 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 1, - 576 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "dense_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 1, - 16, - 256 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['logits_dense']/['kernel']": { - "partition_spec": [ - "embed_vocab", - "vocab" - ], - "shape": [ - 2048, - 102400 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { - "partition_spec": [ - "embed_moe", - "moe_layers", - null - ], - "shape": [ - 2048, - 26, - 64 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { - "partition_spec": [ - "exp", - "moe_layers", - "mlp_moe", - "embed_moe" - ], - "shape": [ - 64, - 26, - 1408, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "moe_layers", - "embed" - ], - "shape": [ - 2816, - 26, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 512, - 26 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "moe_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 26, - 128, - 2048 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 26, - 16, - 192 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 26, - 576 - ] - }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "moe_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 26, - 16, - 256 - ] - }, - ".opt_state/[0]/.mu/['params']/['token_embedder']/['embedding']": { - "partition_spec": [ - "vocab", - "embed_vocab" - ], - "shape": [ - 102400, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['decoder_norm']/['scale']": { - "partition_spec": [ - "norm" - ], - "shape": [ - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "dense_layers", - "mlp" - ], - "shape": [ - 2048, - 1, - 10944 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "dense_layers", - "embed" - ], - "shape": [ - 10944, - 1, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 2048, - 1 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "dense_layers" - ], - "shape": [ - 512, - 1 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "dense_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 1, - 128, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 1, - 16, - 192 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "dense_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 1, - 576 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "dense_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 1, - 16, - 256 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['logits_dense']/['kernel']": { - "partition_spec": [ - "embed_vocab", - "vocab" - ], - "shape": [ - 2048, - 102400 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { - "partition_spec": [ - "embed_moe", - "moe_layers", - null - ], - "shape": [ - 2048, - 26, - 64 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { - "partition_spec": [ - "exp", - "moe_layers", - "embed_moe", - "mlp_moe" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { - "partition_spec": [ - "exp", - "moe_layers", - "mlp_moe", - "embed_moe" - ], - "shape": [ - 64, - 26, - 1408, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { - "partition_spec": [ - "embed", - "moe_layers", - "mlp" - ], - "shape": [ - 2048, - 26, - 2816 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { - "partition_spec": [ - "mlp", - "moe_layers", - "embed" - ], - "shape": [ - 2816, - 26, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 2048, - 26 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { - "partition_spec": [ - "norm", - "moe_layers" - ], - "shape": [ - 512, - 26 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { - "partition_spec": [ - "heads", - "moe_layers", - "kv", - "embed_attn" - ], - "shape": [ - 16, - 26, - 128, - 2048 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "q_heads", - "kv" - ], - "shape": [ - 2048, - 26, - 16, - 192 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { - "partition_spec": [ - "embed_attn", - "moe_layers", - "kv_lora_up_proj" - ], - "shape": [ - 2048, - 26, - 576 - ] - }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { - "partition_spec": [ - "kv_lora", - "moe_layers", - "kv_heads", - "kv_head_dim" - ], - "shape": [ - 512, - 26, - 16, - 256 - ] - }, - ".opt_state/[0]/.nu/['params']/['token_embedder']/['embedding']": { - "partition_spec": [ - "vocab", - "embed_vocab" - ], - "shape": [ - 102400, - 2048 - ] - }, - ".opt_state/[2]/.count": { - "partition_spec": [], - "shape": [] - } -} \ No newline at end of file diff --git a/tests/utils/sharding_info/deepseek2-16b/tpu7x-16/slice_1/rule_default/input_shardings.json b/tests/utils/sharding_info/deepseek3-test/tpu7x-16/slice_1/rule_default/input_shardings.json similarity index 71% rename from tests/utils/sharding_info/deepseek2-16b/tpu7x-16/slice_1/rule_default/input_shardings.json rename to tests/utils/sharding_info/deepseek3-test/tpu7x-16/slice_1/rule_default/input_shardings.json index e44cb3ff94..cf6947cc99 100644 --- a/tests/utils/sharding_info/deepseek2-16b/tpu7x-16/slice_1/rule_default/input_shardings.json +++ b/tests/utils/sharding_info/deepseek3-test/tpu7x-16/slice_1/rule_default/input_shardings.json @@ -1,67 +1,67 @@ { "Activation Sharding Dump": [ { - "deepseek/inputs: bfloat16[192,2048,2048]": { + "deepseek/inputs: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P('fsdp', None, None)" } }, { - "deepseek/pre_attention_norm: bfloat16[192,2048,2048]": { + "deepseek/pre_attention_norm: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P('fsdp', None, None)" } }, { - "attention_mla/inputs_q: bfloat16[192,2048,2048]": { + "attention_mla/inputs_q: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch_attn', 'activation_length', 'activation_embed')", "PartitionSpec": "P('fsdp', None, None)" } }, { - "attention_mla/inputs_kv: bfloat16[192,2048,2048]": { + "attention_mla/inputs_kv: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch_attn', 'activation_length', 'activation_embed')", "PartitionSpec": "P('fsdp', None, None)" } }, { - "attention_mla/q_nope: bfloat16[192,2048,16,128]": { + "attention_mla/q_nope: bfloat16[192,2048,128,128]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P('fsdp', None, None, None)" } }, { - "attention_mla/q_pe: bfloat16[192,2048,16,64]": { + "attention_mla/q_pe: bfloat16[192,2048,128,64]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P('fsdp', None, None, None)" } }, { - "attention_mla/query: bfloat16[192,2048,16,192]": { + "attention_mla/query: bfloat16[192,2048,128,192]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P('fsdp', None, None, None)" } }, { - "attention_mla/key_nope: bfloat16[192,2048,16,128]": { + "attention_mla/key_nope: bfloat16[192,2048,128,128]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P('fsdp', None, None, None)" } }, { - "attention_mla/key_rope: bfloat16[192,2048,16,64]": { + "attention_mla/key_rope: bfloat16[192,2048,128,64]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P('fsdp', None, None, None)" } }, { - "attention_mla/key: bfloat16[192,2048,16,192]": { + "attention_mla/key: bfloat16[192,2048,128,192]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P('fsdp', None, None, None)" } }, { - "attention_mla/value: bfloat16[192,2048,16,128]": { + "attention_mla/value: bfloat16[192,2048,128,128]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P('fsdp', None, None, None)" } @@ -79,97 +79,97 @@ } }, { - "attention_op/query: bfloat16[192,16,2048,192]": { + "attention_op/query: bfloat16[192,128,2048,192]": { "logic_axes": "Unknown", "PartitionSpec": "P('fsdp', None, None, None)" } }, { - "attention_op/key: bfloat16[192,16,2048,192]": { + "attention_op/key: bfloat16[192,128,2048,192]": { "logic_axes": "Unknown", "PartitionSpec": "P('fsdp', None, None, None)" } }, { - "attention_op/value: bfloat16[192,16,2048,128]": { + "attention_op/value: bfloat16[192,128,2048,128]": { "logic_axes": "Unknown", "PartitionSpec": "P('fsdp', None, None, None)" } }, { - "attention_mla/out: bfloat16[192,2048,16,128]": { + "attention_op/decoder_segment_ids: int32[192,2048]": { + "logic_axes": "Unknown", + "PartitionSpec": "P('fsdp', None)" + } + }, + { + "attention_mla/out: bfloat16[192,2048,128,128]": { "logic_axes": "('activation_batch_attn', 'activation_length', 'activation_heads', 'activation_kv')", "PartitionSpec": "P('fsdp', None, None, None)" } }, { - "deepseek/attention_result: bfloat16[192,2048,2048]": { + "deepseek/attention_result: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P('fsdp', None, None)" } }, { - "deepseek/post_attention_norm: bfloat16[192,2048,2048]": { + "deepseek/post_attention_norm: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P('fsdp', None, None)" } }, { - "linears/x: bfloat16[192,2048,10944]": { + "linears/x: bfloat16[192,2048,18432]": { "logic_axes": "('activation_batch', 'activation_length', 'activation_mlp')", "PartitionSpec": "P('fsdp', None, None)" } }, { - "deepseek/mlp: bfloat16[192,2048,2048]": { + "deepseek/mlp: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P('fsdp', None, None)" } }, { - "deepseek/x: bfloat16[192,2048,2048]": { + "deepseek/x: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P('fsdp', None, None)" } }, { - "moe/inputs: bfloat16[192,2048,2048]": { - "logic_axes": "('activation_batch', 'activation_norm_length', None)", + "moe/inputs: bfloat16[192,2048,512]": { + "logic_axes": "('activation_batch', 'activation_norm_length_routed', None)", "PartitionSpec": "P('fsdp', None, None)" } }, { - "moe/gate_logits: bfloat16[192,2048,64]": { - "logic_axes": "('activation_batch', 'activation_norm_length', None)", + "moe/gate_logits: bfloat16[192,2048,256]": { + "logic_axes": "('activation_batch', 'activation_norm_length_routed', None)", "PartitionSpec": "P('fsdp', None, None)" } }, { - "moe/w0_kernel: bfloat16[64,2048,1408]": { - "logic_axes": "Unknown", - "PartitionSpec": "P(None, None, None)" - } - }, - { - "moe/w1_kernel: bfloat16[64,2048,1408]": { - "logic_axes": "Unknown", - "PartitionSpec": "P(None, None, None)" + "moe/pre_bias_logits: bfloat16[192,2048,256]": { + "logic_axes": "('activation_batch', 'activation_norm_length_routed', None)", + "PartitionSpec": "P('fsdp', None, None)" } }, { - "moe/wo_kernel: bfloat16[64,1408,2048]": { + "moe/w: bfloat16[256,512,512]": { "logic_axes": "Unknown", "PartitionSpec": "P(None, None, None)" } }, { - "linears/x: bfloat16[192,2048,2816]": { + "linears/x: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_length', 'activation_mlp')", "PartitionSpec": "P('fsdp', None, None)" } }, { - "deepseek/mlp_lnx: bfloat16[192,2048,2048]": { + "deepseek/mlp_lnx: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P('fsdp', None, None)" } diff --git a/tests/utils/sharding_info/deepseek3-test/tpu7x-16/slice_1/rule_default/logical_shardings.json b/tests/utils/sharding_info/deepseek3-test/tpu7x-16/slice_1/rule_default/logical_shardings.json new file mode 100644 index 0000000000..d7f45f5385 --- /dev/null +++ b/tests/utils/sharding_info/deepseek3-test/tpu7x-16/slice_1/rule_default/logical_shardings.json @@ -0,0 +1,1218 @@ +{ + "['model']/['decoder']/['decoder_norm']/['scale']/.value": { + "partition_spec": [ + null + ], + "shape": [ + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + "fsdp" + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['count']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['key']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['count']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['key']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['count']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['key']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['model']/['decoder']/['logits_dense']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null + ], + "shape": [ + 512, + 129280 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['bias']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 58, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['count']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['key']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['count']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['key']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['count']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['key']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "partition_spec": [ + null, + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "partition_spec": [ + null, + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + "fsdp" + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['model']/['decoder']/['rngs']/['aqt']/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['aqt']/['key']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['dropout']/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['dropout']/['key']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['params']/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['params']/['key']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['token_embedder']/['embedding']/.value": { + "partition_spec": [ + null, + "fsdp" + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['decoder_norm']/['scale']/.value": { + "partition_spec": [ + null + ], + "shape": [ + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + "fsdp" + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['logits_dense']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null + ], + "shape": [ + 512, + 129280 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "partition_spec": [ + null, + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "partition_spec": [ + null, + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + "fsdp" + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['token_embedder']/['embedding']/.value": { + "partition_spec": [ + null, + "fsdp" + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['decoder_norm']/['scale']/.value": { + "partition_spec": [ + null + ], + "shape": [ + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + "fsdp" + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['logits_dense']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null + ], + "shape": [ + 512, + 129280 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "partition_spec": [ + null, + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "partition_spec": [ + null, + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + "fsdp" + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['token_embedder']/['embedding']/.value": { + "partition_spec": [ + null, + "fsdp" + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[2]/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['optimizer']/['step']/.value": { + "partition_spec": [], + "shape": [] + } +} \ No newline at end of file diff --git a/tests/utils/sharding_info/deepseek2-16b/tpu7x-16/slice_1/rule_default/named_shardings.json b/tests/utils/sharding_info/deepseek3-test/tpu7x-16/slice_1/rule_default/named_shardings.json similarity index 66% rename from tests/utils/sharding_info/deepseek2-16b/tpu7x-16/slice_1/rule_default/named_shardings.json rename to tests/utils/sharding_info/deepseek3-test/tpu7x-16/slice_1/rule_default/named_shardings.json index c18b427c7a..b51a105f00 100644 --- a/tests/utils/sharding_info/deepseek2-16b/tpu7x-16/slice_1/rule_default/named_shardings.json +++ b/tests/utils/sharding_info/deepseek3-test/tpu7x-16/slice_1/rule_default/named_shardings.json @@ -1,5 +1,883 @@ { - ".step": { + "['model']/['decoder']/['decoder_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 16, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 1, + "autoregressive": 1 + } + }, + "partition_spec": [ + null + ], + "shape": [ + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 16, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 1, + "autoregressive": 1 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 16, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 1, + "autoregressive": 1 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 16, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 1, + "autoregressive": 1 + } + }, + "partition_spec": [ + null, + null, + "fsdp" + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 16, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 1, + "autoregressive": 1 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 16, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 1, + "autoregressive": 1 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['count']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 16, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 1, + "autoregressive": 1 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['key']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 16, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 1, + "autoregressive": 1 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['count']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 16, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 1, + "autoregressive": 1 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['key']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 16, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 1, + "autoregressive": 1 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['count']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 16, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 1, + "autoregressive": 1 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['key']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 16, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 1, + "autoregressive": 1 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 16, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 1, + "autoregressive": 1 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 16, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 1, + "autoregressive": 1 + } + }, + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 16, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 1, + "autoregressive": 1 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 16, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 1, + "autoregressive": 1 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 16, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 1, + "autoregressive": 1 + } + }, + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 16, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 1, + "autoregressive": 1 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 16, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 1, + "autoregressive": 1 + } + }, + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['model']/['decoder']/['logits_dense']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 16, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 1, + "autoregressive": 1 + } + }, + "partition_spec": [ + "fsdp", + null + ], + "shape": [ + 512, + 129280 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['bias']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 16, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 1, + "autoregressive": 1 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 58, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 16, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 1, + "autoregressive": 1 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['count']/.value": { "mesh": { "axis_names": [ "diloco", @@ -31,9 +909,279 @@ } }, "partition_spec": [], - "shape": [] + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['key']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 16, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 1, + "autoregressive": 1 + } + }, + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['count']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 16, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 1, + "autoregressive": 1 + } + }, + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['key']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 16, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 1, + "autoregressive": 1 + } + }, + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['count']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 16, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 1, + "autoregressive": 1 + } + }, + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['key']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 16, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 1, + "autoregressive": 1 + } + }, + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 16, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 1, + "autoregressive": 1 + } + }, + "partition_spec": [ + null, + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 16, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 1, + "autoregressive": 1 + } + }, + "partition_spec": [ + null, + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] }, - ".params/['params']/['decoder']/['decoder_norm']/['scale']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { "mesh": { "axis_names": [ "diloco", @@ -65,13 +1213,19 @@ } }, "partition_spec": [ - "tensor" + null, + null, + null, + "fsdp" ], "shape": [ - 2048 + 256, + 58, + 512, + 512 ] }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -103,27 +1257,17 @@ } }, "partition_spec": [ - [ - "fsdp", - "context", - "context_usp_ulysses", - "expert" - ], + "fsdp", null, - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ] + null ], "shape": [ - 2048, - 1, - 10944 + 512, + 58, + 512 ] }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -155,27 +1299,17 @@ } }, "partition_spec": [ - [ - "fsdp", - "context", - "context_usp_ulysses", - "expert" - ], + "fsdp", null, - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ] + null ], "shape": [ - 2048, - 1, - 10944 + 512, + 58, + 512 ] }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -207,27 +1341,17 @@ } }, "partition_spec": [ - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ], null, - [ - "fsdp", - "context", - "context_usp_ulysses", - "expert" - ] + null, + "fsdp" ], "shape": [ - 10944, - 1, - 2048 + 512, + 58, + 512 ] }, - ".params/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { + "['model']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -259,15 +1383,15 @@ } }, "partition_spec": [ - "tensor", + null, null ], "shape": [ - 2048, - 1 + 512, + 58 ] }, - ".params/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['model']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -299,15 +1423,15 @@ } }, "partition_spec": [ - "tensor", + null, null ], "shape": [ - 2048, - 1 + 512, + 58 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { + "['model']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -339,15 +1463,15 @@ } }, "partition_spec": [ - "tensor", + null, null ], "shape": [ 512, - 1 + 58 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -379,29 +1503,19 @@ } }, "partition_spec": [ - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], null, null, - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ] + null, + "fsdp" ], "shape": [ - 16, - 1, 128, - 2048 + 58, + 128, + 512 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -433,29 +1547,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ], null, - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], null ], "shape": [ - 2048, - 1, - 16, - 192 + 1536, + 58 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -487,23 +1587,17 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ], + "fsdp", null, null ], "shape": [ - 2048, - 1, + 512, + 58, 576 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -535,29 +1629,19 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ], + "fsdp", + null, null, - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], null ], "shape": [ 512, - 1, - 16, + 58, + 128, 256 ] }, - ".params/['params']/['decoder']/['logits_dense']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -589,25 +1673,17 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ], - [ - "tensor", - "tensor_sequence", - "autoregressive" - ] + "fsdp", + null, + null ], "shape": [ - 2048, - 102400 + 512, + 58, + 1536 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -639,22 +1715,19 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses" - ], + "fsdp", + null, null, null ], "shape": [ - 2048, - 26, - 64 + 1536, + 58, + 128, + 192 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { + "['model']/['decoder']/['rngs']/['aqt']/['count']/.value": { "mesh": { "axis_names": [ "diloco", @@ -685,29 +1758,10 @@ "autoregressive": 1 } }, - "partition_spec": [ - "expert", - null, - [ - "fsdp", - "context", - "context_usp_ulysses" - ], - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ] - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] + "partition_spec": [], + "shape": [] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { + "['model']/['decoder']/['rngs']/['aqt']/['key']/.value": { "mesh": { "axis_names": [ "diloco", @@ -738,29 +1792,10 @@ "autoregressive": 1 } }, - "partition_spec": [ - "expert", - null, - [ - "fsdp", - "context", - "context_usp_ulysses" - ], - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ] - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] + "partition_spec": [], + "shape": [] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { + "['model']/['decoder']/['rngs']/['dropout']/['count']/.value": { "mesh": { "axis_names": [ "diloco", @@ -791,29 +1826,44 @@ "autoregressive": 1 } }, - "partition_spec": [ - "expert", - null, - [ + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['dropout']/['key']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", "tensor", "tensor_sequence", + "expert", "autoregressive" ], - [ - "fsdp", - "context", - "context_usp_ulysses" - ] - ], - "shape": [ - 64, - 26, - 1408, - 2048 - ] + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 16, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 1, + "autoregressive": 1 + } + }, + "partition_spec": [], + "shape": [] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { + "['model']/['decoder']/['rngs']/['params']/['count']/.value": { "mesh": { "axis_names": [ "diloco", @@ -844,28 +1894,44 @@ "autoregressive": 1 } }, - "partition_spec": [ - [ + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['params']/['key']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", "fsdp", + "fsdp_transpose", "context", "context_usp_ulysses", - "expert" - ], - null, - [ - "fsdp_transpose", + "context_autoregressive", "tensor", "tensor_sequence", + "expert", "autoregressive" - ] - ], - "shape": [ - 2048, - 26, - 2816 - ] + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 16, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 1, + "autoregressive": 1 + } + }, + "partition_spec": [], + "shape": [] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { + "['model']/['token_embedder']/['embedding']/.value": { "mesh": { "axis_names": [ "diloco", @@ -897,27 +1963,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "context", - "context_usp_ulysses", - "expert" - ], null, - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ] + "fsdp" ], "shape": [ - 2048, - 26, - 2816 + 129280, + 512 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['count']/.value": { "mesh": { "axis_names": [ "diloco", @@ -948,28 +2002,10 @@ "autoregressive": 1 } }, - "partition_spec": [ - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ], - null, - [ - "fsdp", - "context", - "context_usp_ulysses", - "expert" - ] - ], - "shape": [ - 2816, - 26, - 2048 - ] + "partition_spec": [], + "shape": [] }, - ".params/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['decoder_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1001,15 +2037,13 @@ } }, "partition_spec": [ - "tensor", null ], "shape": [ - 2048, - 26 + 512 ] }, - ".params/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1041,15 +2075,17 @@ } }, "partition_spec": [ - "tensor", + "fsdp", + null, null ], "shape": [ - 2048, - 26 + 512, + 3, + 18432 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1081,15 +2117,17 @@ } }, "partition_spec": [ - "tensor", + "fsdp", + null, null ], "shape": [ 512, - 26 + 3, + 18432 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1121,29 +2159,17 @@ } }, "partition_spec": [ - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], - null, null, - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ] + null, + "fsdp" ], "shape": [ - 16, - 26, - 128, - 2048 + 18432, + 3, + 512 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1175,29 +2201,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ], null, - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], null ], "shape": [ - 2048, - 26, - 16, - 192 + 512, + 3 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1229,23 +2241,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ], null, null ], "shape": [ - 2048, - 26, - 576 + 512, + 3 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1277,29 +2281,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ], null, - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], null ], "shape": [ 512, - 26, - 16, - 256 + 3 ] }, - ".params/['params']/['token_embedder']/['embedding']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1331,25 +2321,19 @@ } }, "partition_spec": [ - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ] + null, + null, + null, + "fsdp" ], "shape": [ - 102400, - 2048 + 128, + 3, + 128, + 512 ] }, - ".opt_state/[0]/.count": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1380,10 +2364,16 @@ "autoregressive": 1 } }, - "partition_spec": [], - "shape": [] + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 + ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['decoder_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1415,13 +2405,17 @@ } }, "partition_spec": [ - "tensor" + "fsdp", + null, + null ], "shape": [ - 2048 + 512, + 3, + 576 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1453,27 +2447,19 @@ } }, "partition_spec": [ - [ - "fsdp", - "context", - "context_usp_ulysses", - "expert" - ], + "fsdp", null, - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ] + null, + null ], "shape": [ - 2048, - 1, - 10944 + 512, + 3, + 128, + 256 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1505,27 +2491,17 @@ } }, "partition_spec": [ - [ - "fsdp", - "context", - "context_usp_ulysses", - "expert" - ], + "fsdp", null, - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ] + null ], "shape": [ - 2048, - 1, - 10944 + 512, + 3, + 1536 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1557,27 +2533,19 @@ } }, "partition_spec": [ - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ], + "fsdp", null, - [ - "fsdp", - "context", - "context_usp_ulysses", - "expert" - ] + null, + null ], "shape": [ - 10944, - 1, - 2048 + 1536, + 3, + 128, + 192 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['logits_dense']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1609,15 +2577,15 @@ } }, "partition_spec": [ - "tensor", + "fsdp", null ], "shape": [ - 2048, - 1 + 512, + 129280 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1649,15 +2617,17 @@ } }, "partition_spec": [ - "tensor", + "fsdp", + null, null ], "shape": [ - 2048, - 1 + 512, + 58, + 256 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1689,15 +2659,19 @@ } }, "partition_spec": [ - "tensor", + null, + null, + "fsdp", null ], "shape": [ + 256, + 58, 512, - 1 + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1729,29 +2703,19 @@ } }, "partition_spec": [ - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], null, null, - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ] + "fsdp", + null ], "shape": [ - 16, - 1, - 128, - 2048 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1783,29 +2747,19 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ], null, - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], - null + null, + null, + "fsdp" ], "shape": [ - 2048, - 1, - 16, - 192 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1837,23 +2791,17 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ], + "fsdp", null, null ], "shape": [ - 2048, - 1, - 576 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1885,29 +2833,17 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ], + "fsdp", null, - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], null ], "shape": [ 512, - 1, - 16, - 256 + 58, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['logits_dense']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1939,25 +2875,17 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ], - [ - "tensor", - "tensor_sequence", - "autoregressive" - ] + null, + null, + "fsdp" ], "shape": [ - 2048, - 102400 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1989,22 +2917,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses" - ], null, null ], "shape": [ - 2048, - 26, - 64 + 512, + 58 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2036,28 +2957,15 @@ } }, "partition_spec": [ - "expert", null, - [ - "fsdp", - "context", - "context_usp_ulysses" - ], - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ] + null ], "shape": [ - 64, - 26, - 2048, - 1408 + 512, + 58 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2089,28 +2997,15 @@ } }, "partition_spec": [ - "expert", null, - [ - "fsdp", - "context", - "context_usp_ulysses" - ], - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ] + null ], "shape": [ - 64, - 26, - 2048, - 1408 + 512, + 58 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2142,28 +3037,19 @@ } }, "partition_spec": [ - "expert", null, - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ], - [ - "fsdp", - "context", - "context_usp_ulysses" - ] + null, + null, + "fsdp" ], "shape": [ - 64, - 26, - 1408, - 2048 + 128, + 58, + 128, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2195,27 +3081,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "context", - "context_usp_ulysses", - "expert" - ], null, - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ] + null ], "shape": [ - 2048, - 26, - 2816 + 1536, + 58 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2247,27 +3121,17 @@ } }, "partition_spec": [ - [ - "fsdp", - "context", - "context_usp_ulysses", - "expert" - ], + "fsdp", null, - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ] + null ], "shape": [ - 2048, - 26, - 2816 + 512, + 58, + 576 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2299,27 +3163,19 @@ } }, "partition_spec": [ - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ], + "fsdp", null, - [ - "fsdp", - "context", - "context_usp_ulysses", - "expert" - ] + null, + null ], "shape": [ - 2816, - 26, - 2048 + 512, + 58, + 128, + 256 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2351,15 +3207,17 @@ } }, "partition_spec": [ - "tensor", + "fsdp", + null, null ], "shape": [ - 2048, - 26 + 512, + 58, + 1536 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2391,15 +3249,19 @@ } }, "partition_spec": [ - "tensor", + "fsdp", + null, + null, null ], "shape": [ - 2048, - 26 + 1536, + 58, + 128, + 192 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['token_embedder']/['embedding']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2431,15 +3293,15 @@ } }, "partition_spec": [ - "tensor", - null + null, + "fsdp" ], "shape": [ - 512, - 26 + 129280, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['decoder_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2471,29 +3333,13 @@ } }, "partition_spec": [ - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], - null, - null, - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ] + null ], "shape": [ - 16, - 26, - 128, - 2048 + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2525,29 +3371,17 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ], + "fsdp", null, - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], null ], "shape": [ - 2048, - 26, - 16, - 192 + 512, + 3, + 18432 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2576,26 +3410,20 @@ "tensor_sequence": 1, "expert": 1, "autoregressive": 1 - } - }, - "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ], + } + }, + "partition_spec": [ + "fsdp", null, null ], "shape": [ - 2048, - 26, - 576 + 512, + 3, + 18432 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2627,29 +3455,17 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ], null, - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], - null + null, + "fsdp" ], "shape": [ - 512, - 26, - 16, - 256 + 18432, + 3, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['token_embedder']/['embedding']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2681,25 +3497,15 @@ } }, "partition_spec": [ - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ] + null, + null ], "shape": [ - 102400, - 2048 + 512, + 3 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['decoder_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2731,13 +3537,15 @@ } }, "partition_spec": [ - "tensor" + null, + null ], "shape": [ - 2048 + 512, + 3 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2769,27 +3577,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "context", - "context_usp_ulysses", - "expert" - ], null, - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ] + null ], "shape": [ - 2048, - 1, - 10944 + 512, + 3 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2821,27 +3617,19 @@ } }, "partition_spec": [ - [ - "fsdp", - "context", - "context_usp_ulysses", - "expert" - ], null, - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ] + null, + null, + "fsdp" ], "shape": [ - 2048, - 1, - 10944 + 128, + 3, + 128, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2873,27 +3661,15 @@ } }, "partition_spec": [ - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ], null, - [ - "fsdp", - "context", - "context_usp_ulysses", - "expert" - ] + null ], "shape": [ - 10944, - 1, - 2048 + 1536, + 3 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2925,15 +3701,17 @@ } }, "partition_spec": [ - "tensor", + "fsdp", + null, null ], "shape": [ - 2048, - 1 + 512, + 3, + 576 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2965,15 +3743,19 @@ } }, "partition_spec": [ - "tensor", + "fsdp", + null, + null, null ], "shape": [ - 2048, - 1 + 512, + 3, + 128, + 256 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3005,15 +3787,17 @@ } }, "partition_spec": [ - "tensor", + "fsdp", + null, null ], "shape": [ 512, - 1 + 3, + 1536 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3045,29 +3829,19 @@ } }, "partition_spec": [ - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], + "fsdp", null, null, - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ] + null ], "shape": [ - 16, - 1, + 1536, + 3, 128, - 2048 + 192 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['logits_dense']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3099,29 +3873,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ], - null, - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], + "fsdp", null ], "shape": [ - 2048, - 1, - 16, - 192 + 512, + 129280 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3153,23 +3913,17 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ], + "fsdp", null, null ], "shape": [ - 2048, - 1, - 576 + 512, + 58, + 256 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3201,29 +3955,19 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ], null, - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], + null, + "fsdp", null ], "shape": [ + 256, + 58, 512, - 1, - 16, - 256 + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['logits_dense']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3255,25 +3999,19 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ], - [ - "tensor", - "tensor_sequence", - "autoregressive" - ] + null, + null, + "fsdp", + null ], "shape": [ - 2048, - 102400 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3305,22 +4043,19 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses" - ], null, - null + null, + null, + "fsdp" ], "shape": [ - 2048, - 26, - 64 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3352,28 +4087,17 @@ } }, "partition_spec": [ - "expert", + "fsdp", null, - [ - "fsdp", - "context", - "context_usp_ulysses" - ], - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ] + null ], "shape": [ - 64, - 26, - 2048, - 1408 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3405,28 +4129,17 @@ } }, "partition_spec": [ - "expert", + "fsdp", null, - [ - "fsdp", - "context", - "context_usp_ulysses" - ], - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ] + null ], "shape": [ - 64, - 26, - 2048, - 1408 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3458,28 +4171,17 @@ } }, "partition_spec": [ - "expert", null, - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ], - [ - "fsdp", - "context", - "context_usp_ulysses" - ] + null, + "fsdp" ], "shape": [ - 64, - 26, - 1408, - 2048 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3511,27 +4213,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "context", - "context_usp_ulysses", - "expert" - ], null, - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ] + null ], "shape": [ - 2048, - 26, - 2816 + 512, + 58 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3563,27 +4253,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "context", - "context_usp_ulysses", - "expert" - ], null, - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ] + null ], "shape": [ - 2048, - 26, - 2816 + 512, + 58 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3615,27 +4293,15 @@ } }, "partition_spec": [ - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ], null, - [ - "fsdp", - "context", - "context_usp_ulysses", - "expert" - ] + null ], "shape": [ - 2816, - 26, - 2048 + 512, + 58 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3667,15 +4333,19 @@ } }, "partition_spec": [ - "tensor", - null + null, + null, + null, + "fsdp" ], "shape": [ - 2048, - 26 + 128, + 58, + 128, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3707,15 +4377,15 @@ } }, "partition_spec": [ - "tensor", + null, null ], "shape": [ - 2048, - 26 + 1536, + 58 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3747,15 +4417,17 @@ } }, "partition_spec": [ - "tensor", + "fsdp", + null, null ], "shape": [ 512, - 26 + 58, + 576 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3787,29 +4459,19 @@ } }, "partition_spec": [ - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], + "fsdp", null, null, - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ] + null ], "shape": [ - 16, - 26, + 512, + 58, 128, - 2048 + 256 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3841,29 +4503,17 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ], + "fsdp", null, - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], null ], "shape": [ - 2048, - 26, - 16, - 192 + 512, + 58, + 1536 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3895,23 +4545,19 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ], + "fsdp", + null, null, null ], "shape": [ - 2048, - 26, - 576 + 1536, + 58, + 128, + 192 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['token_embedder']/['embedding']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3943,29 +4589,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ], null, - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], - null + "fsdp" ], "shape": [ - 512, - 26, - 16, - 256 + 129280, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['token_embedder']/['embedding']": { + "['optimizer']/['opt_state']/[2]/['count']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3996,26 +4628,10 @@ "autoregressive": 1 } }, - "partition_spec": [ - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ] - ], - "shape": [ - 102400, - 2048 - ] + "partition_spec": [], + "shape": [] }, - ".opt_state/[2]/.count": { + "['optimizer']/['step']/.value": { "mesh": { "axis_names": [ "diloco", diff --git a/tests/utils/sharding_info/deepseek2-16b/v6e-16/slice_1/rule_default_ici_fsdp_parallelism=-1_ici_expert_parallelism=4/input_shardings.json b/tests/utils/sharding_info/deepseek3-test/tpu7x-16/slice_1/rule_ep-as-dp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2_use_ring_of_experts=true/input_shardings.json similarity index 72% rename from tests/utils/sharding_info/deepseek2-16b/v6e-16/slice_1/rule_default_ici_fsdp_parallelism=-1_ici_expert_parallelism=4/input_shardings.json rename to tests/utils/sharding_info/deepseek3-test/tpu7x-16/slice_1/rule_ep-as-dp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2_use_ring_of_experts=true/input_shardings.json index f556d60c89..850add1a70 100644 --- a/tests/utils/sharding_info/deepseek2-16b/v6e-16/slice_1/rule_default_ici_fsdp_parallelism=-1_ici_expert_parallelism=4/input_shardings.json +++ b/tests/utils/sharding_info/deepseek3-test/tpu7x-16/slice_1/rule_ep-as-dp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2_use_ring_of_experts=true/input_shardings.json @@ -1,67 +1,67 @@ { "Activation Sharding Dump": [ { - "deepseek/inputs: bfloat16[192,2048,2048]": { + "deepseek/inputs: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "deepseek/pre_attention_norm: bfloat16[192,2048,2048]": { + "deepseek/pre_attention_norm: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "attention_mla/inputs_q: bfloat16[192,2048,2048]": { + "attention_mla/inputs_q: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch_attn', 'activation_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "attention_mla/inputs_kv: bfloat16[192,2048,2048]": { + "attention_mla/inputs_kv: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch_attn', 'activation_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "attention_mla/q_nope: bfloat16[192,2048,16,128]": { + "attention_mla/q_nope: bfloat16[192,2048,128,128]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_mla/q_pe: bfloat16[192,2048,16,64]": { + "attention_mla/q_pe: bfloat16[192,2048,128,64]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_mla/query: bfloat16[192,2048,16,192]": { + "attention_mla/query: bfloat16[192,2048,128,192]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_mla/key_nope: bfloat16[192,2048,16,128]": { + "attention_mla/key_nope: bfloat16[192,2048,128,128]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_mla/key_rope: bfloat16[192,2048,16,64]": { + "attention_mla/key_rope: bfloat16[192,2048,128,64]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_mla/key: bfloat16[192,2048,16,192]": { + "attention_mla/key: bfloat16[192,2048,128,192]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_mla/value: bfloat16[192,2048,16,128]": { + "attention_mla/value: bfloat16[192,2048,128,128]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } @@ -79,97 +79,97 @@ } }, { - "attention_op/query: bfloat16[192,16,2048,192]": { + "attention_op/query: bfloat16[192,128,2048,192]": { "logic_axes": "Unknown", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_op/key: bfloat16[192,16,2048,192]": { + "attention_op/key: bfloat16[192,128,2048,192]": { "logic_axes": "Unknown", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_op/value: bfloat16[192,16,2048,128]": { + "attention_op/value: bfloat16[192,128,2048,128]": { "logic_axes": "Unknown", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_mla/out: bfloat16[192,2048,16,128]": { + "attention_op/decoder_segment_ids: int32[192,2048]": { + "logic_axes": "Unknown", + "PartitionSpec": "P(('fsdp', 'expert'), None)" + } + }, + { + "attention_mla/out: bfloat16[192,2048,128,128]": { "logic_axes": "('activation_batch_attn', 'activation_length', 'activation_heads', 'activation_kv')", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "deepseek/attention_result: bfloat16[192,2048,2048]": { + "deepseek/attention_result: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "deepseek/post_attention_norm: bfloat16[192,2048,2048]": { + "deepseek/post_attention_norm: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "linears/x: bfloat16[192,2048,10944]": { + "linears/x: bfloat16[192,2048,18432]": { "logic_axes": "('activation_batch', 'activation_length', 'activation_mlp')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "deepseek/mlp: bfloat16[192,2048,2048]": { + "deepseek/mlp: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "deepseek/x: bfloat16[192,2048,2048]": { + "deepseek/x: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "moe/inputs: bfloat16[192,2048,2048]": { - "logic_axes": "('activation_batch', 'activation_norm_length', None)", + "moe/inputs: bfloat16[192,2048,512]": { + "logic_axes": "('activation_batch', 'activation_norm_length_routed', None)", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "moe/gate_logits: bfloat16[192,2048,64]": { - "logic_axes": "('activation_batch', 'activation_norm_length', None)", + "moe/gate_logits: bfloat16[192,2048,256]": { + "logic_axes": "('activation_batch', 'activation_norm_length_routed', None)", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "moe/w0_kernel: bfloat16[64,2048,1408]": { - "logic_axes": "Unknown", - "PartitionSpec": "P('expert', None, None)" - } - }, - { - "moe/w1_kernel: bfloat16[64,2048,1408]": { - "logic_axes": "Unknown", - "PartitionSpec": "P('expert', None, None)" + "moe/pre_bias_logits: bfloat16[192,2048,256]": { + "logic_axes": "('activation_batch', 'activation_norm_length_routed', None)", + "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "moe/wo_kernel: bfloat16[64,1408,2048]": { + "moe/w: bfloat16[256,512,512]": { "logic_axes": "Unknown", "PartitionSpec": "P('expert', None, None)" } }, { - "linears/x: bfloat16[192,2048,2816]": { + "linears/x: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_length', 'activation_mlp')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "deepseek/mlp_lnx: bfloat16[192,2048,2048]": { + "deepseek/mlp_lnx: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } diff --git a/tests/utils/sharding_info/deepseek3-test/tpu7x-16/slice_1/rule_ep-as-dp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2_use_ring_of_experts=true/logical_shardings.json b/tests/utils/sharding_info/deepseek3-test/tpu7x-16/slice_1/rule_ep-as-dp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2_use_ring_of_experts=true/logical_shardings.json new file mode 100644 index 0000000000..377294b0a5 --- /dev/null +++ b/tests/utils/sharding_info/deepseek3-test/tpu7x-16/slice_1/rule_ep-as-dp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2_use_ring_of_experts=true/logical_shardings.json @@ -0,0 +1,1218 @@ +{ + "['model']/['decoder']/['decoder_norm']/['scale']/.value": { + "partition_spec": [ + null + ], + "shape": [ + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + "fsdp" + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['count']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['key']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['count']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['key']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['count']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['key']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['model']/['decoder']/['logits_dense']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null + ], + "shape": [ + 512, + 129280 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['bias']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 58, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['count']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['key']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['count']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['key']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['count']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['key']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "partition_spec": [ + "expert", + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "partition_spec": [ + "expert", + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "partition_spec": [ + "expert", + null, + null, + "fsdp" + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + "fsdp" + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['model']/['decoder']/['rngs']/['aqt']/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['aqt']/['key']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['dropout']/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['dropout']/['key']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['params']/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['params']/['key']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['token_embedder']/['embedding']/.value": { + "partition_spec": [ + null, + "fsdp" + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['decoder_norm']/['scale']/.value": { + "partition_spec": [ + null + ], + "shape": [ + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + "fsdp" + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['logits_dense']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null + ], + "shape": [ + 512, + 129280 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "partition_spec": [ + "expert", + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "partition_spec": [ + "expert", + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "partition_spec": [ + "expert", + null, + null, + "fsdp" + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + "fsdp" + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['token_embedder']/['embedding']/.value": { + "partition_spec": [ + null, + "fsdp" + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['decoder_norm']/['scale']/.value": { + "partition_spec": [ + null + ], + "shape": [ + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + "fsdp" + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['logits_dense']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null + ], + "shape": [ + 512, + 129280 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "partition_spec": [ + "expert", + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "partition_spec": [ + "expert", + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "partition_spec": [ + "expert", + null, + null, + "fsdp" + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + "fsdp" + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['token_embedder']/['embedding']/.value": { + "partition_spec": [ + null, + "fsdp" + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[2]/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['optimizer']/['step']/.value": { + "partition_spec": [], + "shape": [] + } +} \ No newline at end of file diff --git a/tests/utils/sharding_info/deepseek2-16b/tpu7x-16/slice_1/rule_ep-as-dp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2_use_ring_of_experts=true/named_shardings.json b/tests/utils/sharding_info/deepseek3-test/tpu7x-16/slice_1/rule_ep-as-dp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2_use_ring_of_experts=true/named_shardings.json similarity index 53% rename from tests/utils/sharding_info/deepseek2-16b/tpu7x-16/slice_1/rule_ep-as-dp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2_use_ring_of_experts=true/named_shardings.json rename to tests/utils/sharding_info/deepseek3-test/tpu7x-16/slice_1/rule_ep-as-dp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2_use_ring_of_experts=true/named_shardings.json index a19c528418..0402a12b04 100644 --- a/tests/utils/sharding_info/deepseek2-16b/tpu7x-16/slice_1/rule_ep-as-dp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2_use_ring_of_experts=true/named_shardings.json +++ b/tests/utils/sharding_info/deepseek3-test/tpu7x-16/slice_1/rule_ep-as-dp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2_use_ring_of_experts=true/named_shardings.json @@ -1,5 +1,5 @@ { - ".step": { + "['model']/['decoder']/['decoder_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -14,10 +14,14 @@ "expert": 2 } }, - "partition_spec": [], - "shape": [] + "partition_spec": [ + null + ], + "shape": [ + 512 + ] }, - ".params/['params']/['decoder']/['decoder_norm']/['scale']": { + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -33,13 +37,17 @@ } }, "partition_spec": [ + "fsdp", + null, null ], "shape": [ - 2048 + 512, + 3, + 18432 ] }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -57,15 +65,15 @@ "partition_spec": [ "fsdp", null, - "fsdp_transpose" + null ], "shape": [ - 2048, - 1, - 10944 + 512, + 3, + 18432 ] }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -81,17 +89,17 @@ } }, "partition_spec": [ - "fsdp", null, - "fsdp_transpose" + null, + "fsdp" ], "shape": [ - 2048, - 1, - 10944 + 18432, + 3, + 512 ] }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -107,17 +115,15 @@ } }, "partition_spec": [ - "fsdp_transpose", null, - "fsdp" + null ], "shape": [ - 10944, - 1, - 2048 + 512, + 3 ] }, - ".params/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { + "['model']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -137,11 +143,11 @@ null ], "shape": [ - 2048, - 1 + 512, + 3 ] }, - ".params/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['count']/.value": { "mesh": { "axis_names": [ "data", @@ -156,16 +162,112 @@ "expert": 2 } }, - "partition_spec": [ - null, - null - ], + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "fsdp", + "fsdp_transpose", + "expert" + ], + "shape": { + "data": 1, + "fsdp": 8, + "fsdp_transpose": 1, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "fsdp", + "fsdp_transpose", + "expert" + ], + "shape": { + "data": 1, + "fsdp": 8, + "fsdp_transpose": 1, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "fsdp", + "fsdp_transpose", + "expert" + ], + "shape": { + "data": 1, + "fsdp": 8, + "fsdp_transpose": 1, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "fsdp", + "fsdp_transpose", + "expert" + ], + "shape": { + "data": 1, + "fsdp": 8, + "fsdp_transpose": 1, + "expert": 2 + } + }, + "partition_spec": [], "shape": [ - 2048, - 1 + 3 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "fsdp", + "fsdp_transpose", + "expert" + ], + "shape": { + "data": 1, + "fsdp": 8, + "fsdp_transpose": 1, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -186,10 +288,10 @@ ], "shape": [ 512, - 1 + 3 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -208,19 +310,16 @@ null, null, null, - [ - "fsdp", - "fsdp_transpose" - ] + "fsdp" ], "shape": [ - 16, - 1, 128, - 2048 + 3, + 128, + 512 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -236,22 +335,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose" - ], - null, null, null ], "shape": [ - 2048, - 1, - 16, - 192 + 1536, + 3 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -267,20 +359,17 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose" - ], + "fsdp", null, null ], "shape": [ - 2048, - 1, + 512, + 3, 576 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -303,12 +392,12 @@ ], "shape": [ 512, - 1, - 16, + 3, + 128, 256 ] }, - ".params/['params']/['decoder']/['logits_dense']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -324,18 +413,17 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose" - ], + "fsdp", + null, null ], "shape": [ - 2048, - 102400 + 512, + 3, + 1536 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -353,15 +441,17 @@ "partition_spec": [ "fsdp", null, + null, null ], "shape": [ - 2048, - 26, - 64 + 1536, + 3, + 128, + 192 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { + "['model']/['decoder']/['logits_dense']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -377,19 +467,15 @@ } }, "partition_spec": [ - "expert", - null, "fsdp", - "fsdp_transpose" + null ], "shape": [ - 64, - 26, - 2048, - 1408 + 512, + 129280 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['bias']/.value": { "mesh": { "axis_names": [ "data", @@ -405,19 +491,15 @@ } }, "partition_spec": [ - "expert", null, - "fsdp", - "fsdp_transpose" + null ], "shape": [ - 64, - 26, - 2048, - 1408 + 58, + 256 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -433,19 +515,17 @@ } }, "partition_spec": [ - "expert", + "fsdp", null, - "fsdp_transpose", - "fsdp" + null ], "shape": [ - 64, - 26, - 1408, - 2048 + 512, + 58, + 256 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['count']/.value": { "mesh": { "axis_names": [ "data", @@ -460,18 +540,12 @@ "expert": 2 } }, - "partition_spec": [ - "fsdp", - null, - "fsdp_transpose" - ], + "partition_spec": [], "shape": [ - 2048, - 26, - 2816 + 58 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['key']/.value": { "mesh": { "axis_names": [ "data", @@ -486,18 +560,92 @@ "expert": 2 } }, - "partition_spec": [ - "fsdp", - null, - "fsdp_transpose" - ], + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "fsdp", + "fsdp_transpose", + "expert" + ], + "shape": { + "data": 1, + "fsdp": 8, + "fsdp_transpose": 1, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "fsdp", + "fsdp_transpose", + "expert" + ], + "shape": { + "data": 1, + "fsdp": 8, + "fsdp_transpose": 1, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "fsdp", + "fsdp_transpose", + "expert" + ], + "shape": { + "data": 1, + "fsdp": 8, + "fsdp_transpose": 1, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "fsdp", + "fsdp_transpose", + "expert" + ], + "shape": { + "data": 1, + "fsdp": 8, + "fsdp_transpose": 1, + "expert": 2 + } + }, + "partition_spec": [], "shape": [ - 2048, - 26, - 2816 + 58 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { "mesh": { "axis_names": [ "data", @@ -513,17 +661,19 @@ } }, "partition_spec": [ - "fsdp_transpose", + "expert", null, - "fsdp" + "fsdp", + null ], "shape": [ - 2816, - 26, - 2048 + 256, + 58, + 512, + 512 ] }, - ".params/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { "mesh": { "axis_names": [ "data", @@ -539,15 +689,19 @@ } }, "partition_spec": [ + "expert", null, + "fsdp", null ], "shape": [ - 2048, - 26 + 256, + 58, + 512, + 512 ] }, - ".params/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { "mesh": { "axis_names": [ "data", @@ -563,15 +717,19 @@ } }, "partition_spec": [ + "expert", null, - null + null, + "fsdp" ], "shape": [ - 2048, - 26 + 256, + 58, + 512, + 512 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -587,15 +745,17 @@ } }, "partition_spec": [ + "fsdp", null, null ], "shape": [ 512, - 26 + 58, + 512 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -611,22 +771,17 @@ } }, "partition_spec": [ + "fsdp", null, - null, - null, - [ - "fsdp", - "fsdp_transpose" - ] + null ], "shape": [ - 16, - 26, - 128, - 2048 + 512, + 58, + 512 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -642,22 +797,523 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose" - ], null, null, - null + "fsdp" ], "shape": [ - 2048, - 26, - 16, - 192 + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "fsdp", + "fsdp_transpose", + "expert" + ], + "shape": { + "data": 1, + "fsdp": 8, + "fsdp_transpose": 1, + "expert": 2 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "fsdp", + "fsdp_transpose", + "expert" + ], + "shape": { + "data": 1, + "fsdp": 8, + "fsdp_transpose": 1, + "expert": 2 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "fsdp", + "fsdp_transpose", + "expert" + ], + "shape": { + "data": 1, + "fsdp": 8, + "fsdp_transpose": 1, + "expert": 2 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "fsdp", + "fsdp_transpose", + "expert" + ], + "shape": { + "data": 1, + "fsdp": 8, + "fsdp_transpose": 1, + "expert": 2 + } + }, + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "fsdp", + "fsdp_transpose", + "expert" + ], + "shape": { + "data": 1, + "fsdp": 8, + "fsdp_transpose": 1, + "expert": 2 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "fsdp", + "fsdp_transpose", + "expert" + ], + "shape": { + "data": 1, + "fsdp": 8, + "fsdp_transpose": 1, + "expert": 2 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "fsdp", + "fsdp_transpose", + "expert" + ], + "shape": { + "data": 1, + "fsdp": 8, + "fsdp_transpose": 1, + "expert": 2 + } + }, + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "fsdp", + "fsdp_transpose", + "expert" + ], + "shape": { + "data": 1, + "fsdp": 8, + "fsdp_transpose": 1, + "expert": 2 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "fsdp", + "fsdp_transpose", + "expert" + ], + "shape": { + "data": 1, + "fsdp": 8, + "fsdp_transpose": 1, + "expert": 2 + } + }, + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['model']/['decoder']/['rngs']/['aqt']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "fsdp", + "fsdp_transpose", + "expert" + ], + "shape": { + "data": 1, + "fsdp": 8, + "fsdp_transpose": 1, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['aqt']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "fsdp", + "fsdp_transpose", + "expert" + ], + "shape": { + "data": 1, + "fsdp": 8, + "fsdp_transpose": 1, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['dropout']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "fsdp", + "fsdp_transpose", + "expert" + ], + "shape": { + "data": 1, + "fsdp": 8, + "fsdp_transpose": 1, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['dropout']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "fsdp", + "fsdp_transpose", + "expert" + ], + "shape": { + "data": 1, + "fsdp": 8, + "fsdp_transpose": 1, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['params']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "fsdp", + "fsdp_transpose", + "expert" + ], + "shape": { + "data": 1, + "fsdp": 8, + "fsdp_transpose": 1, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['params']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "fsdp", + "fsdp_transpose", + "expert" + ], + "shape": { + "data": 1, + "fsdp": 8, + "fsdp_transpose": 1, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['token_embedder']/['embedding']/.value": { + "mesh": { + "axis_names": [ + "data", + "fsdp", + "fsdp_transpose", + "expert" + ], + "shape": { + "data": 1, + "fsdp": 8, + "fsdp_transpose": 1, + "expert": 2 + } + }, + "partition_spec": [ + null, + "fsdp" + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "fsdp", + "fsdp_transpose", + "expert" + ], + "shape": { + "data": 1, + "fsdp": 8, + "fsdp_transpose": 1, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['decoder_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "fsdp", + "fsdp_transpose", + "expert" + ], + "shape": { + "data": 1, + "fsdp": 8, + "fsdp_transpose": 1, + "expert": 2 + } + }, + "partition_spec": [ + null + ], + "shape": [ + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "fsdp", + "fsdp_transpose", + "expert" + ], + "shape": { + "data": 1, + "fsdp": 8, + "fsdp_transpose": 1, + "expert": 2 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "fsdp", + "fsdp_transpose", + "expert" + ], + "shape": { + "data": 1, + "fsdp": 8, + "fsdp_transpose": 1, + "expert": 2 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "fsdp", + "fsdp_transpose", + "expert" + ], + "shape": { + "data": 1, + "fsdp": 8, + "fsdp_transpose": 1, + "expert": 2 + } + }, + "partition_spec": [ + null, + null, + "fsdp" + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "fsdp", + "fsdp_transpose", + "expert" + ], + "shape": { + "data": 1, + "fsdp": 8, + "fsdp_transpose": 1, + "expert": 2 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -673,20 +1329,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose" - ], null, null ], "shape": [ - 2048, - 26, - 576 + 512, + 3 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -702,19 +1353,15 @@ } }, "partition_spec": [ - "fsdp", - null, null, null ], "shape": [ 512, - 26, - 16, - 256 + 3 ] }, - ".params/['params']/['token_embedder']/['embedding']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -731,17 +1378,18 @@ }, "partition_spec": [ null, - [ - "fsdp", - "fsdp_transpose" - ] + null, + null, + "fsdp" ], "shape": [ - 102400, - 2048 + 128, + 3, + 128, + 512 ] }, - ".opt_state/[0]/.count": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -756,10 +1404,16 @@ "expert": 2 } }, - "partition_spec": [], - "shape": [] + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 + ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['decoder_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -775,13 +1429,17 @@ } }, "partition_spec": [ + "fsdp", + null, null ], "shape": [ - 2048 + 512, + 3, + 576 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -799,15 +1457,17 @@ "partition_spec": [ "fsdp", null, - "fsdp_transpose" + null, + null ], "shape": [ - 2048, - 1, - 10944 + 512, + 3, + 128, + 256 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -825,15 +1485,15 @@ "partition_spec": [ "fsdp", null, - "fsdp_transpose" + null ], "shape": [ - 2048, - 1, - 10944 + 512, + 3, + 1536 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -849,17 +1509,19 @@ } }, "partition_spec": [ - "fsdp_transpose", + "fsdp", null, - "fsdp" + null, + null ], "shape": [ - 10944, - 1, - 2048 + 1536, + 3, + 128, + 192 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['logits_dense']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -875,15 +1537,15 @@ } }, "partition_spec": [ - null, + "fsdp", null ], "shape": [ - 2048, - 1 + 512, + 129280 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -899,15 +1561,17 @@ } }, "partition_spec": [ + "fsdp", null, null ], "shape": [ - 2048, - 1 + 512, + 58, + 256 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { "mesh": { "axis_names": [ "data", @@ -923,15 +1587,19 @@ } }, "partition_spec": [ + "expert", null, + "fsdp", null ], "shape": [ + 256, + 58, 512, - 1 + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { "mesh": { "axis_names": [ "data", @@ -947,22 +1615,19 @@ } }, "partition_spec": [ + "expert", null, - null, - null, - [ - "fsdp", - "fsdp_transpose" - ] + "fsdp", + null ], "shape": [ - 16, - 1, - 128, - 2048 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { "mesh": { "axis_names": [ "data", @@ -978,22 +1643,19 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose" - ], + "expert", null, null, - null + "fsdp" ], "shape": [ - 2048, - 1, - 16, - 192 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1009,20 +1671,17 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose" - ], + "fsdp", null, null ], "shape": [ - 2048, - 1, - 576 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1040,17 +1699,15 @@ "partition_spec": [ "fsdp", null, - null, null ], "shape": [ 512, - 1, - 16, - 256 + 58, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['logits_dense']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1066,18 +1723,17 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose" - ], - null + null, + null, + "fsdp" ], "shape": [ - 2048, - 102400 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1093,17 +1749,15 @@ } }, "partition_spec": [ - "fsdp", null, null ], "shape": [ - 2048, - 26, - 64 + 512, + 58 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1119,19 +1773,15 @@ } }, "partition_spec": [ - "expert", null, - "fsdp", - "fsdp_transpose" + null ], "shape": [ - 64, - 26, - 2048, - 1408 + 512, + 58 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1147,19 +1797,15 @@ } }, "partition_spec": [ - "expert", null, - "fsdp", - "fsdp_transpose" + null ], "shape": [ - 64, - 26, - 2048, - 1408 + 512, + 58 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1175,19 +1821,19 @@ } }, "partition_spec": [ - "expert", null, - "fsdp_transpose", + null, + null, "fsdp" ], "shape": [ - 64, - 26, - 1408, - 2048 + 128, + 58, + 128, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1203,17 +1849,15 @@ } }, "partition_spec": [ - "fsdp", null, - "fsdp_transpose" + null ], "shape": [ - 2048, - 26, - 2816 + 1536, + 58 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1231,15 +1875,15 @@ "partition_spec": [ "fsdp", null, - "fsdp_transpose" + null ], "shape": [ - 2048, - 26, - 2816 + 512, + 58, + 576 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1255,17 +1899,19 @@ } }, "partition_spec": [ - "fsdp_transpose", + "fsdp", null, - "fsdp" + null, + null ], "shape": [ - 2816, - 26, - 2048 + 512, + 58, + 128, + 256 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1281,15 +1927,17 @@ } }, "partition_spec": [ + "fsdp", null, null ], "shape": [ - 2048, - 26 + 512, + 58, + 1536 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1305,15 +1953,19 @@ } }, "partition_spec": [ + "fsdp", + null, null, null ], "shape": [ - 2048, - 26 + 1536, + 58, + 128, + 192 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['token_embedder']/['embedding']/.value": { "mesh": { "axis_names": [ "data", @@ -1330,14 +1982,14 @@ }, "partition_spec": [ null, - null + "fsdp" ], "shape": [ - 512, - 26 + 129280, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['decoder_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1353,22 +2005,13 @@ } }, "partition_spec": [ - null, - null, - null, - [ - "fsdp", - "fsdp_transpose" - ] + null ], "shape": [ - 16, - 26, - 128, - 2048 + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1384,22 +2027,17 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose" - ], - null, + "fsdp", null, null ], "shape": [ - 2048, - 26, - 16, - 192 + 512, + 3, + 18432 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1415,20 +2053,17 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose" - ], + "fsdp", null, null ], "shape": [ - 2048, - 26, - 576 + 512, + 3, + 18432 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1444,19 +2079,17 @@ } }, "partition_spec": [ - "fsdp", null, null, - null + "fsdp" ], "shape": [ - 512, - 26, - 16, - 256 + 18432, + 3, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['token_embedder']/['embedding']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1473,17 +2106,14 @@ }, "partition_spec": [ null, - [ - "fsdp", - "fsdp_transpose" - ] + null ], "shape": [ - 102400, - 2048 + 512, + 3 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['decoder_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1499,13 +2129,15 @@ } }, "partition_spec": [ + null, null ], "shape": [ - 2048 + 512, + 3 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1521,17 +2153,15 @@ } }, "partition_spec": [ - "fsdp", null, - "fsdp_transpose" + null ], "shape": [ - 2048, - 1, - 10944 + 512, + 3 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1547,17 +2177,19 @@ } }, "partition_spec": [ - "fsdp", null, - "fsdp_transpose" + null, + null, + "fsdp" ], "shape": [ - 2048, - 1, - 10944 + 128, + 3, + 128, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1573,17 +2205,15 @@ } }, "partition_spec": [ - "fsdp_transpose", null, - "fsdp" + null ], "shape": [ - 10944, - 1, - 2048 + 1536, + 3 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1599,15 +2229,17 @@ } }, "partition_spec": [ + "fsdp", null, null ], "shape": [ - 2048, - 1 + 512, + 3, + 576 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1623,15 +2255,19 @@ } }, "partition_spec": [ + "fsdp", + null, null, null ], "shape": [ - 2048, - 1 + 512, + 3, + 128, + 256 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1647,15 +2283,17 @@ } }, "partition_spec": [ + "fsdp", null, null ], "shape": [ 512, - 1 + 3, + 1536 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1671,22 +2309,19 @@ } }, "partition_spec": [ + "fsdp", null, null, - null, - [ - "fsdp", - "fsdp_transpose" - ] + null ], "shape": [ - 16, - 1, + 1536, + 3, 128, - 2048 + 192 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['logits_dense']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1702,22 +2337,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose" - ], - null, - null, + "fsdp", null ], "shape": [ - 2048, - 1, - 16, - 192 + 512, + 129280 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1733,20 +2361,17 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose" - ], + "fsdp", null, null ], "shape": [ - 2048, - 1, - 576 + 512, + 58, + 256 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { "mesh": { "axis_names": [ "data", @@ -1762,19 +2387,19 @@ } }, "partition_spec": [ - "fsdp", - null, + "expert", null, + "fsdp", null ], "shape": [ + 256, + 58, 512, - 1, - 16, - 256 + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['logits_dense']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { "mesh": { "axis_names": [ "data", @@ -1790,18 +2415,19 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose" - ], + "expert", + null, + "fsdp", null ], "shape": [ - 2048, - 102400 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { "mesh": { "axis_names": [ "data", @@ -1817,17 +2443,19 @@ } }, "partition_spec": [ - "fsdp", + "expert", null, - null + null, + "fsdp" ], "shape": [ - 2048, - 26, - 64 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1843,19 +2471,17 @@ } }, "partition_spec": [ - "expert", - null, "fsdp", - "fsdp_transpose" + null, + null ], "shape": [ - 64, - 26, - 2048, - 1408 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1871,19 +2497,17 @@ } }, "partition_spec": [ - "expert", - null, "fsdp", - "fsdp_transpose" + null, + null ], "shape": [ - 64, - 26, - 2048, - 1408 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1899,19 +2523,17 @@ } }, "partition_spec": [ - "expert", null, - "fsdp_transpose", + null, "fsdp" ], "shape": [ - 64, - 26, - 1408, - 2048 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1927,17 +2549,15 @@ } }, "partition_spec": [ - "fsdp", null, - "fsdp_transpose" + null ], "shape": [ - 2048, - 26, - 2816 + 512, + 58 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1953,17 +2573,15 @@ } }, "partition_spec": [ - "fsdp", null, - "fsdp_transpose" + null ], "shape": [ - 2048, - 26, - 2816 + 512, + 58 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1979,17 +2597,15 @@ } }, "partition_spec": [ - "fsdp_transpose", null, - "fsdp" + null ], "shape": [ - 2816, - 26, - 2048 + 512, + 58 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2006,14 +2622,18 @@ }, "partition_spec": [ null, - null + null, + null, + "fsdp" ], "shape": [ - 2048, - 26 + 128, + 58, + 128, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -2033,11 +2653,11 @@ null ], "shape": [ - 2048, - 26 + 1536, + 58 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2053,15 +2673,17 @@ } }, "partition_spec": [ + "fsdp", null, null ], "shape": [ 512, - 26 + 58, + 576 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2077,22 +2699,19 @@ } }, "partition_spec": [ + "fsdp", null, null, - null, - [ - "fsdp", - "fsdp_transpose" - ] + null ], "shape": [ - 16, - 26, + 512, + 58, 128, - 2048 + 256 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2108,22 +2727,17 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose" - ], - null, + "fsdp", null, null ], "shape": [ - 2048, - 26, - 16, - 192 + 512, + 58, + 1536 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2139,20 +2753,19 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose" - ], + "fsdp", + null, null, null ], "shape": [ - 2048, - 26, - 576 + 1536, + 58, + 128, + 192 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['token_embedder']/['embedding']/.value": { "mesh": { "axis_names": [ "data", @@ -2168,19 +2781,15 @@ } }, "partition_spec": [ - "fsdp", - null, null, - null + "fsdp" ], "shape": [ - 512, - 26, - 16, - 256 + 129280, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['token_embedder']/['embedding']": { + "['optimizer']/['opt_state']/[2]/['count']/.value": { "mesh": { "axis_names": [ "data", @@ -2195,19 +2804,10 @@ "expert": 2 } }, - "partition_spec": [ - null, - [ - "fsdp", - "fsdp_transpose" - ] - ], - "shape": [ - 102400, - 2048 - ] + "partition_spec": [], + "shape": [] }, - ".opt_state/[2]/.count": { + "['optimizer']/['step']/.value": { "mesh": { "axis_names": [ "data", diff --git a/tests/utils/sharding_info/deepseek2-16b/tpu7x-16/slice_1/rule_pure-fsdp/input_shardings.json b/tests/utils/sharding_info/deepseek3-test/tpu7x-16/slice_1/rule_pure-fsdp/input_shardings.json similarity index 71% rename from tests/utils/sharding_info/deepseek2-16b/tpu7x-16/slice_1/rule_pure-fsdp/input_shardings.json rename to tests/utils/sharding_info/deepseek3-test/tpu7x-16/slice_1/rule_pure-fsdp/input_shardings.json index e44cb3ff94..cf6947cc99 100644 --- a/tests/utils/sharding_info/deepseek2-16b/tpu7x-16/slice_1/rule_pure-fsdp/input_shardings.json +++ b/tests/utils/sharding_info/deepseek3-test/tpu7x-16/slice_1/rule_pure-fsdp/input_shardings.json @@ -1,67 +1,67 @@ { "Activation Sharding Dump": [ { - "deepseek/inputs: bfloat16[192,2048,2048]": { + "deepseek/inputs: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P('fsdp', None, None)" } }, { - "deepseek/pre_attention_norm: bfloat16[192,2048,2048]": { + "deepseek/pre_attention_norm: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P('fsdp', None, None)" } }, { - "attention_mla/inputs_q: bfloat16[192,2048,2048]": { + "attention_mla/inputs_q: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch_attn', 'activation_length', 'activation_embed')", "PartitionSpec": "P('fsdp', None, None)" } }, { - "attention_mla/inputs_kv: bfloat16[192,2048,2048]": { + "attention_mla/inputs_kv: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch_attn', 'activation_length', 'activation_embed')", "PartitionSpec": "P('fsdp', None, None)" } }, { - "attention_mla/q_nope: bfloat16[192,2048,16,128]": { + "attention_mla/q_nope: bfloat16[192,2048,128,128]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P('fsdp', None, None, None)" } }, { - "attention_mla/q_pe: bfloat16[192,2048,16,64]": { + "attention_mla/q_pe: bfloat16[192,2048,128,64]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P('fsdp', None, None, None)" } }, { - "attention_mla/query: bfloat16[192,2048,16,192]": { + "attention_mla/query: bfloat16[192,2048,128,192]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P('fsdp', None, None, None)" } }, { - "attention_mla/key_nope: bfloat16[192,2048,16,128]": { + "attention_mla/key_nope: bfloat16[192,2048,128,128]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P('fsdp', None, None, None)" } }, { - "attention_mla/key_rope: bfloat16[192,2048,16,64]": { + "attention_mla/key_rope: bfloat16[192,2048,128,64]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P('fsdp', None, None, None)" } }, { - "attention_mla/key: bfloat16[192,2048,16,192]": { + "attention_mla/key: bfloat16[192,2048,128,192]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P('fsdp', None, None, None)" } }, { - "attention_mla/value: bfloat16[192,2048,16,128]": { + "attention_mla/value: bfloat16[192,2048,128,128]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P('fsdp', None, None, None)" } @@ -79,97 +79,97 @@ } }, { - "attention_op/query: bfloat16[192,16,2048,192]": { + "attention_op/query: bfloat16[192,128,2048,192]": { "logic_axes": "Unknown", "PartitionSpec": "P('fsdp', None, None, None)" } }, { - "attention_op/key: bfloat16[192,16,2048,192]": { + "attention_op/key: bfloat16[192,128,2048,192]": { "logic_axes": "Unknown", "PartitionSpec": "P('fsdp', None, None, None)" } }, { - "attention_op/value: bfloat16[192,16,2048,128]": { + "attention_op/value: bfloat16[192,128,2048,128]": { "logic_axes": "Unknown", "PartitionSpec": "P('fsdp', None, None, None)" } }, { - "attention_mla/out: bfloat16[192,2048,16,128]": { + "attention_op/decoder_segment_ids: int32[192,2048]": { + "logic_axes": "Unknown", + "PartitionSpec": "P('fsdp', None)" + } + }, + { + "attention_mla/out: bfloat16[192,2048,128,128]": { "logic_axes": "('activation_batch_attn', 'activation_length', 'activation_heads', 'activation_kv')", "PartitionSpec": "P('fsdp', None, None, None)" } }, { - "deepseek/attention_result: bfloat16[192,2048,2048]": { + "deepseek/attention_result: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P('fsdp', None, None)" } }, { - "deepseek/post_attention_norm: bfloat16[192,2048,2048]": { + "deepseek/post_attention_norm: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P('fsdp', None, None)" } }, { - "linears/x: bfloat16[192,2048,10944]": { + "linears/x: bfloat16[192,2048,18432]": { "logic_axes": "('activation_batch', 'activation_length', 'activation_mlp')", "PartitionSpec": "P('fsdp', None, None)" } }, { - "deepseek/mlp: bfloat16[192,2048,2048]": { + "deepseek/mlp: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P('fsdp', None, None)" } }, { - "deepseek/x: bfloat16[192,2048,2048]": { + "deepseek/x: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P('fsdp', None, None)" } }, { - "moe/inputs: bfloat16[192,2048,2048]": { - "logic_axes": "('activation_batch', 'activation_norm_length', None)", + "moe/inputs: bfloat16[192,2048,512]": { + "logic_axes": "('activation_batch', 'activation_norm_length_routed', None)", "PartitionSpec": "P('fsdp', None, None)" } }, { - "moe/gate_logits: bfloat16[192,2048,64]": { - "logic_axes": "('activation_batch', 'activation_norm_length', None)", + "moe/gate_logits: bfloat16[192,2048,256]": { + "logic_axes": "('activation_batch', 'activation_norm_length_routed', None)", "PartitionSpec": "P('fsdp', None, None)" } }, { - "moe/w0_kernel: bfloat16[64,2048,1408]": { - "logic_axes": "Unknown", - "PartitionSpec": "P(None, None, None)" - } - }, - { - "moe/w1_kernel: bfloat16[64,2048,1408]": { - "logic_axes": "Unknown", - "PartitionSpec": "P(None, None, None)" + "moe/pre_bias_logits: bfloat16[192,2048,256]": { + "logic_axes": "('activation_batch', 'activation_norm_length_routed', None)", + "PartitionSpec": "P('fsdp', None, None)" } }, { - "moe/wo_kernel: bfloat16[64,1408,2048]": { + "moe/w: bfloat16[256,512,512]": { "logic_axes": "Unknown", "PartitionSpec": "P(None, None, None)" } }, { - "linears/x: bfloat16[192,2048,2816]": { + "linears/x: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_length', 'activation_mlp')", "PartitionSpec": "P('fsdp', None, None)" } }, { - "deepseek/mlp_lnx: bfloat16[192,2048,2048]": { + "deepseek/mlp_lnx: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P('fsdp', None, None)" } diff --git a/tests/utils/sharding_info/deepseek3-test/tpu7x-16/slice_1/rule_pure-fsdp/logical_shardings.json b/tests/utils/sharding_info/deepseek3-test/tpu7x-16/slice_1/rule_pure-fsdp/logical_shardings.json new file mode 100644 index 0000000000..d7f45f5385 --- /dev/null +++ b/tests/utils/sharding_info/deepseek3-test/tpu7x-16/slice_1/rule_pure-fsdp/logical_shardings.json @@ -0,0 +1,1218 @@ +{ + "['model']/['decoder']/['decoder_norm']/['scale']/.value": { + "partition_spec": [ + null + ], + "shape": [ + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + "fsdp" + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['count']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['key']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['count']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['key']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['count']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['key']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['model']/['decoder']/['logits_dense']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null + ], + "shape": [ + 512, + 129280 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['bias']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 58, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['count']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['key']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['count']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['key']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['count']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['key']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "partition_spec": [ + null, + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "partition_spec": [ + null, + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + "fsdp" + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['model']/['decoder']/['rngs']/['aqt']/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['aqt']/['key']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['dropout']/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['dropout']/['key']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['params']/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['params']/['key']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['token_embedder']/['embedding']/.value": { + "partition_spec": [ + null, + "fsdp" + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['decoder_norm']/['scale']/.value": { + "partition_spec": [ + null + ], + "shape": [ + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + "fsdp" + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['logits_dense']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null + ], + "shape": [ + 512, + 129280 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "partition_spec": [ + null, + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "partition_spec": [ + null, + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + "fsdp" + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['token_embedder']/['embedding']/.value": { + "partition_spec": [ + null, + "fsdp" + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['decoder_norm']/['scale']/.value": { + "partition_spec": [ + null + ], + "shape": [ + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + "fsdp" + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['logits_dense']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null + ], + "shape": [ + 512, + 129280 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "partition_spec": [ + null, + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "partition_spec": [ + null, + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + "fsdp" + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['token_embedder']/['embedding']/.value": { + "partition_spec": [ + null, + "fsdp" + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[2]/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['optimizer']/['step']/.value": { + "partition_spec": [], + "shape": [] + } +} \ No newline at end of file diff --git a/tests/utils/sharding_info/deepseek3-test/tpu7x-16/slice_1/rule_pure-fsdp/named_shardings.json b/tests/utils/sharding_info/deepseek3-test/tpu7x-16/slice_1/rule_pure-fsdp/named_shardings.json new file mode 100644 index 0000000000..627ac471a6 --- /dev/null +++ b/tests/utils/sharding_info/deepseek3-test/tpu7x-16/slice_1/rule_pure-fsdp/named_shardings.json @@ -0,0 +1,2138 @@ +{ + "['model']/['decoder']/['decoder_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null + ], + "shape": [ + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null, + "fsdp" + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['count']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['key']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['count']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['key']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['count']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['key']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['model']/['decoder']/['logits_dense']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null + ], + "shape": [ + 512, + 129280 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['bias']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 58, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['count']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['key']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['count']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['key']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['count']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['key']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null, + "fsdp" + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['model']/['decoder']/['rngs']/['aqt']/['count']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['aqt']/['key']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['dropout']/['count']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['dropout']/['key']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['params']/['count']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['params']/['key']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['token_embedder']/['embedding']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + "fsdp" + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['count']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [], + "shape": [] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['decoder_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null + ], + "shape": [ + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null, + "fsdp" + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['logits_dense']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null + ], + "shape": [ + 512, + 129280 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null, + "fsdp" + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['token_embedder']/['embedding']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + "fsdp" + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['decoder_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null + ], + "shape": [ + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null, + "fsdp" + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['logits_dense']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null + ], + "shape": [ + 512, + 129280 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null, + "fsdp" + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null, + null, + "fsdp" + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['token_embedder']/['embedding']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [ + null, + "fsdp" + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[2]/['count']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [], + "shape": [] + }, + "['optimizer']/['step']/.value": { + "mesh": { + "axis_names": [ + "fsdp" + ], + "shape": { + "fsdp": 16 + } + }, + "partition_spec": [], + "shape": [] + } +} \ No newline at end of file diff --git a/tests/utils/sharding_info/deepseek2-16b/tpu7x-8/slice_1/rule_cp-as-ep_ici_fsdp_parallelism=-1_ici_context_parallelism=2_ici_expert_parallelism=2/input_shardings.json b/tests/utils/sharding_info/deepseek3-test/tpu7x-8/slice_1/rule_cp-as-ep_ici_fsdp_parallelism=-1_ici_context_parallelism=2_ici_expert_parallelism=2/input_shardings.json similarity index 72% rename from tests/utils/sharding_info/deepseek2-16b/tpu7x-8/slice_1/rule_cp-as-ep_ici_fsdp_parallelism=-1_ici_context_parallelism=2_ici_expert_parallelism=2/input_shardings.json rename to tests/utils/sharding_info/deepseek3-test/tpu7x-8/slice_1/rule_cp-as-ep_ici_fsdp_parallelism=-1_ici_context_parallelism=2_ici_expert_parallelism=2/input_shardings.json index e8578a5ce9..563b1c0909 100644 --- a/tests/utils/sharding_info/deepseek2-16b/tpu7x-8/slice_1/rule_cp-as-ep_ici_fsdp_parallelism=-1_ici_context_parallelism=2_ici_expert_parallelism=2/input_shardings.json +++ b/tests/utils/sharding_info/deepseek3-test/tpu7x-8/slice_1/rule_cp-as-ep_ici_fsdp_parallelism=-1_ici_context_parallelism=2_ici_expert_parallelism=2/input_shardings.json @@ -1,67 +1,67 @@ { "Activation Sharding Dump": [ { - "deepseek/inputs: bfloat16[96,2048,2048]": { + "deepseek/inputs: bfloat16[96,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), 'context', None)" } }, { - "deepseek/pre_attention_norm: bfloat16[96,2048,2048]": { + "deepseek/pre_attention_norm: bfloat16[96,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), 'context', None)" } }, { - "attention_mla/inputs_q: bfloat16[96,2048,2048]": { + "attention_mla/inputs_q: bfloat16[96,2048,512]": { "logic_axes": "('activation_batch_attn', 'activation_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), 'context', None)" } }, { - "attention_mla/inputs_kv: bfloat16[96,2048,2048]": { + "attention_mla/inputs_kv: bfloat16[96,2048,512]": { "logic_axes": "('activation_batch_attn', 'activation_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), 'context', None)" } }, { - "attention_mla/q_nope: bfloat16[96,2048,16,128]": { + "attention_mla/q_nope: bfloat16[96,2048,128,128]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), 'context', None, None)" } }, { - "attention_mla/q_pe: bfloat16[96,2048,16,64]": { + "attention_mla/q_pe: bfloat16[96,2048,128,64]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), 'context', None, None)" } }, { - "attention_mla/query: bfloat16[96,2048,16,192]": { + "attention_mla/query: bfloat16[96,2048,128,192]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), 'context', None, None)" } }, { - "attention_mla/key_nope: bfloat16[96,2048,16,128]": { + "attention_mla/key_nope: bfloat16[96,2048,128,128]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), 'context', None, None)" } }, { - "attention_mla/key_rope: bfloat16[96,2048,16,64]": { + "attention_mla/key_rope: bfloat16[96,2048,128,64]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), 'context', None, None)" } }, { - "attention_mla/key: bfloat16[96,2048,16,192]": { + "attention_mla/key: bfloat16[96,2048,128,192]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), 'context', None, None)" } }, { - "attention_mla/value: bfloat16[96,2048,16,128]": { + "attention_mla/value: bfloat16[96,2048,128,128]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), 'context', None, None)" } @@ -79,97 +79,103 @@ } }, { - "attention_op/query: bfloat16[96,16,2048,192]": { + "attention_op/query: bfloat16[96,128,2048,192]": { "logic_axes": "Unknown", "PartitionSpec": "P(('fsdp', 'expert'), None, 'context', None)" } }, { - "attention_op/key: bfloat16[96,16,2048,192]": { + "attention_op/key: bfloat16[96,128,2048,192]": { "logic_axes": "Unknown", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_op/value: bfloat16[96,16,2048,128]": { + "attention_op/value: bfloat16[96,128,2048,128]": { "logic_axes": "Unknown", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_mla/out: bfloat16[96,2048,16,128]": { + "attention_op/decoder_segment_ids: int32[96,2048]": { + "logic_axes": "Unknown", + "PartitionSpec": "P(('fsdp', 'expert'), 'context')" + } + }, + { + "attention_op/decoder_segment_ids: int32[96,2048]": { + "logic_axes": "Unknown", + "PartitionSpec": "P(('fsdp', 'expert'), None)" + } + }, + { + "attention_mla/out: bfloat16[96,2048,128,128]": { "logic_axes": "('activation_batch_attn', 'activation_length', 'activation_heads', 'activation_kv')", "PartitionSpec": "P(('fsdp', 'expert'), 'context', None, None)" } }, { - "deepseek/attention_result: bfloat16[96,2048,2048]": { + "deepseek/attention_result: bfloat16[96,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), 'context', None)" } }, { - "deepseek/post_attention_norm: bfloat16[96,2048,2048]": { + "deepseek/post_attention_norm: bfloat16[96,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), 'context', None)" } }, { - "linears/x: bfloat16[96,2048,10944]": { + "linears/x: bfloat16[96,2048,18432]": { "logic_axes": "('activation_batch', 'activation_length', 'activation_mlp')", "PartitionSpec": "P(('fsdp', 'expert'), 'context', None)" } }, { - "deepseek/mlp: bfloat16[96,2048,2048]": { + "deepseek/mlp: bfloat16[96,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), 'context', None)" } }, { - "deepseek/x: bfloat16[96,2048,2048]": { + "deepseek/x: bfloat16[96,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), 'context', None)" } }, { - "moe/inputs: bfloat16[96,2048,2048]": { - "logic_axes": "('activation_batch', 'activation_norm_length', None)", + "moe/inputs: bfloat16[96,2048,512]": { + "logic_axes": "('activation_batch', 'activation_norm_length_routed', None)", "PartitionSpec": "P(('fsdp', 'expert'), 'context', None)" } }, { - "moe/gate_logits: bfloat16[96,2048,64]": { - "logic_axes": "('activation_batch', 'activation_norm_length', None)", + "moe/gate_logits: bfloat16[96,2048,256]": { + "logic_axes": "('activation_batch', 'activation_norm_length_routed', None)", "PartitionSpec": "P(('fsdp', 'expert'), 'context', None)" } }, { - "moe/w0_kernel: bfloat16[64,2048,1408]": { - "logic_axes": "Unknown", - "PartitionSpec": "P(('context', 'expert'), None, None)" - } - }, - { - "moe/w1_kernel: bfloat16[64,2048,1408]": { - "logic_axes": "Unknown", - "PartitionSpec": "P(('context', 'expert'), None, None)" + "moe/pre_bias_logits: bfloat16[96,2048,256]": { + "logic_axes": "('activation_batch', 'activation_norm_length_routed', None)", + "PartitionSpec": "P(('fsdp', 'expert'), 'context', None)" } }, { - "moe/wo_kernel: bfloat16[64,1408,2048]": { + "moe/w: bfloat16[256,512,512]": { "logic_axes": "Unknown", "PartitionSpec": "P(('context', 'expert'), None, None)" } }, { - "linears/x: bfloat16[96,2048,2816]": { + "linears/x: bfloat16[96,2048,512]": { "logic_axes": "('activation_batch', 'activation_length', 'activation_mlp')", "PartitionSpec": "P(('fsdp', 'expert'), 'context', None)" } }, { - "deepseek/mlp_lnx: bfloat16[96,2048,2048]": { + "deepseek/mlp_lnx: bfloat16[96,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), 'context', None)" } diff --git a/tests/utils/sharding_info/deepseek3-test/tpu7x-8/slice_1/rule_cp-as-ep_ici_fsdp_parallelism=-1_ici_context_parallelism=2_ici_expert_parallelism=2/logical_shardings.json b/tests/utils/sharding_info/deepseek3-test/tpu7x-8/slice_1/rule_cp-as-ep_ici_fsdp_parallelism=-1_ici_context_parallelism=2_ici_expert_parallelism=2/logical_shardings.json new file mode 100644 index 0000000000..4e1ac3cca6 --- /dev/null +++ b/tests/utils/sharding_info/deepseek3-test/tpu7x-8/slice_1/rule_cp-as-ep_ici_fsdp_parallelism=-1_ici_context_parallelism=2_ici_expert_parallelism=2/logical_shardings.json @@ -0,0 +1,1461 @@ +{ + "['model']/['decoder']/['decoder_norm']/['scale']/.value": { + "partition_spec": [ + null + ], + "shape": [ + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + [ + "fsdp", + "context", + "expert" + ] + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['count']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['key']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['count']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['key']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['count']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['key']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "context", + "expert" + ] + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null, + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null, + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['model']/['decoder']/['logits_dense']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null + ], + "shape": [ + 512, + 129280 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['bias']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 58, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['count']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['key']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['count']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['key']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['count']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['key']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "partition_spec": [ + [ + "context", + "expert" + ], + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "partition_spec": [ + [ + "context", + "expert" + ], + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "partition_spec": [ + [ + "context", + "expert" + ], + null, + null, + "fsdp" + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + [ + "fsdp", + "context", + "expert" + ] + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "context", + "expert" + ] + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null, + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null, + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['model']/['decoder']/['rngs']/['aqt']/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['aqt']/['key']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['dropout']/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['dropout']/['key']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['params']/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['params']/['key']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['token_embedder']/['embedding']/.value": { + "partition_spec": [ + null, + [ + "fsdp", + "context", + "expert" + ] + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['decoder_norm']/['scale']/.value": { + "partition_spec": [ + null + ], + "shape": [ + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + [ + "fsdp", + "context", + "expert" + ] + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "context", + "expert" + ] + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null, + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null, + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['logits_dense']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null + ], + "shape": [ + 512, + 129280 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "partition_spec": [ + [ + "context", + "expert" + ], + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "partition_spec": [ + [ + "context", + "expert" + ], + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "partition_spec": [ + [ + "context", + "expert" + ], + null, + null, + "fsdp" + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + [ + "fsdp", + "context", + "expert" + ] + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "context", + "expert" + ] + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null, + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null, + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['token_embedder']/['embedding']/.value": { + "partition_spec": [ + null, + [ + "fsdp", + "context", + "expert" + ] + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['decoder_norm']/['scale']/.value": { + "partition_spec": [ + null + ], + "shape": [ + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + [ + "fsdp", + "context", + "expert" + ] + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "context", + "expert" + ] + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null, + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null, + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['logits_dense']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null + ], + "shape": [ + 512, + 129280 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "partition_spec": [ + [ + "context", + "expert" + ], + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "partition_spec": [ + [ + "context", + "expert" + ], + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "partition_spec": [ + [ + "context", + "expert" + ], + null, + null, + "fsdp" + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + [ + "fsdp", + "context", + "expert" + ] + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "context", + "expert" + ] + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null, + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null, + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['token_embedder']/['embedding']/.value": { + "partition_spec": [ + null, + [ + "fsdp", + "context", + "expert" + ] + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[2]/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['optimizer']/['step']/.value": { + "partition_spec": [], + "shape": [] + } +} \ No newline at end of file diff --git a/tests/utils/sharding_info/deepseek2-16b/tpu7x-8/slice_1/rule_cp-as-ep_ici_fsdp_parallelism=-1_ici_context_parallelism=2_ici_expert_parallelism=2/named_shardings.json b/tests/utils/sharding_info/deepseek3-test/tpu7x-8/slice_1/rule_cp-as-ep_ici_fsdp_parallelism=-1_ici_context_parallelism=2_ici_expert_parallelism=2/named_shardings.json similarity index 56% rename from tests/utils/sharding_info/deepseek2-16b/tpu7x-8/slice_1/rule_cp-as-ep_ici_fsdp_parallelism=-1_ici_context_parallelism=2_ici_expert_parallelism=2/named_shardings.json rename to tests/utils/sharding_info/deepseek3-test/tpu7x-8/slice_1/rule_cp-as-ep_ici_fsdp_parallelism=-1_ici_context_parallelism=2_ici_expert_parallelism=2/named_shardings.json index 43e03a274f..49f87f42a4 100644 --- a/tests/utils/sharding_info/deepseek2-16b/tpu7x-8/slice_1/rule_cp-as-ep_ici_fsdp_parallelism=-1_ici_context_parallelism=2_ici_expert_parallelism=2/named_shardings.json +++ b/tests/utils/sharding_info/deepseek3-test/tpu7x-8/slice_1/rule_cp-as-ep_ici_fsdp_parallelism=-1_ici_context_parallelism=2_ici_expert_parallelism=2/named_shardings.json @@ -1,25 +1,5 @@ { - ".step": { - "mesh": { - "axis_names": [ - "data", - "stage", - "fsdp", - "context", - "expert" - ], - "shape": { - "data": 1, - "stage": 1, - "fsdp": 2, - "context": 2, - "expert": 2 - } - }, - "partition_spec": [], - "shape": [] - }, - ".params/['params']/['decoder']/['decoder_norm']/['scale']": { + "['model']/['decoder']/['decoder_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -40,10 +20,10 @@ null ], "shape": [ - 2048 + 512 ] }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -70,12 +50,12 @@ null ], "shape": [ - 2048, - 1, - 10944 + 512, + 3, + 18432 ] }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -102,12 +82,12 @@ null ], "shape": [ - 2048, - 1, - 10944 + 512, + 3, + 18432 ] }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -134,12 +114,12 @@ ] ], "shape": [ - 10944, - 1, - 2048 + 18432, + 3, + 512 ] }, - ".params/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { + "['model']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -161,11 +141,11 @@ null ], "shape": [ - 2048, - 1 + 512, + 3 ] }, - ".params/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['model']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -187,11 +167,11 @@ null ], "shape": [ - 2048, - 1 + 512, + 3 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['count']/.value": { "mesh": { "axis_names": [ "data", @@ -208,16 +188,12 @@ "expert": 2 } }, - "partition_spec": [ - null, - null - ], + "partition_spec": [], "shape": [ - 512, - 1 + 3 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['key']/.value": { "mesh": { "axis_names": [ "data", @@ -234,24 +210,12 @@ "expert": 2 } }, - "partition_spec": [ - null, - null, - null, - [ - "fsdp", - "context", - "expert" - ] - ], + "partition_spec": [], "shape": [ - 16, - 1, - 128, - 2048 + 3 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['count']/.value": { "mesh": { "axis_names": [ "data", @@ -268,24 +232,34 @@ "expert": 2 } }, - "partition_spec": [ - [ + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", "fsdp", "context", "expert" ], - null, - null, - null - ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 2, + "context": 2, + "expert": 2 + } + }, + "partition_spec": [], "shape": [ - 2048, - 1, - 16, - 192 + 3 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['count']/.value": { "mesh": { "axis_names": [ "data", @@ -302,22 +276,34 @@ "expert": 2 } }, - "partition_spec": [ - [ + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", "fsdp", "context", "expert" ], - null, - null - ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 2, + "context": 2, + "expert": 2 + } + }, + "partition_spec": [], "shape": [ - 2048, - 1, - 576 + 3 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -335,23 +321,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "context", - "expert" - ], - null, null, null ], "shape": [ 512, - 1, - 16, - 256 + 3 ] }, - ".params/['params']/['decoder']/['logits_dense']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -369,19 +347,23 @@ } }, "partition_spec": [ + null, + null, + null, [ "fsdp", "context", "expert" - ], - null + ] ], "shape": [ - 2048, - 102400 + 128, + 3, + 128, + 512 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -399,17 +381,15 @@ } }, "partition_spec": [ - "fsdp", null, null ], "shape": [ - 2048, - 26, - 64 + 1536, + 3 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -428,21 +408,20 @@ }, "partition_spec": [ [ + "fsdp", "context", "expert" ], null, - "fsdp", null ], "shape": [ - 64, - 26, - 2048, - 1408 + 512, + 3, + 576 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -461,21 +440,22 @@ }, "partition_spec": [ [ + "fsdp", "context", "expert" ], null, - "fsdp", + null, null ], "shape": [ - 64, - 26, - 2048, - 1408 + 512, + 3, + 128, + 256 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -494,21 +474,20 @@ }, "partition_spec": [ [ + "fsdp", "context", "expert" ], null, - null, - "fsdp" + null ], "shape": [ - 64, - 26, - 1408, - 2048 + 512, + 3, + 1536 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -532,15 +511,17 @@ "expert" ], null, + null, null ], "shape": [ - 2048, - 26, - 2816 + 1536, + 3, + 128, + 192 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { + "['model']/['decoder']/['logits_dense']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -563,16 +544,14 @@ "context", "expert" ], - null, null ], "shape": [ - 2048, - 26, - 2816 + 512, + 129280 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['bias']/.value": { "mesh": { "axis_names": [ "data", @@ -591,20 +570,14 @@ }, "partition_spec": [ null, - null, - [ - "fsdp", - "context", - "expert" - ] + null ], "shape": [ - 2816, - 26, - 2048 + 58, + 256 ] }, - ".params/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -622,15 +595,17 @@ } }, "partition_spec": [ + "fsdp", null, null ], "shape": [ - 2048, - 26 + 512, + 58, + 256 ] }, - ".params/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['count']/.value": { "mesh": { "axis_names": [ "data", @@ -647,16 +622,12 @@ "expert": 2 } }, - "partition_spec": [ - null, - null - ], + "partition_spec": [], "shape": [ - 2048, - 26 + 58 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['key']/.value": { "mesh": { "axis_names": [ "data", @@ -673,16 +644,12 @@ "expert": 2 } }, - "partition_spec": [ - null, - null - ], + "partition_spec": [], "shape": [ - 512, - 26 + 58 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['count']/.value": { "mesh": { "axis_names": [ "data", @@ -699,24 +666,12 @@ "expert": 2 } }, - "partition_spec": [ - null, - null, - null, - [ - "fsdp", - "context", - "expert" - ] - ], + "partition_spec": [], "shape": [ - 16, - 26, - 128, - 2048 + 58 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['key']/.value": { "mesh": { "axis_names": [ "data", @@ -733,24 +688,12 @@ "expert": 2 } }, - "partition_spec": [ - [ - "fsdp", - "context", - "expert" - ], - null, - null, - null - ], + "partition_spec": [], "shape": [ - 2048, - 26, - 16, - 192 + 58 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['count']/.value": { "mesh": { "axis_names": [ "data", @@ -767,22 +710,34 @@ "expert": 2 } }, - "partition_spec": [ - [ + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", "fsdp", "context", "expert" ], - null, - null - ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 2, + "context": 2, + "expert": 2 + } + }, + "partition_spec": [], "shape": [ - 2048, - 26, - 576 + 58 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { "mesh": { "axis_names": [ "data", @@ -801,22 +756,21 @@ }, "partition_spec": [ [ - "fsdp", "context", "expert" ], null, - null, + "fsdp", null ], "shape": [ + 256, + 58, 512, - 26, - 16, - 256 + 512 ] }, - ".params/['params']/['token_embedder']/['embedding']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { "mesh": { "axis_names": [ "data", @@ -834,19 +788,22 @@ } }, "partition_spec": [ - null, [ - "fsdp", "context", "expert" - ] + ], + null, + "fsdp", + null ], "shape": [ - 102400, - 2048 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.count": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { "mesh": { "axis_names": [ "data", @@ -863,10 +820,23 @@ "expert": 2 } }, - "partition_spec": [], - "shape": [] + "partition_spec": [ + [ + "context", + "expert" + ], + null, + null, + "fsdp" + ], + "shape": [ + 256, + 58, + 512, + 512 + ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['decoder_norm']/['scale']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -884,13 +854,21 @@ } }, "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, null ], "shape": [ - 2048 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -917,12 +895,12 @@ null ], "shape": [ - 2048, - 1, - 10944 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -940,21 +918,21 @@ } }, "partition_spec": [ + null, + null, [ "fsdp", "context", "expert" - ], - null, - null + ] ], "shape": [ - 2048, - 1, - 10944 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -973,20 +951,14 @@ }, "partition_spec": [ null, - null, - [ - "fsdp", - "context", - "expert" - ] + null ], "shape": [ - 10944, - 1, - 2048 + 512, + 58 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { + "['model']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1008,11 +980,11 @@ null ], "shape": [ - 2048, - 1 + 512, + 58 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['model']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1034,11 +1006,11 @@ null ], "shape": [ - 2048, - 1 + 512, + 58 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { + "['model']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1057,14 +1029,22 @@ }, "partition_spec": [ null, - null + null, + null, + [ + "fsdp", + "context", + "expert" + ] ], "shape": [ - 512, - 1 + 128, + 58, + 128, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1083,22 +1063,606 @@ }, "partition_spec": [ null, + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 2, + "context": 2, + "expert": 2 + } + }, + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 2, + "context": 2, + "expert": 2 + } + }, + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null, + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 2, + "context": 2, + "expert": 2 + } + }, + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 2, + "context": 2, + "expert": 2 + } + }, + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null, + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['model']/['decoder']/['rngs']/['aqt']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 2, + "context": 2, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['aqt']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 2, + "context": 2, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['dropout']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 2, + "context": 2, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['dropout']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 2, + "context": 2, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['params']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 2, + "context": 2, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['params']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 2, + "context": 2, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['token_embedder']/['embedding']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 2, + "context": 2, + "expert": 2 + } + }, + "partition_spec": [ + null, + [ + "fsdp", + "context", + "expert" + ] + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 2, + "context": 2, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['decoder_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 2, + "context": 2, + "expert": 2 + } + }, + "partition_spec": [ + null + ], + "shape": [ + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 2, + "context": 2, + "expert": 2 + } + }, + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 2, + "context": 2, + "expert": 2 + } + }, + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 2, + "context": 2, + "expert": 2 + } + }, + "partition_spec": [ + null, + null, + [ + "fsdp", + "context", + "expert" + ] + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 2, + "context": 2, + "expert": 2 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 2, + "context": 2, + "expert": 2 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 2, + "context": 2, + "expert": 2 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 2, + "context": 2, + "expert": 2 + } + }, + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "context", + "expert" + ] + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 2, + "context": 2, + "expert": 2 + } + }, + "partition_spec": [ null, - null, + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 2, + "context": 2, + "expert": 2 + } + }, + "partition_spec": [ [ "fsdp", "context", "expert" - ] + ], + null, + null ], "shape": [ - 16, - 1, - 128, - 2048 + 512, + 3, + 576 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1126,13 +1690,13 @@ null ], "shape": [ - 2048, - 1, - 16, - 192 + 512, + 3, + 128, + 256 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1159,12 +1723,12 @@ null ], "shape": [ - 2048, - 1, - 576 + 512, + 3, + 1536 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1192,13 +1756,13 @@ null ], "shape": [ - 512, - 1, - 16, - 256 + 1536, + 3, + 128, + 192 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['logits_dense']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['logits_dense']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1224,11 +1788,11 @@ null ], "shape": [ - 2048, - 102400 + 512, + 129280 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1251,12 +1815,12 @@ null ], "shape": [ - 2048, - 26, - 64 + 512, + 58, + 256 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { "mesh": { "axis_names": [ "data", @@ -1283,13 +1847,13 @@ null ], "shape": [ - 64, - 26, - 2048, - 1408 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { "mesh": { "axis_names": [ "data", @@ -1316,13 +1880,13 @@ null ], "shape": [ - 64, - 26, - 2048, - 1408 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { "mesh": { "axis_names": [ "data", @@ -1349,13 +1913,13 @@ "fsdp" ], "shape": [ - 64, - 26, - 1408, - 2048 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1382,12 +1946,12 @@ null ], "shape": [ - 2048, - 26, - 2816 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1414,12 +1978,12 @@ null ], "shape": [ - 2048, - 26, - 2816 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1446,12 +2010,12 @@ ] ], "shape": [ - 2816, - 26, - 2048 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1473,11 +2037,11 @@ null ], "shape": [ - 2048, - 26 + 512, + 58 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1499,11 +2063,11 @@ null ], "shape": [ - 2048, - 26 + 512, + 58 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1526,10 +2090,10 @@ ], "shape": [ 512, - 26 + 58 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1557,13 +2121,71 @@ ] ], "shape": [ - 16, - 26, 128, - 2048 + 58, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 2, + "context": 2, + "expert": 2 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 2, + "context": 2, + "expert": 2 + } + }, + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 576 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1591,13 +2213,13 @@ null ], "shape": [ - 2048, - 26, - 16, - 192 + 512, + 58, + 128, + 256 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1624,12 +2246,12 @@ null ], "shape": [ - 2048, - 26, - 576 + 512, + 58, + 1536 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1657,13 +2279,13 @@ null ], "shape": [ - 512, - 26, - 16, - 256 + 1536, + 58, + 128, + 192 ] }, - ".opt_state/[0]/.mu/['params']/['token_embedder']/['embedding']": { + "['optimizer']/['opt_state']/[0]/['mu']/['token_embedder']/['embedding']/.value": { "mesh": { "axis_names": [ "data", @@ -1689,11 +2311,11 @@ ] ], "shape": [ - 102400, - 2048 + 129280, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['decoder_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['decoder_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1714,10 +2336,10 @@ null ], "shape": [ - 2048 + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1744,12 +2366,12 @@ null ], "shape": [ - 2048, - 1, - 10944 + 512, + 3, + 18432 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1776,12 +2398,12 @@ null ], "shape": [ - 2048, - 1, - 10944 + 512, + 3, + 18432 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1808,12 +2430,12 @@ ] ], "shape": [ - 10944, - 1, - 2048 + 18432, + 3, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1835,11 +2457,11 @@ null ], "shape": [ - 2048, - 1 + 512, + 3 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1861,11 +2483,11 @@ null ], "shape": [ - 2048, - 1 + 512, + 3 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1888,10 +2510,10 @@ ], "shape": [ 512, - 1 + 3 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1919,13 +2541,71 @@ ] ], "shape": [ - 16, - 1, 128, - 2048 + 3, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 2, + "context": 2, + "expert": 2 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 2, + "context": 2, + "expert": 2 + } + }, + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 576 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1953,13 +2633,13 @@ null ], "shape": [ - 2048, - 1, - 16, - 192 + 512, + 3, + 128, + 256 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1986,12 +2666,12 @@ null ], "shape": [ - 2048, - 1, - 576 + 512, + 3, + 1536 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2019,13 +2699,13 @@ null ], "shape": [ - 512, - 1, - 16, - 256 + 1536, + 3, + 128, + 192 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['logits_dense']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['logits_dense']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2051,11 +2731,11 @@ null ], "shape": [ - 2048, - 102400 + 512, + 129280 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2078,12 +2758,12 @@ null ], "shape": [ - 2048, - 26, - 64 + 512, + 58, + 256 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { "mesh": { "axis_names": [ "data", @@ -2110,13 +2790,13 @@ null ], "shape": [ - 64, - 26, - 2048, - 1408 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { "mesh": { "axis_names": [ "data", @@ -2143,13 +2823,13 @@ null ], "shape": [ - 64, - 26, - 2048, - 1408 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { "mesh": { "axis_names": [ "data", @@ -2176,13 +2856,13 @@ "fsdp" ], "shape": [ - 64, - 26, - 1408, - 2048 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2209,12 +2889,12 @@ null ], "shape": [ - 2048, - 26, - 2816 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2241,12 +2921,12 @@ null ], "shape": [ - 2048, - 26, - 2816 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2273,12 +2953,12 @@ ] ], "shape": [ - 2816, - 26, - 2048 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -2300,11 +2980,11 @@ null ], "shape": [ - 2048, - 26 + 512, + 58 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -2326,11 +3006,11 @@ null ], "shape": [ - 2048, - 26 + 512, + 58 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -2353,10 +3033,10 @@ ], "shape": [ 512, - 26 + 58 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2384,13 +3064,71 @@ ] ], "shape": [ - 16, - 26, 128, - 2048 + 58, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 2, + "context": 2, + "expert": 2 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 2, + "context": 2, + "expert": 2 + } + }, + "partition_spec": [ + [ + "fsdp", + "context", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 576 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2418,13 +3156,13 @@ null ], "shape": [ - 2048, - 26, - 16, - 192 + 512, + 58, + 128, + 256 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2451,12 +3189,12 @@ null ], "shape": [ - 2048, - 26, - 576 + 512, + 58, + 1536 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2484,13 +3222,13 @@ null ], "shape": [ - 512, - 26, - 16, - 256 + 1536, + 58, + 128, + 192 ] }, - ".opt_state/[0]/.nu/['params']/['token_embedder']/['embedding']": { + "['optimizer']/['opt_state']/[0]/['nu']/['token_embedder']/['embedding']/.value": { "mesh": { "axis_names": [ "data", @@ -2516,11 +3254,31 @@ ] ], "shape": [ - 102400, - 2048 + 129280, + 512 ] }, - ".opt_state/[2]/.count": { + "['optimizer']/['opt_state']/[2]/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 2, + "context": 2, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [] + }, + "['optimizer']/['step']/.value": { "mesh": { "axis_names": [ "data", diff --git a/tests/utils/sharding_info/deepseek2-16b/tpu7x-8/slice_1/rule_ep-as-cp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/input_shardings.json b/tests/utils/sharding_info/deepseek3-test/tpu7x-8/slice_1/rule_ep-as-cp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/input_shardings.json similarity index 71% rename from tests/utils/sharding_info/deepseek2-16b/tpu7x-8/slice_1/rule_ep-as-cp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/input_shardings.json rename to tests/utils/sharding_info/deepseek3-test/tpu7x-8/slice_1/rule_ep-as-cp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/input_shardings.json index 0b392bf89b..d9898e2e76 100644 --- a/tests/utils/sharding_info/deepseek2-16b/tpu7x-8/slice_1/rule_ep-as-cp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/input_shardings.json +++ b/tests/utils/sharding_info/deepseek3-test/tpu7x-8/slice_1/rule_ep-as-cp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/input_shardings.json @@ -1,67 +1,67 @@ { "Activation Sharding Dump": [ { - "deepseek/inputs: bfloat16[96,2048,2048]": { + "deepseek/inputs: bfloat16[96,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P('fsdp', 'expert', None)" } }, { - "deepseek/pre_attention_norm: bfloat16[96,2048,2048]": { + "deepseek/pre_attention_norm: bfloat16[96,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P('fsdp', 'expert', None)" } }, { - "attention_mla/inputs_q: bfloat16[96,2048,2048]": { + "attention_mla/inputs_q: bfloat16[96,2048,512]": { "logic_axes": "('activation_batch_attn', 'activation_length', 'activation_embed')", "PartitionSpec": "P('fsdp', 'expert', None)" } }, { - "attention_mla/inputs_kv: bfloat16[96,2048,2048]": { + "attention_mla/inputs_kv: bfloat16[96,2048,512]": { "logic_axes": "('activation_batch_attn', 'activation_length', 'activation_embed')", "PartitionSpec": "P('fsdp', 'expert', None)" } }, { - "attention_mla/q_nope: bfloat16[96,2048,16,128]": { + "attention_mla/q_nope: bfloat16[96,2048,128,128]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P('fsdp', 'expert', None, None)" } }, { - "attention_mla/q_pe: bfloat16[96,2048,16,64]": { + "attention_mla/q_pe: bfloat16[96,2048,128,64]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P('fsdp', 'expert', None, None)" } }, { - "attention_mla/query: bfloat16[96,2048,16,192]": { + "attention_mla/query: bfloat16[96,2048,128,192]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P('fsdp', 'expert', None, None)" } }, { - "attention_mla/key_nope: bfloat16[96,2048,16,128]": { + "attention_mla/key_nope: bfloat16[96,2048,128,128]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P('fsdp', 'expert', None, None)" } }, { - "attention_mla/key_rope: bfloat16[96,2048,16,64]": { + "attention_mla/key_rope: bfloat16[96,2048,128,64]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P('fsdp', 'expert', None, None)" } }, { - "attention_mla/key: bfloat16[96,2048,16,192]": { + "attention_mla/key: bfloat16[96,2048,128,192]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P('fsdp', 'expert', None, None)" } }, { - "attention_mla/value: bfloat16[96,2048,16,128]": { + "attention_mla/value: bfloat16[96,2048,128,128]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P('fsdp', 'expert', None, None)" } @@ -79,97 +79,103 @@ } }, { - "attention_op/query: bfloat16[96,16,2048,192]": { + "attention_op/query: bfloat16[96,128,2048,192]": { "logic_axes": "Unknown", "PartitionSpec": "P('fsdp', None, 'expert', None)" } }, { - "attention_op/key: bfloat16[96,16,2048,192]": { + "attention_op/key: bfloat16[96,128,2048,192]": { "logic_axes": "Unknown", "PartitionSpec": "P('fsdp', None, None, None)" } }, { - "attention_op/value: bfloat16[96,16,2048,128]": { + "attention_op/value: bfloat16[96,128,2048,128]": { "logic_axes": "Unknown", "PartitionSpec": "P('fsdp', None, None, None)" } }, { - "attention_mla/out: bfloat16[96,2048,16,128]": { + "attention_op/decoder_segment_ids: int32[96,2048]": { + "logic_axes": "Unknown", + "PartitionSpec": "P('fsdp', 'expert')" + } + }, + { + "attention_op/decoder_segment_ids: int32[96,2048]": { + "logic_axes": "Unknown", + "PartitionSpec": "P('fsdp', None)" + } + }, + { + "attention_mla/out: bfloat16[96,2048,128,128]": { "logic_axes": "('activation_batch_attn', 'activation_length', 'activation_heads', 'activation_kv')", "PartitionSpec": "P('fsdp', 'expert', None, None)" } }, { - "deepseek/attention_result: bfloat16[96,2048,2048]": { + "deepseek/attention_result: bfloat16[96,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P('fsdp', 'expert', None)" } }, { - "deepseek/post_attention_norm: bfloat16[96,2048,2048]": { + "deepseek/post_attention_norm: bfloat16[96,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P('fsdp', 'expert', None)" } }, { - "linears/x: bfloat16[96,2048,10944]": { + "linears/x: bfloat16[96,2048,18432]": { "logic_axes": "('activation_batch', 'activation_length', 'activation_mlp')", "PartitionSpec": "P('fsdp', 'expert', None)" } }, { - "deepseek/mlp: bfloat16[96,2048,2048]": { + "deepseek/mlp: bfloat16[96,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P('fsdp', 'expert', None)" } }, { - "deepseek/x: bfloat16[96,2048,2048]": { + "deepseek/x: bfloat16[96,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P('fsdp', 'expert', None)" } }, { - "moe/inputs: bfloat16[96,2048,2048]": { - "logic_axes": "('activation_batch', 'activation_norm_length', None)", + "moe/inputs: bfloat16[96,2048,512]": { + "logic_axes": "('activation_batch', 'activation_norm_length_routed', None)", "PartitionSpec": "P('fsdp', 'expert', None)" } }, { - "moe/gate_logits: bfloat16[96,2048,64]": { - "logic_axes": "('activation_batch', 'activation_norm_length', None)", + "moe/gate_logits: bfloat16[96,2048,256]": { + "logic_axes": "('activation_batch', 'activation_norm_length_routed', None)", "PartitionSpec": "P('fsdp', 'expert', None)" } }, { - "moe/w0_kernel: bfloat16[64,2048,1408]": { - "logic_axes": "Unknown", - "PartitionSpec": "P('expert', None, None)" - } - }, - { - "moe/w1_kernel: bfloat16[64,2048,1408]": { - "logic_axes": "Unknown", - "PartitionSpec": "P('expert', None, None)" + "moe/pre_bias_logits: bfloat16[96,2048,256]": { + "logic_axes": "('activation_batch', 'activation_norm_length_routed', None)", + "PartitionSpec": "P('fsdp', 'expert', None)" } }, { - "moe/wo_kernel: bfloat16[64,1408,2048]": { + "moe/w: bfloat16[256,512,512]": { "logic_axes": "Unknown", "PartitionSpec": "P('expert', None, None)" } }, { - "linears/x: bfloat16[96,2048,2816]": { + "linears/x: bfloat16[96,2048,512]": { "logic_axes": "('activation_batch', 'activation_length', 'activation_mlp')", "PartitionSpec": "P('fsdp', 'expert', None)" } }, { - "deepseek/mlp_lnx: bfloat16[96,2048,2048]": { + "deepseek/mlp_lnx: bfloat16[96,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P('fsdp', 'expert', None)" } diff --git a/tests/utils/sharding_info/deepseek3-test/tpu7x-8/slice_1/rule_ep-as-cp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/logical_shardings.json b/tests/utils/sharding_info/deepseek3-test/tpu7x-8/slice_1/rule_ep-as-cp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/logical_shardings.json new file mode 100644 index 0000000000..e64ed069c8 --- /dev/null +++ b/tests/utils/sharding_info/deepseek3-test/tpu7x-8/slice_1/rule_ep-as-cp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/logical_shardings.json @@ -0,0 +1,1344 @@ +{ + "['model']/['decoder']/['decoder_norm']/['scale']/.value": { + "partition_spec": [ + null + ], + "shape": [ + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['count']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['key']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['count']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['key']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['count']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['key']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['model']/['decoder']/['logits_dense']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null + ], + "shape": [ + 512, + 129280 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['bias']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 58, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['count']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['key']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['count']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['key']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['count']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['key']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "partition_spec": [ + "expert", + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "partition_spec": [ + "expert", + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "partition_spec": [ + "expert", + null, + null, + "fsdp" + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['model']/['decoder']/['rngs']/['aqt']/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['aqt']/['key']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['dropout']/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['dropout']/['key']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['params']/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['params']/['key']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['token_embedder']/['embedding']/.value": { + "partition_spec": [ + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['decoder_norm']/['scale']/.value": { + "partition_spec": [ + null + ], + "shape": [ + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['logits_dense']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null + ], + "shape": [ + 512, + 129280 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "partition_spec": [ + "expert", + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "partition_spec": [ + "expert", + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "partition_spec": [ + "expert", + null, + null, + "fsdp" + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['token_embedder']/['embedding']/.value": { + "partition_spec": [ + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['decoder_norm']/['scale']/.value": { + "partition_spec": [ + null + ], + "shape": [ + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['logits_dense']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null + ], + "shape": [ + 512, + 129280 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "partition_spec": [ + "expert", + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "partition_spec": [ + "expert", + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "partition_spec": [ + "expert", + null, + null, + "fsdp" + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['token_embedder']/['embedding']/.value": { + "partition_spec": [ + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[2]/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['optimizer']/['step']/.value": { + "partition_spec": [], + "shape": [] + } +} \ No newline at end of file diff --git a/tests/utils/sharding_info/deepseek2-16b/tpu7x-8/slice_1/rule_ep-as-cp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/named_shardings.json b/tests/utils/sharding_info/deepseek3-test/tpu7x-8/slice_1/rule_ep-as-cp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/named_shardings.json similarity index 54% rename from tests/utils/sharding_info/deepseek2-16b/tpu7x-8/slice_1/rule_ep-as-cp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/named_shardings.json rename to tests/utils/sharding_info/deepseek3-test/tpu7x-8/slice_1/rule_ep-as-cp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/named_shardings.json index b8bdbc2c58..d97bb27246 100644 --- a/tests/utils/sharding_info/deepseek2-16b/tpu7x-8/slice_1/rule_ep-as-cp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/named_shardings.json +++ b/tests/utils/sharding_info/deepseek3-test/tpu7x-8/slice_1/rule_ep-as-cp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/named_shardings.json @@ -1,5 +1,5 @@ { - ".step": { + "['model']/['decoder']/['decoder_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -14,10 +14,14 @@ "expert": 2 } }, - "partition_spec": [], - "shape": [] + "partition_spec": [ + null + ], + "shape": [ + 512 + ] }, - ".params/['params']/['decoder']/['decoder_norm']/['scale']": { + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -33,13 +37,20 @@ } }, "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, null ], "shape": [ - 2048 + 512, + 3, + 18432 ] }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -63,12 +74,12 @@ null ], "shape": [ - 2048, - 1, - 10944 + 512, + 3, + 18432 ] }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -84,20 +95,20 @@ } }, "partition_spec": [ + null, + null, [ "fsdp", "expert" - ], - null, - null + ] ], "shape": [ - 2048, - 1, - 10944 + 18432, + 3, + 512 ] }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -114,19 +125,14 @@ }, "partition_spec": [ null, - null, - [ - "fsdp", - "expert" - ] + null ], "shape": [ - 10944, - 1, - 2048 + 512, + 3 ] }, - ".params/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { + "['model']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -146,11 +152,11 @@ null ], "shape": [ - 2048, - 1 + 512, + 3 ] }, - ".params/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['count']/.value": { "mesh": { "axis_names": [ "data", @@ -165,16 +171,112 @@ "expert": 2 } }, - "partition_spec": [ - null, - null - ], + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "expert": 2 + } + }, + "partition_spec": [], "shape": [ - 2048, - 1 + 3 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { + "['model']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -195,10 +297,10 @@ ], "shape": [ 512, - 1 + 3 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -223,13 +325,13 @@ ] ], "shape": [ - 16, - 1, 128, - 2048 + 3, + 128, + 512 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -245,22 +347,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "expert" - ], - null, null, null ], "shape": [ - 2048, - 1, - 16, - 192 + 1536, + 3 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -284,12 +379,12 @@ null ], "shape": [ - 2048, - 1, + 512, + 3, 576 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -312,12 +407,12 @@ ], "shape": [ 512, - 1, - 16, + 3, + 128, 256 ] }, - ".params/['params']/['decoder']/['logits_dense']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -337,14 +432,16 @@ "fsdp", "expert" ], + null, null ], "shape": [ - 2048, - 102400 + 512, + 3, + 1536 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -362,15 +459,17 @@ "partition_spec": [ "fsdp", null, + null, null ], "shape": [ - 2048, - 26, - 64 + 1536, + 3, + 128, + 192 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { + "['model']/['decoder']/['logits_dense']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -386,19 +485,18 @@ } }, "partition_spec": [ - "expert", - null, - "fsdp", + [ + "fsdp", + "expert" + ], null ], "shape": [ - 64, - 26, - 2048, - 1408 + 512, + 129280 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['bias']/.value": { "mesh": { "axis_names": [ "data", @@ -414,19 +512,15 @@ } }, "partition_spec": [ - "expert", null, - "fsdp", null ], "shape": [ - 64, - 26, - 2048, - 1408 + 58, + 256 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -442,19 +536,17 @@ } }, "partition_spec": [ - "expert", - null, + "fsdp", null, - "fsdp" + null ], "shape": [ - 64, - 26, - 1408, - 2048 + 512, + 58, + 256 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['count']/.value": { "mesh": { "axis_names": [ "data", @@ -469,21 +561,12 @@ "expert": 2 } }, - "partition_spec": [ - [ - "fsdp", - "expert" - ], - null, - null - ], + "partition_spec": [], "shape": [ - 2048, - 26, - 2816 + 58 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['key']/.value": { "mesh": { "axis_names": [ "data", @@ -498,21 +581,32 @@ "expert": 2 } }, - "partition_spec": [ - [ + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", "fsdp", "expert" ], - null, - null - ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "expert": 2 + } + }, + "partition_spec": [], "shape": [ - 2048, - 26, - 2816 + 58 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['key']/.value": { "mesh": { "axis_names": [ "data", @@ -527,21 +621,12 @@ "expert": 2 } }, - "partition_spec": [ - null, - null, - [ - "fsdp", - "expert" - ] - ], + "partition_spec": [], "shape": [ - 2816, - 26, - 2048 + 58 ] }, - ".params/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['count']/.value": { "mesh": { "axis_names": [ "data", @@ -556,16 +641,12 @@ "expert": 2 } }, - "partition_spec": [ - null, - null - ], + "partition_spec": [], "shape": [ - 2048, - 26 + 58 ] }, - ".params/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['key']/.value": { "mesh": { "axis_names": [ "data", @@ -580,16 +661,12 @@ "expert": 2 } }, - "partition_spec": [ - null, - null - ], + "partition_spec": [], "shape": [ - 2048, - 26 + 58 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { "mesh": { "axis_names": [ "data", @@ -605,15 +682,19 @@ } }, "partition_spec": [ + "expert", null, + "fsdp", null ], "shape": [ + 256, + 58, 512, - 26 + 512 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { "mesh": { "axis_names": [ "data", @@ -629,22 +710,19 @@ } }, "partition_spec": [ + "expert", null, - null, - null, - [ - "fsdp", - "expert" - ] + "fsdp", + null ], "shape": [ - 16, - 26, - 128, - 2048 + 256, + 58, + 512, + 512 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { "mesh": { "axis_names": [ "data", @@ -660,22 +738,19 @@ } }, "partition_spec": [ - [ - "fsdp", - "expert" - ], + "expert", null, null, - null + "fsdp" ], "shape": [ - 2048, - 26, - 16, - 192 + 256, + 58, + 512, + 512 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -699,12 +774,12 @@ null ], "shape": [ - 2048, - 26, - 576 + 512, + 58, + 512 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -720,19 +795,20 @@ } }, "partition_spec": [ - "fsdp", - null, + [ + "fsdp", + "expert" + ], null, null ], "shape": [ 512, - 26, - 16, - 256 + 58, + 512 ] }, - ".params/['params']/['token_embedder']/['embedding']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -748,6 +824,7 @@ } }, "partition_spec": [ + null, null, [ "fsdp", @@ -755,11 +832,12 @@ ] ], "shape": [ - 102400, - 2048 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.count": { + "['model']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -774,10 +852,16 @@ "expert": 2 } }, - "partition_spec": [], - "shape": [] + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['decoder_norm']/['scale']": { + "['model']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -793,13 +877,15 @@ } }, "partition_spec": [ + null, null ], "shape": [ - 2048 + 512, + 58 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -815,20 +901,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "expert" - ], null, null ], "shape": [ - 2048, - 1, - 10944 + 512, + 58 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -844,20 +925,446 @@ } }, "partition_spec": [ + null, + null, + null, [ "fsdp", "expert" + ] + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "expert" ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "expert": 2 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "expert": 2 + } + }, + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "expert": 2 + } + }, + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "expert": 2 + } + }, + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "expert": 2 + } + }, + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['model']/['decoder']/['rngs']/['aqt']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['aqt']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['dropout']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['dropout']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['params']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['params']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['token_embedder']/['embedding']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "expert": 2 + } + }, + "partition_spec": [ + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['decoder_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "expert": 2 + } + }, + "partition_spec": [ + null + ], + "shape": [ + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "expert": 2 + } + }, + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "expert": 2 + } + }, + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "expert": 2 + } + }, + "partition_spec": [ + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "expert": 2 + } + }, + "partition_spec": [ null, null ], "shape": [ - 2048, - 1, - 10944 + 512, + 3 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -874,19 +1381,14 @@ }, "partition_spec": [ null, - null, - [ - "fsdp", - "expert" - ] + null ], "shape": [ - 10944, - 1, - 2048 + 512, + 3 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -906,11 +1408,11 @@ null ], "shape": [ - 2048, - 1 + 512, + 3 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -927,14 +1429,21 @@ }, "partition_spec": [ null, - null + null, + null, + [ + "fsdp", + "expert" + ] ], "shape": [ - 2048, - 1 + 128, + 3, + 128, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -954,11 +1463,11 @@ null ], "shape": [ - 512, - 1 + 1536, + 3 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -974,22 +1483,20 @@ } }, "partition_spec": [ - null, - null, - null, [ "fsdp", "expert" - ] + ], + null, + null ], "shape": [ - 16, - 1, - 128, - 2048 + 512, + 3, + 576 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1005,22 +1512,19 @@ } }, "partition_spec": [ - [ - "fsdp", - "expert" - ], + "fsdp", null, null, null ], "shape": [ - 2048, - 1, - 16, - 192 + 512, + 3, + 128, + 256 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1044,12 +1548,12 @@ null ], "shape": [ - 2048, - 1, - 576 + 512, + 3, + 1536 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1071,13 +1575,13 @@ null ], "shape": [ - 512, - 1, - 16, - 256 + 1536, + 3, + 128, + 192 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['logits_dense']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['logits_dense']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1100,11 +1604,11 @@ null ], "shape": [ - 2048, - 102400 + 512, + 129280 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1125,12 +1629,12 @@ null ], "shape": [ - 2048, - 26, - 64 + 512, + 58, + 256 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { "mesh": { "axis_names": [ "data", @@ -1152,13 +1656,13 @@ null ], "shape": [ - 64, - 26, - 2048, - 1408 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { "mesh": { "axis_names": [ "data", @@ -1180,13 +1684,13 @@ null ], "shape": [ - 64, - 26, - 2048, - 1408 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { "mesh": { "axis_names": [ "data", @@ -1208,13 +1712,13 @@ "fsdp" ], "shape": [ - 64, - 26, - 1408, - 2048 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1238,12 +1742,12 @@ null ], "shape": [ - 2048, - 26, - 2816 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1267,12 +1771,12 @@ null ], "shape": [ - 2048, - 26, - 2816 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1296,12 +1800,12 @@ ] ], "shape": [ - 2816, - 26, - 2048 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1321,11 +1825,11 @@ null ], "shape": [ - 2048, - 26 + 512, + 58 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1345,11 +1849,11 @@ null ], "shape": [ - 2048, - 26 + 512, + 58 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1370,10 +1874,10 @@ ], "shape": [ 512, - 26 + 58 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1398,13 +1902,37 @@ ] ], "shape": [ - 16, - 26, 128, - 2048 + 58, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "expert": 2 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1425,17 +1953,43 @@ "expert" ], null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "expert": 2 + } + }, + "partition_spec": [ + "fsdp", + null, null, null ], "shape": [ - 2048, - 26, - 16, - 192 + 512, + 58, + 128, + 256 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1459,12 +2013,12 @@ null ], "shape": [ - 2048, - 26, - 576 + 512, + 58, + 1536 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1486,13 +2040,13 @@ null ], "shape": [ - 512, - 26, - 16, - 256 + 1536, + 58, + 128, + 192 ] }, - ".opt_state/[0]/.mu/['params']/['token_embedder']/['embedding']": { + "['optimizer']/['opt_state']/[0]/['mu']/['token_embedder']/['embedding']/.value": { "mesh": { "axis_names": [ "data", @@ -1515,11 +2069,11 @@ ] ], "shape": [ - 102400, - 2048 + 129280, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['decoder_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['decoder_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1538,10 +2092,10 @@ null ], "shape": [ - 2048 + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1565,12 +2119,12 @@ null ], "shape": [ - 2048, - 1, - 10944 + 512, + 3, + 18432 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1594,12 +2148,12 @@ null ], "shape": [ - 2048, - 1, - 10944 + 512, + 3, + 18432 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1623,12 +2177,12 @@ ] ], "shape": [ - 10944, - 1, - 2048 + 18432, + 3, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1648,11 +2202,11 @@ null ], "shape": [ - 2048, - 1 + 512, + 3 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1672,11 +2226,11 @@ null ], "shape": [ - 2048, - 1 + 512, + 3 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1697,10 +2251,10 @@ ], "shape": [ 512, - 1 + 3 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1725,13 +2279,37 @@ ] ], "shape": [ - 16, - 1, 128, - 2048 + 3, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "expert": 2 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1752,17 +2330,43 @@ "expert" ], null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "expert": 2 + } + }, + "partition_spec": [ + "fsdp", + null, null, null ], "shape": [ - 2048, - 1, - 16, - 192 + 512, + 3, + 128, + 256 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1786,12 +2390,12 @@ null ], "shape": [ - 2048, - 1, - 576 + 512, + 3, + 1536 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1813,13 +2417,13 @@ null ], "shape": [ - 512, - 1, - 16, - 256 + 1536, + 3, + 128, + 192 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['logits_dense']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['logits_dense']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1842,11 +2446,11 @@ null ], "shape": [ - 2048, - 102400 + 512, + 129280 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1867,12 +2471,12 @@ null ], "shape": [ - 2048, - 26, - 64 + 512, + 58, + 256 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { "mesh": { "axis_names": [ "data", @@ -1894,13 +2498,13 @@ null ], "shape": [ - 64, - 26, - 2048, - 1408 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { "mesh": { "axis_names": [ "data", @@ -1922,13 +2526,13 @@ null ], "shape": [ - 64, - 26, - 2048, - 1408 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { "mesh": { "axis_names": [ "data", @@ -1950,13 +2554,13 @@ "fsdp" ], "shape": [ - 64, - 26, - 1408, - 2048 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1980,12 +2584,12 @@ null ], "shape": [ - 2048, - 26, - 2816 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2009,12 +2613,12 @@ null ], "shape": [ - 2048, - 26, - 2816 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2038,12 +2642,12 @@ ] ], "shape": [ - 2816, - 26, - 2048 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -2063,11 +2667,11 @@ null ], "shape": [ - 2048, - 26 + 512, + 58 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -2087,11 +2691,11 @@ null ], "shape": [ - 2048, - 26 + 512, + 58 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -2112,10 +2716,10 @@ ], "shape": [ 512, - 26 + 58 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2140,13 +2744,37 @@ ] ], "shape": [ - 16, - 26, 128, - 2048 + 58, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "expert": 2 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2167,17 +2795,43 @@ "expert" ], null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "expert": 2 + } + }, + "partition_spec": [ + "fsdp", + null, null, null ], "shape": [ - 2048, - 26, - 16, - 192 + 512, + 58, + 128, + 256 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2201,12 +2855,12 @@ null ], "shape": [ - 2048, - 26, - 576 + 512, + 58, + 1536 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2228,13 +2882,13 @@ null ], "shape": [ - 512, - 26, - 16, - 256 + 1536, + 58, + 128, + 192 ] }, - ".opt_state/[0]/.nu/['params']/['token_embedder']/['embedding']": { + "['optimizer']/['opt_state']/[0]/['nu']/['token_embedder']/['embedding']/.value": { "mesh": { "axis_names": [ "data", @@ -2257,11 +2911,29 @@ ] ], "shape": [ - 102400, - 2048 + 129280, + 512 ] }, - ".opt_state/[2]/.count": { + "['optimizer']/['opt_state']/[2]/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [] + }, + "['optimizer']/['step']/.value": { "mesh": { "axis_names": [ "data", diff --git a/tests/utils/sharding_info/deepseek2-16b/tpu7x-8/slice_1/rule_shard-exp-on-fsdp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/input_shardings.json b/tests/utils/sharding_info/deepseek3-test/tpu7x-8/slice_1/rule_shard-exp-on-fsdp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/input_shardings.json similarity index 73% rename from tests/utils/sharding_info/deepseek2-16b/tpu7x-8/slice_1/rule_shard-exp-on-fsdp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/input_shardings.json rename to tests/utils/sharding_info/deepseek3-test/tpu7x-8/slice_1/rule_shard-exp-on-fsdp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/input_shardings.json index 1aa17e9725..130d8e291d 100644 --- a/tests/utils/sharding_info/deepseek2-16b/tpu7x-8/slice_1/rule_shard-exp-on-fsdp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/input_shardings.json +++ b/tests/utils/sharding_info/deepseek3-test/tpu7x-8/slice_1/rule_shard-exp-on-fsdp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/input_shardings.json @@ -1,67 +1,67 @@ { "Activation Sharding Dump": [ { - "deepseek/inputs: bfloat16[96,2048,2048]": { + "deepseek/inputs: bfloat16[96,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "deepseek/pre_attention_norm: bfloat16[96,2048,2048]": { + "deepseek/pre_attention_norm: bfloat16[96,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "attention_mla/inputs_q: bfloat16[96,2048,2048]": { + "attention_mla/inputs_q: bfloat16[96,2048,512]": { "logic_axes": "('activation_batch_attn', 'activation_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "attention_mla/inputs_kv: bfloat16[96,2048,2048]": { + "attention_mla/inputs_kv: bfloat16[96,2048,512]": { "logic_axes": "('activation_batch_attn', 'activation_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "attention_mla/q_nope: bfloat16[96,2048,16,128]": { + "attention_mla/q_nope: bfloat16[96,2048,128,128]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_mla/q_pe: bfloat16[96,2048,16,64]": { + "attention_mla/q_pe: bfloat16[96,2048,128,64]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_mla/query: bfloat16[96,2048,16,192]": { + "attention_mla/query: bfloat16[96,2048,128,192]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_mla/key_nope: bfloat16[96,2048,16,128]": { + "attention_mla/key_nope: bfloat16[96,2048,128,128]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_mla/key_rope: bfloat16[96,2048,16,64]": { + "attention_mla/key_rope: bfloat16[96,2048,128,64]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_mla/key: bfloat16[96,2048,16,192]": { + "attention_mla/key: bfloat16[96,2048,128,192]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_mla/value: bfloat16[96,2048,16,128]": { + "attention_mla/value: bfloat16[96,2048,128,128]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } @@ -79,97 +79,97 @@ } }, { - "attention_op/query: bfloat16[96,16,2048,192]": { + "attention_op/query: bfloat16[96,128,2048,192]": { "logic_axes": "Unknown", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_op/key: bfloat16[96,16,2048,192]": { + "attention_op/key: bfloat16[96,128,2048,192]": { "logic_axes": "Unknown", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_op/value: bfloat16[96,16,2048,128]": { + "attention_op/value: bfloat16[96,128,2048,128]": { "logic_axes": "Unknown", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_mla/out: bfloat16[96,2048,16,128]": { + "attention_op/decoder_segment_ids: int32[96,2048]": { + "logic_axes": "Unknown", + "PartitionSpec": "P(('fsdp', 'expert'), None)" + } + }, + { + "attention_mla/out: bfloat16[96,2048,128,128]": { "logic_axes": "('activation_batch_attn', 'activation_length', 'activation_heads', 'activation_kv')", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "deepseek/attention_result: bfloat16[96,2048,2048]": { + "deepseek/attention_result: bfloat16[96,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "deepseek/post_attention_norm: bfloat16[96,2048,2048]": { + "deepseek/post_attention_norm: bfloat16[96,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "linears/x: bfloat16[96,2048,10944]": { + "linears/x: bfloat16[96,2048,18432]": { "logic_axes": "('activation_batch', 'activation_length', 'activation_mlp')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "deepseek/mlp: bfloat16[96,2048,2048]": { + "deepseek/mlp: bfloat16[96,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "deepseek/x: bfloat16[96,2048,2048]": { + "deepseek/x: bfloat16[96,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "moe/inputs: bfloat16[96,2048,2048]": { - "logic_axes": "('activation_batch', 'activation_norm_length', None)", + "moe/inputs: bfloat16[96,2048,512]": { + "logic_axes": "('activation_batch', 'activation_norm_length_routed', None)", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "moe/gate_logits: bfloat16[96,2048,64]": { - "logic_axes": "('activation_batch', 'activation_norm_length', None)", + "moe/gate_logits: bfloat16[96,2048,256]": { + "logic_axes": "('activation_batch', 'activation_norm_length_routed', None)", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "moe/w0_kernel: bfloat16[64,2048,1408]": { - "logic_axes": "Unknown", - "PartitionSpec": "P('expert', None, None)" - } - }, - { - "moe/w1_kernel: bfloat16[64,2048,1408]": { - "logic_axes": "Unknown", - "PartitionSpec": "P('expert', None, None)" + "moe/pre_bias_logits: bfloat16[96,2048,256]": { + "logic_axes": "('activation_batch', 'activation_norm_length_routed', None)", + "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "moe/wo_kernel: bfloat16[64,1408,2048]": { + "moe/w: bfloat16[256,512,512]": { "logic_axes": "Unknown", "PartitionSpec": "P('expert', None, None)" } }, { - "linears/x: bfloat16[96,2048,2816]": { + "linears/x: bfloat16[96,2048,512]": { "logic_axes": "('activation_batch', 'activation_length', 'activation_mlp')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "deepseek/mlp_lnx: bfloat16[96,2048,2048]": { + "deepseek/mlp_lnx: bfloat16[96,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } diff --git a/tests/utils/sharding_info/deepseek3-test/tpu7x-8/slice_1/rule_shard-exp-on-fsdp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/logical_shardings.json b/tests/utils/sharding_info/deepseek3-test/tpu7x-8/slice_1/rule_shard-exp-on-fsdp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/logical_shardings.json new file mode 100644 index 0000000000..5ae5b62842 --- /dev/null +++ b/tests/utils/sharding_info/deepseek3-test/tpu7x-8/slice_1/rule_shard-exp-on-fsdp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/logical_shardings.json @@ -0,0 +1,1407 @@ +{ + "['model']/['decoder']/['decoder_norm']/['scale']/.value": { + "partition_spec": [ + null + ], + "shape": [ + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['count']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['key']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['count']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['key']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['count']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['key']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null, + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null, + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['model']/['decoder']/['logits_dense']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null + ], + "shape": [ + 512, + 129280 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['bias']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 58, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['count']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['key']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['count']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['key']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['count']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['key']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "partition_spec": [ + [ + "expert", + "fsdp" + ], + null, + null, + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "partition_spec": [ + [ + "expert", + "fsdp" + ], + null, + null, + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "partition_spec": [ + [ + "expert", + "fsdp" + ], + null, + null, + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null, + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null, + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['model']/['decoder']/['rngs']/['aqt']/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['aqt']/['key']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['dropout']/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['dropout']/['key']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['params']/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['params']/['key']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['token_embedder']/['embedding']/.value": { + "partition_spec": [ + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['decoder_norm']/['scale']/.value": { + "partition_spec": [ + null + ], + "shape": [ + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null, + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null, + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['logits_dense']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null + ], + "shape": [ + 512, + 129280 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "partition_spec": [ + [ + "expert", + "fsdp" + ], + null, + null, + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "partition_spec": [ + [ + "expert", + "fsdp" + ], + null, + null, + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "partition_spec": [ + [ + "expert", + "fsdp" + ], + null, + null, + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null, + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null, + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['token_embedder']/['embedding']/.value": { + "partition_spec": [ + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['decoder_norm']/['scale']/.value": { + "partition_spec": [ + null + ], + "shape": [ + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null, + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null, + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['logits_dense']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null + ], + "shape": [ + 512, + 129280 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "partition_spec": [ + [ + "expert", + "fsdp" + ], + null, + null, + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "partition_spec": [ + [ + "expert", + "fsdp" + ], + null, + null, + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "partition_spec": [ + [ + "expert", + "fsdp" + ], + null, + null, + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null, + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null, + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['token_embedder']/['embedding']/.value": { + "partition_spec": [ + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[2]/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['optimizer']/['step']/.value": { + "partition_spec": [], + "shape": [] + } +} \ No newline at end of file diff --git a/tests/utils/sharding_info/deepseek2-16b/tpu7x-8/slice_1/rule_shard-exp-on-fsdp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/named_shardings.json b/tests/utils/sharding_info/deepseek3-test/tpu7x-8/slice_1/rule_shard-exp-on-fsdp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/named_shardings.json similarity index 58% rename from tests/utils/sharding_info/deepseek2-16b/tpu7x-8/slice_1/rule_shard-exp-on-fsdp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/named_shardings.json rename to tests/utils/sharding_info/deepseek3-test/tpu7x-8/slice_1/rule_shard-exp-on-fsdp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/named_shardings.json index c35db11b7c..29bccee4f1 100644 --- a/tests/utils/sharding_info/deepseek2-16b/tpu7x-8/slice_1/rule_shard-exp-on-fsdp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/named_shardings.json +++ b/tests/utils/sharding_info/deepseek3-test/tpu7x-8/slice_1/rule_shard-exp-on-fsdp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2/named_shardings.json @@ -1,5 +1,646 @@ { - ".step": { + "['model']/['decoder']/['decoder_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "expert": 2 + } + }, + "partition_spec": [ + null + ], + "shape": [ + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "expert": 2 + } + }, + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "expert": 2 + } + }, + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "expert": 2 + } + }, + "partition_spec": [ + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "expert": 2 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "expert": 2 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "expert": 2 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "expert": 2 + } + }, + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "expert": 2 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "expert": 2 + } + }, + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "expert": 2 + } + }, + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null, + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "expert": 2 + } + }, + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "expert": 2 + } + }, + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null, + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['model']/['decoder']/['logits_dense']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "expert": 2 + } + }, + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null + ], + "shape": [ + 512, + 129280 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['bias']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "expert": 2 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 58, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "expert": 2 + } + }, + "partition_spec": [ + null, + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['count']/.value": { "mesh": { "axis_names": [ "data", @@ -19,9 +660,201 @@ } }, "partition_spec": [], - "shape": [] + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "expert": 2 + } + }, + "partition_spec": [ + [ + "expert", + "fsdp" + ], + null, + null, + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "expert": 2 + } + }, + "partition_spec": [ + [ + "expert", + "fsdp" + ], + null, + null, + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] }, - ".params/['params']/['decoder']/['decoder_norm']/['scale']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { "mesh": { "axis_names": [ "data", @@ -41,13 +874,22 @@ } }, "partition_spec": [ + [ + "expert", + "fsdp" + ], + null, + null, null ], "shape": [ - 2048 + 256, + 58, + 512, + 512 ] }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -69,19 +911,18 @@ "partition_spec": [ [ "fsdp", - "context", "expert" ], null, - "fsdp_transpose" + null ], "shape": [ - 2048, - 1, - 10944 + 512, + 58, + 512 ] }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -103,19 +944,18 @@ "partition_spec": [ [ "fsdp", - "context", "expert" ], null, - "fsdp_transpose" + null ], "shape": [ - 2048, - 1, - 10944 + 512, + 58, + 512 ] }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -135,21 +975,20 @@ } }, "partition_spec": [ - "fsdp_transpose", + null, null, [ "fsdp", - "context", "expert" ] ], "shape": [ - 10944, - 1, - 2048 + 512, + 58, + 512 ] }, - ".params/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { + "['model']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -173,11 +1012,11 @@ null ], "shape": [ - 2048, - 1 + 512, + 58 ] }, - ".params/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['model']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -201,11 +1040,11 @@ null ], "shape": [ - 2048, - 1 + 512, + 58 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { + "['model']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -230,10 +1069,10 @@ ], "shape": [ 512, - 1 + 58 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -258,18 +1097,17 @@ null, [ "fsdp", - "context", "expert" ] ], "shape": [ - 16, - 1, 128, - 2048 + 58, + 128, + 512 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -289,23 +1127,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "context", - "expert" - ], - null, null, null ], "shape": [ - 2048, - 1, - 16, - 192 + 1536, + 58 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -327,19 +1157,18 @@ "partition_spec": [ [ "fsdp", - "context", "expert" ], null, null ], "shape": [ - 2048, - 1, + 512, + 58, 576 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -361,7 +1190,6 @@ "partition_spec": [ [ "fsdp", - "fsdp_transpose", "expert" ], null, @@ -370,12 +1198,12 @@ ], "shape": [ 512, - 1, - 16, + 58, + 128, 256 ] }, - ".params/['params']/['decoder']/['logits_dense']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -397,18 +1225,18 @@ "partition_spec": [ [ "fsdp", - "fsdp_transpose", - "context", "expert" ], + null, null ], "shape": [ - 2048, - 102400 + 512, + 58, + 1536 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -428,17 +1256,22 @@ } }, "partition_spec": [ - "context", + [ + "fsdp", + "expert" + ], + null, null, null ], "shape": [ - 2048, - 26, - 64 + 1536, + 58, + 128, + 192 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { + "['model']/['decoder']/['rngs']/['aqt']/['count']/.value": { "mesh": { "axis_names": [ "data", @@ -457,23 +1290,10 @@ "expert": 2 } }, - "partition_spec": [ - [ - "expert", - "fsdp" - ], - null, - "context", - "fsdp_transpose" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] + "partition_spec": [], + "shape": [] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { + "['model']/['decoder']/['rngs']/['aqt']/['key']/.value": { "mesh": { "axis_names": [ "data", @@ -492,23 +1312,10 @@ "expert": 2 } }, - "partition_spec": [ - [ - "expert", - "fsdp" - ], - null, - "context", - "fsdp_transpose" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] + "partition_spec": [], + "shape": [] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { + "['model']/['decoder']/['rngs']/['dropout']/['count']/.value": { "mesh": { "axis_names": [ "data", @@ -527,23 +1334,10 @@ "expert": 2 } }, - "partition_spec": [ - [ - "expert", - "fsdp" - ], - null, - "fsdp_transpose", - "context" - ], - "shape": [ - 64, - 26, - 1408, - 2048 - ] + "partition_spec": [], + "shape": [] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { + "['model']/['decoder']/['rngs']/['dropout']/['key']/.value": { "mesh": { "axis_names": [ "data", @@ -562,22 +1356,10 @@ "expert": 2 } }, - "partition_spec": [ - [ - "fsdp", - "context", - "expert" - ], - null, - "fsdp_transpose" - ], - "shape": [ - 2048, - 26, - 2816 - ] + "partition_spec": [], + "shape": [] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { + "['model']/['decoder']/['rngs']/['params']/['count']/.value": { "mesh": { "axis_names": [ "data", @@ -596,22 +1378,32 @@ "expert": 2 } }, - "partition_spec": [ - [ + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['params']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", "fsdp", + "fsdp_transpose", "context", "expert" ], - null, - "fsdp_transpose" - ], - "shape": [ - 2048, - 26, - 2816 - ] + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { + "['model']/['token_embedder']/['embedding']/.value": { "mesh": { "axis_names": [ "data", @@ -631,21 +1423,18 @@ } }, "partition_spec": [ - "fsdp_transpose", null, [ "fsdp", - "context", "expert" ] ], "shape": [ - 2816, - 26, - 2048 + 129280, + 512 ] }, - ".params/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['count']/.value": { "mesh": { "axis_names": [ "data", @@ -664,16 +1453,10 @@ "expert": 2 } }, - "partition_spec": [ - null, - null - ], - "shape": [ - 2048, - 26 - ] + "partition_spec": [], + "shape": [] }, - ".params/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['decoder_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -693,15 +1476,13 @@ } }, "partition_spec": [ - null, null ], "shape": [ - 2048, - 26 + 512 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -721,15 +1502,20 @@ } }, "partition_spec": [ + [ + "fsdp", + "expert" + ], null, null ], "shape": [ 512, - 26 + 3, + 18432 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -749,23 +1535,20 @@ } }, "partition_spec": [ - null, - null, - null, [ "fsdp", - "context", "expert" - ] + ], + null, + null ], "shape": [ - 16, - 26, - 128, - 2048 + 512, + 3, + 18432 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -785,23 +1568,20 @@ } }, "partition_spec": [ + null, + null, [ "fsdp", - "context", "expert" - ], - null, - null, - null + ] ], "shape": [ - 2048, - 26, - 16, - 192 + 18432, + 3, + 512 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -821,21 +1601,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "context", - "expert" - ], null, null ], "shape": [ - 2048, - 26, - 576 + 512, + 3 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -855,23 +1629,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "expert" - ], - null, null, null ], "shape": [ 512, - 26, - 16, - 256 + 3 ] }, - ".params/['params']/['token_embedder']/['embedding']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -892,19 +1658,14 @@ }, "partition_spec": [ null, - [ - "fsdp", - "fsdp_transpose", - "context", - "expert" - ] + null ], "shape": [ - 102400, - 2048 + 512, + 3 ] }, - ".opt_state/[0]/.count": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -923,10 +1684,23 @@ "expert": 2 } }, - "partition_spec": [], - "shape": [] + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 128, + 3, + 128, + 512 + ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['decoder_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -946,13 +1720,15 @@ } }, "partition_spec": [ + null, null ], "shape": [ - 2048 + 1536, + 3 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -974,19 +1750,18 @@ "partition_spec": [ [ "fsdp", - "context", "expert" ], null, - "fsdp_transpose" + null ], "shape": [ - 2048, - 1, - 10944 + 512, + 3, + 576 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1008,19 +1783,20 @@ "partition_spec": [ [ "fsdp", - "context", "expert" ], null, - "fsdp_transpose" + null, + null ], "shape": [ - 2048, - 1, - 10944 + 512, + 3, + 128, + 256 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1040,21 +1816,20 @@ } }, "partition_spec": [ - "fsdp_transpose", - null, [ "fsdp", - "context", "expert" - ] + ], + null, + null ], "shape": [ - 10944, - 1, - 2048 + 512, + 3, + 1536 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1074,15 +1849,22 @@ } }, "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, null, null ], "shape": [ - 2048, - 1 + 1536, + 3, + 128, + 192 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['logits_dense']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1102,15 +1884,18 @@ } }, "partition_spec": [ - null, + [ + "fsdp", + "expert" + ], null ], "shape": [ - 2048, - 1 + 512, + 129280 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1130,15 +1915,17 @@ } }, "partition_spec": [ + null, null, null ], "shape": [ 512, - 1 + 58, + 256 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { "mesh": { "axis_names": [ "data", @@ -1158,23 +1945,22 @@ } }, "partition_spec": [ + [ + "expert", + "fsdp" + ], null, null, - null, - [ - "fsdp", - "context", - "expert" - ] + null ], "shape": [ - 16, - 1, - 128, - 2048 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { "mesh": { "axis_names": [ "data", @@ -1195,22 +1981,21 @@ }, "partition_spec": [ [ - "fsdp", - "context", - "expert" + "expert", + "fsdp" ], null, null, null ], "shape": [ - 2048, - 1, - 16, - 192 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { "mesh": { "axis_names": [ "data", @@ -1231,20 +2016,21 @@ }, "partition_spec": [ [ - "fsdp", - "context", - "expert" + "expert", + "fsdp" ], null, + null, null ], "shape": [ - 2048, - 1, - 576 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1266,21 +2052,18 @@ "partition_spec": [ [ "fsdp", - "fsdp_transpose", "expert" ], null, - null, null ], "shape": [ 512, - 1, - 16, - 256 + 58, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['logits_dense']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1302,18 +2085,18 @@ "partition_spec": [ [ "fsdp", - "fsdp_transpose", - "context", "expert" ], + null, null ], "shape": [ - 2048, - 102400 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1333,17 +2116,20 @@ } }, "partition_spec": [ - "context", null, - null + null, + [ + "fsdp", + "expert" + ] ], "shape": [ - 2048, - 26, - 64 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1363,22 +2149,15 @@ } }, "partition_spec": [ - [ - "expert", - "fsdp" - ], null, - "context", - "fsdp_transpose" + null ], "shape": [ - 64, - 26, - 2048, - 1408 + 512, + 58 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1398,22 +2177,15 @@ } }, "partition_spec": [ - [ - "expert", - "fsdp" - ], null, - "context", - "fsdp_transpose" + null ], "shape": [ - 64, - 26, - 2048, - 1408 + 512, + 58 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1433,22 +2205,15 @@ } }, "partition_spec": [ - [ - "expert", - "fsdp" - ], null, - "fsdp_transpose", - "context" + null ], "shape": [ - 64, - 26, - 1408, - 2048 + 512, + 58 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1468,21 +2233,22 @@ } }, "partition_spec": [ + null, + null, + null, [ "fsdp", - "context", "expert" - ], - null, - "fsdp_transpose" + ] ], "shape": [ - 2048, - 26, - 2816 + 128, + 58, + 128, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1502,21 +2268,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "context", - "expert" - ], null, - "fsdp_transpose" + null ], "shape": [ - 2048, - 26, - 2816 + 1536, + 58 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1536,21 +2296,20 @@ } }, "partition_spec": [ - "fsdp_transpose", - null, [ "fsdp", - "context", "expert" - ] + ], + null, + null ], "shape": [ - 2816, - 26, - 2048 + 512, + 58, + 576 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1570,15 +2329,22 @@ } }, "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, null, null ], "shape": [ - 2048, - 26 + 512, + 58, + 128, + 256 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1598,15 +2364,20 @@ } }, "partition_spec": [ + [ + "fsdp", + "expert" + ], null, null ], "shape": [ - 2048, - 26 + 512, + 58, + 1536 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1626,15 +2397,22 @@ } }, "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, null, null ], "shape": [ - 512, - 26 + 1536, + 58, + 128, + 192 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['token_embedder']/['embedding']/.value": { "mesh": { "axis_names": [ "data", @@ -1654,23 +2432,18 @@ } }, "partition_spec": [ - null, - null, null, [ "fsdp", - "context", "expert" ] ], "shape": [ - 16, - 26, - 128, - 2048 + 129280, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['decoder_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1690,23 +2463,13 @@ } }, "partition_spec": [ - [ - "fsdp", - "context", - "expert" - ], - null, - null, null ], "shape": [ - 2048, - 26, - 16, - 192 + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1728,19 +2491,18 @@ "partition_spec": [ [ "fsdp", - "context", "expert" ], null, null ], "shape": [ - 2048, - 26, - 576 + 512, + 3, + 18432 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1762,21 +2524,18 @@ "partition_spec": [ [ "fsdp", - "fsdp_transpose", "expert" ], null, - null, null ], "shape": [ 512, - 26, - 16, - 256 + 3, + 18432 ] }, - ".opt_state/[0]/.mu/['params']/['token_embedder']/['embedding']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1796,20 +2555,20 @@ } }, "partition_spec": [ + null, null, [ "fsdp", - "fsdp_transpose", - "context", "expert" ] ], "shape": [ - 102400, - 2048 + 18432, + 3, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['decoder_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1829,13 +2588,15 @@ } }, "partition_spec": [ + null, null ], "shape": [ - 2048 + 512, + 3 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1855,21 +2616,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "context", - "expert" - ], null, - "fsdp_transpose" + null ], "shape": [ - 2048, - 1, - 10944 + 512, + 3 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1889,21 +2644,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "context", - "expert" - ], null, - "fsdp_transpose" + null ], "shape": [ - 2048, - 1, - 10944 + 512, + 3 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1923,21 +2672,22 @@ } }, "partition_spec": [ - "fsdp_transpose", + null, + null, null, [ "fsdp", - "context", "expert" ] ], "shape": [ - 10944, - 1, - 2048 + 128, + 3, + 128, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1961,11 +2711,11 @@ null ], "shape": [ - 2048, - 1 + 1536, + 3 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1985,15 +2735,20 @@ } }, "partition_spec": [ + [ + "fsdp", + "expert" + ], null, null ], "shape": [ - 2048, - 1 + 512, + 3, + 576 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2013,15 +2768,22 @@ } }, "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, null, null ], "shape": [ 512, - 1 + 3, + 128, + 256 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2041,23 +2803,20 @@ } }, "partition_spec": [ - null, - null, - null, [ "fsdp", - "context", "expert" - ] + ], + null, + null ], "shape": [ - 16, - 1, - 128, - 2048 + 512, + 3, + 1536 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2079,7 +2838,6 @@ "partition_spec": [ [ "fsdp", - "context", "expert" ], null, @@ -2087,13 +2845,13 @@ null ], "shape": [ - 2048, - 1, - 16, + 1536, + 3, + 128, 192 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['logits_dense']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2115,19 +2873,16 @@ "partition_spec": [ [ "fsdp", - "context", "expert" ], - null, null ], "shape": [ - 2048, - 1, - 576 + 512, + 129280 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2147,23 +2902,17 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "expert" - ], null, null, null ], "shape": [ 512, - 1, - 16, + 58, 256 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['logits_dense']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { "mesh": { "axis_names": [ "data", @@ -2184,19 +2933,21 @@ }, "partition_spec": [ [ - "fsdp", - "fsdp_transpose", - "context", - "expert" + "expert", + "fsdp" ], + null, + null, null ], "shape": [ - 2048, - 102400 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { "mesh": { "axis_names": [ "data", @@ -2216,17 +2967,22 @@ } }, "partition_spec": [ - "context", + [ + "expert", + "fsdp" + ], + null, null, null ], "shape": [ - 2048, - 26, - 64 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { "mesh": { "axis_names": [ "data", @@ -2251,17 +3007,17 @@ "fsdp" ], null, - "context", - "fsdp_transpose" + null, + null ], "shape": [ - 64, - 26, - 2048, - 1408 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2282,21 +3038,19 @@ }, "partition_spec": [ [ - "expert", - "fsdp" + "fsdp", + "expert" ], null, - "context", - "fsdp_transpose" + null ], "shape": [ - 64, - 26, - 2048, - 1408 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2317,21 +3071,19 @@ }, "partition_spec": [ [ - "expert", - "fsdp" + "fsdp", + "expert" ], null, - "fsdp_transpose", - "context" + null ], "shape": [ - 64, - 26, - 1408, - 2048 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2351,21 +3103,20 @@ } }, "partition_spec": [ + null, + null, [ "fsdp", - "context", "expert" - ], - null, - "fsdp_transpose" + ] ], "shape": [ - 2048, - 26, - 2816 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -2385,21 +3136,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "context", - "expert" - ], null, - "fsdp_transpose" + null ], "shape": [ - 2048, - 26, - 2816 + 512, + 58 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -2419,21 +3164,15 @@ } }, "partition_spec": [ - "fsdp_transpose", null, - [ - "fsdp", - "context", - "expert" - ] + null ], "shape": [ - 2816, - 26, - 2048 + 512, + 58 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -2457,11 +3196,11 @@ null ], "shape": [ - 2048, - 26 + 512, + 58 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2482,14 +3221,21 @@ }, "partition_spec": [ null, - null + null, + null, + [ + "fsdp", + "expert" + ] ], "shape": [ - 2048, - 26 + 128, + 58, + 128, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -2513,11 +3259,11 @@ null ], "shape": [ - 512, - 26 + 1536, + 58 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2537,23 +3283,20 @@ } }, "partition_spec": [ - null, - null, - null, [ "fsdp", - "context", "expert" - ] + ], + null, + null ], "shape": [ - 16, - 26, - 128, - 2048 + 512, + 58, + 576 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2575,7 +3318,6 @@ "partition_spec": [ [ "fsdp", - "context", "expert" ], null, @@ -2583,13 +3325,13 @@ null ], "shape": [ - 2048, - 26, - 16, - 192 + 512, + 58, + 128, + 256 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2611,19 +3353,18 @@ "partition_spec": [ [ "fsdp", - "context", "expert" ], null, null ], "shape": [ - 2048, - 26, - 576 + 512, + 58, + 1536 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2645,7 +3386,6 @@ "partition_spec": [ [ "fsdp", - "fsdp_transpose", "expert" ], null, @@ -2653,13 +3393,13 @@ null ], "shape": [ - 512, - 26, - 16, - 256 + 1536, + 58, + 128, + 192 ] }, - ".opt_state/[0]/.nu/['params']/['token_embedder']/['embedding']": { + "['optimizer']/['opt_state']/[0]/['nu']/['token_embedder']/['embedding']/.value": { "mesh": { "axis_names": [ "data", @@ -2682,17 +3422,37 @@ null, [ "fsdp", - "fsdp_transpose", - "context", "expert" ] ], "shape": [ - 102400, - 2048 + 129280, + 512 ] }, - ".opt_state/[2]/.count": { + "['optimizer']/['opt_state']/[2]/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "expert": 2 + } + }, + "partition_spec": [], + "shape": [] + }, + "['optimizer']/['step']/.value": { "mesh": { "axis_names": [ "data", diff --git a/tests/utils/sharding_info/deepseek2-16b/v6e-16/slice_1/rule_pipeline-large-moe_ici_fsdp_parallelism=-1_ici_expert_parallelism=4_use_ring_of_experts=true/input_shardings.json b/tests/utils/sharding_info/deepseek3-test/v6e-16/slice_1/rule_default_ici_fsdp_parallelism=-1_ici_expert_parallelism=4/input_shardings.json similarity index 72% rename from tests/utils/sharding_info/deepseek2-16b/v6e-16/slice_1/rule_pipeline-large-moe_ici_fsdp_parallelism=-1_ici_expert_parallelism=4_use_ring_of_experts=true/input_shardings.json rename to tests/utils/sharding_info/deepseek3-test/v6e-16/slice_1/rule_default_ici_fsdp_parallelism=-1_ici_expert_parallelism=4/input_shardings.json index f556d60c89..850add1a70 100644 --- a/tests/utils/sharding_info/deepseek2-16b/v6e-16/slice_1/rule_pipeline-large-moe_ici_fsdp_parallelism=-1_ici_expert_parallelism=4_use_ring_of_experts=true/input_shardings.json +++ b/tests/utils/sharding_info/deepseek3-test/v6e-16/slice_1/rule_default_ici_fsdp_parallelism=-1_ici_expert_parallelism=4/input_shardings.json @@ -1,67 +1,67 @@ { "Activation Sharding Dump": [ { - "deepseek/inputs: bfloat16[192,2048,2048]": { + "deepseek/inputs: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "deepseek/pre_attention_norm: bfloat16[192,2048,2048]": { + "deepseek/pre_attention_norm: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "attention_mla/inputs_q: bfloat16[192,2048,2048]": { + "attention_mla/inputs_q: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch_attn', 'activation_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "attention_mla/inputs_kv: bfloat16[192,2048,2048]": { + "attention_mla/inputs_kv: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch_attn', 'activation_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "attention_mla/q_nope: bfloat16[192,2048,16,128]": { + "attention_mla/q_nope: bfloat16[192,2048,128,128]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_mla/q_pe: bfloat16[192,2048,16,64]": { + "attention_mla/q_pe: bfloat16[192,2048,128,64]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_mla/query: bfloat16[192,2048,16,192]": { + "attention_mla/query: bfloat16[192,2048,128,192]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_mla/key_nope: bfloat16[192,2048,16,128]": { + "attention_mla/key_nope: bfloat16[192,2048,128,128]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_mla/key_rope: bfloat16[192,2048,16,64]": { + "attention_mla/key_rope: bfloat16[192,2048,128,64]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_mla/key: bfloat16[192,2048,16,192]": { + "attention_mla/key: bfloat16[192,2048,128,192]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_mla/value: bfloat16[192,2048,16,128]": { + "attention_mla/value: bfloat16[192,2048,128,128]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } @@ -79,97 +79,97 @@ } }, { - "attention_op/query: bfloat16[192,16,2048,192]": { + "attention_op/query: bfloat16[192,128,2048,192]": { "logic_axes": "Unknown", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_op/key: bfloat16[192,16,2048,192]": { + "attention_op/key: bfloat16[192,128,2048,192]": { "logic_axes": "Unknown", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_op/value: bfloat16[192,16,2048,128]": { + "attention_op/value: bfloat16[192,128,2048,128]": { "logic_axes": "Unknown", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_mla/out: bfloat16[192,2048,16,128]": { + "attention_op/decoder_segment_ids: int32[192,2048]": { + "logic_axes": "Unknown", + "PartitionSpec": "P(('fsdp', 'expert'), None)" + } + }, + { + "attention_mla/out: bfloat16[192,2048,128,128]": { "logic_axes": "('activation_batch_attn', 'activation_length', 'activation_heads', 'activation_kv')", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "deepseek/attention_result: bfloat16[192,2048,2048]": { + "deepseek/attention_result: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "deepseek/post_attention_norm: bfloat16[192,2048,2048]": { + "deepseek/post_attention_norm: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "linears/x: bfloat16[192,2048,10944]": { + "linears/x: bfloat16[192,2048,18432]": { "logic_axes": "('activation_batch', 'activation_length', 'activation_mlp')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "deepseek/mlp: bfloat16[192,2048,2048]": { + "deepseek/mlp: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "deepseek/x: bfloat16[192,2048,2048]": { + "deepseek/x: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "moe/inputs: bfloat16[192,2048,2048]": { - "logic_axes": "('activation_batch', 'activation_norm_length', None)", + "moe/inputs: bfloat16[192,2048,512]": { + "logic_axes": "('activation_batch', 'activation_norm_length_routed', None)", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "moe/gate_logits: bfloat16[192,2048,64]": { - "logic_axes": "('activation_batch', 'activation_norm_length', None)", + "moe/gate_logits: bfloat16[192,2048,256]": { + "logic_axes": "('activation_batch', 'activation_norm_length_routed', None)", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "moe/w0_kernel: bfloat16[64,2048,1408]": { - "logic_axes": "Unknown", - "PartitionSpec": "P('expert', None, None)" - } - }, - { - "moe/w1_kernel: bfloat16[64,2048,1408]": { - "logic_axes": "Unknown", - "PartitionSpec": "P('expert', None, None)" + "moe/pre_bias_logits: bfloat16[192,2048,256]": { + "logic_axes": "('activation_batch', 'activation_norm_length_routed', None)", + "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "moe/wo_kernel: bfloat16[64,1408,2048]": { + "moe/w: bfloat16[256,512,512]": { "logic_axes": "Unknown", "PartitionSpec": "P('expert', None, None)" } }, { - "linears/x: bfloat16[192,2048,2816]": { + "linears/x: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_length', 'activation_mlp')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "deepseek/mlp_lnx: bfloat16[192,2048,2048]": { + "deepseek/mlp_lnx: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } diff --git a/tests/utils/sharding_info/deepseek3-test/v6e-16/slice_1/rule_default_ici_fsdp_parallelism=-1_ici_expert_parallelism=4/logical_shardings.json b/tests/utils/sharding_info/deepseek3-test/v6e-16/slice_1/rule_default_ici_fsdp_parallelism=-1_ici_expert_parallelism=4/logical_shardings.json new file mode 100644 index 0000000000..0748fe0e5a --- /dev/null +++ b/tests/utils/sharding_info/deepseek3-test/v6e-16/slice_1/rule_default_ici_fsdp_parallelism=-1_ici_expert_parallelism=4/logical_shardings.json @@ -0,0 +1,1380 @@ +{ + "['model']/['decoder']/['decoder_norm']/['scale']/.value": { + "partition_spec": [ + null + ], + "shape": [ + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['count']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['key']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['count']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['key']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['count']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['key']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null, + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null, + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['model']/['decoder']/['logits_dense']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null + ], + "shape": [ + 512, + 129280 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['bias']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 58, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['count']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['key']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['count']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['key']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['count']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['key']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "partition_spec": [ + "expert", + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "partition_spec": [ + "expert", + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "partition_spec": [ + "expert", + null, + null, + "fsdp" + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null, + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null, + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['model']/['decoder']/['rngs']/['aqt']/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['aqt']/['key']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['dropout']/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['dropout']/['key']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['params']/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['params']/['key']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['token_embedder']/['embedding']/.value": { + "partition_spec": [ + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['decoder_norm']/['scale']/.value": { + "partition_spec": [ + null + ], + "shape": [ + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null, + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null, + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['logits_dense']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null + ], + "shape": [ + 512, + 129280 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "partition_spec": [ + "expert", + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "partition_spec": [ + "expert", + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "partition_spec": [ + "expert", + null, + null, + "fsdp" + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null, + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null, + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['token_embedder']/['embedding']/.value": { + "partition_spec": [ + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['decoder_norm']/['scale']/.value": { + "partition_spec": [ + null + ], + "shape": [ + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null, + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null, + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['logits_dense']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null + ], + "shape": [ + 512, + 129280 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "partition_spec": [ + "expert", + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "partition_spec": [ + "expert", + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "partition_spec": [ + "expert", + null, + null, + "fsdp" + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null, + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null, + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['token_embedder']/['embedding']/.value": { + "partition_spec": [ + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[2]/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['optimizer']/['step']/.value": { + "partition_spec": [], + "shape": [] + } +} \ No newline at end of file diff --git a/tests/utils/sharding_info/deepseek2-16b/v6e-16/slice_1/rule_default_ici_fsdp_parallelism=-1_ici_expert_parallelism=4/named_shardings.json b/tests/utils/sharding_info/deepseek3-test/v6e-16/slice_1/rule_default_ici_fsdp_parallelism=-1_ici_expert_parallelism=4/named_shardings.json similarity index 67% rename from tests/utils/sharding_info/deepseek2-16b/v6e-16/slice_1/rule_default_ici_fsdp_parallelism=-1_ici_expert_parallelism=4/named_shardings.json rename to tests/utils/sharding_info/deepseek3-test/v6e-16/slice_1/rule_default_ici_fsdp_parallelism=-1_ici_expert_parallelism=4/named_shardings.json index faafd2aab5..05335c30c8 100644 --- a/tests/utils/sharding_info/deepseek2-16b/v6e-16/slice_1/rule_default_ici_fsdp_parallelism=-1_ici_expert_parallelism=4/named_shardings.json +++ b/tests/utils/sharding_info/deepseek3-test/v6e-16/slice_1/rule_default_ici_fsdp_parallelism=-1_ici_expert_parallelism=4/named_shardings.json @@ -1,5 +1,5 @@ { - ".step": { + "['model']/['decoder']/['decoder_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -30,10 +30,14 @@ "autoregressive": 1 } }, - "partition_spec": [], - "shape": [] + "partition_spec": [ + null + ], + "shape": [ + 512 + ] }, - ".params/['params']/['decoder']/['decoder_norm']/['scale']": { + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -65,13 +69,20 @@ } }, "partition_spec": [ - "tensor" + [ + "fsdp", + "expert" + ], + null, + null ], "shape": [ - 2048 + 512, + 3, + 18432 ] }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -105,25 +116,18 @@ "partition_spec": [ [ "fsdp", - "context", - "context_usp_ulysses", "expert" ], null, - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ] + null ], "shape": [ - 2048, - 1, - 10944 + 512, + 3, + 18432 ] }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -155,27 +159,20 @@ } }, "partition_spec": [ + null, + null, [ "fsdp", - "context", - "context_usp_ulysses", "expert" - ], - null, - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" ] ], "shape": [ - 2048, - 1, - 10944 + 18432, + 3, + 512 ] }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -207,27 +204,15 @@ } }, "partition_spec": [ - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ], null, - [ - "fsdp", - "context", - "context_usp_ulysses", - "expert" - ] + null ], "shape": [ - 10944, - 1, - 2048 + 512, + 3 ] }, - ".params/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { + "['model']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -259,15 +244,15 @@ } }, "partition_spec": [ - "tensor", + null, null ], "shape": [ - 2048, - 1 + 512, + 3 ] }, - ".params/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['count']/.value": { "mesh": { "axis_names": [ "diloco", @@ -298,16 +283,12 @@ "autoregressive": 1 } }, - "partition_spec": [ - "tensor", - null - ], + "partition_spec": [], "shape": [ - 2048, - 1 + 3 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['key']/.value": { "mesh": { "axis_names": [ "diloco", @@ -338,16 +319,12 @@ "autoregressive": 1 } }, - "partition_spec": [ - "tensor", - null - ], + "partition_spec": [], "shape": [ - 512, - 1 + 3 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['count']/.value": { "mesh": { "axis_names": [ "diloco", @@ -378,30 +355,12 @@ "autoregressive": 1 } }, - "partition_spec": [ - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], - null, - null, - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ] - ], + "partition_spec": [], "shape": [ - 16, - 1, - 128, - 2048 + 3 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['key']/.value": { "mesh": { "axis_names": [ "diloco", @@ -432,30 +391,48 @@ "autoregressive": 1 } }, - "partition_spec": [ - [ + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['count']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", "fsdp", "fsdp_transpose", "context", "context_usp_ulysses", - "expert" - ], - null, - [ + "context_autoregressive", "tensor", "tensor_sequence", + "expert", "autoregressive" ], - null - ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 4, + "autoregressive": 1 + } + }, + "partition_spec": [], "shape": [ - 2048, - 1, - 16, - 192 + 3 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['key']/.value": { "mesh": { "axis_names": [ "diloco", @@ -486,24 +463,12 @@ "autoregressive": 1 } }, - "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ], - null, - null - ], + "partition_spec": [], "shape": [ - 2048, - 1, - 576 + 3 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -535,29 +500,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ], null, - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], null ], "shape": [ 512, - 1, - 16, - 256 + 3 ] }, - ".params/['params']/['decoder']/['logits_dense']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -589,25 +540,22 @@ } }, "partition_spec": [ + null, + null, + null, [ "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", "expert" - ], - [ - "tensor", - "tensor_sequence", - "autoregressive" ] ], "shape": [ - 2048, - 102400 + 128, + 3, + 128, + 512 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -639,22 +587,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses" - ], null, null ], "shape": [ - 2048, - 26, - 64 + 1536, + 3 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -686,28 +627,20 @@ } }, "partition_spec": [ - "expert", - null, [ "fsdp", - "context", - "context_usp_ulysses" + "expert" ], - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ] + null, + null ], "shape": [ - 64, - 26, - 2048, - 1408 + 512, + 3, + 576 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -739,28 +672,22 @@ } }, "partition_spec": [ - "expert", - null, [ "fsdp", - "context", - "context_usp_ulysses" + "expert" ], - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ] + null, + null, + null ], "shape": [ - 64, - 26, - 2048, - 1408 + 512, + 3, + 128, + 256 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -792,28 +719,20 @@ } }, "partition_spec": [ - "expert", - null, - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ], [ "fsdp", - "context", - "context_usp_ulysses" - ] + "expert" + ], + null, + null ], "shape": [ - 64, - 26, - 1408, - 2048 + 512, + 3, + 1536 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -847,25 +766,20 @@ "partition_spec": [ [ "fsdp", - "context", - "context_usp_ulysses", "expert" ], null, - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ] + null, + null ], "shape": [ - 2048, - 26, - 2816 + 1536, + 3, + 128, + 192 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { + "['model']/['decoder']/['logits_dense']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -899,25 +813,16 @@ "partition_spec": [ [ "fsdp", - "context", - "context_usp_ulysses", "expert" ], - null, - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ] + null ], "shape": [ - 2048, - 26, - 2816 + 512, + 129280 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['bias']/.value": { "mesh": { "axis_names": [ "diloco", @@ -949,27 +854,15 @@ } }, "partition_spec": [ - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ], null, - [ - "fsdp", - "context", - "context_usp_ulysses", - "expert" - ] + null ], "shape": [ - 2816, - 26, - 2048 + 58, + 256 ] }, - ".params/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1001,15 +894,17 @@ } }, "partition_spec": [ - "tensor", + "fsdp", + null, null ], "shape": [ - 2048, - 26 + 512, + 58, + 256 ] }, - ".params/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['count']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1040,16 +935,12 @@ "autoregressive": 1 } }, - "partition_spec": [ - "tensor", - null - ], + "partition_spec": [], "shape": [ - 2048, - 26 + 58 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['key']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1080,16 +971,12 @@ "autoregressive": 1 } }, - "partition_spec": [ - "tensor", - null - ], + "partition_spec": [], "shape": [ - 512, - 26 + 58 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['count']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1120,30 +1007,1097 @@ "autoregressive": 1 } }, - "partition_spec": [ - [ - "tensor", - "tensor_sequence", + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['key']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 4, + "autoregressive": 1 + } + }, + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['count']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 4, + "autoregressive": 1 + } + }, + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['key']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 4, + "autoregressive": 1 + } + }, + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 4, + "autoregressive": 1 + } + }, + "partition_spec": [ + "expert", + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 4, + "autoregressive": 1 + } + }, + "partition_spec": [ + "expert", + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", "autoregressive" ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 4, + "autoregressive": 1 + } + }, + "partition_spec": [ + "expert", null, null, + "fsdp" + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 4, + "autoregressive": 1 + } + }, + "partition_spec": [ [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 4, + "autoregressive": 1 + } + }, + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 4, + "autoregressive": 1 + } + }, + "partition_spec": [ + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 4, + "autoregressive": 1 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 4, + "autoregressive": 1 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 4, + "autoregressive": 1 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 4, + "autoregressive": 1 + } + }, + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 4, + "autoregressive": 1 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 4, + "autoregressive": 1 + } + }, + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 4, + "autoregressive": 1 + } + }, + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null, + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 4, + "autoregressive": 1 + } + }, + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 4, + "autoregressive": 1 + } + }, + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null, + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['model']/['decoder']/['rngs']/['aqt']/['count']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 4, + "autoregressive": 1 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['aqt']/['key']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 4, + "autoregressive": 1 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['dropout']/['count']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 4, + "autoregressive": 1 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['dropout']/['key']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 4, + "autoregressive": 1 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['params']/['count']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 4, + "autoregressive": 1 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['params']/['key']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 4, + "autoregressive": 1 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['token_embedder']/['embedding']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 4, + "autoregressive": 1 + } + }, + "partition_spec": [ + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['count']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", + "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 4, + "autoregressive": 1 + } + }, + "partition_spec": [], + "shape": [] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['decoder_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", "fsdp", "fsdp_transpose", "context", "context_usp_ulysses", - "expert" - ] + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 4, + "autoregressive": 1 + } + }, + "partition_spec": [ + null ], "shape": [ - 16, - 26, - 128, - 2048 + 512 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1177,27 +2131,18 @@ "partition_spec": [ [ "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", "expert" ], null, - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], null ], "shape": [ - 2048, - 26, - 16, - 192 + 512, + 3, + 18432 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1231,21 +2176,18 @@ "partition_spec": [ [ "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", "expert" ], null, null ], "shape": [ - 2048, - 26, - 576 + 512, + 3, + 18432 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1277,29 +2219,60 @@ } }, "partition_spec": [ + null, + null, [ + "fsdp", + "expert" + ] + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", "fsdp", "fsdp_transpose", "context", "context_usp_ulysses", - "expert" - ], - null, - [ + "context_autoregressive", "tensor", "tensor_sequence", + "expert", "autoregressive" ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 4, + "autoregressive": 1 + } + }, + "partition_spec": [ + null, null ], "shape": [ 512, - 26, - 16, - 256 + 3 ] }, - ".params/['params']/['token_embedder']/['embedding']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1331,25 +2304,55 @@ } }, "partition_spec": [ - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], - [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", "fsdp", "fsdp_transpose", "context", "context_usp_ulysses", - "expert" - ] + "context_autoregressive", + "tensor", + "tensor_sequence", + "expert", + "autoregressive" + ], + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 4, + "autoregressive": 1 + } + }, + "partition_spec": [ + null, + null ], "shape": [ - 102400, - 2048 + 512, + 3 ] }, - ".opt_state/[0]/.count": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1380,10 +2383,23 @@ "autoregressive": 1 } }, - "partition_spec": [], - "shape": [] + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 128, + 3, + 128, + 512 + ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['decoder_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1415,13 +2431,15 @@ } }, "partition_spec": [ - "tensor" + null, + null ], "shape": [ - 2048 + 1536, + 3 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1455,25 +2473,18 @@ "partition_spec": [ [ "fsdp", - "context", - "context_usp_ulysses", "expert" ], null, - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ] + null ], "shape": [ - 2048, - 1, - 10944 + 512, + 3, + 576 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1507,25 +2518,20 @@ "partition_spec": [ [ "fsdp", - "context", - "context_usp_ulysses", "expert" ], null, - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ] + null, + null ], "shape": [ - 2048, - 1, - 10944 + 512, + 3, + 128, + 256 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1558,26 +2564,66 @@ }, "partition_spec": [ [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "diloco", + "data", + "stage", + "fsdp", "fsdp_transpose", + "context", + "context_usp_ulysses", + "context_autoregressive", "tensor", "tensor_sequence", + "expert", "autoregressive" ], - null, + "shape": { + "diloco": 1, + "data": 1, + "stage": 1, + "fsdp": 4, + "fsdp_transpose": 1, + "context": 1, + "context_usp_ulysses": 1, + "context_autoregressive": 1, + "tensor": 1, + "tensor_sequence": 1, + "expert": 4, + "autoregressive": 1 + } + }, + "partition_spec": [ [ "fsdp", - "context", - "context_usp_ulysses", "expert" - ] + ], + null, + null, + null ], "shape": [ - 10944, - 1, - 2048 + 1536, + 3, + 128, + 192 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['logits_dense']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1609,15 +2655,18 @@ } }, "partition_spec": [ - "tensor", + [ + "fsdp", + "expert" + ], null ], "shape": [ - 2048, - 1 + 512, + 129280 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1649,15 +2698,17 @@ } }, "partition_spec": [ - "tensor", + "fsdp", + null, null ], "shape": [ - 2048, - 1 + 512, + 58, + 256 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1689,15 +2740,19 @@ } }, "partition_spec": [ - "tensor", + "expert", + null, + "fsdp", null ], "shape": [ + 256, + 58, 512, - 1 + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1729,29 +2784,19 @@ } }, "partition_spec": [ - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], - null, + "expert", null, - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ] + "fsdp", + null ], "shape": [ - 16, - 1, - 128, - 2048 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1783,29 +2828,19 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ], + "expert", null, - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], - null + null, + "fsdp" ], "shape": [ - 2048, - 1, - 16, - 192 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1839,21 +2874,18 @@ "partition_spec": [ [ "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", "expert" ], null, null ], "shape": [ - 2048, - 1, - 576 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1887,27 +2919,18 @@ "partition_spec": [ [ "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", "expert" ], null, - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], null ], "shape": [ 512, - 1, - 16, - 256 + 58, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['logits_dense']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1939,25 +2962,20 @@ } }, "partition_spec": [ + null, + null, [ "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", "expert" - ], - [ - "tensor", - "tensor_sequence", - "autoregressive" ] ], "shape": [ - 2048, - 102400 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -1989,22 +3007,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses" - ], null, null ], "shape": [ - 2048, - 26, - 64 + 512, + 58 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2036,28 +3047,15 @@ } }, "partition_spec": [ - "expert", null, - [ - "fsdp", - "context", - "context_usp_ulysses" - ], - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ] + null ], "shape": [ - 64, - 26, - 2048, - 1408 + 512, + 58 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2089,28 +3087,15 @@ } }, "partition_spec": [ - "expert", null, - [ - "fsdp", - "context", - "context_usp_ulysses" - ], - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ] + null ], "shape": [ - 64, - 26, - 2048, - 1408 + 512, + 58 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2142,28 +3127,22 @@ } }, "partition_spec": [ - "expert", null, - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ], + null, + null, [ "fsdp", - "context", - "context_usp_ulysses" + "expert" ] ], "shape": [ - 64, - 26, - 1408, - 2048 + 128, + 58, + 128, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2195,27 +3174,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "context", - "context_usp_ulysses", - "expert" - ], null, - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ] + null ], "shape": [ - 2048, - 26, - 2816 + 1536, + 58 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2249,25 +3216,18 @@ "partition_spec": [ [ "fsdp", - "context", - "context_usp_ulysses", "expert" ], null, - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ] + null ], "shape": [ - 2048, - 26, - 2816 + 512, + 58, + 576 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2299,27 +3259,22 @@ } }, "partition_spec": [ - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ], - null, [ "fsdp", - "context", - "context_usp_ulysses", "expert" - ] + ], + null, + null, + null ], "shape": [ - 2816, - 26, - 2048 + 512, + 58, + 128, + 256 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2351,15 +3306,20 @@ } }, "partition_spec": [ - "tensor", + [ + "fsdp", + "expert" + ], + null, null ], "shape": [ - 2048, - 26 + 512, + 58, + 1536 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2391,15 +3351,22 @@ } }, "partition_spec": [ - "tensor", + [ + "fsdp", + "expert" + ], + null, + null, null ], "shape": [ - 2048, - 26 + 1536, + 58, + 128, + 192 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['token_embedder']/['embedding']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2431,15 +3398,18 @@ } }, "partition_spec": [ - "tensor", - null + null, + [ + "fsdp", + "expert" + ] ], "shape": [ - 512, - 26 + 129280, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['decoder_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2471,29 +3441,13 @@ } }, "partition_spec": [ - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], - null, - null, - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ] + null ], "shape": [ - 16, - 26, - 128, - 2048 + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2527,27 +3481,18 @@ "partition_spec": [ [ "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", "expert" ], null, - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], null ], "shape": [ - 2048, - 26, - 16, - 192 + 512, + 3, + 18432 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2581,21 +3526,18 @@ "partition_spec": [ [ "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", "expert" ], null, null ], "shape": [ - 2048, - 26, - 576 + 512, + 3, + 18432 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2627,29 +3569,20 @@ } }, "partition_spec": [ + null, + null, [ "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", "expert" - ], - null, - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], - null + ] ], "shape": [ - 512, - 26, - 16, - 256 + 18432, + 3, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['token_embedder']/['embedding']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2681,25 +3614,15 @@ } }, "partition_spec": [ - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ] + null, + null ], "shape": [ - 102400, - 2048 + 512, + 3 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['decoder_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2731,13 +3654,15 @@ } }, "partition_spec": [ - "tensor" + null, + null ], "shape": [ - 2048 + 512, + 3 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2769,27 +3694,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "context", - "context_usp_ulysses", - "expert" - ], null, - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ] + null ], "shape": [ - 2048, - 1, - 10944 + 512, + 3 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2821,27 +3734,22 @@ } }, "partition_spec": [ + null, + null, + null, [ "fsdp", - "context", - "context_usp_ulysses", "expert" - ], - null, - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" ] ], "shape": [ - 2048, - 1, - 10944 + 128, + 3, + 128, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2873,27 +3781,15 @@ } }, "partition_spec": [ - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ], null, - [ - "fsdp", - "context", - "context_usp_ulysses", - "expert" - ] + null ], "shape": [ - 10944, - 1, - 2048 + 1536, + 3 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2925,15 +3821,20 @@ } }, "partition_spec": [ - "tensor", + [ + "fsdp", + "expert" + ], + null, null ], "shape": [ - 2048, - 1 + 512, + 3, + 576 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -2965,15 +3866,22 @@ } }, "partition_spec": [ - "tensor", + [ + "fsdp", + "expert" + ], + null, + null, null ], "shape": [ - 2048, - 1 + 512, + 3, + 128, + 256 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3005,15 +3913,20 @@ } }, "partition_spec": [ - "tensor", + [ + "fsdp", + "expert" + ], + null, null ], "shape": [ 512, - 1 + 3, + 1536 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3046,28 +3959,21 @@ }, "partition_spec": [ [ - "tensor", - "tensor_sequence", - "autoregressive" + "fsdp", + "expert" ], null, null, - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ] + null ], "shape": [ - 16, - 1, + 1536, + 3, 128, - 2048 + 192 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['logits_dense']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3101,27 +4007,16 @@ "partition_spec": [ [ "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", "expert" ], - null, - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], null ], "shape": [ - 2048, - 1, - 16, - 192 + 512, + 129280 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3153,23 +4048,17 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ], + "fsdp", null, null ], "shape": [ - 2048, - 1, - 576 + 512, + 58, + 256 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3201,29 +4090,19 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ], + "expert", null, - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], + "fsdp", null ], "shape": [ + 256, + 58, 512, - 1, - 16, - 256 + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['logits_dense']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3255,25 +4134,19 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ], - [ - "tensor", - "tensor_sequence", - "autoregressive" - ] + "expert", + null, + "fsdp", + null ], "shape": [ - 2048, - 102400 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3305,22 +4178,19 @@ } }, "partition_spec": [ - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses" - ], + "expert", null, - null + null, + "fsdp" ], "shape": [ - 2048, - 26, - 64 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3352,28 +4222,20 @@ } }, "partition_spec": [ - "expert", - null, [ "fsdp", - "context", - "context_usp_ulysses" + "expert" ], - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ] + null, + null ], "shape": [ - 64, - 26, - 2048, - 1408 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3405,28 +4267,20 @@ } }, "partition_spec": [ - "expert", - null, [ "fsdp", - "context", - "context_usp_ulysses" + "expert" ], - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ] + null, + null ], "shape": [ - 64, - 26, - 2048, - 1408 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3458,28 +4312,20 @@ } }, "partition_spec": [ - "expert", null, - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ], + null, [ "fsdp", - "context", - "context_usp_ulysses" + "expert" ] ], "shape": [ - 64, - 26, - 1408, - 2048 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3511,27 +4357,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "context", - "context_usp_ulysses", - "expert" - ], null, - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ] + null ], "shape": [ - 2048, - 26, - 2816 + 512, + 58 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3563,27 +4397,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "context", - "context_usp_ulysses", - "expert" - ], null, - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ] + null ], "shape": [ - 2048, - 26, - 2816 + 512, + 58 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3615,27 +4437,15 @@ } }, "partition_spec": [ - [ - "fsdp_transpose", - "tensor", - "tensor_sequence", - "autoregressive" - ], null, - [ - "fsdp", - "context", - "context_usp_ulysses", - "expert" - ] + null ], "shape": [ - 2816, - 26, - 2048 + 512, + 58 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3667,15 +4477,22 @@ } }, "partition_spec": [ - "tensor", - null + null, + null, + null, + [ + "fsdp", + "expert" + ] ], "shape": [ - 2048, - 26 + 128, + 58, + 128, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3707,15 +4524,15 @@ } }, "partition_spec": [ - "tensor", + null, null ], "shape": [ - 2048, - 26 + 1536, + 58 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3747,15 +4564,20 @@ } }, "partition_spec": [ - "tensor", + [ + "fsdp", + "expert" + ], + null, null ], "shape": [ 512, - 26 + 58, + 576 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3788,28 +4610,21 @@ }, "partition_spec": [ [ - "tensor", - "tensor_sequence", - "autoregressive" + "fsdp", + "expert" ], null, null, - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ] + null ], "shape": [ - 16, - 26, + 512, + 58, 128, - 2048 + 256 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3843,27 +4658,18 @@ "partition_spec": [ [ "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", "expert" ], null, - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], null ], "shape": [ - 2048, - 26, - 16, - 192 + 512, + 58, + 1536 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3897,21 +4703,20 @@ "partition_spec": [ [ "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", "expert" ], null, + null, null ], "shape": [ - 2048, - 26, - 576 + 1536, + 58, + 128, + 192 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['token_embedder']/['embedding']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3943,29 +4748,18 @@ } }, "partition_spec": [ + null, [ "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", "expert" - ], - null, - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], - null + ] ], "shape": [ - 512, - 26, - 16, - 256 + 129280, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['token_embedder']/['embedding']": { + "['optimizer']/['opt_state']/[2]/['count']/.value": { "mesh": { "axis_names": [ "diloco", @@ -3996,26 +4790,10 @@ "autoregressive": 1 } }, - "partition_spec": [ - [ - "tensor", - "tensor_sequence", - "autoregressive" - ], - [ - "fsdp", - "fsdp_transpose", - "context", - "context_usp_ulysses", - "expert" - ] - ], - "shape": [ - 102400, - 2048 - ] + "partition_spec": [], + "shape": [] }, - ".opt_state/[2]/.count": { + "['optimizer']/['step']/.value": { "mesh": { "axis_names": [ "diloco", diff --git a/tests/utils/sharding_info/deepseek2-16b/tpu7x-16/slice_1/rule_ep-as-dp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2_use_ring_of_experts=true/input_shardings.json b/tests/utils/sharding_info/deepseek3-test/v6e-16/slice_1/rule_pipeline-large-moe_ici_fsdp_parallelism=-1_ici_expert_parallelism=4_use_ring_of_experts=true/input_shardings.json similarity index 72% rename from tests/utils/sharding_info/deepseek2-16b/tpu7x-16/slice_1/rule_ep-as-dp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2_use_ring_of_experts=true/input_shardings.json rename to tests/utils/sharding_info/deepseek3-test/v6e-16/slice_1/rule_pipeline-large-moe_ici_fsdp_parallelism=-1_ici_expert_parallelism=4_use_ring_of_experts=true/input_shardings.json index f556d60c89..850add1a70 100644 --- a/tests/utils/sharding_info/deepseek2-16b/tpu7x-16/slice_1/rule_ep-as-dp_ici_fsdp_parallelism=-1_ici_expert_parallelism=2_use_ring_of_experts=true/input_shardings.json +++ b/tests/utils/sharding_info/deepseek3-test/v6e-16/slice_1/rule_pipeline-large-moe_ici_fsdp_parallelism=-1_ici_expert_parallelism=4_use_ring_of_experts=true/input_shardings.json @@ -1,67 +1,67 @@ { "Activation Sharding Dump": [ { - "deepseek/inputs: bfloat16[192,2048,2048]": { + "deepseek/inputs: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "deepseek/pre_attention_norm: bfloat16[192,2048,2048]": { + "deepseek/pre_attention_norm: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "attention_mla/inputs_q: bfloat16[192,2048,2048]": { + "attention_mla/inputs_q: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch_attn', 'activation_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "attention_mla/inputs_kv: bfloat16[192,2048,2048]": { + "attention_mla/inputs_kv: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch_attn', 'activation_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "attention_mla/q_nope: bfloat16[192,2048,16,128]": { + "attention_mla/q_nope: bfloat16[192,2048,128,128]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_mla/q_pe: bfloat16[192,2048,16,64]": { + "attention_mla/q_pe: bfloat16[192,2048,128,64]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_mla/query: bfloat16[192,2048,16,192]": { + "attention_mla/query: bfloat16[192,2048,128,192]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_mla/key_nope: bfloat16[192,2048,16,128]": { + "attention_mla/key_nope: bfloat16[192,2048,128,128]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_mla/key_rope: bfloat16[192,2048,16,64]": { + "attention_mla/key_rope: bfloat16[192,2048,128,64]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_mla/key: bfloat16[192,2048,16,192]": { + "attention_mla/key: bfloat16[192,2048,128,192]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_mla/value: bfloat16[192,2048,16,128]": { + "attention_mla/value: bfloat16[192,2048,128,128]": { "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } @@ -79,97 +79,97 @@ } }, { - "attention_op/query: bfloat16[192,16,2048,192]": { + "attention_op/query: bfloat16[192,128,2048,192]": { "logic_axes": "Unknown", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_op/key: bfloat16[192,16,2048,192]": { + "attention_op/key: bfloat16[192,128,2048,192]": { "logic_axes": "Unknown", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_op/value: bfloat16[192,16,2048,128]": { + "attention_op/value: bfloat16[192,128,2048,128]": { "logic_axes": "Unknown", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "attention_mla/out: bfloat16[192,2048,16,128]": { + "attention_op/decoder_segment_ids: int32[192,2048]": { + "logic_axes": "Unknown", + "PartitionSpec": "P(('fsdp', 'expert'), None)" + } + }, + { + "attention_mla/out: bfloat16[192,2048,128,128]": { "logic_axes": "('activation_batch_attn', 'activation_length', 'activation_heads', 'activation_kv')", "PartitionSpec": "P(('fsdp', 'expert'), None, None, None)" } }, { - "deepseek/attention_result: bfloat16[192,2048,2048]": { + "deepseek/attention_result: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "deepseek/post_attention_norm: bfloat16[192,2048,2048]": { + "deepseek/post_attention_norm: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "linears/x: bfloat16[192,2048,10944]": { + "linears/x: bfloat16[192,2048,18432]": { "logic_axes": "('activation_batch', 'activation_length', 'activation_mlp')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "deepseek/mlp: bfloat16[192,2048,2048]": { + "deepseek/mlp: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "deepseek/x: bfloat16[192,2048,2048]": { + "deepseek/x: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "moe/inputs: bfloat16[192,2048,2048]": { - "logic_axes": "('activation_batch', 'activation_norm_length', None)", + "moe/inputs: bfloat16[192,2048,512]": { + "logic_axes": "('activation_batch', 'activation_norm_length_routed', None)", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "moe/gate_logits: bfloat16[192,2048,64]": { - "logic_axes": "('activation_batch', 'activation_norm_length', None)", + "moe/gate_logits: bfloat16[192,2048,256]": { + "logic_axes": "('activation_batch', 'activation_norm_length_routed', None)", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "moe/w0_kernel: bfloat16[64,2048,1408]": { - "logic_axes": "Unknown", - "PartitionSpec": "P('expert', None, None)" - } - }, - { - "moe/w1_kernel: bfloat16[64,2048,1408]": { - "logic_axes": "Unknown", - "PartitionSpec": "P('expert', None, None)" + "moe/pre_bias_logits: bfloat16[192,2048,256]": { + "logic_axes": "('activation_batch', 'activation_norm_length_routed', None)", + "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "moe/wo_kernel: bfloat16[64,1408,2048]": { + "moe/w: bfloat16[256,512,512]": { "logic_axes": "Unknown", "PartitionSpec": "P('expert', None, None)" } }, { - "linears/x: bfloat16[192,2048,2816]": { + "linears/x: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_length', 'activation_mlp')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } }, { - "deepseek/mlp_lnx: bfloat16[192,2048,2048]": { + "deepseek/mlp_lnx: bfloat16[192,2048,512]": { "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", "PartitionSpec": "P(('fsdp', 'expert'), None, None)" } diff --git a/tests/utils/sharding_info/deepseek3-test/v6e-16/slice_1/rule_pipeline-large-moe_ici_fsdp_parallelism=-1_ici_expert_parallelism=4_use_ring_of_experts=true/logical_shardings.json b/tests/utils/sharding_info/deepseek3-test/v6e-16/slice_1/rule_pipeline-large-moe_ici_fsdp_parallelism=-1_ici_expert_parallelism=4_use_ring_of_experts=true/logical_shardings.json new file mode 100644 index 0000000000..e64ed069c8 --- /dev/null +++ b/tests/utils/sharding_info/deepseek3-test/v6e-16/slice_1/rule_pipeline-large-moe_ici_fsdp_parallelism=-1_ici_expert_parallelism=4_use_ring_of_experts=true/logical_shardings.json @@ -0,0 +1,1344 @@ +{ + "['model']/['decoder']/['decoder_norm']/['scale']/.value": { + "partition_spec": [ + null + ], + "shape": [ + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['count']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['key']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['count']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['key']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['count']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['key']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['model']/['decoder']/['logits_dense']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null + ], + "shape": [ + 512, + 129280 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['bias']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 58, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['count']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['key']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['count']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['key']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['count']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['key']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "partition_spec": [ + "expert", + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "partition_spec": [ + "expert", + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "partition_spec": [ + "expert", + null, + null, + "fsdp" + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['model']/['decoder']/['rngs']/['aqt']/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['aqt']/['key']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['dropout']/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['dropout']/['key']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['params']/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['params']/['key']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['token_embedder']/['embedding']/.value": { + "partition_spec": [ + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['decoder_norm']/['scale']/.value": { + "partition_spec": [ + null + ], + "shape": [ + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['logits_dense']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null + ], + "shape": [ + 512, + 129280 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "partition_spec": [ + "expert", + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "partition_spec": [ + "expert", + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "partition_spec": [ + "expert", + null, + null, + "fsdp" + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['token_embedder']/['embedding']/.value": { + "partition_spec": [ + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['decoder_norm']/['scale']/.value": { + "partition_spec": [ + null + ], + "shape": [ + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['logits_dense']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null + ], + "shape": [ + 512, + 129280 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "partition_spec": [ + "expert", + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "partition_spec": [ + "expert", + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "partition_spec": [ + "expert", + null, + null, + "fsdp" + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "partition_spec": [ + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['token_embedder']/['embedding']/.value": { + "partition_spec": [ + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[2]/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['optimizer']/['step']/.value": { + "partition_spec": [], + "shape": [] + } +} \ No newline at end of file diff --git a/tests/utils/sharding_info/deepseek2-16b/v6e-16/slice_1/rule_pipeline-large-moe_ici_fsdp_parallelism=-1_ici_expert_parallelism=4_use_ring_of_experts=true/named_shardings.json b/tests/utils/sharding_info/deepseek3-test/v6e-16/slice_1/rule_pipeline-large-moe_ici_fsdp_parallelism=-1_ici_expert_parallelism=4_use_ring_of_experts=true/named_shardings.json similarity index 55% rename from tests/utils/sharding_info/deepseek2-16b/v6e-16/slice_1/rule_pipeline-large-moe_ici_fsdp_parallelism=-1_ici_expert_parallelism=4_use_ring_of_experts=true/named_shardings.json rename to tests/utils/sharding_info/deepseek3-test/v6e-16/slice_1/rule_pipeline-large-moe_ici_fsdp_parallelism=-1_ici_expert_parallelism=4_use_ring_of_experts=true/named_shardings.json index e9ce13caba..cd443e0d81 100644 --- a/tests/utils/sharding_info/deepseek2-16b/v6e-16/slice_1/rule_pipeline-large-moe_ici_fsdp_parallelism=-1_ici_expert_parallelism=4_use_ring_of_experts=true/named_shardings.json +++ b/tests/utils/sharding_info/deepseek3-test/v6e-16/slice_1/rule_pipeline-large-moe_ici_fsdp_parallelism=-1_ici_expert_parallelism=4_use_ring_of_experts=true/named_shardings.json @@ -1,5 +1,640 @@ { - ".step": { + "['model']/['decoder']/['decoder_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "context": 1, + "tensor": 1, + "expert": 4 + } + }, + "partition_spec": [ + null + ], + "shape": [ + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "context": 1, + "tensor": 1, + "expert": 4 + } + }, + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "context": 1, + "tensor": 1, + "expert": 4 + } + }, + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "context": 1, + "tensor": 1, + "expert": 4 + } + }, + "partition_spec": [ + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "context": 1, + "tensor": 1, + "expert": 4 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "context": 1, + "tensor": 1, + "expert": 4 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "context": 1, + "tensor": 1, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "context": 1, + "tensor": 1, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "context": 1, + "tensor": 1, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "context": 1, + "tensor": 1, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "context": 1, + "tensor": 1, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "context": 1, + "tensor": 1, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "context": 1, + "tensor": 1, + "expert": 4 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "context": 1, + "tensor": 1, + "expert": 4 + } + }, + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "context": 1, + "tensor": 1, + "expert": 4 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "context": 1, + "tensor": 1, + "expert": 4 + } + }, + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "context": 1, + "tensor": 1, + "expert": 4 + } + }, + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "context": 1, + "tensor": 1, + "expert": 4 + } + }, + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "context": 1, + "tensor": 1, + "expert": 4 + } + }, + "partition_spec": [ + "fsdp", + null, + null, + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['model']/['decoder']/['logits_dense']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "context": 1, + "tensor": 1, + "expert": 4 + } + }, + "partition_spec": [ + [ + "fsdp", + "expert" + ], + null + ], + "shape": [ + 512, + 129280 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['bias']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "context": 1, + "tensor": 1, + "expert": 4 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 58, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "context": 1, + "tensor": 1, + "expert": 4 + } + }, + "partition_spec": [ + "fsdp", + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['count']/.value": { "mesh": { "axis_names": [ "data", @@ -19,9 +654,195 @@ } }, "partition_spec": [], - "shape": [] + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "context": 1, + "tensor": 1, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "context": 1, + "tensor": 1, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "context": 1, + "tensor": 1, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "context": 1, + "tensor": 1, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "context": 1, + "tensor": 1, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "context": 1, + "tensor": 1, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "context": 1, + "tensor": 1, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + "fsdp", + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] }, - ".params/['params']/['decoder']/['decoder_norm']/['scale']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { "mesh": { "axis_names": [ "data", @@ -41,13 +862,19 @@ } }, "partition_spec": [ - "tensor" + "expert", + null, + null, + "fsdp" ], "shape": [ - 2048 + 256, + 58, + 512, + 512 ] }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -72,15 +899,15 @@ "expert" ], null, - "tensor" + null ], "shape": [ - 2048, - 1, - 10944 + 512, + 58, + 512 ] }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -105,15 +932,15 @@ "expert" ], null, - "tensor" + null ], "shape": [ - 2048, - 1, - 10944 + 512, + 58, + 512 ] }, - ".params/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -133,7 +960,7 @@ } }, "partition_spec": [ - "tensor", + null, null, [ "fsdp", @@ -141,12 +968,12 @@ ] ], "shape": [ - 10944, - 1, - 2048 + 512, + 58, + 512 ] }, - ".params/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { + "['model']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -166,15 +993,15 @@ } }, "partition_spec": [ - "tensor", + null, null ], "shape": [ - 2048, - 1 + 512, + 58 ] }, - ".params/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['model']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -194,15 +1021,15 @@ } }, "partition_spec": [ - "tensor", + null, null ], "shape": [ - 2048, - 1 + 512, + 58 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { + "['model']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -222,15 +1049,15 @@ } }, "partition_spec": [ - "tensor", + null, null ], "shape": [ 512, - 1 + 58 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -250,7 +1077,7 @@ } }, "partition_spec": [ - "tensor", + null, null, null, [ @@ -259,13 +1086,13 @@ ] ], "shape": [ - 16, - 1, 128, - 2048 + 58, + 128, + 512 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -285,22 +1112,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "expert" - ], null, - "tensor", null ], "shape": [ - 2048, - 1, - 16, - 192 + 1536, + 58 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -328,12 +1148,12 @@ null ], "shape": [ - 2048, - 1, + 512, + 58, 576 ] }, - ".params/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -355,17 +1175,17 @@ "partition_spec": [ "fsdp", null, - "tensor", + null, null ], "shape": [ 512, - 1, - 16, + 58, + 128, 256 ] }, - ".params/['params']/['decoder']/['logits_dense']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -389,14 +1209,16 @@ "fsdp", "expert" ], - "tensor" + null, + null ], "shape": [ - 2048, - 102400 + 512, + 58, + 1536 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -418,15 +1240,17 @@ "partition_spec": [ "fsdp", null, + null, null ], "shape": [ - 2048, - 26, - 64 + 1536, + 58, + 128, + 192 ] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { + "['model']/['decoder']/['rngs']/['aqt']/['count']/.value": { "mesh": { "axis_names": [ "data", @@ -445,20 +1269,10 @@ "expert": 4 } }, - "partition_spec": [ - "expert", - null, - "fsdp", - "tensor" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] + "partition_spec": [], + "shape": [] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { + "['model']/['decoder']/['rngs']/['aqt']/['key']/.value": { "mesh": { "axis_names": [ "data", @@ -477,20 +1291,10 @@ "expert": 4 } }, - "partition_spec": [ - "expert", - null, - "fsdp", - "tensor" - ], - "shape": [ - 64, - 26, - 2048, - 1408 - ] + "partition_spec": [], + "shape": [] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { + "['model']/['decoder']/['rngs']/['dropout']/['count']/.value": { "mesh": { "axis_names": [ "data", @@ -509,20 +1313,10 @@ "expert": 4 } }, - "partition_spec": [ - "expert", - null, - "tensor", - "fsdp" - ], - "shape": [ - 64, - 26, - 1408, - 2048 - ] + "partition_spec": [], + "shape": [] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { + "['model']/['decoder']/['rngs']/['dropout']/['key']/.value": { "mesh": { "axis_names": [ "data", @@ -541,21 +1335,10 @@ "expert": 4 } }, - "partition_spec": [ - [ - "fsdp", - "expert" - ], - null, - "tensor" - ], - "shape": [ - 2048, - 26, - 2816 - ] + "partition_spec": [], + "shape": [] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { + "['model']/['decoder']/['rngs']/['params']/['count']/.value": { "mesh": { "axis_names": [ "data", @@ -574,21 +1357,32 @@ "expert": 4 } }, - "partition_spec": [ - [ + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['params']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", "fsdp", + "context", + "tensor", "expert" ], - null, - "tensor" - ], - "shape": [ - 2048, - 26, - 2816 - ] + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "context": 1, + "tensor": 1, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [] }, - ".params/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { + "['model']/['token_embedder']/['embedding']/.value": { "mesh": { "axis_names": [ "data", @@ -608,7 +1402,6 @@ } }, "partition_spec": [ - "tensor", null, [ "fsdp", @@ -616,12 +1409,11 @@ ] ], "shape": [ - 2816, - 26, - 2048 + 129280, + 512 ] }, - ".params/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['count']/.value": { "mesh": { "axis_names": [ "data", @@ -640,16 +1432,10 @@ "expert": 4 } }, - "partition_spec": [ - "tensor", - null - ], - "shape": [ - 2048, - 26 - ] + "partition_spec": [], + "shape": [] }, - ".params/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['decoder_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -669,15 +1455,13 @@ } }, "partition_spec": [ - "tensor", null ], "shape": [ - 2048, - 26 + 512 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -697,15 +1481,20 @@ } }, "partition_spec": [ - "tensor", + [ + "fsdp", + "expert" + ], + null, null ], "shape": [ 512, - 26 + 3, + 18432 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -725,22 +1514,20 @@ } }, "partition_spec": [ - "tensor", - null, - null, [ "fsdp", "expert" - ] + ], + null, + null ], "shape": [ - 16, - 26, - 128, - 2048 + 512, + 3, + 18432 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -760,22 +1547,20 @@ } }, "partition_spec": [ + null, + null, [ "fsdp", "expert" - ], - null, - "tensor", - null + ] ], "shape": [ - 2048, - 26, - 16, - 192 + 18432, + 3, + 512 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -795,20 +1580,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "expert" - ], null, null ], "shape": [ - 2048, - 26, - 576 + 512, + 3 ] }, - ".params/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -828,19 +1608,15 @@ } }, "partition_spec": [ - "fsdp", null, - "tensor", null ], "shape": [ 512, - 26, - 16, - 256 + 3 ] }, - ".params/['params']/['token_embedder']/['embedding']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -860,18 +1636,15 @@ } }, "partition_spec": [ - "tensor", - [ - "fsdp", - "expert" - ] + null, + null ], "shape": [ - 102400, - 2048 + 512, + 3 ] }, - ".opt_state/[0]/.count": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -890,10 +1663,23 @@ "expert": 4 } }, - "partition_spec": [], - "shape": [] + "partition_spec": [ + null, + null, + null, + [ + "fsdp", + "expert" + ] + ], + "shape": [ + 128, + 3, + 128, + 512 + ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['decoder_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -913,13 +1699,15 @@ } }, "partition_spec": [ - "tensor" + null, + null ], "shape": [ - 2048 + 1536, + 3 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -944,15 +1732,15 @@ "expert" ], null, - "tensor" + null ], "shape": [ - 2048, - 1, - 10944 + 512, + 3, + 576 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -972,20 +1760,19 @@ } }, "partition_spec": [ - [ - "fsdp", - "expert" - ], + "fsdp", null, - "tensor" + null, + null ], "shape": [ - 2048, - 1, - 10944 + 512, + 3, + 128, + 256 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1005,20 +1792,20 @@ } }, "partition_spec": [ - "tensor", - null, [ "fsdp", "expert" - ] + ], + null, + null ], "shape": [ - 10944, - 1, - 2048 + 512, + 3, + 1536 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1038,15 +1825,19 @@ } }, "partition_spec": [ - "tensor", + "fsdp", + null, + null, null ], "shape": [ - 2048, - 1 + 1536, + 3, + 128, + 192 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['logits_dense']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1066,15 +1857,18 @@ } }, "partition_spec": [ - "tensor", + [ + "fsdp", + "expert" + ], null ], "shape": [ - 2048, - 1 + 512, + 129280 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1094,15 +1888,17 @@ } }, "partition_spec": [ - "tensor", + "fsdp", + null, null ], "shape": [ 512, - 1 + 58, + 256 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { "mesh": { "axis_names": [ "data", @@ -1122,22 +1918,19 @@ } }, "partition_spec": [ - "tensor", - null, + "expert", null, - [ - "fsdp", - "expert" - ] + "fsdp", + null ], "shape": [ - 16, - 1, - 128, - 2048 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { "mesh": { "axis_names": [ "data", @@ -1157,22 +1950,19 @@ } }, "partition_spec": [ - [ - "fsdp", - "expert" - ], + "expert", null, - "tensor", + "fsdp", null ], "shape": [ - 2048, - 1, - 16, - 192 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { "mesh": { "axis_names": [ "data", @@ -1190,22 +1980,21 @@ "tensor": 1, "expert": 4 } - }, - "partition_spec": [ - [ - "fsdp", - "expert" - ], + }, + "partition_spec": [ + "expert", null, - null + null, + "fsdp" ], "shape": [ - 2048, - 1, - 576 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1225,19 +2014,20 @@ } }, "partition_spec": [ - "fsdp", + [ + "fsdp", + "expert" + ], null, - "tensor", null ], "shape": [ 512, - 1, - 16, - 256 + 58, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['logits_dense']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1261,14 +2051,16 @@ "fsdp", "expert" ], - "tensor" + null, + null ], "shape": [ - 2048, - 102400 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1288,17 +2080,20 @@ } }, "partition_spec": [ - "fsdp", null, - null + null, + [ + "fsdp", + "expert" + ] ], "shape": [ - 2048, - 26, - 64 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1318,19 +2113,15 @@ } }, "partition_spec": [ - "expert", null, - "fsdp", - "tensor" + null ], "shape": [ - 64, - 26, - 2048, - 1408 + 512, + 58 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1350,19 +2141,15 @@ } }, "partition_spec": [ - "expert", null, - "fsdp", - "tensor" + null ], "shape": [ - 64, - 26, - 2048, - 1408 + 512, + 58 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1382,19 +2169,15 @@ } }, "partition_spec": [ - "expert", null, - "tensor", - "fsdp" + null ], "shape": [ - 64, - 26, - 1408, - 2048 + 512, + 58 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1414,20 +2197,22 @@ } }, "partition_spec": [ + null, + null, + null, [ "fsdp", "expert" - ], - null, - "tensor" + ] ], "shape": [ - 2048, - 26, - 2816 + 128, + 58, + 128, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1447,20 +2232,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "expert" - ], null, - "tensor" + null ], "shape": [ - 2048, - 26, - 2816 + 1536, + 58 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1480,20 +2260,20 @@ } }, "partition_spec": [ - "tensor", - null, [ "fsdp", "expert" - ] + ], + null, + null ], "shape": [ - 2816, - 26, - 2048 + 512, + 58, + 576 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1513,15 +2293,19 @@ } }, "partition_spec": [ - "tensor", + "fsdp", + null, + null, null ], "shape": [ - 2048, - 26 + 512, + 58, + 128, + 256 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1541,15 +2325,20 @@ } }, "partition_spec": [ - "tensor", + [ + "fsdp", + "expert" + ], + null, null ], "shape": [ - 2048, - 26 + 512, + 58, + 1536 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1569,15 +2358,19 @@ } }, "partition_spec": [ - "tensor", + "fsdp", + null, + null, null ], "shape": [ - 512, - 26 + 1536, + 58, + 128, + 192 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['mu']/['token_embedder']/['embedding']/.value": { "mesh": { "axis_names": [ "data", @@ -1597,8 +2390,6 @@ } }, "partition_spec": [ - "tensor", - null, null, [ "fsdp", @@ -1606,13 +2397,11 @@ ] ], "shape": [ - 16, - 26, - 128, - 2048 + 129280, + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['decoder_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1632,22 +2421,13 @@ } }, "partition_spec": [ - [ - "fsdp", - "expert" - ], - null, - "tensor", null ], "shape": [ - 2048, - 26, - 16, - 192 + 512 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1675,12 +2455,12 @@ null ], "shape": [ - 2048, - 26, - 576 + 512, + 3, + 18432 ] }, - ".opt_state/[0]/.mu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1700,19 +2480,20 @@ } }, "partition_spec": [ - "fsdp", + [ + "fsdp", + "expert" + ], null, - "tensor", null ], "shape": [ 512, - 26, - 16, - 256 + 3, + 18432 ] }, - ".opt_state/[0]/.mu/['params']/['token_embedder']/['embedding']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1732,18 +2513,20 @@ } }, "partition_spec": [ - "tensor", + null, + null, [ "fsdp", "expert" ] ], "shape": [ - 102400, - 2048 + 18432, + 3, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['decoder_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1763,13 +2546,15 @@ } }, "partition_spec": [ - "tensor" + null, + null ], "shape": [ - 2048 + 512, + 3 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1789,20 +2574,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "expert" - ], null, - "tensor" + null ], "shape": [ - 2048, - 1, - 10944 + 512, + 3 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1822,20 +2602,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "expert" - ], null, - "tensor" + null ], "shape": [ - 2048, - 1, - 10944 + 512, + 3 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1855,7 +2630,8 @@ } }, "partition_spec": [ - "tensor", + null, + null, null, [ "fsdp", @@ -1863,12 +2639,13 @@ ] ], "shape": [ - 10944, - 1, - 2048 + 128, + 3, + 128, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -1888,15 +2665,15 @@ } }, "partition_spec": [ - "tensor", + null, null ], "shape": [ - 2048, - 1 + 1536, + 3 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1916,15 +2693,20 @@ } }, "partition_spec": [ - "tensor", + [ + "fsdp", + "expert" + ], + null, null ], "shape": [ - 2048, - 1 + 512, + 3, + 576 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1944,15 +2726,19 @@ } }, "partition_spec": [ - "tensor", + "fsdp", + null, + null, null ], "shape": [ 512, - 1 + 3, + 128, + 256 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -1972,22 +2758,20 @@ } }, "partition_spec": [ - "tensor", - null, - null, [ "fsdp", "expert" - ] + ], + null, + null ], "shape": [ - 16, - 1, - 128, - 2048 + 512, + 3, + 1536 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['query']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2007,22 +2791,19 @@ } }, "partition_spec": [ - [ - "fsdp", - "expert" - ], + "fsdp", + null, null, - "tensor", null ], "shape": [ - 2048, - 1, - 16, + 1536, + 3, + 128, 192 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['logits_dense']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2046,16 +2827,14 @@ "fsdp", "expert" ], - null, null ], "shape": [ - 2048, - 1, - 576 + 512, + 129280 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2077,17 +2856,15 @@ "partition_spec": [ "fsdp", null, - "tensor", null ], "shape": [ 512, - 1, - 16, + 58, 256 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['logits_dense']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { "mesh": { "axis_names": [ "data", @@ -2107,18 +2884,19 @@ } }, "partition_spec": [ - [ - "fsdp", - "expert" - ], - "tensor" + "expert", + null, + "fsdp", + null ], "shape": [ - 2048, - 102400 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { "mesh": { "axis_names": [ "data", @@ -2138,17 +2916,19 @@ } }, "partition_spec": [ - "fsdp", + "expert", null, + "fsdp", null ], "shape": [ - 2048, - 26, - 64 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { "mesh": { "axis_names": [ "data", @@ -2170,17 +2950,17 @@ "partition_spec": [ "expert", null, - "fsdp", - "tensor" + null, + "fsdp" ], "shape": [ - 64, - 26, - 2048, - 1408 + 256, + 58, + 512, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2200,19 +2980,20 @@ } }, "partition_spec": [ - "expert", + [ + "fsdp", + "expert" + ], null, - "fsdp", - "tensor" + null ], "shape": [ - 64, - 26, - 2048, - 1408 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2232,19 +3013,20 @@ } }, "partition_spec": [ - "expert", + [ + "fsdp", + "expert" + ], null, - "tensor", - "fsdp" + null ], "shape": [ - 64, - 26, - 1408, - 2048 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2264,20 +3046,20 @@ } }, "partition_spec": [ + null, + null, [ "fsdp", "expert" - ], - null, - "tensor" + ] ], "shape": [ - 2048, - 26, - 2816 + 512, + 58, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -2297,20 +3079,15 @@ } }, "partition_spec": [ - [ - "fsdp", - "expert" - ], null, - "tensor" + null ], "shape": [ - 2048, - 26, - 2816 + 512, + 58 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -2330,20 +3107,15 @@ } }, "partition_spec": [ - "tensor", null, - [ - "fsdp", - "expert" - ] + null ], "shape": [ - 2816, - 26, - 2048 + 512, + 58 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -2363,15 +3135,15 @@ } }, "partition_spec": [ - "tensor", + null, null ], "shape": [ - 2048, - 26 + 512, + 58 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2391,15 +3163,22 @@ } }, "partition_spec": [ - "tensor", - null + null, + null, + null, + [ + "fsdp", + "expert" + ] ], "shape": [ - 2048, - 26 + 128, + 58, + 128, + 512 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { "mesh": { "axis_names": [ "data", @@ -2419,15 +3198,15 @@ } }, "partition_spec": [ - "tensor", + null, null ], "shape": [ - 512, - 26 + 1536, + 58 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2447,22 +3226,20 @@ } }, "partition_spec": [ - "tensor", - null, - null, [ "fsdp", "expert" - ] + ], + null, + null ], "shape": [ - 16, - 26, - 128, - 2048 + 512, + 58, + 576 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['query']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2482,22 +3259,19 @@ } }, "partition_spec": [ - [ - "fsdp", - "expert" - ], + "fsdp", + null, null, - "tensor", null ], "shape": [ - 2048, - 26, - 16, - 192 + 512, + 58, + 128, + 256 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2525,12 +3299,12 @@ null ], "shape": [ - 2048, - 26, - 576 + 512, + 58, + 1536 ] }, - ".opt_state/[0]/.nu/['params']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']": { + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { "mesh": { "axis_names": [ "data", @@ -2552,17 +3326,17 @@ "partition_spec": [ "fsdp", null, - "tensor", + null, null ], "shape": [ - 512, - 26, - 16, - 256 + 1536, + 58, + 128, + 192 ] }, - ".opt_state/[0]/.nu/['params']/['token_embedder']/['embedding']": { + "['optimizer']/['opt_state']/[0]/['nu']/['token_embedder']/['embedding']/.value": { "mesh": { "axis_names": [ "data", @@ -2582,18 +3356,40 @@ } }, "partition_spec": [ - "tensor", + null, [ "fsdp", "expert" ] ], "shape": [ - 102400, - 2048 + 129280, + 512 ] }, - ".opt_state/[2]/.count": { + "['optimizer']/['opt_state']/[2]/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 4, + "context": 1, + "tensor": 1, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [] + }, + "['optimizer']/['step']/.value": { "mesh": { "axis_names": [ "data", diff --git a/tests/utils/sharding_info/deepseek3-test/v6e-16/slice_1/rule_tp-as-ep_ici_fsdp_parallelism=-1_ici_expert_parallelism=4_ici_tensor_parallelism=4/input_shardings.json b/tests/utils/sharding_info/deepseek3-test/v6e-16/slice_1/rule_tp-as-ep_ici_fsdp_parallelism=-1_ici_expert_parallelism=4_ici_tensor_parallelism=4/input_shardings.json new file mode 100644 index 0000000000..660521d50c --- /dev/null +++ b/tests/utils/sharding_info/deepseek3-test/v6e-16/slice_1/rule_tp-as-ep_ici_fsdp_parallelism=-1_ici_expert_parallelism=4_ici_tensor_parallelism=4/input_shardings.json @@ -0,0 +1,178 @@ +{ + "Activation Sharding Dump": [ + { + "deepseek/inputs: bfloat16[192,2048,512]": { + "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", + "PartitionSpec": "P('expert', None, 'tensor')" + } + }, + { + "deepseek/pre_attention_norm: bfloat16[192,2048,512]": { + "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", + "PartitionSpec": "P('expert', None, 'tensor')" + } + }, + { + "attention_mla/inputs_q: bfloat16[192,2048,512]": { + "logic_axes": "('activation_batch_attn', 'activation_length', 'activation_embed')", + "PartitionSpec": "P('expert', None, 'tensor')" + } + }, + { + "attention_mla/inputs_kv: bfloat16[192,2048,512]": { + "logic_axes": "('activation_batch_attn', 'activation_length', 'activation_embed')", + "PartitionSpec": "P('expert', None, 'tensor')" + } + }, + { + "attention_mla/q_nope: bfloat16[192,2048,128,128]": { + "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", + "PartitionSpec": "P('expert', None, 'tensor', None)" + } + }, + { + "attention_mla/q_pe: bfloat16[192,2048,128,64]": { + "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", + "PartitionSpec": "P('expert', None, 'tensor', None)" + } + }, + { + "attention_mla/query: bfloat16[192,2048,128,192]": { + "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", + "PartitionSpec": "P('expert', None, 'tensor', None)" + } + }, + { + "attention_mla/key_nope: bfloat16[192,2048,128,128]": { + "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", + "PartitionSpec": "P('expert', None, 'tensor', None)" + } + }, + { + "attention_mla/key_rope: bfloat16[192,2048,128,64]": { + "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", + "PartitionSpec": "P('expert', None, 'tensor', None)" + } + }, + { + "attention_mla/key: bfloat16[192,2048,128,192]": { + "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", + "PartitionSpec": "P('expert', None, 'tensor', None)" + } + }, + { + "attention_mla/value: bfloat16[192,2048,128,128]": { + "logic_axes": "('activation_kv_batch', 'activation_length', 'activation_kv_heads', 'activation_kv_head_dim')", + "PartitionSpec": "P('expert', None, 'tensor', None)" + } + }, + { + "attention_op/arr: int8[4,4,4]": { + "logic_axes": "Unknown", + "PartitionSpec": "P('tensor', None)" + } + }, + { + "attention_op/arr: int32[2048]": { + "logic_axes": "Unknown", + "PartitionSpec": "P(None,)" + } + }, + { + "attention_op/query: bfloat16[192,128,2048,192]": { + "logic_axes": "Unknown", + "PartitionSpec": "P('expert', 'tensor', None, None)" + } + }, + { + "attention_op/key: bfloat16[192,128,2048,192]": { + "logic_axes": "Unknown", + "PartitionSpec": "P('expert', 'tensor', None, None)" + } + }, + { + "attention_op/value: bfloat16[192,128,2048,128]": { + "logic_axes": "Unknown", + "PartitionSpec": "P('expert', 'tensor', None, None)" + } + }, + { + "attention_op/decoder_segment_ids: int32[192,2048]": { + "logic_axes": "Unknown", + "PartitionSpec": "P('expert', None)" + } + }, + { + "attention_mla/out: bfloat16[192,2048,128,128]": { + "logic_axes": "('activation_batch_attn', 'activation_length', 'activation_heads', 'activation_kv')", + "PartitionSpec": "P('expert', None, 'tensor', None)" + } + }, + { + "deepseek/attention_result: bfloat16[192,2048,512]": { + "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", + "PartitionSpec": "P('expert', None, 'tensor')" + } + }, + { + "deepseek/post_attention_norm: bfloat16[192,2048,512]": { + "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", + "PartitionSpec": "P('expert', None, 'tensor')" + } + }, + { + "linears/x: bfloat16[192,2048,18432]": { + "logic_axes": "('activation_batch', 'activation_length', 'activation_mlp')", + "PartitionSpec": "P('expert', None, 'tensor')" + } + }, + { + "deepseek/mlp: bfloat16[192,2048,512]": { + "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", + "PartitionSpec": "P('expert', None, 'tensor')" + } + }, + { + "deepseek/x: bfloat16[192,2048,512]": { + "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", + "PartitionSpec": "P('expert', None, 'tensor')" + } + }, + { + "moe/inputs: bfloat16[192,2048,512]": { + "logic_axes": "('activation_batch', 'activation_norm_length_routed', None)", + "PartitionSpec": "P('expert', 'tensor', None)" + } + }, + { + "moe/gate_logits: bfloat16[192,2048,256]": { + "logic_axes": "('activation_batch', 'activation_norm_length_routed', None)", + "PartitionSpec": "P('expert', 'tensor', None)" + } + }, + { + "moe/pre_bias_logits: bfloat16[192,2048,256]": { + "logic_axes": "('activation_batch', 'activation_norm_length_routed', None)", + "PartitionSpec": "P('expert', 'tensor', None)" + } + }, + { + "moe/w: bfloat16[256,512,512]": { + "logic_axes": "Unknown", + "PartitionSpec": "P(('expert', 'tensor'), None, None)" + } + }, + { + "linears/x: bfloat16[192,2048,512]": { + "logic_axes": "('activation_batch', 'activation_length', 'activation_mlp')", + "PartitionSpec": "P('expert', None, 'tensor')" + } + }, + { + "deepseek/mlp_lnx: bfloat16[192,2048,512]": { + "logic_axes": "('activation_batch', 'activation_norm_length', 'activation_embed')", + "PartitionSpec": "P('expert', None, 'tensor')" + } + } + ] +} \ No newline at end of file diff --git a/tests/utils/sharding_info/deepseek3-test/v6e-16/slice_1/rule_tp-as-ep_ici_fsdp_parallelism=-1_ici_expert_parallelism=4_ici_tensor_parallelism=4/logical_shardings.json b/tests/utils/sharding_info/deepseek3-test/v6e-16/slice_1/rule_tp-as-ep_ici_fsdp_parallelism=-1_ici_expert_parallelism=4_ici_tensor_parallelism=4/logical_shardings.json new file mode 100644 index 0000000000..e79c6e9e09 --- /dev/null +++ b/tests/utils/sharding_info/deepseek3-test/v6e-16/slice_1/rule_tp-as-ep_ici_fsdp_parallelism=-1_ici_expert_parallelism=4_ici_tensor_parallelism=4/logical_shardings.json @@ -0,0 +1,1245 @@ +{ + "['model']/['decoder']/['decoder_norm']/['scale']/.value": { + "partition_spec": [ + "tensor" + ], + "shape": [ + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + "tensor" + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + "tensor" + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "partition_spec": [ + "tensor", + null, + "expert" + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['count']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['key']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['count']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['key']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['count']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['key']/.value": { + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + "tensor", + null, + null, + "expert" + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + "tensor", + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + "tensor", + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['model']/['decoder']/['logits_dense']/['kernel']/.value": { + "partition_spec": [ + "expert", + "tensor" + ], + "shape": [ + 512, + 129280 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['bias']/.value": { + "partition_spec": [ + null, + null + ], + "shape": [ + 58, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['count']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['key']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['count']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['key']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['count']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['key']/.value": { + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "partition_spec": [ + [ + "expert", + "tensor" + ], + null, + null, + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "partition_spec": [ + [ + "expert", + "tensor" + ], + null, + null, + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "partition_spec": [ + [ + "expert", + "tensor" + ], + null, + null, + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + "tensor" + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + "tensor" + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "partition_spec": [ + "tensor", + null, + "expert" + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + "tensor", + null, + null, + "expert" + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + "tensor", + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + "tensor", + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['model']/['decoder']/['rngs']/['aqt']/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['aqt']/['key']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['dropout']/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['dropout']/['key']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['params']/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['params']/['key']/.value": { + "partition_spec": [], + "shape": [] + }, + "['model']/['token_embedder']/['embedding']/.value": { + "partition_spec": [ + "tensor", + "expert" + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['decoder_norm']/['scale']/.value": { + "partition_spec": [ + "tensor" + ], + "shape": [ + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + "tensor" + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + "tensor" + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "partition_spec": [ + "tensor", + null, + "expert" + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + "tensor", + null, + null, + "expert" + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + "tensor", + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + "tensor", + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['logits_dense']/['kernel']/.value": { + "partition_spec": [ + "expert", + "tensor" + ], + "shape": [ + 512, + 129280 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "partition_spec": [ + [ + "expert", + "tensor" + ], + null, + null, + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "partition_spec": [ + [ + "expert", + "tensor" + ], + null, + null, + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "partition_spec": [ + [ + "expert", + "tensor" + ], + null, + null, + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + "tensor" + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + "tensor" + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "partition_spec": [ + "tensor", + null, + "expert" + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + "tensor", + null, + null, + "expert" + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + "tensor", + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + "tensor", + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['token_embedder']/['embedding']/.value": { + "partition_spec": [ + "tensor", + "expert" + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['decoder_norm']/['scale']/.value": { + "partition_spec": [ + "tensor" + ], + "shape": [ + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + "tensor" + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + "tensor" + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "partition_spec": [ + "tensor", + null, + "expert" + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + "tensor", + null, + null, + "expert" + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + "tensor", + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + "tensor", + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['logits_dense']/['kernel']/.value": { + "partition_spec": [ + "expert", + "tensor" + ], + "shape": [ + 512, + 129280 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "partition_spec": [ + null, + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "partition_spec": [ + [ + "expert", + "tensor" + ], + null, + null, + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "partition_spec": [ + [ + "expert", + "tensor" + ], + null, + null, + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "partition_spec": [ + [ + "expert", + "tensor" + ], + null, + null, + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + "tensor" + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + "tensor" + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "partition_spec": [ + "tensor", + null, + "expert" + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "partition_spec": [ + "tensor", + null, + null, + "expert" + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + "tensor", + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "partition_spec": [ + "expert", + null, + "tensor", + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['token_embedder']/['embedding']/.value": { + "partition_spec": [ + "tensor", + "expert" + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[2]/['count']/.value": { + "partition_spec": [], + "shape": [] + }, + "['optimizer']/['step']/.value": { + "partition_spec": [], + "shape": [] + } +} \ No newline at end of file diff --git a/tests/utils/sharding_info/deepseek3-test/v6e-16/slice_1/rule_tp-as-ep_ici_fsdp_parallelism=-1_ici_expert_parallelism=4_ici_tensor_parallelism=4/named_shardings.json b/tests/utils/sharding_info/deepseek3-test/v6e-16/slice_1/rule_tp-as-ep_ici_fsdp_parallelism=-1_ici_expert_parallelism=4_ici_tensor_parallelism=4/named_shardings.json new file mode 100644 index 0000000000..2068f79b7e --- /dev/null +++ b/tests/utils/sharding_info/deepseek3-test/v6e-16/slice_1/rule_tp-as-ep_ici_fsdp_parallelism=-1_ici_expert_parallelism=4_ici_tensor_parallelism=4/named_shardings.json @@ -0,0 +1,3315 @@ +{ + "['model']/['decoder']/['decoder_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor" + ], + "shape": [ + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + "tensor" + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + "tensor" + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['model']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null, + "expert" + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['aqt']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['dropout']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['rngs']/['params']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [ + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null, + null, + "expert" + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + "tensor", + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['model']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + "tensor", + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['model']/['decoder']/['logits_dense']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + "tensor" + ], + "shape": [ + 512, + 129280 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['bias']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + null, + null + ], + "shape": [ + 58, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + null, + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['aqt']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['dropout']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['rngs']/['params']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [ + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + [ + "expert", + "tensor" + ], + null, + null, + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + [ + "expert", + "tensor" + ], + null, + null, + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + [ + "expert", + "tensor" + ], + null, + null, + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + "tensor" + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + "tensor" + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null, + "expert" + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null, + null, + "expert" + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + "tensor", + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['model']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + "tensor", + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['model']/['decoder']/['rngs']/['aqt']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['aqt']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['dropout']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['dropout']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['params']/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['decoder']/['rngs']/['params']/['key']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [] + }, + "['model']/['token_embedder']/['embedding']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + "expert" + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['decoder_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor" + ], + "shape": [ + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + "tensor" + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + "tensor" + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null, + "expert" + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null, + null, + "expert" + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + "tensor", + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + "tensor", + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['logits_dense']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + "tensor" + ], + "shape": [ + 512, + 129280 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + null, + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + [ + "expert", + "tensor" + ], + null, + null, + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + [ + "expert", + "tensor" + ], + null, + null, + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + [ + "expert", + "tensor" + ], + null, + null, + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + "tensor" + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + "tensor" + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null, + "expert" + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null, + null, + "expert" + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + "tensor", + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + "tensor", + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['mu']/['token_embedder']/['embedding']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + "expert" + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['decoder_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor" + ], + "shape": [ + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_0']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + "tensor" + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wi_1']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + "tensor" + ], + "shape": [ + 512, + 3, + 18432 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['mlp']/['wo']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null, + "expert" + ], + "shape": [ + 18432, + 3, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['out']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null, + null, + "expert" + ], + "shape": [ + 128, + 3, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 1536, + 3 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + null + ], + "shape": [ + 512, + 3, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + "tensor", + null + ], + "shape": [ + 512, + 3, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + null + ], + "shape": [ + 512, + 3, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['dense_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + "tensor", + null + ], + "shape": [ + 1536, + 3, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['logits_dense']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + "tensor" + ], + "shape": [ + 512, + 129280 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['gate']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + null, + null, + null + ], + "shape": [ + 512, + 58, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_0']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + [ + "expert", + "tensor" + ], + null, + null, + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wi_1']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + [ + "expert", + "tensor" + ], + null, + null, + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['MoeBlock_0']/['wo']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + [ + "expert", + "tensor" + ], + null, + null, + null + ], + "shape": [ + 256, + 58, + 512, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_0']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + "tensor" + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wi_1']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + "tensor" + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['DeepSeekMoeBlock_0']/['shared_experts']/['wo']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null, + "expert" + ], + "shape": [ + 512, + 58, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['post_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['pre_self_attention_layer_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['kv_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 512, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['out']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null, + null, + "expert" + ], + "shape": [ + 128, + 58, + 128, + 512 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['q_norm']/['scale']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + null + ], + "shape": [ + 1536, + 58 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + null + ], + "shape": [ + 512, + 58, + 576 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wkv_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + "tensor", + null + ], + "shape": [ + 512, + 58, + 128, + 256 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_a']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + null + ], + "shape": [ + 512, + 58, + 1536 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['decoder']/['moe_layers']/['self_attention']/['wq_b']/['kernel']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "expert", + null, + "tensor", + null + ], + "shape": [ + 1536, + 58, + 128, + 192 + ] + }, + "['optimizer']/['opt_state']/[0]/['nu']/['token_embedder']/['embedding']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [ + "tensor", + "expert" + ], + "shape": [ + 129280, + 512 + ] + }, + "['optimizer']/['opt_state']/[2]/['count']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [] + }, + "['optimizer']/['step']/.value": { + "mesh": { + "axis_names": [ + "data", + "stage", + "fsdp", + "context", + "tensor", + "expert" + ], + "shape": { + "data": 1, + "stage": 1, + "fsdp": 1, + "context": 1, + "tensor": 4, + "expert": 4 + } + }, + "partition_spec": [], + "shape": [] + } +} \ No newline at end of file