From 6a01e4194419db7eed68a065bcca83782b1a65de Mon Sep 17 00:00:00 2001 From: maxtext authors Date: Tue, 27 Oct 2026 11:32:08 -0700 Subject: [PATCH] COPYBARA_INTEGRATE_REVIEW=https://github.com/AI-Hypercomputer/maxtext/pull/5987 from AI-Hypercomputer:emma/fix-rl-local-layers-axis-rule d8b7ef948ee44fd8447f5319ada85e5dc2cc08aa PiperOrigin-RevId: 1008255052 --- src/maxtext/configs/base.yml | 4 + src/maxtext/configs/post_train/rl.yml | 1 + src/maxtext/configs/types.py | 12 + .../integration/vllm/maxtext_vllm_rollout.py | 39 +- src/maxtext/kernels/megablox/ops.py | 106 +- src/maxtext/layers/fsdp_prefetch.py | 211 ++- .../layers/fsdp_prefetch_inhomogeneous.py | 5 + src/maxtext/layers/moe.py | 51 +- .../models/glm5_3_flash/convert_glm5_ckpt.py | 30 + .../m3/models/glm5_3_flash/glm5_ckpt.py | 1228 +++++++++++++++-- .../glm5_3_flash/modeling_glm5_3_flash.py | 125 +- .../unit/rl_logical_axis_rules_test.py | 108 ++ .../unit/vllm_rollout_unroll_test.py | 78 +- tests/unit/fsdp_prefetch_test.py | 140 +- tests/unit/m3/glm5_3_flash_config_test.py | 465 ------- tests/unit/m3/m3_smoke_test.py | 14 +- tests/unit/nnx_decoders_test.py | 7 +- .../unit/pallas_mosaic_tpu_v2_kernel_test.py | 24 +- 18 files changed, 1841 insertions(+), 807 deletions(-) create mode 100644 src/maxtext/m3/models/glm5_3_flash/convert_glm5_ckpt.py create mode 100644 tests/post_training/unit/rl_logical_axis_rules_test.py delete mode 100644 tests/unit/m3/glm5_3_flash_config_test.py diff --git a/src/maxtext/configs/base.yml b/src/maxtext/configs/base.yml index c0af92dacf..9bb9d8ebe0 100644 --- a/src/maxtext/configs/base.yml +++ b/src/maxtext/configs/base.yml @@ -565,6 +565,10 @@ param_scan_axis: 1 # never saved and are re-gathered for the backward pass. See docs/guides/optimization/fsdp_prefetch.md for the # recommended XLA flags. prefetch_fsdp_weights: false +# With prefetch_fsdp_weights, defer each layer's weight-gradient reduce-scatters to the next layer's backward step +# and run them on the SparseCore dedicated to FSDP collectives. Hides them at the cost of extra HBM. Homogeneous +# blocks only for now (ignored for inhomogeneous blocks prefetched per sub-layer). +prefetch_fsdp_defer_grad_reduce_scatter: false # The attention parameter dictates the specific algorithm/methodology used to compute the attention scores # The attention_type parameter determines the variants of attention, e.g. global or local_sliding diff --git a/src/maxtext/configs/post_train/rl.yml b/src/maxtext/configs/post_train/rl.yml index 5b3f2b197c..77947b7eac 100644 --- a/src/maxtext/configs/post_train/rl.yml +++ b/src/maxtext/configs/post_train/rl.yml @@ -366,5 +366,6 @@ logical_axis_rules: [ ['engram_dim', ['tensor']], ['dense_layers', []], ['moe_layers', []], + ['local_layers', []], ['mhc', []], ] diff --git a/src/maxtext/configs/types.py b/src/maxtext/configs/types.py index 2b8fbe3cfa..f38a83c373 100644 --- a/src/maxtext/configs/types.py +++ b/src/maxtext/configs/types.py @@ -2127,6 +2127,18 @@ class LayoutAndSharding(BaseModel): " FSDP mesh axis spans more than one device. See docs/guides/optimization/fsdp_prefetch.md." ), ) + prefetch_fsdp_defer_grad_reduce_scatter: bool = Field( + False, + description=( + "With prefetch_fsdp_weights and shard_mode=explicit, issue each layer's weight-gradient" + " reduce-scatters at the start of the next layer's backward step, tagged for XLA's FSDP scheduler," + " which runs them on the SparseCore dedicated to FSDP collectives" + " (--xla_tpu_explicit_fsdp_dedicated_sparse_core_id) and overlaps them with that layer's backward." + " Keeps one extra layer of unreduced weight gradients live (more HBM). When False, they run untagged" + " on the other SparseCore and complete within their own layer's backward step. Not yet supported" + " (ignored) for inhomogeneous blocks prefetched per sub-layer." + ), + ) class DcnParallelism(BaseModel): diff --git a/src/maxtext/integration/vllm/maxtext_vllm_rollout.py b/src/maxtext/integration/vllm/maxtext_vllm_rollout.py index 137c2d8a5c..4dec36b546 100644 --- a/src/maxtext/integration/vllm/maxtext_vllm_rollout.py +++ b/src/maxtext/integration/vllm/maxtext_vllm_rollout.py @@ -238,12 +238,15 @@ def _find_scanned_layer_idx(key_tuple, container_names=("layers", "scanned_block def validate_direct_sync_layer_coverage(source, target) -> int: - """Fail if an unrolled source would leave MaxText target layers untouched. + """Fail if a direct-sync source would leave MaxText target layers untouched. Tunix intentionally intersects direct-sync trees. For heterogeneous Qwen scans, a schema error can therefore skip every transformer layer without an exception. This check runs on the initial full-parameter load and requires - every unscanned target-layer parameter path to exist in the source. + every unscanned target-layer parameter path to be reachable from the source: + either directly (sources MaxText already unrolled, e.g. Gemma and Qwen) or + through the scanned-layer lookup of Tunix's `transfer_state_directly`, which + unrolls the homogeneous scans MaxText leaves stacked (e.g. Llama). """ def to_pure_params(state): @@ -268,21 +271,31 @@ def unwrap(state, wrapper): source_flat = flatten_dict(source) target_flat = flatten_dict(target) - def is_unscanned_layer_path(path): - return any(isinstance(part, str) and re.fullmatch(r"layers_\d+", part) for part in path) + def unscanned_layer_index(path): + """Position of the first `layers_N` component of `path`, or -1.""" + for i, part in enumerate(path): + if isinstance(part, str) and re.fullmatch(r"layers_\d+", part): + return i + return -1 - source_layer_keys = {key for key in source_flat if is_unscanned_layer_path(key)} - target_layer_keys = {key for key in target_flat if is_unscanned_layer_path(key)} + source_keys = set(source_flat) + target_layer_keys = {key for key in target_flat if unscanned_layer_index(key) != -1} - def source_covers(target_key): - if target_key in source_layer_keys: - return True + def has_split_wi(key): # Tunix fuses split training weights into the inference-only prefused # parameter before transfer. Treat the pair as coverage for target `wi`. - if target_key and target_key[-1] == "wi": - prefix = target_key[:-1] - return prefix + ("wi_0",) in source_layer_keys and prefix + ("wi_1",) in source_layer_keys - return False + return key[-1] == "wi" and key[:-1] + ("wi_0",) in source_keys and key[:-1] + ("wi_1",) in source_keys + + def source_covers(target_key): + if target_key in source_keys or has_split_wi(target_key): + return True + # Mirror the fallback of Tunix's `transfer_state_directly`: a target + # `layers_N` is slice N of a scanned `layers` container (standard MaxText) + # or of an implicit stack without one. Only the former fuses split MoE. + idx = unscanned_layer_index(target_key) + scanned_key = target_key[:idx] + ("layers",) + target_key[idx + 1 :] + implicit_key = target_key[:idx] + target_key[idx + 1 :] + return scanned_key in source_keys or implicit_key in source_keys or has_split_wi(scanned_key) missing = {key for key in target_layer_keys if not source_covers(key)} if not target_layer_keys or missing: diff --git a/src/maxtext/kernels/megablox/ops.py b/src/maxtext/kernels/megablox/ops.py index cd32ac3f92..0b60d8f6a7 100644 --- a/src/maxtext/kernels/megablox/ops.py +++ b/src/maxtext/kernels/megablox/ops.py @@ -77,7 +77,7 @@ def gmm( ), group_offset: jnp.ndarray | None = None, existing_out: jnp.ndarray | None = None, - transpose_rhs: bool = False, + *, interpret: bool | None = None, lhs_quantize_dtype: Literal[jnp.int4, jnp.int8] | None = None, # pyrefly: ignore[invalid-literal] rhs_quantize_dtype: Literal[jnp.int4, jnp.int8] | None = None, # pyrefly: ignore[invalid-literal] @@ -158,7 +158,7 @@ def gmm( gmm_fwd_bwd = lambda *args: _gmm_fwd(*args)[0] # pylint: disable=C3001 gmm_fwd_bwd = jax.custom_vjp( gmm_fwd_bwd, - nondiff_argnums=(3, 4, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22, 23, 24, 26), + nondiff_argnums=(3, 4, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17, 18, 19, 20, 21, 22, 23, 25), ) gmm_fwd_bwd.defvjp(_gmm_fwd, functools.partial(_gmm_bwd, lhs_dtype, rhs_dtype)) out, lhs_out, rhs_out = gmm_fwd_bwd( @@ -169,7 +169,6 @@ def gmm( tiling, group_offset, existing_out, - transpose_rhs, interpret, quantization_rule, use_tokamax_backend, @@ -232,7 +231,6 @@ def _gmm_fwd( ), group_offset: jnp.ndarray | None = None, existing_out: jnp.ndarray | None = None, - transpose_rhs: bool = False, interpret: bool = False, quantization_rule: qwix.QtRule | None = None, use_tokamax_backend: bool = False, @@ -268,7 +266,7 @@ def _gmm_fwd( """Forward function for GMM VJP. - lhs: [m, k] (or [m, k // 128, 128]) - - rhs: [g, k, n] if transpose_rhs=False. [g, n, k] if transpose_rhs=True + - rhs: [g, k, n] - out: [m, n], or [m, n // 128, 128] if out_is_3d Always returns (out, lhs, rhs) so the output structure does not depend on return_lhs/return_rhs; @@ -282,8 +280,8 @@ def _gmm_fwd( lhs_is_qarray = isinstance(lhs, qpl.QArray) rhs_is_qarray = isinstance(rhs, qpl.QArray) if flat_fsdp_shard is not None: - if rhs_is_qarray or transpose_rhs or rhs.ndim != 2 or not weight_gather_axes: - raise NotImplementedError("flat_fsdp_shard needs a plain 2D rhs block, transpose_rhs=False and weight_gather_axes.") + if rhs_is_qarray or rhs.ndim != 2 or not weight_gather_axes: + raise NotImplementedError("flat_fsdp_shard needs a plain 2D rhs block and weight_gather_axes.") if any(axis_idx != 0 for _, axis_idx in weight_gather_axes): raise ValueError("flat_fsdp_shard: the flat rhs is gathered on dim 0 only.") if not ( @@ -303,7 +301,6 @@ def _gmm_fwd( quantization_rule, use_gmm_v2, use_manual_quantization, - transpose_rhs, rhs_is_flat2d=flat_fsdp_shard is not None, ) @@ -321,7 +318,7 @@ def _gmm_fwd( # Backend Execution Routing if use_tokamax_backend and not use_gmm_v2: - out = _fwd_run_tokamax_v1(lhs, rhs, group_sizes, preferred_element_type, transpose_rhs, use_manual_quantization) + out = _fwd_run_tokamax_v1(lhs, rhs, group_sizes, preferred_element_type, use_manual_quantization) elif use_tokamax_backend and use_gmm_v2: out = _fwd_run_tokamax_v2( lhs, @@ -332,7 +329,6 @@ def _gmm_fwd( use_gmm_v2_heuristic_tiling, group_offset, partial_sum, - transpose_rhs, quantization_rule, out_is_3d=out_is_3d, ) @@ -345,7 +341,6 @@ def _gmm_fwd( tiling, group_offset, existing_out, - transpose_rhs, interpret, lhs_vma_axes, ) @@ -381,7 +376,6 @@ def _fwd_quantize_activation_and_weight( quantization_rule: qwix.QtRule, use_gmm_v2: bool, use_manual_quantization: bool, - transpose_rhs: bool, rhs_is_flat2d: bool = False, ) -> tuple[jnp.ndarray | qpl.QArray, jnp.ndarray | qpl.QArray]: """Handles act and weight quantization for GMM forward inputs.""" @@ -405,7 +399,7 @@ def _fwd_quantize_activation_and_weight( channelwise_axes=( [] if quantization_rule.disable_channelwise_axes - else ([1] if (transpose_rhs or rhs_is_flat2d) else [2]) # flat 2D [rows, cols]: cols is the N axis + else ([1] if rhs_is_flat2d else [2]) # flat 2D [rows, cols]: cols is the N axis ), calibration_method=quantization_rule.weight_calibration_method, ) @@ -453,7 +447,6 @@ def _fwd_run_tokamax_v1( rhs: jnp.ndarray | qpl.QArray, group_sizes: jnp.ndarray, preferred_element_type: jnp.dtype, - transpose_rhs: bool, use_manual_quantization: bool, ) -> jnp.ndarray: """Executes the standard Tokamax GMM V1 for forward pass.""" @@ -463,9 +456,6 @@ def _fwd_run_tokamax_v1( # used in batchsplit out_kwargs["manual_axis_type"] = jax.sharding.ManualAxisType(varying=frozenset(["data", "fsdp", "expert"])) - if transpose_rhs: - rhs = rhs.swapaxes(1, 2) - return tokamax.ragged_dot( lhs=lhs, rhs=rhs, @@ -479,17 +469,11 @@ def _fwd_run_tokamax_v1( ) -def _fwd_prepare_rhs_scale(rhs: qpl.QArray, transpose_rhs: bool = False) -> jnp.ndarray: +def _fwd_prepare_rhs_scale(rhs: qpl.QArray) -> jnp.ndarray: """Formats and broadcasts rhs scale for the V2 GMM forward kernel.""" # Target shape: (size_group, num_quant_blocks, 1, size_n) - if transpose_rhs: - G, N, _ = rhs.qvalue.shape - scale = rhs.scale - if scale.ndim == 3: - scale = scale.swapaxes(1, 2) - else: - G, _, N = rhs.qvalue.shape - scale = rhs.scale + G, _, N = rhs.qvalue.shape + scale = rhs.scale if scale.ndim == 2: # Per-Channel quantization rhs_scale = jnp.expand_dims(scale, axis=(1, 2)) @@ -541,20 +525,16 @@ def _fwd_run_tokamax_v2( use_gmm_v2_heuristic_tiling: bool, group_offset: jnp.ndarray | None, partial_sum: jnp.ndarray | None, - transpose_rhs: bool, quantization_rule: qwix.QtRule | None = None, out_is_3d: bool = False, ) -> jnp.ndarray: """Executes the Tokamax GMM V2 backend for forward pass OUT = LHS @ RHS.""" - # if transpose_rhs=False, rhs is [g, k, n], remain unchanged - # if transpose_rhs=True, rhs [g, n, k], explicit transpose to [g, k, n] - rhs_operand = rhs if not transpose_rhs else rhs.swapaxes(1, 2) + rhs_operand = rhs rhs_scale = None if isinstance(rhs, qpl.QArray): - # pyrefly: ignore[missing-attribute] - rhs_operand = rhs_operand.qvalue - rhs_scale = _fwd_prepare_rhs_scale(rhs, transpose_rhs=transpose_rhs) + rhs_operand = rhs.qvalue + rhs_scale = _fwd_prepare_rhs_scale(rhs) # When lhs is already a QArray (e.g. from quantized token all-gather), extract raw # qvalue buffer and skip redundant internal quantization inside gmm_v2. @@ -607,7 +587,6 @@ def _fwd_run_megablox( tiling: tuple, group_offset: jnp.ndarray | None, existing_out: jnp.ndarray | None, - transpose_rhs: bool, interpret: bool, lhs_vma_axes: tuple, ) -> jnp.ndarray: @@ -620,7 +599,6 @@ def _fwd_run_megablox( tiling[:3], group_offset, existing_out, - transpose_rhs=transpose_rhs, interpret=interpret, ) for axis in lhs_vma_axes: @@ -638,7 +616,6 @@ def _gmm_bwd( rhs_dtype: jax.typing.DTypeLike, preferred_element_type: jnp.dtype, tiling: tuple[int, int, int, int, int, int, int, int, int], - transpose_rhs: bool, interpret: bool, quantization_rule: qwix.QtRule | None, use_tokamax_backend: bool, @@ -699,11 +676,8 @@ def _gmm_bwd( # by its downstream consumers (e.g. later MoE token chunks). tgmm adds this drhs onto it in place. if not return_rhs: drhs_partial_sum = None - else: - if isinstance(drhs_partial_sum, qpl.QArray): - drhs_partial_sum = drhs_partial_sum.qvalue - if transpose_rhs: - drhs_partial_sum = drhs_partial_sum.swapaxes(1, 2) + elif isinstance(drhs_partial_sum, qpl.QArray): + drhs_partial_sum = drhs_partial_sum.qvalue # 3D layouts: the incoming grad matches the (possibly 3D) forward output and dlhs is # produced in the layout of the forward lhs. lhs_is_3d = (residual_lhs.qvalue if isinstance(residual_lhs, qpl.QArray) else residual_lhs).ndim == 3 @@ -730,7 +704,6 @@ def _gmm_bwd( residual_rhs, group_sizes, use_gmm_v2, - transpose_rhs, quantization_rule, ) @@ -770,7 +743,6 @@ def _unsort_gather_maybe_qa(x): group_offset, lhs_dtype, tiling, - transpose_rhs, use_tokamax_backend, use_gmm_v2, use_manual_quantization, @@ -807,11 +779,6 @@ def _unsort_gather_maybe_qa(x): ) # 5. Output Formatting - # NOTE: If the rhs transposition is fused into the forward pass we need to - # return the transpose of the rhs gradient that we calculated above. - # - # TODO(tgale, enriqueps, apaske): Fuse this transposition into the tgmm. - drhs = drhs.swapaxes(1, 2) if transpose_rhs else drhs dpartial_sum = grad if partial_sum_fwd is not None else None d_existing_out = None if use_tokamax_backend else grad @@ -909,7 +876,6 @@ def _bwd_prepare_inputs( rhs: jnp.ndarray | qpl.QArray, group_sizes: jnp.ndarray, use_gmm_v2: bool, - transpose_rhs: bool, quantization_rule: qwix.QtRule | None, ) -> tuple[jnp.ndarray | qpl.QArray, jnp.ndarray | qpl.QArray, jnp.ndarray, jnp.ndarray]: """Prepares backward operands.""" @@ -924,7 +890,7 @@ def _bwd_prepare_inputs( # Apply rhs.scale to dlhs_dout to avoid dequantizing or requantizing rhs. # We cannot apply the scale to dlhs because axis n will disappear there. if isinstance(rhs, qpl.QArray): - # rhs - qvalue: [g, k, n] scale: [1, 1, n], assume transpose_rhs=False + # rhs - qvalue: [g, k, n] scale: [1, 1, n] if not use_gmm_v2: dlhs_dout *= rhs.scale.astype(grad.dtype).reshape(1, -1) rhs = rhs.qvalue @@ -932,7 +898,7 @@ def _bwd_prepare_inputs( # NOTE: rhs.scale is for the contracting dimension (N) in DLHS, but gmm_v2 # only supports scaling the output dimension. Thus, we must scale dlhs_dout # beforehand. - dlhs_dout = _dlhs_scale_grad_by_rhs_scale(dlhs_dout, rhs, group_sizes, transpose_rhs) + dlhs_dout = _dlhs_scale_grad_by_rhs_scale(dlhs_dout, rhs, group_sizes) rhs = rhs.qvalue # GMM2 FWD performs lhs quantization inside kernel, lhs is stored as unquantized dtype @@ -1000,7 +966,6 @@ def _compute_dlhs( group_offset: jnp.ndarray | None, lhs_dtype: jax.typing.DTypeLike, tiling: tuple, - transpose_rhs: bool, use_tokamax_backend: bool, use_gmm_v2: bool, use_manual_quantization: bool, @@ -1025,7 +990,6 @@ def _compute_dlhs( rhs, group_sizes, lhs_dtype, - transpose_rhs, use_manual_quantization, ) return dlhs, False @@ -1038,7 +1002,6 @@ def _compute_dlhs( lhs_dtype, tiling, use_gmm_v2_heuristic_tiling, - transpose_rhs, partial_sum=partial_sum, lhs_scale=lhs_scale, fuse_dlhs_scale=fuse_dlhs_scale, @@ -1047,9 +1010,7 @@ def _compute_dlhs( fuse_3d_dlhs_ps=fuse_3d_dlhs_ps, ) else: - dlhs = _dlhs_run_megablox( - dlhs_dout, rhs, group_sizes, group_offset, lhs_dtype, tiling, transpose_rhs, interpret, lhs_vma_axes - ) + dlhs = _dlhs_run_megablox(dlhs_dout, rhs, group_sizes, group_offset, lhs_dtype, tiling, interpret, lhs_vma_axes) return dlhs, False @@ -1058,7 +1019,6 @@ def _dlhs_run_tokamax_v1( rhs: jnp.ndarray, group_sizes: jnp.ndarray, lhs_dtype: jax.typing.DTypeLike, - transpose_rhs: bool, use_manual_quantization: bool, ) -> jnp.ndarray: """Executes DLHS using GMM 1""" @@ -1066,10 +1026,9 @@ def _dlhs_run_tokamax_v1( if use_manual_quantization: dlhs_kwargs["manual_axis_type"] = jax.sharding.ManualAxisType(varying=frozenset(["data", "fsdp", "expert"])) - dlhs_rhs = rhs.swapaxes(1, 2) if transpose_rhs else rhs return tokamax.ragged_dot_general( lhs=dlhs_dout, - rhs=dlhs_rhs, + rhs=rhs, group_sizes=group_sizes, ragged_dot_dimension_numbers=DLHS_RAGGED_DOT_DIM_NUMS, precision=jax.lax.Precision.DEFAULT, @@ -1085,7 +1044,6 @@ def _dlhs_scale_grad_by_rhs_scale( grad: jnp.ndarray, rhs: qpl.QArray, group_sizes: jnp.ndarray, - transpose_rhs: bool = False, ) -> jnp.ndarray: """Squeezes the rhs scale and multiplies it with the incoming gradient. @@ -1093,11 +1051,9 @@ def _dlhs_scale_grad_by_rhs_scale( """ rhs_scale = rhs.scale - # 1. Squeeze the scale to 2D [g, n] based on transpose_rhs - if rhs_scale.ndim == 3: - squeeze_axis = 2 if transpose_rhs else 1 - if rhs_scale.shape[squeeze_axis] == 1: - rhs_scale = rhs_scale.squeeze(axis=squeeze_axis) + # 1. Squeeze the scale [g, 1, n] to 2D [g, n] + if rhs_scale.ndim == 3 and rhs_scale.shape[1] == 1: + rhs_scale = rhs_scale.squeeze(axis=1) # 2. Apply scale (handle shared vs per-expert scales) if rhs_scale.size == 1: @@ -1124,7 +1080,6 @@ def _dlhs_run_tokamax_v2( lhs_dtype: jax.typing.DTypeLike, tiling: tuple, use_gmm_v2_heuristic_tiling: bool, - transpose_rhs: bool, partial_sum: jnp.ndarray | None = None, lhs_scale: jnp.ndarray | None = None, fuse_dlhs_scale: bool = False, @@ -1141,16 +1096,10 @@ def _dlhs_run_tokamax_v2( else: dlhs_tiling = gmm_v2.TileSizes(tile_m=tiling[3], tile_k=tiling[4], tile_n=tiling[5]) - if use_dlhs_transpose_rhs: - kernel_transpose_rhs = not transpose_rhs - dlhs_rhs = rhs - else: - kernel_transpose_rhs = False - dlhs_rhs = rhs if transpose_rhs else rhs.swapaxes(1, 2) - # Sub-byte packing only supports the standard [group, k, n] RHS layout. - if kernel_transpose_rhs and jax.dtypes.itemsize_bits(rhs.dtype) < 8: - dlhs_rhs = rhs.swapaxes(1, 2) - kernel_transpose_rhs = False + # DLHS contracts rhs [g, k, n] over n: the kernel either reads it natively (use_dlhs_transpose_rhs) or takes an + # explicit rhs^T [g, n, k]. Sub-byte rhs always takes rhs^T, since gmm_v2 rejects transpose_rhs for sub-byte dtypes. + kernel_transpose_rhs = use_dlhs_transpose_rhs and jax.dtypes.itemsize_bits(rhs.dtype) >= 8 + dlhs_rhs = rhs if kernel_transpose_rhs else rhs.swapaxes(1, 2) # The kernel adds partial_sum after applying its lhs_scale, so partial_sum can only go into the kernel # when every scale on dlhs (dlhs_dout.scale and, with partial_sum, the pre-quantized lhs scale) is @@ -1206,7 +1155,6 @@ def _dlhs_run_megablox( group_offset: jnp.ndarray | None, lhs_dtype: jax.typing.DTypeLike, tiling: tuple, - transpose_rhs: bool, interpret: bool, lhs_vma_axes: tuple, ) -> jnp.ndarray: @@ -1218,7 +1166,7 @@ def _dlhs_run_megablox( lhs_dtype, tiling[3:6], group_offset, - transpose_rhs=not transpose_rhs, + transpose_rhs=True, interpret=interpret, varying_axes=lhs_vma_axes, ) diff --git a/src/maxtext/layers/fsdp_prefetch.py b/src/maxtext/layers/fsdp_prefetch.py index 88967502e5..a69b5292ed 100644 --- a/src/maxtext/layers/fsdp_prefetch.py +++ b/src/maxtext/layers/fsdp_prefetch.py @@ -58,13 +58,15 @@ def _apply_sharding_hint(w, mesh, shard_mode, debug_sharding=False): # stored parameter (which would materialize the whole gathered stack). w = jax.lax.with_sharding_constraint(w, src_sharding) gathered_sharding = sharding.remove_fsdp_from_sharding(src_sharding) - out = _explicit_fsdp_all_gather(w, mesh, src_sharding.spec, gathered_sharding.spec) + out = _explicit_fsdp_all_gather( + w, mesh, src_sharding.spec, gathered_sharding.spec, reduced=shard_mode == ShardMode.EXPLICIT + ) if out is not None: return out return sharding.maybe_shard_with_name(w, gathered_sharding, shard_mode=shard_mode, debug_sharding=debug_sharding) -def _explicit_fsdp_all_gather(w, mesh, src_spec, gathered_spec): +def _explicit_fsdp_all_gather(w, mesh, src_spec, gathered_spec, reduced=False): """All-gathers `w` over its FSDP mesh axes with an explicit all_gather tagged `xla_explicit_fsdp`. On TPUs that offload all-gathers to the SparseCore, the tag lets XLA's FSDP scheduler @@ -72,6 +74,11 @@ def _explicit_fsdp_all_gather(w, mesh, src_spec, gathered_spec): run these gathers on a dedicated SparseCore. Otherwise they share the SparseCore queue with the layer's own SparseCore work (e.g. MoE token gathers) and must finish before that work can. + With `reduced` (explicit sharding mode, one gathered dimension), the result is typed `reduced` over + the FSDP axes. The gradient of a reduced value is `unreduced` (per-device partial sums), so the + backward pass emits no all-reduce for it and `_explicit_fsdp_reduce_scatter` performs the only + reduction, tagged so XLA's FSDP scheduler can overlap it like the prefetch all-gathers. + Returns None (the caller falls back to resharding) if the FSDP axes of a dimension are not its minor-most mesh axes, since a tiled gather would then not produce the target layout. """ @@ -89,6 +96,33 @@ def _explicit_fsdp_all_gather(w, mesh, src_spec, gathered_spec): if not dims: return None + if reduced and len(dims) == 1: + (dim, ax) = dims[0] + gathered = tuple(gathered_spec) + (None,) * (w.ndim - len(tuple(gathered_spec))) + # Size-1 FSDP (co-)axes (e.g. `fsdp_transpose`, `context`), which `ax` skips but `gathered_spec` + # strips, are first resharded away (a no-op on the data); otherwise the output would be typed as + # varying over axes the out_specs replicate. + in_spec = [] + for entry in spec: + entry_axes = () if entry is None else (tuple(entry) if isinstance(entry, (tuple, list)) else (entry,)) + stripped = set(sharding.fsdp_axes_for_dim(entry_axes)) + kept = tuple(a for a in entry_axes if a not in stripped or mesh.shape[a] > 1) + in_spec.append(kept if len(kept) > 1 else (kept[0] if kept else None)) + in_spec = jax.sharding.PartitionSpec(*in_spec) + if tuple(in_spec) != spec: + w = jax.sharding.reshard(w, NamedSharding(mesh, in_spec)) + + def gather_reduced(x): + return jax.lax.all_gather(x, ax if len(ax) > 1 else ax[0], axis=dim, tiled=True, to="reduced") + + with xla_metadata.set_xla_metadata(xla_explicit_fsdp="yes"): + return jax.shard_map( + gather_reduced, + mesh=mesh, + in_specs=in_spec, + out_specs=jax.sharding.PartitionSpec(*gathered, reduced=frozenset(ax)), + )(w) + def gather(x): for i, ax in dims: x = jax.lax.all_gather(x, ax if len(ax) > 1 else ax[0], axis=i, tiled=True) @@ -100,6 +134,78 @@ def gather(x): )(w) +def _fsdp_scatter_dim(ndim, mesh, src_spec): + """Returns (dim, fsdp_axes) of the single dimension a gradient is reduce-scattered over, else None. + + None if no dimension or more than one is FSDP-sharded (chained FSDP collectives must be independent + for XLA's FSDP scheduler), or if the FSDP axes of the dimension are not its minor-most mesh axes. + """ + spec = tuple(src_spec) + (None,) * (ndim - len(tuple(src_spec))) + dims = [] + for i, entry in enumerate(spec): + axes = () if entry is None else (tuple(entry) if isinstance(entry, (tuple, list)) else (entry,)) + axes = tuple(a for a in axes if mesh.shape[a] > 1) + fsdp = sharding.fsdp_axes_for_dim(axes) + if not fsdp: + continue + if axes[len(axes) - len(fsdp) :] != fsdp: + return None + dims.append((i, fsdp)) + if len(dims) != 1: + return None + dim, fsdp_axes = dims[0] + return dim, fsdp_axes + + +def _is_unreduced_over(g, axes): + g_spec = getattr(getattr(jax.typeof(g), "sharding", None), "spec", None) + return g_spec is not None and frozenset(g_spec.unreduced) == frozenset(axes) + + +def _pin_scatter_dim_major(g, dim): + """Lays `g` out with dimension `dim` major-most before it is reduce-scattered over `dim`. + + XLA does not offload a reduce-scatter over a physically minor dimension to the SparseCore as one and + instead rewrites it into an all-reduce plus slice. That happens for kernels sharded on their last + dimension (e.g. a [F, D/fsdp] output projection) and for gradients of transposed matmuls (laid out + {0, 1}) unless their layout is pinned. + """ + return with_layout_constraint(g, Layout((dim,) + tuple(i for i in range(g.ndim) if i != dim))) + + +def _explicit_fsdp_reduce_scatter(g, mesh, src_spec, gathered_spec): + """Reduce-scatters a gathered-weight gradient `g` over its FSDP mesh axes, tagged `xla_explicit_fsdp`. + + The inverse of the `reduced` `_explicit_fsdp_all_gather`: `g` is `unreduced` over the FSDP axes + (each device holds a partial sum) with the gathered sharding `gathered_spec`, and the result is the + summed gradient with the parameter's FSDP sharding `src_spec`. The tag lets XLA's FSDP scheduler + run the reduce-scatter on the dedicated FSDP SparseCore (next to the prefetch all-gathers) and + complete it with the next FSDP all-gather, so it overlaps the following layer's backward. + + Returns None (the caller falls back to resharding) if `g` is not unreduced over exactly the FSDP + axes of a single dimension: reduce-scattering an already-reduced gradient would sum it again. + """ + scatter = _fsdp_scatter_dim(g.ndim, mesh, src_spec) + if scatter is None or not _is_unreduced_over(g, scatter[1]): + return None + dim, ax = scatter + spec = tuple(src_spec) + (None,) * (g.ndim - len(tuple(src_spec))) + gathered = tuple(gathered_spec) + (None,) * (g.ndim - len(tuple(gathered_spec))) + g_spec = jax.typeof(g).sharding.spec + g = _pin_scatter_dim_major(g, dim) + + def psum_scatter(x): + return jax.lax.psum_scatter(x, ax if len(ax) > 1 else ax[0], scatter_dimension=dim, tiled=True) + + with xla_metadata.set_xla_metadata(xla_explicit_fsdp="yes"): + return jax.shard_map( + psum_scatter, + mesh=mesh, + in_specs=jax.sharding.PartitionSpec(*gathered, unreduced=frozenset(ax), unreduced_kind=g_spec.unreduced_kind), + out_specs=jax.sharding.PartitionSpec(*spec), + )(g) + + def _per_layer_sharding(stacked, mesh): """Sharding of one layer slice `stacked[i]` of a scanned (layer-major) parameter, or None.""" s = getattr(jax.typeof(stacked), "sharding", None) or getattr(stacked, "sharding", None) @@ -134,6 +240,7 @@ def _needs_minor_dim_layout_pin(stacked, layer_sharding, mesh): "kwargs_treedef", "remat_policy", "prevent_cse", + "defer_grad_reduce_scatter", ], ) @@ -320,15 +427,47 @@ def layer_pure(y, layer_weights, layer_state, layer_consts): def layer_fn(y, layer_weights, layer_state): return layer_pure(y, layer_weights, layer_state, consts) - def rs_fn(layer_grads): - """Reduce-scatters one layer's gathered-weight gradients back to the FSDP parameter sharding.""" + def rs_one(g, s, tag): + """Reduce-scatters one gathered-weight gradient leaf back to its FSDP parameter sharding `s`.""" + if s is None: + return g + if tag: + out = _explicit_fsdp_reduce_scatter(g, ctx.mesh, s.spec, sharding.remove_fsdp_from_sharding(s).spec) + if out is not None: + return out + scatter = _fsdp_scatter_dim(g.ndim, ctx.mesh, s.spec) + if scatter is not None and _is_unreduced_over(g, scatter[1]): + g = _pin_scatter_dim_major(g, scatter[0]) + return sharding.maybe_shard_with_name(g, s, ctx.shard_mode, debug_sharding=ctx.debug_sharding) + + tag_cache = {} + + def tag_flags(leaves): + """Per-leaf choice of the tagged, deferred (dedicated SparseCore) reduce-scatter; the same for every layer.""" + if "flags" not in tag_cache: + flags = [] + for g, s in zip(leaves, layer_shardings): + scatter = None if s is None else _fsdp_scatter_dim(g.ndim, ctx.mesh, s.spec) + eligible = scatter is not None and _is_unreduced_over(g, scatter[1]) + flags.append(bool(ctx.defer_grad_reduce_scatter and eligible)) + tag_cache["flags"] = flags + return tag_cache["flags"] + + def rs_fn(layer_grads, which="all"): + """Reduce-scatters one layer's gathered-weight gradients back to the FSDP parameter sharding. + + `which` restricts it to the "tagged" or "untagged" reduce-scatters (the others are returned as is). + """ leaves, treedef = jax.tree_util.tree_flatten(layer_grads) + flags = tag_flags(leaves) out = [ - g if s is None else sharding.maybe_shard_with_name(g, s, ctx.shard_mode, debug_sharding=ctx.debug_sharding) - for g, s in zip(leaves, layer_shardings) + rs_one(g, s, tag) if which == "all" or (which == "tagged") == tag else g + for g, s, tag in zip(leaves, layer_shardings, flags) ] return jax.tree_util.tree_unflatten(treedef, out) + rs_fn.tag_flags = tag_flags + # Full remat (no policy): save only each layer's input and recompute the whole layer in the backward pass. remat_layer = None if ctx.remat_policy is not None: @@ -475,49 +614,66 @@ def _custom_vjp_prefetch_pipeline_bwd(graphdef, length, ctx, res, g_out): layout = layout_fn(y_last_in, w_last, _slice_at(state, last)) w_prev = gather_fn(params_prev_init) if length > 1 else None # AG(L-2) || bwd(L-1). g, dw = layer_vjp_fn(y_last_in, w_last, _slice_at(state, last), g, saved_last, layout) - d_params = jax.tree.map( - lambda p, d: jax.lax.dynamic_update_index_in_dim(jnp.zeros_like(p), d, last, axis=0), - params, - rs_fn(dw), - ) - if length > 1: + + def write(d_acc, dw_tree, idx, which): + """Writes the `which` ("tagged"/"untagged"/"all") leaves of one layer's reduced dW into row `idx`.""" + acc_leaves, treedef = jax.tree_util.tree_flatten(d_acc) + d_leaves = jax.tree_util.tree_leaves(dw_tree) + flags = rs_fn.tag_flags(d_leaves) if which != "all" else [None] * len(d_leaves) + out = [ + jax.lax.dynamic_update_index_in_dim(a, d, idx, axis=0) if which == "all" or (which == "tagged") == f else a + for a, d, f in zip(acc_leaves, d_leaves, flags) + ] + return jax.tree_util.tree_unflatten(treedef, out) + + def deferred(dw_tree): + """The unreduced tagged leaves of dW, carried to the next step (placeholders for the others).""" + leaves, treedef = jax.tree_util.tree_flatten(dw_tree) + flags = rs_fn.tag_flags(leaves) + return jax.tree_util.tree_unflatten(treedef, [x if f else jnp.zeros((0,), x.dtype) for x, f in zip(leaves, flags)]) + + # Tagged (dedicated SparseCore) reduce-scatters of layer k's dW are issued at the start of the next + # backward step (layer k-1), so each completes with that step's FSDP all-gather and overlaps its whole + # backward. Issued at the end of their own step, XLA's FSDP scheduler would have to complete those of + # the last step of each loop iteration before the iteration ends, exposing them. Untagged ones complete + # within their own step. + d_params = jax.tree.map(jnp.zeros_like, params) + if length == 1: + d_params = write(d_params, rs_fn(dw), last, "all") + else: + d_params = write(d_params, rs_fn(dw, "untagged"), last, "untagged") + pending = deferred(dw) if length > 2: ordered = (length - 2) // 2 >= 2 def body(carry, xs): - g_k, w_k, d_acc = carry + g_k, w_k, d_acc, pending = carry y_k, saved_k, state_k, k_idx = xs params_prev = _slice_at(params, k_idx - 1) if ordered: # In the 2nd unrolled step of each iteration, waits for the 1st step's g_k (which is # barrier-tied to dw_k) so W(k) and dW(k) are dead before AG(k-2) and recompute(k-1) start. g_k, y_k, saved_k, params_prev = jax.lax.optimization_barrier((g_k, y_k, saved_k, params_prev)) + d_acc = write(d_acc, rs_fn(pending, "tagged"), k_idx + 1, "tagged") # RS(k+1) || bwd(k). w_km1 = gather_fn(params_prev) # AG(k-1) || bwd(k). g_km1, dw_k = layer_vjp_fn(y_k, w_k, state_k, g_k, saved_k, layout) - dw_k = rs_fn(dw_k) + dw_k = rs_fn(dw_k, "untagged") if ordered: g_km1, dw_k = jax.lax.optimization_barrier((g_km1, dw_k)) - d_acc = jax.tree.map( - lambda acc, d: jax.lax.dynamic_update_index_in_dim(acc, d, k_idx, axis=0), - d_acc, - dw_k, - ) - return (g_km1, w_km1, d_acc), None + d_acc = write(d_acc, dw_k, k_idx, "untagged") + return (g_km1, w_km1, d_acc, deferred(dw_k)), None # Iterates k = length-2 .. 1; element j of xs is layer k = j + 1. - (g, w_prev, d_params), _ = jax.lax.scan( + (g, w_prev, d_params, pending), _ = jax.lax.scan( body, - (g, w_prev, d_params), + (g, w_prev, d_params, pending), (ys_mid, saved_mid, _slice_range(state, 1, length - 1), jnp.arange(1, length - 1, dtype=jnp.int32)), reverse=True, unroll=2, ) + d_params = write(d_params, rs_fn(pending, "tagged"), 1, "tagged") # RS(1) || bwd(0). g, dw0 = layer_vjp_fn(x_in, w_prev, _slice_at(state, 0), g, saved_0, layout) - d_params = jax.tree.map( - lambda acc, d: jax.lax.dynamic_update_index_in_dim(acc, d, 0, axis=0), - d_params, - rs_fn(dw0), - ) + d_params = write(d_params, rs_fn(dw0), 0, "all") params_bar_leaves = jax.tree_util.tree_leaves(d_params) if ctx.scan_axis != 0: @@ -621,6 +777,7 @@ def _is_diff_leaf(x): kwargs_treedef=kwargs_treedef, remat_policy=remat_policy, prevent_cse=prevent_cse, + defer_grad_reduce_scatter=bool(getattr(config, "prefetch_fsdp_defer_grad_reduce_scatter", False)), ) return graphdef, tuple(params_leaves), tuple(state_leaves), diff_args_leaves, diff_kwargs_leaves, ctx diff --git a/src/maxtext/layers/fsdp_prefetch_inhomogeneous.py b/src/maxtext/layers/fsdp_prefetch_inhomogeneous.py index 0307406052..1c73e52416 100644 --- a/src/maxtext/layers/fsdp_prefetch_inhomogeneous.py +++ b/src/maxtext/layers/fsdp_prefetch_inhomogeneous.py @@ -427,6 +427,11 @@ def apply_layers_with_fsdp_prefetch( _, _, _, diff_args_leaves, diff_kwargs_leaves, ctx = fsdp_prefetch._prepare_prefetch_call( layers, args, kwargs, length=length, mesh=mesh, config=config, remat_policy=remat_policy, prevent_cse=prevent_cse ) + if ctx.defer_grad_reduce_scatter: + # The per-sub-layer backward reduce-scatters each gradient within its own step; tagging them without + # deferring to the next sub-layer is slower than untagged, so deferral is only supported for homogeneous blocks. + max_logging.log("prefetch_fsdp_defer_grad_reduce_scatter is not supported for inhomogeneous blocks yet; ignoring it.") + ctx = ctx._replace(defer_grad_reduce_scatter=False) # Groups consecutive sub-layers of one structure (e.g. the unstacked `local_layers`), stacked # `[length, n, ...]`, so each group can be scanned by sub-layer. groups = [] # [graphdef, params_treedef, state_treedef, [params leaves per sub-layer], [state leaves ...]] diff --git a/src/maxtext/layers/moe.py b/src/maxtext/layers/moe.py index b509aacecd..4f1bff98b2 100644 --- a/src/maxtext/layers/moe.py +++ b/src/maxtext/layers/moe.py @@ -84,6 +84,17 @@ } +def _reduced_axes(w) -> frozenset: + """Mesh axes a weight is typed `reduced` over (all-gathered by fsdp_prefetch); empty for non-JAX values. + + Expert kernels may be quantized containers (e.g. qwix QArrays) rather than arrays, which have no such type. + """ + if not isinstance(w, jax.Array): # Includes tracers. + return frozenset() + spec = getattr(getattr(jax.typeof(w), "sharding", None), "spec", None) + return frozenset(spec.reduced) if spec is not None else frozenset() + + def _get_te_moe_checkpoint_kwargs(moe_fn) -> dict[str, str]: """Return supported TE MoE output remat names.""" parameters = inspect.signature(moe_fn).parameters @@ -2751,6 +2762,7 @@ def get_routed_moe_shardings(is_batch_sharded_by_expert, has_input_ids): decoder_tokens_pspec, ) = get_routed_moe_shardings(is_batch_sharded_by_expert, input_ids is not None) w0_pspec, w1_pspec, wo_pspec = maybe_aqt_partition(w0_kernel, w0_pspec, w1_kernel, w1_pspec, wo_kernel, wo_pspec) + output_pspec = self._logical_to_mesh_axes( ( batch_logical_axis, @@ -3816,13 +3828,22 @@ def _cn(t): # mesh axes, so it is threaded down as an argument instead of read off the module. per_expert_scale = None if self.per_expert_scale is None else self.per_expert_scale[...] + def keep_reduced(pspec, w): + # Weights all-gathered by fsdp_prefetch arrive typed `reduced` over the FSDP axes; shard_map requires + # in_specs to match, and keeping the type makes their gradient leave the shard_map `unreduced`, so the + # FSDP reduce-scatter happens once outside it instead of an all-reduce here. + reduced = _reduced_axes(w) + if pspec is None or not reduced or not isinstance(pspec, P): + return pspec + return P(*pspec, unreduced=pspec.unreduced, reduced=reduced, unreduced_kind=pspec.unreduced_kind) + in_specs = ( input_partition_pspec, gate_logits_pspec, pre_bias_logits_pspec, - w0_pspec, - w1_pspec, - wo_pspec, + keep_reduced(w0_pspec, w0_kernel), + keep_reduced(w1_pspec, w1_kernel), + keep_reduced(wo_pspec, wo_kernel), w0_bias_pspec, w1_bias_pspec, wo_bias_pspec, @@ -3836,7 +3857,13 @@ def _cn(t): P() if per_expert_scale is not None else None, ) # Not every mesh axis: shard_map rejects axes an enclosing vmap mapped away ("stage", "diloco"). - input_mesh_axes = {axis for spec in in_specs if spec is not None for dim in spec for axis in mesh_axes_for_dim(dim)} + input_mesh_axes = { + axis + for spec in in_specs + if spec is not None + for dim in (spec.partitions if isinstance(spec, P) else spec) + for axis in mesh_axes_for_dim(dim) + } overflow_mesh_axes = tuple(axis for axis in self.mesh.axis_names if axis in input_mesh_axes) @functools.partial( @@ -4113,13 +4140,21 @@ def _fsdp_all_gather(w, pspec): return w return self._maybe_shard_with_pspec(w, pspec) - if jnp.dtype(self.weight_dtype) == jnp.dtype(self.dtype) and self.config.scan_layers: + if ( + jnp.dtype(self.weight_dtype) == jnp.dtype(self.dtype) + and self.config.scan_layers + and not getattr(self.config, "prefetch_fsdp_weights", False) + ): # Without this, XLA rewrites all_gather(dynamic_slice(w)) into dynamic_slice(all_gather(w)) and hoists the # gather out of the scanned decoder, keeping every layer's experts live; the dtype cast normally blocks that. + # Skipped with prefetch_fsdp_weights: the weights arrive already gathered, and the barrier's transpose would + # sit between the dW all-reduce and its FSDP dynamic-slice, blocking their fusion into a reduce-scatter. w0_kernel, w1_kernel, wo_kernel = jax.lax.optimization_barrier((w0_kernel, w1_kernel, wo_kernel)) - w0_kernel = _fsdp_all_gather(w0_kernel, w0_pspec) - w1_kernel = _fsdp_all_gather(w1_kernel, w1_pspec) - wo_kernel = _fsdp_all_gather(wo_kernel, wo_pspec) + + # Weights already all-gathered (typed `reduced`) by fsdp_prefetch skip the reshard, which would drop the type. + w0_kernel = w0_kernel if _reduced_axes(w0_kernel) else _fsdp_all_gather(w0_kernel, w0_pspec) + w1_kernel = w1_kernel if _reduced_axes(w1_kernel) else _fsdp_all_gather(w1_kernel, w1_pspec) + wo_kernel = wo_kernel if _reduced_axes(wo_kernel) else _fsdp_all_gather(wo_kernel, wo_pspec) if w0_bias is not None: w0_bias = _fsdp_all_gather(w0_bias, w0_bias_pspec) if w1_bias is not None: diff --git a/src/maxtext/m3/models/glm5_3_flash/convert_glm5_ckpt.py b/src/maxtext/m3/models/glm5_3_flash/convert_glm5_ckpt.py new file mode 100644 index 0000000000..8e833e8880 --- /dev/null +++ b/src/maxtext/m3/models/glm5_3_flash/convert_glm5_ckpt.py @@ -0,0 +1,30 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Command-line entrypoint for GLM-5.3-Flash HuggingFace-to-Orbax checkpoint conversion.""" + +from collections.abc import Sequence + +from absl import app +from absl import flags +from maxtext.m3.models.glm5_3_flash import glm5_ckpt + + +def _parse_flags(argv: Sequence[str]) -> list[str]: + flags.FLAGS.mark_as_parsed() + return list(argv) + + +if __name__ == "__main__": + app.run(lambda argv: glm5_ckpt.run_cli(argv[1:]), flags_parser=_parse_flags) diff --git a/src/maxtext/m3/models/glm5_3_flash/glm5_ckpt.py b/src/maxtext/m3/models/glm5_3_flash/glm5_ckpt.py index aa5dac9a22..f5cdf1d110 100644 --- a/src/maxtext/m3/models/glm5_3_flash/glm5_ckpt.py +++ b/src/maxtext/m3/models/glm5_3_flash/glm5_ckpt.py @@ -14,14 +14,20 @@ """Checkpoint loading and conversion for GLM-5.3-Flash.""" -from collections.abc import Callable, Mapping +import argparse +import collections +from collections.abc import Callable, Mapping, Sequence +from concurrent import futures import dataclasses import functools -import glob import json import math import os +import sys +import threading +import time from typing import Any +from etils import epath from flax import nnx import jax import jax.numpy as jnp @@ -32,10 +38,54 @@ import numpy as np import orbax.checkpoint as ocp import safetensors +import tensorstore as ts import torch MAX_IN_MEMORY_CONVERSION_LAYERS = 6 MAX_IN_MEMORY_CONVERSION_PARAMS = 25_000_000_000 +_MAX_CONCURRENT_OPEN_SHARDS = 2 +# math.gcd(_MAX_MOE_EXPERT_CHUNK_SIZE, num_experts) ensures expert_chunk evenly +# divides num_experts so TensorStore writes full Zarr chunks without +# partial-chunk read-modify-write. +_MAX_MOE_EXPERT_CHUNK_SIZE = 16 +_STANDARD_CHECKPOINT_HANDLER = "orbax.checkpoint._src.handlers.standard_checkpoint_handler" ".StandardCheckpointHandler" +_BYTE_SIZE_SUFFIXES = { + "GiB": 1024**3, + "GB": 1000**3, + "MiB": 1024**2, + "MB": 1000**2, + "KiB": 1024, + "KB": 1000, + "B": 1, +} + + +def _parse_byte_size(val: str, flag_name: str) -> int: + """Parses a byte size string with optional suffixes into an integer.""" + if not isinstance(val, str): + raise ValueError(f"Invalid {flag_name} value: expected string, got {type(val)}") + val = val.strip() + for suffix, multiplier in sorted(_BYTE_SIZE_SUFFIXES.items(), key=lambda kv: len(kv[0]), reverse=True): + if val.endswith(suffix): + num_str = val[: -len(suffix)].strip() + if suffix == "B" and any(c.isalpha() for c in num_str): + raise ValueError(f"Invalid format for {flag_name} (unknown unit suffix): {val!r}") + try: + total_bytes = int(num_str) * multiplier + except ValueError as e: + raise ValueError(f"Invalid numeric part in {flag_name}: {num_str}") from e + if total_bytes < 1: + raise ValueError(f"{flag_name} must be >= 1, got {total_bytes}") + return total_bytes + if any(c.isalpha() for c in val): + raise ValueError(f"Invalid format for {flag_name} (unknown unit suffix): {val!r}") + try: + total_bytes = int(val) + except ValueError as e: + raise ValueError(f"Invalid format for {flag_name}: {val}") from e + if total_bytes < 1: + raise ValueError(f"{flag_name} must be >= 1, got {total_bytes}") + return total_bytes def _is_fp8(val: Any) -> bool: @@ -56,6 +106,24 @@ def _is_fp8(val: Any) -> bool: np.dtype(ml_dtypes.bfloat16): (torch.float32, np.dtype(np.float32)), } +_SUPPORTED_CLI_DTYPES: frozenset[str] = frozenset(dt.name for dt in _TARGET_TO_INTERMEDIATE_DTYPE) +_CLI_OWNED_CONFIG_KEYS: frozenset[str] = frozenset( + { + "num_hidden_layers", + "num_nextn_predict_layers", + "weight_dtype", + } +) +_ALLOWED_CONFIG_JSON_KEYS: frozenset[str] = ( + frozenset(f.name for f in dataclasses.fields(glm5_config.Glm5Config)) - _CLI_OWNED_CONFIG_KEYS +) +_TUPLE_CONFIG_KEYS: frozenset[str] = frozenset( + { + "layer_types", + "indexer_types", + } +) + def _to_target_np_array(val: Any, target_dtype: Any) -> np.ndarray: """Converts a tensor or array to an owned NumPy array of target_dtype.""" @@ -118,7 +186,8 @@ def dequantize_fp8_block128( if s_np.shape != exp_scale: raise ValueError(f"scale_inv shape {s_np.shape} != expected {exp_scale} for weight shape" f" {w_np.shape}.") scale = np.repeat(np.repeat(s_np, bm, axis=-2), bn, axis=-1)[..., :m, :n] - return np.asarray(w_np * scale, dtype=target_np_dt) + w_np *= scale + return np.asarray(w_np, dtype=target_np_dt) def _normalize_key(key: str) -> str: @@ -160,6 +229,154 @@ def _fetch_and_convert_tensor( return _to_target_np_array(val, target_dtype) +class _MemoryBudgetSemaphore: + """Thread-safe memory budget semaphore tracking active inflight bytes.""" + + def __init__(self, max_bytes: int): + if max_bytes <= 0: + raise ValueError(f"max_bytes must be positive, got {max_bytes}") + self._max_bytes = max_bytes + self._current_bytes = 0 + self._peak_bytes = 0 + self._active_acquisitions = 0 + self._max_concurrent_acquisitions = 0 + self._cond = threading.Condition() + + def acquire(self, required_bytes: int) -> None: + """Acquires required_bytes from the memory budget, blocking until available.""" + if required_bytes <= 0: + raise ValueError(f"required_bytes must be > 0, got {required_bytes}") + if required_bytes > self._max_bytes: + raise ValueError( + f"Required work unit memory {required_bytes} B exceeds total memory" f" budget {self._max_bytes} B." + ) + with self._cond: + while self._current_bytes + required_bytes > self._max_bytes: + self._cond.wait() + self._current_bytes += required_bytes + self._peak_bytes = max(self._peak_bytes, self._current_bytes) + self._active_acquisitions += 1 + self._max_concurrent_acquisitions = max(self._max_concurrent_acquisitions, self._active_acquisitions) + + def release(self, freed_bytes: int) -> None: + with self._cond: + self._current_bytes -= freed_bytes + self._active_acquisitions -= 1 + self._cond.notify_all() + + @property + def current_bytes(self) -> int: + with self._cond: + return self._current_bytes + + @property + def peak_bytes(self) -> int: + with self._cond: + return self._peak_bytes + + @property + def max_concurrent_acquisitions(self) -> int: + with self._cond: + return self._max_concurrent_acquisitions + + +class _ThreadSafeShardLRU: + """Thread-safe LRU handle cache for safetensors shards with reference leases.""" + + def __init__(self, max_open: int): + if max_open <= 0: + raise ValueError(f"max_open must be positive, got {max_open}") + self._max_open = max_open + self._handles: collections.OrderedDict[str, Any] = collections.OrderedDict() + self._leases: collections.Counter[str] = collections.Counter() + self._opening: set[str] = set() + self._cond = threading.Condition() + + def _acquire_shard(self, shard_path: str) -> Any: + """Acquires a shard handle under reference lease.""" + while True: + with self._cond: + if shard_path in self._handles: + self._handles.move_to_end(shard_path) + self._leases[shard_path] += 1 + return self._handles[shard_path] + if shard_path in self._opening: + self._cond.wait() + continue + if len(self._handles) + len(self._opening) < self._max_open: + self._opening.add(shard_path) + break + evicted = False + for sp in list(self._handles.keys()): + if self._leases[sp] == 0: + h = self._handles.pop(sp) + # pylint: disable=unnecessary-dunder-call + h.__exit__(None, None, None) + del self._leases[sp] + evicted = True + break + if evicted: + self._opening.add(shard_path) + break + self._cond.wait() + + try: + handle = safetensors.safe_open(shard_path, framework="pt", device="cpu") + # pylint: disable=unnecessary-dunder-call + handle.__enter__() + except BaseException: + with self._cond: + self._opening.discard(shard_path) + self._cond.notify_all() + raise + + with self._cond: + self._opening.discard(shard_path) + self._handles[shard_path] = handle + self._leases[shard_path] = 1 + self._cond.notify_all() + return handle + + def _release_shard(self, shard_path: str) -> None: + with self._cond: + self._leases[shard_path] -= 1 + if self._leases[shard_path] <= 0: + self._leases[shard_path] = 0 + self._cond.notify_all() + + def load_tensor(self, shard_path: str, orig_k: str, shard_file: str) -> Any: + """Loads a tensor from a safetensors shard under concurrency-bounded LRU lease.""" + handle = self._acquire_shard(shard_path) + try: + return handle.get_tensor(orig_k) + except (safetensors.SafetensorError, KeyError) as e: + raise ValueError(f"Tensor {orig_k!r} listed in index for shard {shard_file!r} is" " absent from shard file.") from e + finally: + self._release_shard(shard_path) + + def get_slice_shape(self, shard_path: str, orig_k: str, shard_file: str) -> tuple[int, ...]: + """Inspects a tensor slice shape from a safetensors shard header under LRU lease.""" + handle = self._acquire_shard(shard_path) + try: + s = handle.get_slice(orig_k) + return tuple(int(x) for x in s.get_shape()) + except (safetensors.SafetensorError, KeyError) as e: + raise ValueError(f"Tensor {orig_k!r} listed in index for shard {shard_file!r} is" " absent from shard file.") from e + finally: + self._release_shard(shard_path) + + def close_all(self) -> None: + """Closes all active and cached shard handles.""" + with self._cond: + while self._handles: + _, h = self._handles.popitem(last=False) + # pylint: disable=unnecessary-dunder-call + h.__exit__(None, None, None) + self._leases.clear() + self._opening.clear() + self._cond.notify_all() + + @dataclasses.dataclass(frozen=True) class ConversionSpec: """Specification for mapping HuggingFace tensor(s) to a MaxText model parameter.""" @@ -172,6 +389,18 @@ class ConversionSpec: target_dtype: Any # jnp.dtype +@dataclasses.dataclass(frozen=True) +class _ConversionWorkUnit: + """Metadata for a parallel conversion work unit.""" + + sort_key: tuple[str, int, int] + spec: ConversionSpec + slot: int + e_start: int + e_end: int + byte_weight: int + + def iter_conversion_specs( config: glm5_config.Glm5Config, *, @@ -201,8 +430,8 @@ def _add( kind: str, *, dtype: Any, - stack_idx: int | None = None, - num_sbs: int | None = None, + stack_idx: int | None, + num_sbs: int | None, ) -> None: specs.append( ConversionSpec( @@ -355,18 +584,24 @@ def _add_l( ("embed_tokens", "embedding"), "direct", dtype=p_dt, + stack_idx=None, + num_sbs=None, ) _add( ("model.language_model.norm.weight",), ("norm", "weight"), "direct", dtype=p_dt, + stack_idx=None, + num_sbs=None, ) _add( ("lm_head.weight",), ("lm_head", "kernel"), "transpose_2d", dtype=p_dt, + stack_idx=None, + num_sbs=None, ) def _add_layer_spec( @@ -471,18 +706,24 @@ def _add_layer_spec( (*tgt_base, pre, "weight"), "direct", dtype=p_dt, + stack_idx=None, + num_sbs=None, ) _add( (f"{lp}eh_proj.weight",), (*tgt_base, "eh_proj", "kernel"), "transpose_2d", dtype=p_dt, + stack_idx=None, + num_sbs=None, ) _add( (f"{lp}shared_head.norm.weight",), (*tgt_base, "shared_head_norm", "weight"), "direct", dtype=p_dt, + stack_idx=None, + num_sbs=None, ) _emit_mla(lp, tgt_base, None, None) _emit_moe(lp, tgt_base, None, None) @@ -587,25 +828,52 @@ def _assign( f"Dtype mismatch for {key_name}: spec produced {val.dtype}, model" f" variable expects {var.value.dtype}" ) if is_stacked: + if not 0 <= b_idx < var.value.shape[0]: + raise ValueError(f"b_idx {b_idx} out of bounds [0, {var.value.shape[0]}) for stacked" f" {key_name}") var.value = var.value.at[b_idx].set(val) else: + if b_idx != 0: + raise ValueError(f"b_idx must be 0 when is_stacked=False for {key_name}, got {b_idx}") var.value = val specs = iter_conversion_specs(model.config, include_mtp=not ignore_mtp_layer) + all_vars: list[tuple[tuple[Any, ...], nnx.Variable]] = [] + sb_leaves: set[int] = set() + unstacked_leaves: set[int] = set() + for path, node in nnx.iter_graph(model): + if isinstance(node, (nnx.Param, modeling_glm5_3_flash.Glm5RouterBias)): + var_id = id(node) + all_vars.append((path, node)) + if path and path[0] == "super_block": + sb_leaves.add(var_id) + else: + unstacked_leaves.add(var_id) + + if not sb_leaves.isdisjoint(unstacked_leaves) or (len(sb_leaves) + len(unstacked_leaves) != len(all_vars)): + raise ValueError("Model variable leaf classification mismatch between stacked and" " unstacked sets.") + for spec in specs: + arr = _transform_for_spec(spec, _get) target_var = resolve_target_var(model, spec.target_path) - val = _transform_for_spec(spec, _get) - is_stacked = spec.stack_index is not None + var_id = id(target_var) + if (var_id in sb_leaves) == (var_id in unstacked_leaves): + raise ValueError( + f"Target variable {spec.target_path} was not classified into" " exactly one of stacked or unstacked leaves." + ) + is_stacked = var_id in sb_leaves + if is_stacked and spec.stack_index is None: + raise ValueError( + f"Stacked parameter {'.'.join(str(p) for p in spec.target_path)}" " requires a non-None stack_index" + ) b_idx = 0 if spec.stack_index is None else spec.stack_index _assign( target_var, - val, + arr, b_idx=b_idx, is_stacked=is_stacked, key_name=spec.hf_keys[0], ) - for orig_k in hf_tensors.keys(): if orig_k.startswith("model.visual.") or orig_k.startswith("visual."): continue @@ -615,25 +883,12 @@ def _assign( continue raise ValueError(f"Unmapped key in hf_tensors: {orig_k!r}") - def _is_var(x: Any) -> bool: - return isinstance(x, nnx.Variable) - num_prefix = min(model.config.num_hidden_layers, model.config.first_k_dense_replace) num_sb = (model.config.num_hidden_layers - num_prefix) // 4 - sb_leaves = set() - if model.super_block is not None: - for s in range(4): - sl = getattr(model.super_block, f"slot_{s}") - for v in jax.tree.leaves(nnx.state(sl), is_leaf=_is_var): - sb_leaves.add(id(v)) - for p in jax.tree.leaves( - nnx.state(model, (nnx.Param, modeling_glm5_3_flash.Glm5RouterBias)), - is_leaf=_is_var, - ): - if isinstance(p, nnx.Variable): - for b in range(num_sb) if id(p) in sb_leaves else (0,): - if (id(p), b) not in assigned_slots: - raise ValueError(f"Unpopulated model parameter slot ({b}): {p}") + for path, p in all_vars: + for b in range(num_sb) if id(p) in sb_leaves else (0,): + if (id(p), b) not in assigned_slots: + raise ValueError(f"Unpopulated model parameter slot ({b}) at" f" {'.'.join(str(k) for k in path)}: {p}") def save_orbax_checkpoint(model: modeling_glm5_3_flash.Glm5Model, orbax_dir: str) -> None: @@ -643,124 +898,877 @@ def save_orbax_checkpoint(model: modeling_glm5_3_flash.Glm5Model, orbax_dir: str checkpointer.wait_until_finished() -def restore_orbax_checkpoint(model: modeling_glm5_3_flash.Glm5Model, orbax_dir: str) -> None: - """Restores Orbax checkpoint state into an existing Glm5Model instance.""" - target = nnx.to_pure_dict(nnx.state(model)) +def _to_str_paths(tree: Any) -> Any: + """Converts dict keys to strings so target PyTree matches Orbax on-disk keys.""" + if isinstance(tree, dict): + return {str(k): _to_str_paths(v) for k, v in tree.items()} + return tree + + +def restore_orbax_checkpoint( + model: Any, + orbax_dir: str, + *, + mesh: jax.sharding.Mesh | None, +) -> None: + """Restores Orbax checkpoint state directly into sharded model variables. + + Args: + model: Glm5Model instance (or nnx.eval_shape abstract model) to restore + into. + orbax_dir: Directory containing the Orbax checkpoint. + mesh: Must be None. Passing a mesh to restore_orbax_checkpoint is rejected; + pass mesh to Glm5Model(..., mesh=mesh) at construction time so + model.sharding is populated. + + Raises: + ValueError: If mesh is not None, if model.sharding is set but model.mesh is + None or empty, if checkpoint metadata is missing, or if leaf dtype/shape + mismatches. + TypeError: If variable value or sharding metadata has an unexpected type. + """ + var_state = nnx.state(model) + pure_target = nnx.to_pure_dict(var_state) + str_pure_target = _to_str_paths(pure_target) + with ocp.StandardCheckpointer() as checkpointer: - restored = checkpointer.restore(orbax_dir, target) - nnx.update( - model, - nnx.restore_int_paths(restored), + step_metadata = checkpointer.metadata(orbax_dir) + if step_metadata.item_metadata is None: + raise ValueError(f"Missing item_metadata in Orbax checkpoint at {orbax_dir!r}.") + meta_tree = step_metadata.item_metadata.tree + + def _validate_meta_leaf(meta_leaf: Any, target_leaf: Any) -> None: + if meta_leaf is None or not isinstance(meta_leaf, ocp.metadata.value.ArrayMetadata): + raise ValueError(f"Expected Orbax ArrayMetadata leaf, got {type(meta_leaf)}") + saved_dt = meta_leaf.dtype + probe = jnp.asarray(np.empty(0, dtype=saved_dt)) + if np.dtype(probe.dtype) != np.dtype(saved_dt): + raise ValueError(f"Restored leaf dtype changed during conversion: saved {saved_dt}," f" got {probe.dtype}") + if np.dtype(probe.dtype) != np.dtype(target_leaf.dtype): + raise ValueError(f"Restored leaf dtype {probe.dtype} does not match model variable" f" dtype {target_leaf.dtype}") + if tuple(meta_leaf.shape) != tuple(target_leaf.shape): + raise ValueError( + f"Restored leaf shape {tuple(meta_leaf.shape)} does not match model" + f" variable shape {tuple(target_leaf.shape)}" + ) + + jax.tree.map(_validate_meta_leaf, meta_tree, str_pure_target) + + if mesh is not None: + raise ValueError( + "Passing mesh to restore_orbax_checkpoint is not supported; pass" + " mesh to Glm5Model(..., mesh=mesh) at construction time so" + " model.sharding is populated." + ) + + if model.sharding is not None: + model_mesh = model.mesh + if model_mesh is None or model_mesh.empty: + raise ValueError("model.sharding is set, but model.mesh is None or empty.") + + def _sharding_from_model_metadata(var: nnx.Variable) -> nnx.Variable: + if not var.has_metadata("out_sharding") or (var.get_metadata("out_sharding") is None): + raise ValueError(f"Missing out_sharding metadata on variable {var!r} when" " model.sharding is set.") + out_sh = var.get_metadata("out_sharding") + if isinstance(out_sh, jax.sharding.NamedSharding): + spec = out_sh.spec + elif isinstance(out_sh, jax.sharding.PartitionSpec): + spec = out_sh + else: + raise TypeError(f"Unsupported out_sharding metadata type: {type(out_sh)}") + return var.replace(jax.sharding.NamedSharding(model_mesh, spec)) + + named_sharding_state = jax.tree.map( + _sharding_from_model_metadata, + var_state, + is_leaf=lambda x: isinstance(x, nnx.Variable), + ) + else: + for leaf in jax.tree.leaves(pure_target): + if isinstance(leaf, (jax.Array, jax.ShapeDtypeStruct)) and isinstance(leaf.sharding, jax.sharding.NamedSharding): + raise ValueError( + "mesh=None with model.sharding=None cannot be used when model" " leaves already carry NamedSharding." + ) + single_sharding = jax.sharding.SingleDeviceSharding(jax.local_devices(backend="cpu")[0]) + named_sharding_state = jax.tree.map( + lambda var: var.replace(single_sharding), + var_state, + is_leaf=lambda x: isinstance(x, nnx.Variable), + ) + + sharding_pure = _to_str_paths(nnx.to_pure_dict(named_sharding_state)) + target_struct_tree = jax.tree.map( + lambda leaf, sh: jax.ShapeDtypeStruct(shape=leaf.shape, dtype=leaf.dtype, sharding=sh), + str_pure_target, + sharding_pure, + ) + restored = checkpointer.restore(orbax_dir, target=target_struct_tree) + nnx.update(model, nnx.restore_int_paths(restored)) + + +def _discover_and_validate_checkpoint( + hf_checkpoint_dir: str, + config: glm5_config.Glm5Config, + *, + slice_to_config_layers: bool, + ignore_mtp_layer: bool, +) -> tuple[dict[str, str], list[str]]: + """Discovers and validates safetensors checkpoint files via intent-keyed discovery. + + Args: + hf_checkpoint_dir: Directory containing index or single-shard safetensors. + config: Model configuration. + slice_to_config_layers: Whether to filter keys to config.num_hidden_layers. + ignore_mtp_layer: Whether to exclude MTP layer 46 keys. + + Returns: + Tuple of (active_weight_map, sorted_shard_paths). + Raises: + FileNotFoundError: If checkpoint files or referenced shards are missing. + ValueError: If model.safetensors.index.json is missing 'weight_map'. + """ + idx_p = os.path.join(hf_checkpoint_dir, "model.safetensors.index.json") + single_p = os.path.join(hf_checkpoint_dir, "model.safetensors") + if os.path.isfile(idx_p): + with open(idx_p, "r", encoding="utf-8") as f: + idx_data = json.load(f) + if not (isinstance(idx_data, Mapping) and "weight_map" in idx_data and isinstance(idx_data["weight_map"], Mapping)): + raise ValueError(f"Invalid safetensors index in {idx_p!r}: missing 'weight_map'" " mapping.") + weight_map: dict[str, str] = {str(k): str(v) for k, v in idx_data["weight_map"].items()} + elif os.path.isfile(single_p): + st_files = [f for f in os.listdir(hf_checkpoint_dir) if f.endswith(".safetensors")] + if len(st_files) > 1: + raise FileNotFoundError( + "Multiple safetensors files found without" + f" model.safetensors.index.json in {hf_checkpoint_dir!r}:" + f" {sorted(st_files)}" + ) + with safetensors.safe_open(single_p, framework="pt", device="cpu") as f: + weight_map = {k: "model.safetensors" for k in f.keys()} + else: + contents = sorted(os.listdir(hf_checkpoint_dir)) if os.path.isdir(hf_checkpoint_dir) else "directory does not exist" + raise FileNotFoundError( + "Expected model.safetensors.index.json or single-file" + f" model.safetensors in {hf_checkpoint_dir!r}; found: {contents}" + ) + + mtp_idx = None if ignore_mtp_layer else glm5_config.MTP_LAYER_INDEX + active_map: dict[str, str] = ( + create_mini_slice.extract_mini_slice_keys(weight_map, config.num_hidden_layers, include_mtp_layer=mtp_idx) + if slice_to_config_layers + else weight_map ) + unique_shards = sorted(set(active_map.values())) + missing = [s for s in unique_shards if not os.path.isfile(os.path.join(hf_checkpoint_dir, s))] + if missing: + raise FileNotFoundError(f"Missing referenced safetensors shards in {hf_checkpoint_dir!r}:" f" {missing}") + return active_map, [os.path.join(hf_checkpoint_dir, s) for s in unique_shards] + + +def _extract_valid_hf_keys(specs: list[ConversionSpec]) -> set[str]: + return { + k + for s in specs + for hk in s.hf_keys + for k in ( + hk, + f"{hk[:-7]}.weight_scale_inv" if hk.endswith(".weight") else f"{hk}_scale_inv", + ) + } -def convert_hf_checkpoint_to_orbax( +def _fill_moe_expert_chunk( + spec: ConversionSpec, + e_start: int, + e_end: int, + per_expert_shape: tuple[int, ...], + dt: np.dtype, + get_tensor_fn: Callable[[str, Any], np.ndarray], +) -> np.ndarray: + """Fills an MoE expert chunk buffer for gate_up or down projection.""" + chunk_buf = np.empty((e_end - e_start, *per_expert_shape), dtype=dt) + if spec.kind == "moe_gate_up": + expected_proj_shape = (per_expert_shape[0], per_expert_shape[2]) + for e in range(e_start, e_end): + gate_t = get_tensor_fn(spec.hf_keys[2 * e], spec.target_dtype).T + up_t = get_tensor_fn(spec.hf_keys[2 * e + 1], spec.target_dtype).T + if gate_t.shape != expected_proj_shape or up_t.shape != expected_proj_shape: + raise ValueError( + f"Shape mismatch for {spec.target_path} expert {e}:" + f" expected {expected_proj_shape}, got gate={gate_t.shape}," + f" up={up_t.shape}." + ) + chunk_buf[e - e_start, :, 0, :] = gate_t + chunk_buf[e - e_start, :, 1, :] = up_t + del gate_t, up_t + elif spec.kind == "moe_down": + for e in range(e_start, e_end): + down_t = get_tensor_fn(spec.hf_keys[e], spec.target_dtype).T + if down_t.shape != per_expert_shape: + raise ValueError( + f"Shape mismatch for {spec.target_path} expert {e}:" f" expected {per_expert_shape}, got {down_t.shape}." + ) + chunk_buf[e - e_start] = down_t + del down_t + else: + raise ValueError(f"Unknown conversion spec kind: {spec.kind!r}") + return chunk_buf + + +def _convert_hf_checkpoint_to_orbax_in_memory( hf_checkpoint_dir: str, orbax_output_dir: str, config: glm5_config.Glm5Config, *, - ignore_mtp_layer: bool = False, - slice_to_config_layers: bool = False, + ignore_mtp_layer: bool, + slice_to_config_layers: bool, ) -> None: - """Converts HuggingFace FP8/BF16 safetensors checkpoint to an Orbax checkpoint. - - convert_hf_checkpoint_to_orbax is an in-memory converter scoped strictly to - mini-slice checkpoints (num_hidden_layers <= MAX_IN_MEMORY_CONVERSION_LAYERS - (6) and <= MAX_IN_MEMORY_CONVERSION_PARAMS (25B) parameters). One MoE layer - is ~7.25B params, so a 5-layer real slice is ~16.9B params and fits under - 25B, whereas >= 7 layers exceeds 25B and requires - convert_hf_checkpoint_to_orbax_streaming. - """ + """In-memory converter reference oracle scoped strictly to mini-slice checkpoints.""" if config.num_hidden_layers > MAX_IN_MEMORY_CONVERSION_LAYERS: raise ValueError( - "convert_hf_checkpoint_to_orbax is scoped to mini-slice checkpoints" - f" with at most {MAX_IN_MEMORY_CONVERSION_LAYERS} layers (got" - f" num_hidden_layers={config.num_hidden_layers}); full-model streaming" - " conversion is tracked under F33." + "_convert_hf_checkpoint_to_orbax_in_memory is scoped to mini-slice" + f" checkpoints with at most {MAX_IN_MEMORY_CONVERSION_LAYERS} layers" + f" (got num_hidden_layers={config.num_hidden_layers}); use" + " convert_hf_checkpoint_to_orbax for streaming conversion." ) with jax.default_device(jax.devices("cpu")[0]): - hf_dir = hf_checkpoint_dir - orbax_dir = orbax_output_dir - index_path = os.path.join(hf_dir, "model.safetensors.index.json") - all_tensors: dict[str, Any] = {} - needed: dict[str, str] | None = None - - if os.path.exists(index_path): - with open(index_path, "r", encoding="utf-8") as f: - weight_map = json.load(f)["weight_map"] - if slice_to_config_layers: - needed = create_mini_slice.extract_mini_slice_keys( - weight_map, - config.num_hidden_layers, - include_mtp_layer=None if ignore_mtp_layer else glm5_config.MTP_LAYER_INDEX, - ) - shard_paths = [os.path.join(hf_dir, s) for s in sorted(set(needed.values()))] - else: - shard_paths = [os.path.join(hf_dir, s) for s in sorted(set(weight_map.values()))] - else: - shard_paths = sorted(glob.glob(os.path.join(hf_dir, "*.safetensors"))) - if not shard_paths: - raise ValueError(f"No safetensors files found in {hf_dir}") - if slice_to_config_layers: - temp_map: dict[str, str] = {} - for sp in shard_paths: - with safetensors.safe_open(sp, framework="pt", device="cpu") as f: - for k in f.keys(): - temp_map[k] = sp - needed = create_mini_slice.extract_mini_slice_keys( - temp_map, - config.num_hidden_layers, - include_mtp_layer=None if ignore_mtp_layer else glm5_config.MTP_LAYER_INDEX, - ) - + active_weight_map, shard_paths = _discover_and_validate_checkpoint( + hf_checkpoint_dir, + config, + slice_to_config_layers=slice_to_config_layers, + ignore_mtp_layer=ignore_mtp_layer, + ) specs = iter_conversion_specs(config, include_mtp=not ignore_mtp_layer) - valid_hf_keys: set[str] = set() - for spec in specs: - for hk in spec.hf_keys: - valid_hf_keys.add(hk) - valid_hf_keys.add(f"{hk[:-7]}.weight_scale_inv" if hk.endswith(".weight") else f"{hk}_scale_inv") + valid_hf_keys = _extract_valid_hf_keys(specs) # First pass: check headers and keys before materializing tensors total_params = 0 + all_tensors: dict[str, Any] = {} for sp in shard_paths: with safetensors.safe_open(sp, framework="pt", device="cpu") as f: for k in f.keys(): - if k.startswith("model.visual.") or k.startswith("visual."): - continue - if needed is not None and k not in needed: - continue - if ignore_mtp_layer and k.startswith(f"model.language_model.layers.{glm5_config.MTP_LAYER_INDEX}."): + if ( + k.startswith(("model.visual.", "visual.")) + or k not in active_weight_map + or (ignore_mtp_layer and k.startswith(f"model.language_model.layers.{glm5_config.MTP_LAYER_INDEX}.")) + ): continue if not (k.startswith("model.language_model.") or k in ("lm_head.weight", "lm_head.weight_scale_inv")): raise ValueError(f"Unknown HF key prefix: {k!r}") if k not in valid_hf_keys: raise ValueError(f"Unmapped key in hf_tensors: {k!r}") - header_shape = f.get_slice(k).get_shape() - total_params += int(np.prod(header_shape)) + total_params += int(np.prod(f.get_slice(k).get_shape())) if total_params > MAX_IN_MEMORY_CONVERSION_PARAMS: raise ValueError( - "convert_hf_checkpoint_to_orbax is scoped to mini-slice" - f" checkpoints with at most {MAX_IN_MEMORY_CONVERSION_PARAMS}" - f" parameters (got total_params={total_params}); full-model" - " streaming conversion is tracked under F33." + "_convert_hf_checkpoint_to_orbax_in_memory is scoped to" + " mini-slice checkpoints with at most" + f" {MAX_IN_MEMORY_CONVERSION_PARAMS} parameters (got" + f" total_params={total_params}); use" + " convert_hf_checkpoint_to_orbax for streaming conversion." ) # Second pass: materialize tensors for sp in shard_paths: with safetensors.safe_open(sp, framework="pt", device="cpu") as f: for k in f.keys(): - if k.startswith("model.visual.") or k.startswith("visual."): - continue - if needed is not None and k not in needed: - continue - if ignore_mtp_layer and k.startswith(f"model.language_model.layers.{glm5_config.MTP_LAYER_INDEX}."): + if ( + k.startswith(("model.visual.", "visual.")) + or k not in active_weight_map + or (ignore_mtp_layer and k.startswith(f"model.language_model.layers.{glm5_config.MTP_LAYER_INDEX}.")) + ): continue all_tensors[k] = f.get_tensor(k) - model = modeling_glm5_3_flash.Glm5Model( - config, - rngs=nnx.Rngs(0), - place_on_devices=False, - ) + model = modeling_glm5_3_flash.Glm5Model(config, rngs=nnx.Rngs(0), place_on_devices=False) load_hf_state_into_model(model, all_tensors, ignore_mtp_layer=ignore_mtp_layer) - save_orbax_checkpoint(model, orbax_dir) + save_orbax_checkpoint(model, orbax_output_dir) + + +def convert_hf_checkpoint_to_orbax( + hf_checkpoint_dir: str, + orbax_output_dir: str, + config: glm5_config.Glm5Config, + *, + ignore_mtp_layer: bool, + slice_to_config_layers: bool, + num_workers: int, + max_inflight_bytes: int, +) -> None: + """Streams a multi-shard HF safetensors checkpoint to an Orbax checkpoint. + + Bounds host RAM by writing each parameter slice directly to TensorStore/OCDBT + on disk rather than accumulating the full parameter tree in memory, using a + bounded-inflight thread pool. + + Args: + hf_checkpoint_dir: Directory containing HuggingFace safetensors checkpoint. + orbax_output_dir: Destination directory for the saved Orbax checkpoint. + config: Glm5Config model configuration. + ignore_mtp_layer: Whether to omit MTP layer 46 weights. + slice_to_config_layers: Whether to slice checkpoint keys to config layers. + num_workers: Number of concurrent worker threads (must be >= 1). + max_inflight_bytes: Memory budget for concurrent work units. + + Returns: + None. + Raises: + FileNotFoundError: If checkpoint files or referenced shards are missing. + ValueError: If config, keys, dtypes, or parameter slots are invalid. + FileExistsError: If orbax_output_dir exists and is non-empty. + """ + if isinstance(num_workers, bool) or not isinstance(num_workers, int) or num_workers < 1: + raise ValueError(f"num_workers must be an integer >= 1, got {num_workers!r}") + if isinstance(max_inflight_bytes, bool) or not isinstance(max_inflight_bytes, int) or max_inflight_bytes < 1: + raise ValueError("max_inflight_bytes must be an integer >= 1, got" f" {max_inflight_bytes!r}") + if ignore_mtp_layer and config.num_nextn_predict_layers > 0: + raise ValueError( + "ignore_mtp_layer=True cannot be used when" + " config.num_nextn_predict_layers > 0; set num_nextn_predict_layers=0" + " on the config instead." + ) + if not ignore_mtp_layer and config.num_nextn_predict_layers == 0: + raise ValueError("ignore_mtp_layer=False requires config.num_nextn_predict_layers > 0.") + + active_weight_map, _ = _discover_and_validate_checkpoint( + hf_checkpoint_dir, + config, + slice_to_config_layers=slice_to_config_layers, + ignore_mtp_layer=ignore_mtp_layer, + ) + + specs = iter_conversion_specs( + config, + include_mtp=not ignore_mtp_layer and config.num_nextn_predict_layers > 0, + ) + valid_hf_keys = _extract_valid_hf_keys(specs) + + orig_key_map: dict[str, str] = {} + for orig_k in active_weight_map.keys(): + if orig_k.startswith("model.visual.") or orig_k.startswith("visual."): + continue + if slice_to_config_layers and orig_k not in valid_hf_keys: + continue + norm_k = _normalize_key(orig_k) + hub_k = norm_k if norm_k.startswith("lm_head.") else f"model.language_model.{norm_k}" + if hub_k in orig_key_map: + raise ValueError(f"Duplicate normalized key found: {norm_k!r}") + orig_key_map[hub_k] = orig_k + + for spec in specs: + for hk in spec.hf_keys: + if hk not in orig_key_map: + raise ValueError(f"Required parameter missing from HF state: {hk!r}") + + final_path = epath.Path(orbax_output_dir) + if final_path.exists() and (not final_path.is_dir() or any(final_path.iterdir())): + raise FileExistsError("Refusing to write into existing non-empty orbax_output_dir:" f" {orbax_output_dir!r}") + + shard_lru = _ThreadSafeShardLRU(_MAX_CONCURRENT_OPEN_SHARDS) + write_dir: epath.Path | None = None + ts_arrays: dict[tuple[str | int, ...], tuple[Any, tuple[int, ...]]] = {} + try: + init_timestamp_nsecs = time.time_ns() + tmp_cls = ocp.path.atomicity_defaults.get_default_temporary_path_class(final_path) + tmp_path = tmp_cls.from_final(final_path) + write_dir = ocp.checkpoint_utils.asyncio_utils.run_sync(tmp_path.create()) + ts_context = ocp.serialization.tensorstore_utils.get_ts_context(use_ocdbt=True) + + abs_model = nnx.eval_shape(lambda: modeling_glm5_3_flash.Glm5Model(config, rngs=nnx.Rngs(0), place_on_devices=False)) + + # Phase 1: Planning and work unit generation + work_units: list[_ConversionWorkUnit] = [] + moe_populated_experts: dict[tuple[tuple[str | int, ...], int], set[int]] = collections.defaultdict(set) + stacked_shapes: dict[tuple[str | int, ...], int] = collections.defaultdict(int) + + for spec_idx, spec in enumerate(specs): + if spec.stack_index is not None and spec.num_super_blocks is not None: + stacked_shapes[spec.target_path] = max(stacked_shapes[spec.target_path], spec.num_super_blocks) + dt = np.dtype(spec.target_dtype) + target_itemsize = dt.itemsize + compute_itemsize = _TARGET_TO_INTERMEDIATE_DTYPE[dt][1].itemsize + slot = 0 if spec.stack_index is None else spec.stack_index + target_var = resolve_target_var(abs_model, spec.target_path) + expected_shape = target_var.shape + expected_slot_shape = expected_shape[1:] if spec.stack_index is not None else expected_shape + + if spec.kind in ("direct", "transpose_2d"): + orig_k = orig_key_map[spec.hf_keys[0]] + shard_file = active_weight_map[orig_k] + primary_shard = shard_file + slice_shape = shard_lru.get_slice_shape(os.path.join(hf_checkpoint_dir, shard_file), orig_k, shard_file) + in_shape = slice_shape if spec.kind == "direct" else (slice_shape[1], slice_shape[0]) + if in_shape != expected_slot_shape: + raise ValueError( + f"Shape mismatch for {spec.target_path} (slot {slot}): expected" f" {expected_slot_shape}, got {in_shape}." + ) + elems = int(np.prod(expected_slot_shape)) + output_slice_bytes = elems * target_itemsize + norm_k = _normalize_key(orig_k) + scale_norm_k = norm_k[:-7] + ".weight_scale_inv" if norm_k.endswith(".weight") else norm_k + "_scale_inv" + scale_hub_k = scale_norm_k if scale_norm_k.startswith("lm_head.") else f"model.language_model.{scale_norm_k}" + if scale_hub_k in orig_key_map: + byte_weight = 3 * output_slice_bytes + 2 * elems * compute_itemsize + elif dt == np.dtype(ml_dtypes.bfloat16): + byte_weight = 3 * output_slice_bytes + elems * compute_itemsize + else: + byte_weight = 3 * output_slice_bytes + work_units.append( + _ConversionWorkUnit( + sort_key=(primary_shard, spec_idx, 0), + spec=spec, + slot=slot, + e_start=0, + e_end=0, + byte_weight=byte_weight, + ) + ) + elif spec.kind in ("moe_gate_up", "moe_down"): + per_expert_shape = expected_shape[1:] if spec.stack_index is None else expected_shape[2:] + num_experts = len(spec.hf_keys) // 2 if spec.kind == "moe_gate_up" else len(spec.hf_keys) + expert_chunk = math.gcd(_MAX_MOE_EXPERT_CHUNK_SIZE, num_experts) if num_experts > 0 else 1 + single_proj_elems = config.hidden_size * config.moe_intermediate_size + for e_start in range(0, num_experts, expert_chunk): + e_end = min(e_start + expert_chunk, num_experts) + first_hk = spec.hf_keys[2 * e_start] if spec.kind == "moe_gate_up" else spec.hf_keys[e_start] + primary_shard = active_weight_map[orig_key_map[first_hk]] + output_slice_bytes = (e_end - e_start) * int(np.prod(per_expert_shape)) * target_itemsize + byte_weight = 3 * output_slice_bytes + 2 * single_proj_elems * compute_itemsize + work_units.append( + _ConversionWorkUnit( + sort_key=(primary_shard, spec_idx, e_start), + spec=spec, + slot=slot, + e_start=e_start, + e_end=e_end, + byte_weight=byte_weight, + ) + ) + else: + raise ValueError(f"Unknown conversion spec kind: {spec.kind!r}") + + if not work_units: + raise ValueError("No conversion work units generated.") + largest_work_unit_bytes = max(u.byte_weight for u in work_units) + if max_inflight_bytes < largest_work_unit_bytes: + raise ValueError( + f"max_inflight_bytes ({max_inflight_bytes} B) is smaller than the" + f" largest single work unit ({largest_work_unit_bytes} B);" + " deadlock would occur." + ) + + work_units.sort(key=lambda u: u.sort_key) + + # Phase 2: Open TensorStore arrays helper + tree_metadata_entries: dict[str, Any] = {} + + def _open_ts_array( + target_path: tuple[str | int, ...], + global_shape: tuple[int, ...], + chunk_shape: tuple[int, ...], + dt: np.dtype, + ) -> Any: + param_name = ".".join(str(k) for k in target_path) + write_spec = ocp._src.serialization.tensorstore_utils.ArrayWriteSpec( # pylint: disable=protected-access + str(write_dir), + param_name, + global_shape=global_shape, + write_shape=global_shape, + dtype=dt, + use_compression=True, + use_zarr3=False, + use_ocdbt=True, + process_id=0, + ) + tspec = dict(write_spec.json) + tspec["metadata"] = dict(tspec["metadata"]) + tspec["metadata"]["chunks"] = [int(d) for d in chunk_shape] + if "compressor" not in tspec["metadata"] or tspec["metadata"]["compressor"] not in ( + {"id": "zstd"}, + {"id": "zstd", "level": 1}, + ): + raise ValueError("Expected default Orbax zstd level 1 compressor in metadata:" f" {tspec['metadata']!r}") + tspec["metadata"]["compressor"] = {"id": "zstd", "level": 1} + store = ts.open( + ts.Spec(tspec), + create=True, + open=True, + context=ts_context, + ).result() + tuple_key_str = str(tuple(str(k) for k in target_path)) + tree_metadata_entries[tuple_key_str] = { + "key_metadata": [{"key": str(k), "key_type": 2} for k in target_path], + "value_metadata": { + "value_type": "np.ndarray", + "skip_deserialize": False, + }, + } + return store + + # Phase 3: Parallel worker execution + semaphore = _MemoryBudgetSemaphore(max_inflight_bytes) + consumed_lock = threading.Lock() + consumed: set[str] = set() + populated_slots_lock = threading.Lock() + populated_slots: set[tuple[tuple[str | int, ...], int]] = set() + ts_arrays_lock = threading.Lock() + abort_event = threading.Event() + + def _load_from_shard(orig_k: str) -> Any: + shard_file = active_weight_map[orig_k] + shard_path = os.path.join(hf_checkpoint_dir, shard_file) + return shard_lru.load_tensor(shard_path, orig_k, shard_file) + + def _get_tensor(hub_k: str, target_dtype: Any) -> np.ndarray: + local_consumed: set[str] = set() + val = _fetch_and_convert_tensor( + hub_k, + target_dtype, + orig_key_map, + local_consumed, + _load_from_shard, + ) + with consumed_lock: + consumed.update(local_consumed) + return val + + def _run_work_unit(unit: _ConversionWorkUnit) -> None: + if abort_event.is_set(): + return + semaphore.acquire(unit.byte_weight) + try: + if abort_event.is_set(): + return + spec = unit.spec + dt = np.dtype(spec.target_dtype) + slot = unit.slot + slot_key = (spec.target_path, slot) + target_var = resolve_target_var(abs_model, spec.target_path) + expected_shape = target_var.shape + expected_slot_shape = expected_shape[1:] if spec.stack_index is not None else expected_shape + + if spec.kind in ("direct", "transpose_2d"): + raw_v = _get_tensor(spec.hf_keys[0], spec.target_dtype) + slot_val = np.ascontiguousarray(raw_v if spec.kind == "direct" else raw_v.T) + del raw_v + if slot_val.shape != expected_slot_shape: + raise ValueError( + f"Shape mismatch for {spec.target_path} (slot {slot}): expected" + f" {expected_slot_shape}, got {slot_val.shape}." + ) + if spec.stack_index is None: + store = _open_ts_array(spec.target_path, expected_shape, slot_val.shape, dt) + store.write(slot_val).result() + del store + else: + with ts_arrays_lock: + if spec.target_path not in ts_arrays: + c_shape = (1, *slot_val.shape) + ts_arrays[spec.target_path] = ( + _open_ts_array(spec.target_path, expected_shape, c_shape, dt), + expected_shape, + ) + stacked_shapes[spec.target_path] = max(stacked_shapes[spec.target_path], expected_shape[0]) + store, _ = ts_arrays[spec.target_path] + store[slot].write(slot_val).result() + del slot_val + with populated_slots_lock: + if slot_key in populated_slots: + raise ValueError(f"Duplicate write to parameter slot {slot_key}.") + populated_slots.add(slot_key) + elif spec.kind in ("moe_gate_up", "moe_down"): + num_experts = len(spec.hf_keys) // 2 if spec.kind == "moe_gate_up" else len(spec.hf_keys) + expert_chunk = math.gcd(_MAX_MOE_EXPERT_CHUNK_SIZE, num_experts) if num_experts > 0 else 1 + g_shape = expected_shape + per_expert_shape = g_shape[1:] if spec.stack_index is None else g_shape[2:] + chunk_buf = _fill_moe_expert_chunk( + spec, + unit.e_start, + unit.e_end, + per_expert_shape, + dt, + _get_tensor, + ) + if spec.stack_index is None: + c_shape = (expert_chunk, *g_shape[1:]) + store = _open_ts_array(spec.target_path, g_shape, c_shape, dt) + store[unit.e_start : unit.e_end].write(chunk_buf).result() + del store + else: + with ts_arrays_lock: + if spec.target_path not in ts_arrays: + c_shape = (1, expert_chunk, *g_shape[2:]) + ts_arrays[spec.target_path] = ( + _open_ts_array(spec.target_path, g_shape, c_shape, dt), + g_shape, + ) + stacked_shapes[spec.target_path] = max(stacked_shapes[spec.target_path], expected_shape[0]) + store, _ = ts_arrays[spec.target_path] + store[slot, unit.e_start : unit.e_end].write(chunk_buf).result() + del chunk_buf + chunk_experts = set(range(unit.e_start, unit.e_end)) + with populated_slots_lock: + if not moe_populated_experts[slot_key].isdisjoint(chunk_experts): + raise ValueError(f"Duplicate write to parameter slot {slot_key}.") + moe_populated_experts[slot_key].update(chunk_experts) + if len(moe_populated_experts[slot_key]) == num_experts: + if slot_key in populated_slots: + raise ValueError(f"Duplicate write to parameter slot {slot_key}.") + populated_slots.add(slot_key) + else: + raise ValueError(f"Unknown conversion spec kind: {spec.kind!r}") + except BaseException: + abort_event.set() + raise + finally: + semaphore.release(unit.byte_weight) + + with futures.ThreadPoolExecutor(max_workers=num_workers) as executor: + unit_futures = [executor.submit(_run_work_unit, u) for u in work_units] + try: + for fut in futures.as_completed(unit_futures): + fut.result() + except BaseException: + abort_event.set() + executor.shutdown(wait=True, cancel_futures=True) + raise + + unfilled = [ + (k, s) for k, num_slots in stacked_shapes.items() for s in range(num_slots) if (k, s) not in populated_slots + ] + if unfilled: + raise ValueError(f"Unpopulated model parameter slot ({unfilled[0][1]}) for" f" {unfilled[0][0]}.") + + unconsumed = [ + k + for k in orig_key_map.values() + if k not in consumed + and not (ignore_mtp_layer and _normalize_key(k).startswith(f"layers.{glm5_config.MTP_LAYER_INDEX}.")) + ] + if unconsumed: + raise ValueError(f"Unmapped key in hf_tensors: {unconsumed[0]!r}") + + ts_arrays.clear() + shard_lru.close_all() + ocp.checkpoint_utils.asyncio_utils.run_sync( + ocp.type_handlers.merge_ocdbt_per_process_files( + write_dir, + ts_context=ts_context, + use_zarr3=False, + enable_validation=True, + ) + ) + metadata_content = { + "tree_metadata": tree_metadata_entries, + "use_ocdbt": True, + "use_zarr3": False, + "store_array_data_equal_to_fill_value": True, + "custom_metadata": None, + } + (write_dir / "_METADATA").write_text(json.dumps(metadata_content)) + ckpt_metadata_content = { + "item_handlers": _STANDARD_CHECKPOINT_HANDLER, + "metrics": {}, + "performance_metrics": {}, + "init_timestamp_nsecs": init_timestamp_nsecs, + "commit_timestamp_nsecs": time.time_ns(), + "custom_metadata": {}, + } + (write_dir / "_CHECKPOINT_METADATA").write_text(json.dumps(ckpt_metadata_content)) + if write_dir != final_path and final_path.exists(): + final_path.rmdir() + ocp.checkpoint_utils.asyncio_utils.run_sync(tmp_path.finalize()) + except BaseException: + shard_lru.close_all() + if write_dir is not None and write_dir.exists(): + write_dir.rmtree(missing_ok=True) + raise + finally: + shard_lru.close_all() + + +def _parse_bool(val: str, flag_name: str) -> bool: + """Parses explicit boolean flag values without silent defaults.""" + if not isinstance(val, str): + raise ValueError(f"Invalid boolean value {val!r} for --{flag_name}; expected 'true' or" " 'false'.") + lower = val.strip().lower() + if lower == "true": + return True + elif lower == "false": + return False + else: + raise ValueError(f"Invalid boolean value {val!r} for --{flag_name}; expected 'true' or" " 'false'.") + + +def build_arg_parser() -> argparse.ArgumentParser: + """Builds the argument parser for GLM-5.3-Flash HF-to-Orbax conversion.""" + parser = argparse.ArgumentParser(description="GLM-5.3-Flash HF-to-Orbax Conversion CLI.") + parser.add_argument( + "--hf_checkpoint_dir", + type=str, + required=True, + help=("Path to local directory containing HuggingFace Safetensors" " checkpoint shards."), + ) + parser.add_argument( + "--orbax_output_dir", + type=str, + required=True, + help="Destination path or gs:// URI for output Orbax checkpoint.", + ) + parser.add_argument( + "--weight_dtype", + type=str, + required=True, + help=("Target dtype for dequantized weights (e.g. bfloat16, float32," " float64)."), + ) + parser.add_argument( + "--ignore_mtp_layer", + type=str, + required=True, + help=("Explicit boolean ('true'/'false'): whether to ignore MTP NextN" " prediction layer."), + ) + parser.add_argument( + "--slice_to_config_layers", + type=str, + required=True, + help=("Explicit boolean ('true'/'false'): whether to slice hidden layers to" " num_hidden_layers."), + ) + parser.add_argument( + "--num_hidden_layers", + type=int, + required=True, + help="Number of hidden decoder layers to convert.", + ) + parser.add_argument( + "--num_nextn_predict_layers", + type=int, + required=True, + help="Number of NextN predict layers (0 or 1).", + ) + parser.add_argument( + "--config_json", + type=str, + required=True, + help=( + "Path to JSON file containing Glm5Config field overrides (rather than" + " an HF config.json), or 'none' to use default config fields." + ), + ) + parser.add_argument( + "--num_workers", + type=int, + required=True, + help=("Number of concurrent worker threads for streaming conversion (must" " be >= 1)."), + ) + parser.add_argument( + "--max_inflight_bytes", + type=str, + required=True, + help=("Memory budget for concurrent work units; accepts positive byte count" " or unit suffix."), + ) + return parser + + +def create_glm5_config_from_cli_args( + args: argparse.Namespace, +) -> glm5_config.Glm5Config: + """Validates parsed CLI arguments and constructs a Glm5Config instance.""" + if args.num_hidden_layers < 1: + raise ValueError(f"--num_hidden_layers must be >= 1, got {args.num_hidden_layers}") + if args.num_nextn_predict_layers not in (0, 1): + raise ValueError("--num_nextn_predict_layers must be 0 or 1, got" f" {args.num_nextn_predict_layers}") + if args.weight_dtype not in _SUPPORTED_CLI_DTYPES: + raise ValueError( + f"Unsupported --weight_dtype {args.weight_dtype!r}; expected one of" f" {sorted(_SUPPORTED_CLI_DTYPES)}" + ) + + if args.config_json == "none": + return glm5_config.Glm5Config( + num_hidden_layers=args.num_hidden_layers, + num_nextn_predict_layers=args.num_nextn_predict_layers, + weight_dtype=args.weight_dtype, + ) + + if "://" in args.config_json: + raise ValueError( + f"Unsupported URI scheme in --config_json: {args.config_json!r};" " expected a local filesystem path or 'none'." + ) + if not os.path.isfile(args.config_json): + raise FileNotFoundError(f"config_json file not found: {args.config_json!r}") + + with open(args.config_json, "r", encoding="utf-8") as f: + raw_overrides = json.load(f) + + if not isinstance(raw_overrides, dict) or len(raw_overrides) == 0: + raise ValueError("config_json must contain a non-empty JSON object.") + + overlap = set(raw_overrides.keys()) & _CLI_OWNED_CONFIG_KEYS + if overlap: + raise ValueError(f"config_json must not specify CLI-owned keys {sorted(overlap)};" " pass them via CLI flags only.") + unknown = set(raw_overrides.keys()) - _ALLOWED_CONFIG_JSON_KEYS + if unknown: + raise ValueError(f"Unknown Glm5Config keys in config_json: {sorted(unknown)}") + + overrides = dict(raw_overrides) + for tk in _TUPLE_CONFIG_KEYS: + if tk in overrides: + if not isinstance(overrides[tk], list): + raise TypeError(f"Expected JSON list for {tk!r}, got {type(overrides[tk])}") + overrides[tk] = tuple(overrides[tk]) + + return glm5_config.Glm5Config( + **overrides, + num_hidden_layers=args.num_hidden_layers, + num_nextn_predict_layers=args.num_nextn_predict_layers, + weight_dtype=args.weight_dtype, + ) + + +def run_cli(argv: Sequence[str]) -> None: + """Parses explicit CLI arguments and executes streaming HF-to-Orbax conversion.""" + parser = build_arg_parser() + args = parser.parse_args(list(argv)) + hf_dir = args.hf_checkpoint_dir.strip() + if not hf_dir: + raise ValueError("--hf_checkpoint_dir must be a non-empty path.") + if "://" in hf_dir: + raise ValueError( + "Unsupported URI scheme in --hf_checkpoint_dir:" f" {args.hf_checkpoint_dir!r}; expected a local filesystem path." + ) + if not os.path.isdir(hf_dir): + raise FileNotFoundError(f"HF checkpoint directory not found: {hf_dir!r}") + orbax_out = args.orbax_output_dir.strip() + if not orbax_out: + raise ValueError("--orbax_output_dir must be a non-empty path.") + if "://" in orbax_out and not orbax_out.startswith("gs://"): + raise ValueError( + "Unsupported URI scheme in --orbax_output_dir:" + f" {args.orbax_output_dir!r}; expected a local path or 'gs://' URI." + ) + out_path = epath.Path(orbax_out) + if out_path.exists() and (not out_path.is_dir() or any(out_path.iterdir())): + raise FileExistsError("Refusing to write into existing non-empty --orbax_output_dir:" f" {orbax_out!r}") + ignore_mtp_layer = _parse_bool(args.ignore_mtp_layer, "ignore_mtp_layer") + slice_to_config_layers = _parse_bool(args.slice_to_config_layers, "slice_to_config_layers") + config = create_glm5_config_from_cli_args(args) + if ignore_mtp_layer and config.num_nextn_predict_layers > 0: + raise ValueError("ignore_mtp_layer=True cannot be used when" " config.num_nextn_predict_layers > 0") + if not ignore_mtp_layer and config.num_nextn_predict_layers == 0: + raise ValueError("ignore_mtp_layer=False requires config.num_nextn_predict_layers > 0") + num_workers = int(args.num_workers) + if num_workers < 1: + raise ValueError(f"--num_workers must be >= 1, got {args.num_workers}") + max_inflight_bytes = _parse_byte_size(args.max_inflight_bytes, "max_inflight_bytes") + convert_hf_checkpoint_to_orbax( + hf_dir, + orbax_out, + config, + ignore_mtp_layer=ignore_mtp_layer, + slice_to_config_layers=slice_to_config_layers, + num_workers=num_workers, + max_inflight_bytes=max_inflight_bytes, + ) + + +def main() -> None: + """CLI entrypoint reading sys.argv[1:] with zero parameter defaults.""" + run_cli(sys.argv[1:]) diff --git a/src/maxtext/m3/models/glm5_3_flash/modeling_glm5_3_flash.py b/src/maxtext/m3/models/glm5_3_flash/modeling_glm5_3_flash.py index 159b944837..ffb69f4bc9 100644 --- a/src/maxtext/m3/models/glm5_3_flash/modeling_glm5_3_flash.py +++ b/src/maxtext/m3/models/glm5_3_flash/modeling_glm5_3_flash.py @@ -27,12 +27,32 @@ from jax.typing import DTypeLike from maxtext.m3.core import sharding as m3_sharding from maxtext.m3.configs import glm5_3_flash_config as glm5_config -from maxtext.m3.configs.glm5_3_flash_config import Glm5Config -from maxtext.m3.configs.glm5_3_flash_config import Glm5Sharding import optax from tokamax._src.ops.kda import api as tokamax_kda from tokamax._src.ops.kda import pallas_mosaic_tpu as _tokamax_kda_mosaic # pylint: disable=unused-import # buildcleaner: keep + +def _make_lin(dt: DTypeLike, p_dt: DTypeLike, sharding, rngs: nnx.Rngs): + """Constructs a Linear layer factory closure with sharding metadata.""" + + def _lin(in_f: int, out_f: int, name: str, axes: tuple[str | None, ...]): + km = {"sharding": sharding(name, axes)} if sharding else {} + return nnx.Linear( + in_f, + out_f, + use_bias=False, + dtype=dt, + param_dtype=p_dt, + # Precision.HIGHEST is required on TPU so MXU matmuls match IEEE fp32 + # accumulation and avoid discrete top_k tie-breaking drift vs PyTorch. + precision=jax.lax.Precision.HIGHEST, + kernel_metadata=km, + rngs=rngs, + ) + + return _lin + + # ============================================================================ # Section 1: Linear Attention Primitives (KDA) # ============================================================================ @@ -174,7 +194,7 @@ class Glm5KdaAttention(nnx.Module): def __init__( self, - config: Glm5Config, + config: glm5_config.Glm5Config, *, sharding: m3_sharding.Sharding | None = None, rngs: nnx.Rngs, @@ -199,18 +219,7 @@ def __init__( self.dtype = getattr(jnp, config.dtype) self.param_dtype = getattr(jnp, config.weight_dtype) - def _lin(in_f, out_f, name, axes): - km = {"sharding": sharding(name, axes)} if sharding else {} - return nnx.Linear( - in_features=in_f, - out_features=out_f, - use_bias=False, - dtype=self.dtype, - param_dtype=self.param_dtype, - precision=jax.lax.Precision.HIGHEST, - kernel_metadata=km, - rngs=rngs, - ) + _lin = _make_lin(self.dtype, self.param_dtype, sharding, rngs) def _cv(dim): return Glm5ShortConv1d( @@ -408,7 +417,7 @@ def __init__( ): self.eps = eps spec = sharding("norm", ("norm",)) if sharding else None - self.weight = nnx.Param(jnp.ones((d,), dtype=param_dtype), sharding=spec) + self.weight = nnx.Param(jnp.ones((d,), dtype=param_dtype), out_sharding=spec) def __call__(self, x: jax.Array) -> jax.Array: in_dt, x32 = x.dtype, x.astype(jnp.float32) @@ -430,8 +439,8 @@ def __init__( ): self.eps = eps spec = sharding("norm", ("norm",)) if sharding else None - self.weight = nnx.Param(jnp.ones((d,), dtype=param_dtype), sharding=spec) - self.bias = nnx.Param(jnp.zeros((d,), dtype=param_dtype), sharding=spec) + self.weight = nnx.Param(jnp.ones((d,), dtype=param_dtype), out_sharding=spec) + self.bias = nnx.Param(jnp.zeros((d,), dtype=param_dtype), out_sharding=spec) def __call__(self, x: jax.Array) -> jax.Array: in_dt, x32 = x.dtype, x.astype(jnp.float32) @@ -443,33 +452,12 @@ def __call__(self, x: jax.Array) -> jax.Array: return (normed * w + b).astype(in_dt) -def _make_lin(dt: DTypeLike, p_dt: DTypeLike, sharding, rngs: nnx.Rngs): - """Constructs a Linear layer factory closure with sharding metadata.""" - - def _lin(in_f: int, out_f: int, name: str, axes: tuple[str | None, ...]): - km = {"sharding": sharding(name, axes)} if sharding else {} - return nnx.Linear( - in_f, - out_f, - use_bias=False, - dtype=dt, - param_dtype=p_dt, - # Precision.HIGHEST is required on TPU so MXU matmuls match IEEE fp32 - # accumulation and avoid discrete top_k tie-breaking drift vs PyTorch. - precision=jax.lax.Precision.HIGHEST, - kernel_metadata=km, - rngs=rngs, - ) - - return _lin - - class Glm5Indexer(nnx.Module): """DeepSeek Sparse Attention (DSA) indexer with k-pool compression.""" def __init__( self, - config: Glm5Config, + config: glm5_config.Glm5Config, *, sharding: m3_sharding.Sharding | None = None, rngs: nnx.Rngs, @@ -499,13 +487,13 @@ def __init__( s_ape = (None, None) ape_spec = sharding("index_kpool_compress_ape", s_ape) if sharding else None ape_init = jnp.zeros((self.index_kpool, d), dtype=p_dt) - self.index_kpool_compress_ape = nnx.Param(ape_init, sharding=ape_spec) + self.index_kpool_compress_ape = nnx.Param(ape_init, out_sharding=ape_spec) s_gate = (None, "embed") g_name = "index_kpool_compress_gate" gate_spec = sharding(g_name, s_gate) if sharding else None gate_init = jnp.ones((d, h), dtype=p_dt) - self.index_kpool_compress_gate = nnx.Param(gate_init, sharding=gate_spec) + self.index_kpool_compress_gate = nnx.Param(gate_init, out_sharding=gate_spec) def __call__( self, @@ -615,7 +603,7 @@ class Glm5DsaMlaAttention(nnx.Module): def __init__( self, - config: Glm5Config, + config: glm5_config.Glm5Config, layer_idx: int, *, indexer_type: str, @@ -802,7 +790,7 @@ def __call__( mask = self._build_causal_padding_mask(attn_mask, b, s, q.dtype) topk_idx = ( self.indexer(hidden_states, q_resid, attention_mask=attn_mask) - if (return_topk_indices and self.indexer is not None) + if (return_topk_indices and self.config.use_indexer and self.indexer is not None) else None ) else: @@ -842,7 +830,7 @@ class Glm5MLP(nnx.Module): def __init__( self, - config: Glm5Config, + config: glm5_config.Glm5Config, intermediate_size: int, *, rngs: nnx.Rngs, @@ -855,18 +843,7 @@ def __init__( dt, p_dt = getattr(jnp, config.dtype), getattr(jnp, config.weight_dtype) - def _lin(in_f, out_f, name, axes): - km = {"sharding": sharding(name, axes)} if sharding else {} - return nnx.Linear( - in_f, - out_f, - use_bias=False, - dtype=dt, - param_dtype=p_dt, - precision=jax.lax.Precision.HIGHEST, - kernel_metadata=km, - rngs=rngs, - ) + _lin = _make_lin(dt, p_dt, sharding, rngs) self.gate_proj = _lin(h, inter, "gate_proj", ("embed", "mlp")) self.up_proj = _lin(h, inter, "up_proj", ("embed", "mlp")) @@ -883,7 +860,7 @@ class Glm5TopkRouter(nnx.Module): def __init__( self, - config: Glm5Config, + config: glm5_config.Glm5Config, *, rngs: nnx.Rngs, sharding: m3_sharding.Sharding | None = None, @@ -905,7 +882,7 @@ def __init__( w_spec = sharding("router_weight", ("embed", None)) if sharding else None b_spec = sharding("router_bias", (None,)) if sharding else None w_init = nnx.initializers.normal(0.02)(rngs.params(), (h, e), r_p_dt) - self.weight = nnx.Param(w_init, sharding=w_spec) + self.weight = nnx.Param(w_init, out_sharding=w_spec) self.e_score_correction_bias = Glm5RouterBias( jnp.zeros((e,), dtype=r_p_dt), sharding=b_spec, @@ -959,7 +936,7 @@ class Glm5Experts(nnx.Module): def __init__( self, - config: Glm5Config, + config: glm5_config.Glm5Config, *, rngs: nnx.Rngs, sharding: m3_sharding.Sharding | None = None, @@ -974,8 +951,8 @@ def __init__( self.num_experts, self.top_k = e, config.num_experts_per_tok self.swiglu_limit = config.mlp_clamp_limit self.sharding = sharding - self.ep_axis = sharding.ep_axis if isinstance(sharding, Glm5Sharding) else "expert" - if isinstance(sharding, Glm5Sharding) and sharding.ep_size > 1: + self.ep_axis = sharding.ep_axis if isinstance(sharding, glm5_config.Glm5Sharding) else "expert" + if isinstance(sharding, glm5_config.Glm5Sharding) and sharding.ep_size > 1: raise NotImplementedError( f"Glm5Experts expert parallelism ({self.ep_axis}={sharding.ep_size} >" " 1) requires shard_map EP dispatch (follow-up F34)." @@ -998,7 +975,7 @@ def __init__( self.gate_up_proj = nnx.Param(gu_init, sharding=gu_spec) d_init = nnx.initializers.normal(0.02)(rngs.params(), (e, inter, h), p_dt) - self.down_proj = nnx.Param(d_init, sharding=d_spec) + self.down_proj = nnx.Param(d_init, out_sharding=d_spec) def __call__( self, @@ -1006,7 +983,7 @@ def __call__( topk_indices: jax.Array, topk_weights: jax.Array, ) -> jax.Array: - if isinstance(self.sharding, Glm5Sharding) and self.sharding.ep_size > 1: + if isinstance(self.sharding, glm5_config.Glm5Sharding) and self.sharding.ep_size > 1: raise NotImplementedError( f"Glm5Experts.__call__ with {self.ep_axis} > 1 requires shard_map EP" f" dispatch (follow-up F34); got ep_size={self.sharding.ep_size}." @@ -1073,14 +1050,14 @@ class Glm5MoE(nnx.Module): def __init__( self, - config: Glm5Config, + config: glm5_config.Glm5Config, *, rngs: nnx.Rngs, sharding: m3_sharding.Sharding | None = None, ): self.config = config - self.ep_axis = sharding.ep_axis if isinstance(sharding, Glm5Sharding) else "expert" - if isinstance(sharding, Glm5Sharding) and sharding.ep_size > 1: + self.ep_axis = sharding.ep_axis if isinstance(sharding, glm5_config.Glm5Sharding) else "expert" + if isinstance(sharding, glm5_config.Glm5Sharding) and sharding.ep_size > 1: raise NotImplementedError( f"Glm5MoE expert parallelism ({self.ep_axis}={sharding.ep_size} > 1)" " requires all-to-all dispatch (planned for G8); single-host bringup" @@ -1250,15 +1227,15 @@ def __init__( fn_init = nnx.initializers.normal(0.02)(rngs.params(), (self.hc_mult * config.hidden_size, mix), p_dt) self.fn = nnx.Param( fn_init, - sharding=sharding("mhc_fn", ("embed", None)) if sharding else None, + out_sharding=sharding("mhc_fn", ("embed", None)) if sharding else None, ) self.base = nnx.Param( jnp.zeros((mix,), dtype=p_dt), - sharding=sharding("mhc_base", (None,)) if sharding else None, + out_sharding=sharding("mhc_base", (None,)) if sharding else None, ) self.scale = nnx.Param( jnp.ones((3,), dtype=p_dt), - sharding=sharding("mhc_scale", (None,)) if sharding else None, + out_sharding=sharding("mhc_scale", (None,)) if sharding else None, ) def __call__(self, hidden_streams: jax.Array) -> tuple[jax.Array, jax.Array, jax.Array]: @@ -2120,7 +2097,7 @@ def _run_layers( decoder_segment_ids=decoder_segment_ids, return_aux=return_aux, ) - if idx_topk is not None: + if idx_topk is not None and cfg.use_indexer: dsa_topk_indices_list.append(idx_topk) if layer_0_act is None: layer_0_act = streams @@ -2147,7 +2124,9 @@ def _run_layers( layer_dict["router_indices"] = r_topk layer_dict["routing_weights"] = r_weights layer_dict["router_probs"] = r_probs - if cfg.use_indexer and idx_topk is not None: + if layer.layer_type == "full_attention" and cfg.use_indexer: + if idx_topk is None: + raise RuntimeError(f"Layer {layer.layer_idx} has full_attention but idx is None.") layer_dict["selected_block_indices"] = idx_topk layer_aux_list.append(layer_dict) @@ -2182,7 +2161,7 @@ def _run_layers( return_aux=return_aux, ) - if sb_idx is not None: + if sb_idx is not None and cfg.use_indexer: for block_idx in sb_idx: if block_idx is None: continue diff --git a/tests/post_training/unit/rl_logical_axis_rules_test.py b/tests/post_training/unit/rl_logical_axis_rules_test.py new file mode 100644 index 0000000000..7fcc855024 --- /dev/null +++ b/tests/post_training/unit/rl_logical_axis_rules_test.py @@ -0,0 +1,108 @@ +# Copyright 2023–2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""rl.yml needs a logical-axis rule for every axis of the RL actor's parameters (CPU-only). + +rl.yml replaces base.yml's `logical_axis_rules` instead of extending it. MaxText places parameters +with its own resolver, which leaves an axis without a rule unsharded, so a missing rule goes unnoticed +until Tunix shards the trainer state: `PeftTrainer._shard_optimizer` derives the shardings of the +optimizer state and of the gradient accumulator from `nnx.get_partition_spec` under these rules. That +keeps any name it has no rule for, and `tunix.sft.sharding_utils.get_sharding` then replicates every +buffer whose PartitionSpec names an axis that is not on the mesh. With `local_layers` missing, that +replicated the float32 gradients of all of Qwen3.5-35B-A3B's nested-scanned linear-attention layers +(94 GiB) and ran the Pathways head out of host memory before the first train step. +""" + +import sys +import unittest + +import pytest +from flax import nnx +from flax.linen import partitioning as nn_partitioning + +from maxtext.configs import pyconfig, types +from maxtext.utils import maxtext_utils, model_creation_utils +from tests.utils.test_helpers import get_test_config_path + +pytestmark = [pytest.mark.post_training] + +# A toy Qwen3.5: two cycles of three Gated DeltaNet layers and one full-attention layer. With +# scan_layers=True the linear-attention layers of each block are stacked in a nested +# `local_layers` scan inside the `layers` block scan (the layout Qwen3-Next and Gemma4 share). +_QWEN3_5_OVERRIDES = { + "model_name": "qwen3.5-35b-a3b", + "override_model_config": True, + "scan_layers": True, + "enable_checkpointing": False, + "max_target_length": 8, + "base_emb_dim": 64, + "base_num_decoder_layers": 8, + "base_num_query_heads": 2, + "base_num_kv_heads": 2, + "head_dim": 32, + "base_mlp_dim": 128, + "base_moe_mlp_dim": 32, + "num_experts": 4, + "num_experts_per_tok": 2, + "vocab_size": 32, + "gdn_num_key_heads": 2, + "gdn_num_value_heads": 4, + "gdn_key_head_dim": 16, + "gdn_value_head_dim": 16, + "gdn_chunk_size": 4, + # MRoPE sections must cover rotary_dim / 2, which is 4 here, not the shipped [11, 11, 10]. + "mrope_section": [2, 1, 1], + "sparse_matmul": True, + "megablox": False, + "attention": "dot_product", +} + + +class RLLogicalAxisRulesTest(unittest.TestCase): + """Every axis the RL actor's params are annotated with must resolve to mesh axes under rl.yml's rules.""" + + @pytest.mark.cpu_only + def test_scanned_qwen3_5_param_axes_resolve_to_mesh_axes(self): + config = pyconfig.initialize_pydantic( + [sys.argv[0], get_test_config_path("post_train/rl.yml")], + config_class=types.RLConfig, + run_name="rl_logical_axis_rules_test", + tokenizer_path="Qwen/Qwen3.5-35B-A3B", + **_QWEN3_5_OVERRIDES, + ) + mesh = maxtext_utils.get_mesh_from_config(config) + _, abstract_model = model_creation_utils.create_nnx_abstract_model(config, mesh) + self.assertIn("local_layers", nnx.state(abstract_model, nnx.Param)["decoder"]["layers"]) + + # Tunix's RLCluster runs the actor trainer under these rules, as `nn_partitioning.axis_rules`. + with nn_partitioning.axis_rules(config.logical_axis_rules): + pspecs = nnx.get_partition_spec(nnx.state(abstract_model, nnx.Param)) + + unresolved = {} + for path, spec in nnx.to_flat_state(pspecs): + spec = spec.get_value() if isinstance(spec, nnx.Variable) else spec + for entry in spec or (): + for name in entry if isinstance(entry, tuple) else (entry,): + if name is not None and name not in mesh.axis_names: + unresolved.setdefault(name, "/".join(map(str, path))) + self.assertEqual( + unresolved, + {}, + "rl.yml has no logical_axis_rules entry for these axes (first parameter using each is shown); " + "Tunix would replicate the optimizer state and gradients of every such parameter on all devices.", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/post_training/unit/vllm_rollout_unroll_test.py b/tests/post_training/unit/vllm_rollout_unroll_test.py index 2142cc8cb0..cc558f2915 100644 --- a/tests/post_training/unit/vllm_rollout_unroll_test.py +++ b/tests/post_training/unit/vllm_rollout_unroll_test.py @@ -18,6 +18,7 @@ from types import SimpleNamespace import unittest from unittest import mock +from flax.traverse_util import unflatten_dict import numpy as np import pytest @@ -34,7 +35,7 @@ ) from maxtext.integration.vllm.weight_converter import WeightConverter -from tunix.generate.vllm_sampler import VllmConfig +from tunix.generate.vllm_sampler import VllmConfig, VllmSampler from tunix.rl.rollout import base_rollout pytestmark = pytest.mark.post_training @@ -446,6 +447,81 @@ def test_accepts_split_moe_weights_for_prefused_target(self): self.assertEqual(validate_direct_sync_layer_coverage(source, target), 1) +class ScannedSourceLayerCoverageTest(unittest.TestCase): + """Verify the coverage check accepts scanned sources that Tunix unrolls itself. + + Only Gemma and Qwen are unrolled by MaxText before the direct sync; other + scanned models (e.g. Llama) keep `decoder.layers.*` stacked on the scan axis + and Tunix's `transfer_state_directly` slices layer N out for `layers_N`. + """ + + _PER_LAYER_PARAMS = ( + ("mlp", "wi_0", "kernel"), + ("mlp", "wo", "kernel"), + ("pre_self_attention_layer_norm", "scale"), + ("self_attention", "query", "kernel"), + ) + + def _scanned_llama_trees(self, num_layers, drop_from_source=()): + """Builds a Llama-style scanned source and the matching unscanned rollout target.""" + source_flat = {("base", "decoder", "decoder_norm", "scale"): np.ones((2,))} + target_flat = {("model", "decoder", "decoder_norm", "scale"): np.zeros((2,))} + for param in self._PER_LAYER_PARAMS: + if param not in drop_from_source: + # param_scan_axis=1: the layer axis sits after the leading feature dim. + source_flat[("base", "decoder", "layers") + param] = np.ones((2, num_layers, 3)) + for i in range(num_layers): + target_flat[("model", "decoder", f"layers_{i}") + param] = np.zeros((2, 3)) + return unflatten_dict(source_flat), unflatten_dict(target_flat) + + @pytest.mark.cpu_only + def test_accepts_homogeneous_scanned_source(self): + source, target = self._scanned_llama_trees(num_layers=3) + + self.assertEqual(validate_direct_sync_layer_coverage(source, target), 3 * len(self._PER_LAYER_PARAMS)) + + @pytest.mark.cpu_only + def test_rejects_scanned_source_missing_a_target_parameter(self): + source, target = self._scanned_llama_trees(num_layers=2, drop_from_source=(("mlp", "wo", "kernel"),)) + + with self.assertRaisesRegex(ValueError, r"matched 6/8 target layer parameters.*decoder\.layers_0\.mlp\.wo\.kernel"): + validate_direct_sync_layer_coverage(source, target) + + @pytest.mark.cpu_only + def test_accepts_implicitly_stacked_source(self): + source = {"base": {"decoder": {"probe": np.ones((2, 2, 3))}}} + target = {"model": {"decoder": {"layers_0": {"probe": np.zeros((2, 3))}, "layers_1": {"probe": np.zeros((2, 3))}}}} + + self.assertEqual(validate_direct_sync_layer_coverage(source, target), 2) + + @pytest.mark.cpu_only + def test_accepts_scanned_split_moe_weights_for_prefused_target(self): + experts = {"wi_0": np.ones((4, 2, 3, 5)), "wi_1": np.ones((4, 2, 3, 5))} + source = {"base": {"decoder": {"layers": {"mlp": {"routed_experts": experts}}}}} + target = { + "model": {"decoder": {f"layers_{i}": {"mlp": {"routed_experts": {"wi": np.zeros((4, 3, 10))}}} for i in range(2)}} + } + + self.assertEqual(validate_direct_sync_layer_coverage(source, target), 2) + + @pytest.mark.cpu_only + def test_sampler_leaves_scanned_llama_weights_for_tunix(self): + """Regression: llama3.1-70b direct sync failed with `matched 0/720 target layer parameters`.""" + sampler = object.__new__(MaxTextVllmSampler) + sampler._direct_maxtext_sync = True # pylint: disable=protected-access + sampler._is_gemma = False # pylint: disable=protected-access + sampler._is_qwen = False # pylint: disable=protected-access + source, target = self._scanned_llama_trees(num_layers=2) + + with ( + mock.patch.object(MaxTextVllmSampler, "transformer_state", new_callable=mock.PropertyMock, return_value=target), + mock.patch.object(VllmSampler, "update_params") as tunix_update_params, + ): + sampler.update_params(source, filter_types=None) + + tunix_update_params.assert_called_once_with(source, None) + + class DirectSyncRolloutConfigTest(unittest.TestCase): """Verify TP-sharded MoE rollout targets request the safe fused layout.""" diff --git a/tests/unit/fsdp_prefetch_test.py b/tests/unit/fsdp_prefetch_test.py index 23f1f43aa9..eba00d289c 100644 --- a/tests/unit/fsdp_prefetch_test.py +++ b/tests/unit/fsdp_prefetch_test.py @@ -142,8 +142,9 @@ def test_explicit_mode_gathers_fsdp_axis_with_tagged_all_gather(self): mesh = _abstract_mesh((4, 2), ("fsdp", "tensor")) out, gathers = self._lower_gather(mesh, PartitionSpec("fsdp", "tensor"), (8, 6)) self.assertEqual(out.shape, (8, 6)) - self.assertNotIn("fsdp", str(out.sharding.spec)) - self.assertIn("tensor", str(out.sharding.spec)) # Non-FSDP axes stay sharded. + self.assertEqual(tuple(out.sharding.spec.partitions), (None, "tensor")) # Non-FSDP axes stay sharded. + # Typed `reduced` over the gathered axis, so its gradient is unreduced and reduced once by the tagged RS. + self.assertEqual(out.sharding.spec.reduced, frozenset({"fsdp"})) self.assertEqual(len(gathers), 1) # One all-gather over the 4-way 'fsdp' axis of dim 0; the 2-way 'tensor' shard of dim 1 is kept. self.assertIn("all_gather_dim = 0", gathers[0]) @@ -157,7 +158,8 @@ def test_explicit_mode_gathers_multiple_fsdp_axes_of_one_dim(self): mesh = _abstract_mesh((2, 2, 2), ("tensor", "fsdp", "fsdp_transpose")) out, gathers = self._lower_gather(mesh, PartitionSpec(None, ("tensor", "fsdp", "fsdp_transpose")), (3, 16)) self.assertEqual(out.shape, (3, 16)) - self.assertNotIn("fsdp", str(out.sharding.spec)) + self.assertEqual(tuple(out.sharding.spec.partitions), (None, "tensor")) + self.assertEqual(out.sharding.spec.reduced, frozenset({"fsdp", "fsdp_transpose"})) self.assertEqual(len(gathers), 1) self.assertIn("all_gather_dim = 1", gathers[0]) self.assertIn('name = "fsdp"', gathers[0]) @@ -170,7 +172,7 @@ def test_explicit_mode_gathers_co_sharded_expert_axis_with_fsdp(self): # Non-routed weights (attention, dense MLP, shared experts) are sharded over ('fsdp', 'expert'). out, gathers = self._lower_gather(mesh, PartitionSpec(("fsdp", "expert"), None), (16, 6)) self.assertEqual(out.shape, (16, 6)) - self.assertEqual(tuple(out.sharding.spec), (None, None)) + self.assertEqual(tuple(out.sharding.spec.partitions), (None, None)) self.assertEqual(len(gathers), 1) self.assertIn("all_gather_dim = 0", gathers[0]) self.assertIn('name = "fsdp"', gathers[0]) @@ -183,7 +185,7 @@ def test_explicit_mode_preserves_routed_expert_axis(self): # MoE routed expert weights have 'expert' on dim 0 and 'fsdp' on dim 1; only 'fsdp' is gathered. out, gathers = self._lower_gather(mesh, PartitionSpec("expert", "fsdp", None), (4, 16, 6)) self.assertEqual(out.shape, (4, 16, 6)) - self.assertEqual(tuple(out.sharding.spec), ("expert", None, None)) + self.assertEqual(tuple(out.sharding.spec.partitions), ("expert", None, None)) self.assertEqual(len(gathers), 1) self.assertIn("all_gather_dim = 1", gathers[0]) self.assertIn('name = "fsdp"', gathers[0]) @@ -196,7 +198,7 @@ def test_explicit_mode_preserves_leading_expert_axis_on_compound_dim(self): # Compound ('expert', 'fsdp') on the expert dim (e.g. moe_flat_fsdp_weights) keeps 'expert' sharded. out, gathers = self._lower_gather(mesh, PartitionSpec(("expert", "fsdp"), None), (16, 6)) self.assertEqual(out.shape, (16, 6)) - self.assertEqual(tuple(out.sharding.spec), ("expert", None)) + self.assertEqual(tuple(out.sharding.spec.partitions), ("expert", None)) self.assertEqual(len(gathers), 1) self.assertIn("all_gather_dim = 0", gathers[0]) self.assertIn('name = "fsdp"', gathers[0]) @@ -220,5 +222,131 @@ def test_leaf_without_named_sharding_is_left_alone(self): self.assertIsNone(fsdp_prefetch._per_layer_sharding(leaf, mesh)) # pylint: disable=protected-access +class FsdpPrefetchReduceScatterTest(unittest.TestCase): + """Lowers the tagged dW reduce-scatter against abstract multi-device meshes (no devices needed).""" + + def _lower_scatter(self, mesh, src_spec, shape, unreduced=True): + """Returns (output ShapeDtypeStruct, reduce_scatter StableHLO lines) for a gradient of a `src_spec` weight. + + With `unreduced`, the gradient is typed as the partial sums of a `reduced` gathered weight. + """ + gathered = sharding.remove_fsdp_from_sharding(NamedSharding(mesh, src_spec)).spec + fsdp_axes = frozenset(a for a in mesh.axis_names if a.startswith("fsdp") and mesh.shape[a] > 1) + g_spec = PartitionSpec(*gathered, unreduced=fsdp_axes) if unreduced else gathered + g = jax.ShapeDtypeStruct(shape, jnp.float32, sharding=NamedSharding(mesh, g_spec)) + + def scatter(x): + out = fsdp_prefetch._explicit_fsdp_reduce_scatter(x, mesh, src_spec, gathered) # pylint: disable=protected-access + return x if out is None else out + + with jax.sharding.use_abstract_mesh(mesh): + out = jax.eval_shape(scatter, g) + hlo = jax.jit(scatter).trace(g).lower(lowering_platforms=("tpu",)).as_text().splitlines() + # A reduce_scatter op spans its reduction region; its attributes are on the line closing it (`}) {...}`). + ops = [] + for i, line in enumerate(hlo): + if "stablehlo.reduce_scatter" in line: + end = next(j for j in range(i, len(hlo)) if hlo[j].lstrip().startswith("})")) + ops.append(" ".join(hlo[i : end + 1])) + return out, ops + + def test_scatters_fsdp_axis_with_tagged_reduce_scatter(self): + mesh = _abstract_mesh((4, 2), ("fsdp", "tensor")) + out, scatters = self._lower_scatter(mesh, PartitionSpec("fsdp", "tensor"), (8, 6)) + self.assertEqual(out.shape, (8, 6)) + self.assertEqual(tuple(out.sharding.spec.partitions), ("fsdp", "tensor")) + self.assertEqual(out.sharding.spec.unreduced, frozenset()) + self.assertEqual(len(scatters), 1) + self.assertIn("scatter_dimension = 0", scatters[0]) + self.assertIn('xla_explicit_fsdp = "yes"', scatters[0]) + self.assertIn("(tensor<8x3xf32>) -> tensor<2x3xf32>", scatters[0]) + + def test_flat_expert_weight_scatters_only_fsdp_of_compound_dim(self): + mesh = _abstract_mesh((2, 4), ("expert", "fsdp")) + # moe_flat_fsdp_weights: dim 0 is sharded over ('expert', 'fsdp'); only 'fsdp' is reduce-scattered. + out, scatters = self._lower_scatter(mesh, PartitionSpec(("expert", "fsdp"), None), (16, 6)) + self.assertEqual(tuple(out.sharding.spec.partitions), (("expert", "fsdp"), None)) + self.assertEqual(len(scatters), 1) + self.assertIn("scatter_dimension = 0", scatters[0]) + self.assertIn("(tensor<8x6xf32>) -> tensor<2x6xf32>", scatters[0]) + self.assertIn('xla_explicit_fsdp = "yes"', scatters[0]) + + def test_already_reduced_gradient_falls_back_to_resharding(self): + """A summed (not unreduced) gradient must not be reduce-scattered again: that would scale it by the FSDP size.""" + mesh = _abstract_mesh((4, 2), ("fsdp", "tensor")) + gathered = PartitionSpec(None, "tensor") + g = jax.ShapeDtypeStruct((8, 6), jnp.float32, sharding=NamedSharding(mesh, gathered)) + with jax.sharding.use_abstract_mesh(mesh): + out = jax.eval_shape( + lambda x: fsdp_prefetch._explicit_fsdp_reduce_scatter( # pylint: disable=protected-access + x, mesh, PartitionSpec("fsdp", "tensor"), gathered + ), + g, + ) + self.assertIsNone(out) + + def test_two_scattered_dims_fall_back_to_resharding(self): + mesh = _abstract_mesh((2, 2), ("fsdp", "fsdp_transpose")) + out = fsdp_prefetch._explicit_fsdp_reduce_scatter( # pylint: disable=protected-access + jnp.ones((4, 4)), mesh, PartitionSpec("fsdp", "fsdp_transpose"), PartitionSpec(None, None) + ) + self.assertIsNone(out) + + +class FsdpPrefetchReducedGradTest(unittest.TestCase): + """Gather -> layer -> tagged reduce-scatter on CPU devices: the FSDP reduction happens exactly once.""" + + def test_gradient_matches_reference_with_single_tagged_reduce_scatter(self): + n = jax.device_count() + if n < 2: + self.skipTest("Needs at least 2 devices.") + mesh = jax.make_mesh((2,), ("fsdp",), axis_types=(AxisType.Explicit,), devices=jax.devices()[:2]) + src, gathered = PartitionSpec("fsdp", None), PartitionSpec(None, None) + rng = np.random.default_rng(0) + x_np, wd_np, wm_np = (rng.standard_normal(s).astype(np.float32) for s in ((4, 8), (8, 6), (8, 6))) + + def layer_loss(x, wd, wm): + # A dense consumer and a nested shard_map consumer (like the MoE block) of the gathered weights. + wm_spec = PartitionSpec(None, None, reduced=jax.typeof(wm).sharding.spec.reduced) + m = jax.shard_map( + lambda xs, w: jnp.tanh(xs @ w), + mesh=mesh, + in_specs=(PartitionSpec("fsdp", None), wm_spec), + out_specs=PartitionSpec("fsdp", None), + )(x, wm) + return jnp.sin(jnp.einsum("bd,df->bf", x, wd, out_sharding=PartitionSpec("fsdp", None))).sum() + (m * m).sum() + + def prefetch_grads(x, wd, wm): + gathered_w = [fsdp_prefetch._apply_sharding_hint(w, mesh, ShardMode.EXPLICIT) for w in (wd, wm)] # pylint: disable=protected-access + for w in gathered_w: + self.assertEqual(jax.typeof(w).sharding.spec.reduced, frozenset({"fsdp"})) + _, vjp = jax.vjp(lambda a, b: layer_loss(x, a, b), *gathered_w) + grads = vjp(jnp.ones((), jnp.float32)) + for g in grads: + self.assertEqual(jax.typeof(g).sharding.spec.unreduced, frozenset({"fsdp"})) + return tuple(fsdp_prefetch._explicit_fsdp_reduce_scatter(g, mesh, src, gathered) for g in grads) # pylint: disable=protected-access + + with jax.set_mesh(mesh): + args = ( + jax.device_put(x_np, PartitionSpec("fsdp", None)), + jax.device_put(wd_np, src), + jax.device_put(wm_np, src), + ) + f = jax.jit(prefetch_grads) + out = f(*args) + hlo = f.lower(*args).as_text() + + def ref_loss(wd, wm): + m = jnp.tanh(x_np @ wm) + return jnp.sin(x_np @ wd).sum() + (m * m).sum() + + for got, want in zip(out, jax.grad(ref_loss, argnums=(0, 1))(wd_np, wm_np)): + np.testing.assert_allclose(np.asarray(got), np.asarray(want), rtol=1e-5, atol=1e-5) + self.assertEqual(tuple(got.sharding.spec.partitions), ("fsdp", None)) + # One tagged reduce-scatter per weight and no all-reduce: the gradient is reduced exactly once. + self.assertEqual(hlo.count("stablehlo.reduce_scatter"), 2) + self.assertNotIn("stablehlo.all_reduce", hlo) + + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/m3/glm5_3_flash_config_test.py b/tests/unit/m3/glm5_3_flash_config_test.py deleted file mode 100644 index 51130ae5e2..0000000000 --- a/tests/unit/m3/glm5_3_flash_config_test.py +++ /dev/null @@ -1,465 +0,0 @@ -# Copyright 2023-2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Unit tests for GLM-5.3-Flash configuration, sharding, and validation guards.""" - -import importlib -import json -import pathlib -import sys -from typing import Any, cast -from unittest import mock - -from absl.testing import absltest -import jax -from jax.sharding import PartitionSpec as P -from maxtext.configs import pyconfig -from maxtext.m3.configs.glm5_3_flash_config import Glm5Config -from maxtext.m3.configs.glm5_3_flash_config import Glm5Sharding -from maxtext.m3.configs.glm5_3_flash_config import validate_no_packing -from maxtext.m3.core import sharding as m3_sharding -import numpy as np - - -class Glm5ConfigTest(absltest.TestCase): - """Tests for Glm5Config invariants, loud guards, and pyconfig integration.""" - - def test_default_config_invariants(self): - c = Glm5Config() - self.assertEqual((c.num_hidden_layers, c.first_k_dense_replace), (45, 3)) - self.assertEqual((c.vocab_size, c.intermediate_size), (154880, 12288)) - self.assertEqual((c.hidden_size, c.moe_intermediate_size), (4096, 2048)) - self.assertEqual((c.routed_scaling_factor, c.n_routed_experts), (2.5, 288)) - self.assertEqual((c.num_experts_per_tok, c.n_shared_experts), (8, 1)) - self.assertEqual((c.linear_num_key_heads, c.linear_key_head_dim), (64, 128)) - self.assertEqual((c.linear_num_heads, c.linear_head_dim), (64, 128)) - self.assertEqual((c.linear_conv_kernel_dim, c.linear_lower_bound), (4, -5.0)) - self.assertEqual((c.num_attention_heads, c.num_key_value_heads), (64, 64)) - self.assertEqual((c.q_lora_rank, c.kv_lora_rank), (1536, 512)) - self.assertEqual((c.v_head_dim, c.qk_head_dim), (256, 256)) - self.assertEqual((c.qk_nope_head_dim, c.qk_rope_head_dim), (256, 0)) - self.assertEqual((c.mhc_sinkhorn_iters, c.mhc_expansion_rate), (20, 4)) - self.assertEqual((c.index_topk, c.index_kpool, c.mhc_eps), (2048, 4, 1e-6)) - self.assertTrue(c.index_kpool_always_select_tail) - self.assertEqual((c.indexer_loss_scaling_factor, c.packing), (0.0, False)) - self.assertEqual((c.rms_norm_eps, c.rope_parameters), (1e-5, None)) - self.assertEqual((c.scan_layers, c.kda_implementation), (True, "mosaic")) - self.assertEqual((c.moe_lb_loss_weight, c.mlp_clamp_limit), (0.001, 10.0)) - self.assertEqual(c.remat_policy, "full") - self.assertEqual(c.dsa_attention_implementation, "dense_mask") - self.assertEqual(c.router_param_dtype, "float32") - self.assertEqual(c.weight_dtype, "float32") - assert c.layer_types is not None and c.indexer_types is not None - self.assertEqual((len(c.layer_types), set(c.indexer_types)), (45, {"full"})) - self.assertEqual(c.layer_types.count("linear_attention"), 34) - self.assertEqual(c.layer_types.count("full_attention"), 11) - - def test_import_glm5_config_does_not_import_tokamax(self): - cfg_mod = Glm5Config.__module__ - m3_root = cfg_mod.rsplit(".", 2)[0] - pkg_mod = f"{m3_root}.models.glm5_3_flash" - saved_modules = {k: sys.modules.pop(k) for k in list(sys.modules) if k.startswith((cfg_mod, pkg_mod, "tokamax"))} - try: - importlib.import_module(cfg_mod) - importlib.import_module(pkg_mod) - self.assertFalse( - any(m == "tokamax" or m.startswith("tokamax.") for m in sys.modules), - [m for m in sys.modules if "tokamax" in m], - ) - finally: - sys.modules.update(saved_modules) - - def test_matches_published_hf_text_config(self): - # Provenance: checked-in copy of published zai-org/GLM-5.3-Flash/config.json - config_path = pathlib.Path(__file__).parent / "testdata" / "glm5_3_flash_config.json" - with config_path.open("r", encoding="utf-8") as f: - hf_root = json.load(f) - hf_text_config = hf_root["text_config"] - c = Glm5Config() - self.assertEqual(c.vocab_size, hf_text_config["vocab_size"]) - self.assertEqual(c.hidden_size, hf_text_config["hidden_size"]) - self.assertEqual(c.num_hidden_layers, hf_text_config["num_hidden_layers"]) - expected_mlp = ["dense"] * c.first_k_dense_replace + ["sparse"] * (c.num_hidden_layers - c.first_k_dense_replace) - self.assertEqual(expected_mlp, hf_text_config["mlp_layer_types"]) - self.assertEqual(c.mhc_eps, hf_text_config["hc_eps"]) - self.assertEqual(c.mhc_expansion_rate, hf_text_config["hc_mult"]) - self.assertEqual(c.mhc_sinkhorn_iters, hf_text_config["hc_sinkhorn_iters"]) - self.assertEqual(c.index_head_dim, hf_text_config["index_head_dim"]) - self.assertEqual(c.index_kpool, hf_text_config["index_kpool"]) - self.assertEqual( - c.index_kpool_always_select_tail, - hf_text_config["index_kpool_always_select_tail"], - ) - self.assertEqual(c.index_n_heads, hf_text_config["index_n_heads"]) - self.assertEqual(c.index_topk, hf_text_config["index_topk"]) - self.assertEqual(c.intermediate_size, hf_text_config["intermediate_size"]) - self.assertEqual(c.kv_lora_rank, hf_text_config["kv_lora_rank"]) - self.assertEqual(c.moe_intermediate_size, hf_text_config["moe_intermediate_size"]) - self.assertEqual(c.n_group, hf_text_config["n_group"]) - self.assertEqual(c.n_routed_experts, hf_text_config["n_routed_experts"]) - self.assertEqual(c.n_shared_experts, hf_text_config["n_shared_experts"]) - self.assertEqual(c.norm_topk_prob, hf_text_config["norm_topk_prob"]) - self.assertEqual(c.num_attention_heads, hf_text_config["num_attention_heads"]) - self.assertEqual(c.num_key_value_heads, hf_text_config["num_key_value_heads"]) - self.assertEqual(c.num_experts_per_tok, hf_text_config["num_experts_per_tok"]) - self.assertEqual(c.num_nextn_predict_layers, hf_text_config["num_nextn_predict_layers"]) - self.assertEqual(c.q_lora_rank, hf_text_config["q_lora_rank"]) - self.assertEqual(c.qk_head_dim, hf_text_config["qk_head_dim"]) - self.assertEqual(c.qk_nope_head_dim, hf_text_config["qk_nope_head_dim"]) - self.assertEqual(c.qk_rope_head_dim, hf_text_config["qk_rope_head_dim"]) - self.assertEqual(c.rms_norm_eps, hf_text_config["rms_norm_eps"]) - self.assertEqual(c.routed_scaling_factor, hf_text_config["routed_scaling_factor"]) - self.assertEqual(c.moe_lb_loss_weight, hf_text_config["router_aux_loss_coef"]) - self.assertEqual(c.scoring_func, hf_text_config["scoring_func"]) - self.assertEqual(c.mlp_clamp_limit, hf_text_config["swiglu_limit"]) - self.assertEqual(c.topk_group, hf_text_config["topk_group"]) - self.assertEqual(c.v_head_dim, hf_text_config["v_head_dim"]) - self.assertEqual(c.indexer_types, tuple(hf_text_config["indexer_types"])) - assert c.layer_types is not None - expected_hf_layer_types = [ - "linear_attention" if t == "linear_attention" else "deepseek_sparse_attention" for t in c.layer_types - ] - self.assertEqual(expected_hf_layer_types, hf_text_config["layer_types"]) - - linear_cfg = hf_text_config["linear_attn_config"] - self.assertEqual(c.linear_num_heads, linear_cfg["num_heads"]) - self.assertEqual(c.linear_num_key_heads, linear_cfg["num_heads"]) - self.assertEqual(c.linear_num_value_heads, linear_cfg["num_heads"]) - self.assertEqual(c.linear_head_dim, linear_cfg["head_dim"]) - self.assertEqual(c.linear_key_head_dim, linear_cfg["head_dim"]) - self.assertEqual(c.linear_value_head_dim, linear_cfg["head_dim"]) - self.assertEqual(c.linear_conv_kernel_dim, linear_cfg["short_conv_kernel_size"]) - self.assertEqual(c.linear_lower_bound, linear_cfg["gate_lower_bound"]) - kda_layers = [i for i, t in enumerate(c.layer_types) if t == "linear_attention"] - full_layers = [i for i, t in enumerate(c.layer_types) if t == "full_attention"] - self.assertEqual(kda_layers, linear_cfg["kda_layers"]) - self.assertEqual(full_layers, linear_cfg["full_attn_layers"]) - - def test_mini_config_invariants(self): - exp = ("linear_attention",) * 3 + ("full_attention", "linear_attention") - for cfg in (Glm5Config.mini_config(), Glm5Config(num_hidden_layers=5)): - self.assertEqual((cfg.num_hidden_layers, cfg.layer_types), (5, exp)) - self.assertEqual(cfg.indexer_types, ("full",) * 5) - cfg_norm = Glm5Config( - num_hidden_layers=5, - dtype=cast(Any, jax.numpy.bfloat16), - weight_dtype=cast(Any, np.float32), - layer_types=cast( - Any, - ["linear_attention"] * 3 + ["full_attention", "linear_attention"], - ), - indexer_types=cast(Any, ["full"] * 5), - ) - self.assertEqual(cfg_norm.dtype, "bfloat16") - self.assertEqual(cfg_norm.weight_dtype, "float32") - self.assertIsInstance(cfg_norm.layer_types, tuple) - self.assertIsInstance(cfg_norm.indexer_types, tuple) - self.assertEqual(cfg_norm.layer_types, exp) - self.assertEqual(cfg_norm.indexer_types, ("full",) * 5) - - def test_from_maxtext_config_identity_bypass_with_overrides(self): - base_cfg = Glm5Config() - new_lt = ("linear_attention",) * 5 - new_it = ("full",) * 5 - overridden = Glm5Config.from_maxtext_config( - base_cfg, - num_hidden_layers=5, - layer_types=new_lt, - indexer_types=new_it, - ) - self.assertEqual(overridden.num_hidden_layers, 5) - self.assertEqual(overridden.layer_types, new_lt) - self.assertEqual(overridden.indexer_types, new_it) - overridden_auto = Glm5Config.from_maxtext_config(base_cfg, num_hidden_layers=5) - self.assertEqual(len(overridden_auto.layer_types), 5) - self.assertEqual(len(overridden_auto.indexer_types), 5) - same = Glm5Config.from_maxtext_config(base_cfg) - self.assertIs(same, base_cfg) - - def test_from_maxtext_config_qk_head_dim_derived_on_override(self): - cfg = Glm5Config(qk_nope_head_dim=128) - self.assertEqual(cfg.qk_head_dim, 128) - overridden_qk = Glm5Config.from_maxtext_config(Glm5Config(), qk_nope_head_dim=128) - self.assertEqual(overridden_qk.qk_head_dim, 128) - - def test_dsa_attention_implementation(self): - c_default = Glm5Config() - self.assertEqual(c_default.dsa_attention_implementation, "dense_mask") - c_dense = Glm5Config(dsa_attention_implementation="dense_mask") - self.assertEqual(c_dense.dsa_attention_implementation, "dense_mask") - c_mini = Glm5Config.mini_config(dsa_attention_implementation="dense_mask") - self.assertEqual(c_mini.dsa_attention_implementation, "dense_mask") - - dummy_cfg = pyconfig.initialize(["", "src/maxtext/configs/base.yml", "model_name=glm5.3-flash"]) - c_from = Glm5Config.from_maxtext_config(dummy_cfg, dsa_attention_implementation="dense_mask") - self.assertEqual(c_from.dsa_attention_implementation, "dense_mask") - - def test_loud_guards(self): - for kwargs, pat in ( - ({"packing": True}, "packing=True"), - ({"indexer_types": ("shared",) * 45}, "Shared indexer"), - ({"indexer_types": ("full",) * 44}, r"len\(indexer_types\)"), - ({"layer_types": ("linear_attention",) * 44}, r"len\(layer_types\)"), - ({"layer_types": ("bad",) * 45}, "invalid"), - ({"routed_bias_update_rate": 0.01}, "routed_bias_update_rate"), - ({"indexer_loss_scaling_factor": 0.1}, "indexer_loss_scaling_factor"), - ({"kda_implementation": cast(Any, ["mosaic"])}, "kda_implementation"), - ({"kda_implementation": "bad"}, "kda_implementation"), - ( - {"kda_implementation": "mosaic", "linear_lower_bound": None}, - "linear_lower_bound", - ), - ( - {"kda_implementation": "xla_chunked", "linear_lower_bound": None}, - "linear_lower_bound", - ), - ({"index_kpool": 0}, "index_kpool must be >= 1"), - ( - {"index_topk": 2048, "index_kpool": 3}, - r"index_topk.*divisible by index_kpool", - ), - ({"mlp_clamp_limit": 0.0}, "mlp_clamp_limit must be > 0.0"), - ({"mlp_clamp_limit": -1.0}, "mlp_clamp_limit must be > 0.0"), - ( - {"dsa_attention_implementation": cast(Any, ["dense_mask"])}, - "dsa_attention_implementation must be a str", - ), - ( - {"dsa_attention_implementation": cast(Any, None)}, - "dsa_attention_implementation must be a str", - ), - ( - {"dsa_attention_implementation": cast(Any, 123)}, - "dsa_attention_implementation must be a str", - ), - ( - {"dsa_attention_implementation": "tokamax_splash"}, - "dsa_attention_implementation must be one of", - ), - ( - {"dsa_attention_implementation": "gather_scan"}, - "dsa_attention_implementation must be one of", - ), - ( - {"dsa_attention_implementation": "flash"}, - "dsa_attention_implementation must be one of", - ), - ( - {"dsa_attention_implementation": "bad"}, - "dsa_attention_implementation must be one of", - ), - ({"qk_rope_head_dim": 64}, "NoPE"), - ({"rope_parameters": {"type": "yarn"}}, "NoPE"), - ({"num_key_value_heads": 32}, r"num_key_value_heads.*must equal"), - ({"linear_num_value_heads": 32}, r"linear_num_key_heads.*must equal"), - ({"linear_value_head_dim": 64}, r"linear_key_head_dim.*must equal"), - ({"remat_policy": "bogus"}, "remat_policy must be 'none' or 'full'"), - ( - {"router_param_dtype": "bfloat16"}, - "router_param_dtype must be 'float32'", - ), - ( - {"router_param_dtype": cast(Any, 123)}, - "router_param_dtype must be 'float32'", - ), - ( - {"router_param_dtype": cast(Any, None)}, - "router_param_dtype must be 'float32'", - ), - ({"weight_dtype": "int8"}, "weight_dtype must be one of"), - ({"weight_dtype": cast(Any, 123)}, "weight_dtype must be one of"), - ): - with self.assertRaisesRegex(ValueError, pat): - Glm5Config(**cast(dict[str, Any], kwargs)) - with self.assertRaisesRegex(TypeError, "remat_policy must be a str"): - Glm5Config(remat_policy=cast(Any, 123)) - # Intentional negative test: qk_head_dim is a derived property, not an init field. - with self.assertRaises(TypeError): - Glm5Config(**cast(Any, {"qk_head_dim": 128})) # pylint: disable=unexpected-keyword-arg - - def test_validate_no_packing_helper(self): - validate_no_packing(None) - validate_no_packing(np.array([[0, 1, 1], [0, 0, 1]])) - with self.assertRaisesRegex(ValueError, "Packed sequences unsupported"): - validate_no_packing(np.array([[0, 1, 2]])) - - def test_sharding_resolution(self): - s = Glm5Sharding() - s_tp = Glm5Sharding(use_tp=True) - s_ep = Glm5Sharding(ep_in_batch=True) - act = m3_sharding.TensorType.ACTIVATION - self.assertEqual(s("expert_w", ("expert",)), P("expert")) - self.assertEqual(s("embed_w", ("embed",)), P("fsdp")) - self.assertEqual(s("q_lora_w", ("q_lora",)), P("fsdp")) - self.assertEqual(s("kv_lora_w", ("kv_lora",)), P("fsdp")) - self.assertEqual(s("embed_table", ("vocab", "embed")), P(None, "fsdp")) - self.assertEqual(s("heads_w", ("heads",)), P(None)) - self.assertEqual(s("b", ("batch",), act), P(("data", "fsdp"))) - self.assertEqual(s("h", ("batch", "embed"), act), P(("data", "fsdp"), None)) - self.assertEqual(s("q_act", ("batch", "q_lora"), act), P(("data", "fsdp"), None)) - self.assertEqual(s("kv_act", ("batch", "kv_lora"), act), P(("data", "fsdp"), None)) - self.assertEqual(s_tp("heads_w", ("heads",)), P("tensor")) - self.assertEqual(s_tp("embed_table", ("vocab", "embed")), P("tensor", "fsdp")) - self.assertEqual(s_tp("q_lora_w", ("q_lora",)), P("fsdp")) - self.assertEqual(s_tp("kv_lora_w", ("kv_lora",)), P("fsdp")) - self.assertEqual(s_ep("b", ("batch",), act), P(("data", "fsdp", "expert"))) - self.assertEqual(s_ep("h", ("batch", "embed"), act), P(("data", "fsdp", "expert"), None)) - with self.assertRaises(KeyError): - s("bad", ("unknown_axis",)) - - def test_from_mesh(self): - # Mesh with tp=1 (use_tp=False) - devices = np.array(jax.devices()[:1]).reshape((1, 1, 1, 1)) - mesh_m3_tp1 = jax.sharding.Mesh(devices, ("data", "fsdp", "tensor", "expert")) - s1 = Glm5Sharding.from_mesh(mesh_m3_tp1) - self.assertFalse(s1.use_tp) - self.assertEqual(s1.ep_size, 1) - self.assertEqual( - (s1.dp_axis, s1.fsdp_axis, s1.tp_axis, s1.ep_axis), - ("data", "fsdp", "tensor", "expert"), - ) - - # Mesh with tp=2 (use_tp=True) - mesh_m3_tp2 = mock.create_autospec(jax.sharding.Mesh, instance=True) - mesh_m3_tp2.axis_names = ("data", "fsdp", "tensor", "expert") - mesh_m3_tp2.shape = {"data": 1, "fsdp": 1, "tensor": 2, "expert": 1} - s2 = Glm5Sharding.from_mesh(mesh_m3_tp2) - self.assertTrue(s2.use_tp) - - # MaxText mesh ("data", "fsdp", "tensor", "expert") - mesh_maxtext = mock.create_autospec(jax.sharding.Mesh, instance=True) - mesh_maxtext.axis_names = ("data", "fsdp", "tensor", "expert") - mesh_maxtext.shape = {"data": 1, "fsdp": 1, "tensor": 2, "expert": 1} - s_mt = Glm5Sharding.from_mesh(mesh_maxtext) - self.assertTrue(s_mt.use_tp) - self.assertEqual( - (s_mt.dp_axis, s_mt.fsdp_axis, s_mt.tp_axis, s_mt.ep_axis), - ("data", "fsdp", "tensor", "expert"), - ) - self.assertEqual( - s_mt( - "batch_act", - ("batch", "heads", "expert"), - m3_sharding.TensorType.ACTIVATION, - ), - P(("data", "fsdp"), "tensor", None), - ) - self.assertEqual( - s_mt("w", ("embed", "heads", "expert")), - P("fsdp", "tensor", "expert"), - ) - - # Context axis test (P2-5) - mesh_ctx_1 = mock.create_autospec(jax.sharding.Mesh, instance=True) - mesh_ctx_1.axis_names = ("data", "fsdp", "tensor", "expert", "context") - mesh_ctx_1.shape = { - "data": 1, - "fsdp": 1, - "tensor": 1, - "expert": 1, - "context": 1, - } - s_ctx_1 = Glm5Sharding.from_mesh(mesh_ctx_1) - self.assertEqual(s_ctx_1.ep_size, 1) - - mesh_ctx_2 = mock.create_autospec(jax.sharding.Mesh, instance=True) - mesh_ctx_2.axis_names = ("data", "fsdp", "tensor", "expert", "context") - mesh_ctx_2.shape = { - "data": 1, - "fsdp": 1, - "tensor": 1, - "expert": 1, - "context": 2, - } - with self.assertRaisesRegex(NotImplementedError, "not yet supported in Glm5Sharding"): - Glm5Sharding.from_mesh(mesh_ctx_2) - - # Unsupported mesh axes raising ValueError - mesh_bad = mock.create_autospec(jax.sharding.Mesh, instance=True) - mesh_bad.axis_names = ("x", "y") - mesh_bad.shape = {"x": 1, "y": 1} - with self.assertRaisesRegex(ValueError, "Unsupported mesh axis_names"): - Glm5Sharding.from_mesh(mesh_bad) - - def test_pyconfig_integration(self): - argv = ["", "src/maxtext/configs/base.yml", "model_name=glm5.3-flash"] - raw_cfg = pyconfig.initialize(argv) - self.assertTrue(raw_cfg.use_m3_model) - self.assertEqual(raw_cfg.tokenizer_path, "zai-org/GLM-5.3-Flash") - self.assertEqual(raw_cfg.tokenizer_type, "huggingface") - - c = Glm5Config.from_maxtext_config(raw_cfg) - self.assertEqual((c.vocab_size, c.hidden_size), (154880, 4096)) - self.assertEqual((c.num_hidden_layers, c.num_attention_heads), (45, 64)) - self.assertEqual((c.intermediate_size, c.n_routed_experts), (12288, 288)) - self.assertEqual((c.moe_intermediate_size, c.qk_head_dim), (2048, 256)) - self.assertEqual((c.first_k_dense_replace, c.n_group), (3, 1)) - self.assertEqual((c.num_nextn_predict_layers, c.packing), (1, False)) - self.assertEqual((c.index_n_heads, c.index_head_dim), (32, 128)) - self.assertEqual((c.index_topk, c.mhc_expansion_rate), (2048, 4)) - self.assertEqual((c.mhc_sinkhorn_iters, c.topk_group), (20, 1)) - self.assertEqual((c.mtp_loss_scaling_factor, c.rms_norm_eps), (0.1, 1e-5)) - self.assertEqual((c.norm_topk_prob, c.scoring_func), (True, "sigmoid")) - self.assertEqual(c.routed_bias_update_rate, 0.0) - self.assertEqual(c.indexer_loss_scaling_factor, 0.0) - self.assertEqual(c.moe_lb_loss_weight, 0.001) - self.assertEqual(c.mlp_clamp_limit, 10.0) - - self.assertEqual( - Glm5Config.from_maxtext_config(pyconfig.initialize(argv + ["remat_policy=none"])).remat_policy, - "none", - ) - - cfg_custom = pyconfig.initialize( - argv - + [ - "override_model_config=true", - "load_balance_loss_weight=0.05", - "mlp_activations_limit=8.0", - ] - ) - c_custom = Glm5Config.from_maxtext_config(cfg_custom) - self.assertEqual(c_custom.moe_lb_loss_weight, 0.05) - self.assertEqual(c_custom.mlp_clamp_limit, 8.0) - self.assertEqual(c.dsa_attention_implementation, "dense_mask") - c_dense = Glm5Config.from_maxtext_config(pyconfig.initialize(argv), dsa_attention_implementation="dense_mask") - self.assertEqual(c_dense.dsa_attention_implementation, "dense_mask") - with self.assertRaisesRegex(ValueError, "dsa_attention_implementation must be one of"): - Glm5Config.from_maxtext_config( - pyconfig.initialize(argv), - dsa_attention_implementation="tokamax_splash", - ) - - d4 = ["routed_bias_update_rate=0.01", "decoder_block=deepseek"] - d3 = ["indexer_loss_scaling_factor=0.1", "max_target_length=4096"] - argv_override = argv + ["override_model_config=true"] - for extra, pat in ( - (["packing=true"], "FU-D5"), - (d4, "FU-D4"), - (d3, "FU-D3"), - (["mlp_activations_limit=-1.0"], "mlp_clamp_limit must be > 0.0"), - ): - cfg_err = pyconfig.initialize(argv_override + extra) - with self.assertRaisesRegex(ValueError, pat): - Glm5Config.from_maxtext_config(cfg_err) - - with self.assertRaisesRegex(ValueError, "requires use_m3_model=True"): - pyconfig.initialize(argv_override + ["use_m3_model=false"]) - cfg_no_m3 = mock.MagicMock(use_m3_model=False) - with self.assertRaisesRegex(ValueError, "requires use_m3_model=True"): - Glm5Config.from_maxtext_config(cfg_no_m3) - - -if __name__ == "__main__": - absltest.main() diff --git a/tests/unit/m3/m3_smoke_test.py b/tests/unit/m3/m3_smoke_test.py index aaacb317e0..d07fda2948 100644 --- a/tests/unit/m3/m3_smoke_test.py +++ b/tests/unit/m3/m3_smoke_test.py @@ -14,6 +14,8 @@ """Smoke test for m3 model registry routing and forward pass under Blaze.""" +import re + from absl.testing import absltest from flax import nnx import jax @@ -21,6 +23,7 @@ from jax.sharding import Mesh from maxtext.common.common_types import MODEL_MODE_TRAIN from maxtext.configs import pyconfig +from maxtext.m3 import models as m3_models from maxtext.m3.models.qwen3.modeling_qwen3 import Qwen3Model from maxtext.m3.models.qwen3_vl.modeling_qwen3_vl import Qwen3VLModel from maxtext.utils import model_creation_utils @@ -45,12 +48,12 @@ def test_unregistered_model_raises_value_error(self): "skip_jax_distributed_system=true", ] cfg = pyconfig.initialize_pydantic(config_args) + expected_models = ", ".join(sorted(m3_models.MODEL_REGISTRY)) with self.assertRaisesRegex( ValueError, r"Model 'default' does not support the m3 backend" - r" \(use_m3_model=True\)\. Registered models: glm5\.3-flash," - r" qwen3-0\.6b, qwen3-vl-2b, qwen3\.8-flash-next," - r" qwen3\.8-flash-next-mini\.", + r" \(use_m3_model=True\)\. Registered models:" + rf" {re.escape(expected_models)}\.", ): model_creation_utils.create_model(cfg, self.mesh, model_mode=MODEL_MODE_TRAIN, rngs=nnx.Rngs(0)) @@ -107,7 +110,7 @@ def test_qwen3_vl_routing_and_forward_pass(self): "base_num_query_heads=4", "base_num_kv_heads=2", "head_dim=16", - "vocab_size=256", + "vocab_size=151936", "num_hidden_layers_for_vit=2", "deepstack_visual_indexes_for_vit=[1]", "hidden_size_for_vit=16", @@ -124,7 +127,7 @@ def test_qwen3_vl_routing_and_forward_pass(self): self.assertIsInstance(model, Qwen3VLModel) batch_size, seq_len = 1, 4 - tokens = jnp.ones((batch_size, seq_len), dtype=jnp.int32) + tokens = jnp.array([[1, 151655, 2, 3]], dtype=jnp.int32) images = jnp.zeros((batch_size, 3, 2, 32, 32), dtype=jnp.float32) grid_thw = jnp.array([[1, 2, 2]], dtype=jnp.int32) logits = model( @@ -133,6 +136,7 @@ def test_qwen3_vl_routing_and_forward_pass(self): encoder_video_grid_thw=grid_thw, ) self.assertEqual(logits.shape, (batch_size, seq_len, cfg.vocab_size)) + self.assertTrue(bool(jnp.all(jnp.isfinite(logits)))) if __name__ == "__main__": diff --git a/tests/unit/nnx_decoders_test.py b/tests/unit/nnx_decoders_test.py index e1313ca3b9..aa75084488 100644 --- a/tests/unit/nnx_decoders_test.py +++ b/tests/unit/nnx_decoders_test.py @@ -2865,11 +2865,12 @@ def gather(x): if axes[0] == "exp": checked["routed"] += 1 self.assertNotIn('name = "expert"', gathers[0], name) - self.assertEqual(tuple(out.sharding.spec)[0], "expert", name) + self.assertEqual(out.sharding.spec.partitions[0], "expert", name) else: checked["co_sharded"] += 1 self.assertIn('name = "expert"', gathers[0], name) - self.assertNotIn("expert", str(out.sharding.spec), name) + # The gathered weight is replicated over `expert` (typed `reduced` over the gathered axes). + self.assertNotIn("expert", str(out.sharding.spec.partitions), name) # wi_0/wi_1/wo of both sub-layers; the GDN, attention, router and shared-expert kernels. self.assertEqual(checked["routed"], 6) self.assertGreaterEqual(checked["co_sharded"], 14) @@ -2929,7 +2930,7 @@ def gather(x): self.assertIn('xla_explicit_fsdp = "yes"', gathers[0], name) self.assertIn("all_gather_dim = 0", gathers[0], name) self.assertNotIn('name = "expert"', gathers[0], name) - self.assertEqual(tuple(out.sharding.spec)[0], "expert", name) + self.assertEqual(out.sharding.spec.partitions[0], "expert", name) # The flat weight must not get the minor-dim layout pin that 3D `wo` needs. self.assertFalse( fsdp_prefetch._needs_minor_dim_layout_pin( # pylint: disable=protected-access diff --git a/tests/unit/pallas_mosaic_tpu_v2_kernel_test.py b/tests/unit/pallas_mosaic_tpu_v2_kernel_test.py index 4b373006d7..4c936b4031 100644 --- a/tests/unit/pallas_mosaic_tpu_v2_kernel_test.py +++ b/tests/unit/pallas_mosaic_tpu_v2_kernel_test.py @@ -221,15 +221,12 @@ class GmmDispatchTest(parameterized.TestCase): rhs_dtype=(jnp.int4, jnp.int8, jnp.bfloat16, jnp.float8_e4m3fn), use_gmm_v2_heuristic_tiling=(False, True), use_dlhs_transpose_rhs=(None, False, True), - transpose_rhs=(False, True), ) - def test_dlhs_rhs_transpose_dispatch_by_dtype( - self, rhs_dtype, use_gmm_v2_heuristic_tiling, use_dlhs_transpose_rhs, transpose_rhs - ): + def test_dlhs_rhs_transpose_dispatch_by_dtype(self, rhs_dtype, use_gmm_v2_heuristic_tiling, use_dlhs_transpose_rhs): """DLHS materializes sub-byte RHS transposes before calling GMM v2.""" num_groups, out_size, in_size, batch_size = 2, 16, 32, 8 dlhs_dout = jnp.ones((batch_size, in_size), dtype=jnp.bfloat16) - rhs_shape = (num_groups, in_size, out_size) if transpose_rhs else (num_groups, out_size, in_size) + rhs_shape = (num_groups, out_size, in_size) rhs = (jnp.arange(num_groups * out_size * in_size).reshape(rhs_shape) % 8).astype(rhs_dtype) group_sizes = jnp.array([batch_size // num_groups] * num_groups, dtype=jnp.int32) expected = jnp.zeros((batch_size, out_size), dtype=jnp.bfloat16) @@ -246,15 +243,12 @@ def test_dlhs_rhs_transpose_dispatch_by_dtype( lhs_dtype=jnp.bfloat16, tiling=tiling, use_gmm_v2_heuristic_tiling=use_gmm_v2_heuristic_tiling, - transpose_rhs=transpose_rhs, **kwargs, ) call_kwargs = gmm_v2_mock.call_args.kwargs - expected_transpose_rhs = ( - bool(use_dlhs_transpose_rhs) and not transpose_rhs and jax.dtypes.itemsize_bits(rhs_dtype) >= 8 - ) - expected_rhs = rhs if transpose_rhs or expected_transpose_rhs else rhs.swapaxes(1, 2) + expected_transpose_rhs = bool(use_dlhs_transpose_rhs) and jax.dtypes.itemsize_bits(rhs_dtype) >= 8 + expected_rhs = rhs if expected_transpose_rhs else rhs.swapaxes(1, 2) self.assertIs(actual, expected) self.assertFalse(lhs_scale_applied) self.assertEqual(call_kwargs["transpose_rhs"], expected_transpose_rhs) @@ -267,19 +261,16 @@ def test_dlhs_rhs_transpose_dispatch_by_dtype( @parameterized.product( use_dlhs_transpose_rhs=(None, False, True), - transpose_rhs=(False, True), use_gmm_v2_heuristic_tiling=(False, True), has_partial_sum=(False, True), ) def test_dlhs_transpose_flag_through_custom_vjp( - self, use_dlhs_transpose_rhs, transpose_rhs, use_gmm_v2_heuristic_tiling, has_partial_sum + self, use_dlhs_transpose_rhs, use_gmm_v2_heuristic_tiling, has_partial_sum ): """Check flag routing and cotangents through JIT with reference backends.""" num_groups, batch_size, in_size, out_size = 2, 8, 16, 32 lhs = jnp.linspace(-1.0, 1.0, batch_size * in_size).reshape(batch_size, in_size) rhs = jnp.linspace(-0.5, 0.5, num_groups * in_size * out_size).reshape(num_groups, in_size, out_size) - if transpose_rhs: - rhs = rhs.swapaxes(1, 2) partial_sum = jnp.ones((batch_size, out_size)) if has_partial_sum else None group_sizes = jnp.array([batch_size // num_groups] * num_groups, dtype=jnp.int32) @@ -305,7 +296,6 @@ def loss(lhs, rhs, partial_sum): lhs, rhs, group_sizes, - transpose_rhs=transpose_rhs, use_tokamax_backend=True, use_gmm_v2=True, use_gmm_v2_heuristic_tiling=use_gmm_v2_heuristic_tiling, @@ -315,14 +305,14 @@ def loss(lhs, rhs, partial_sum): return jnp.sum(out**2) def reference_loss(lhs, rhs, partial_sum): - return jnp.sum(mock_gmm(lhs, rhs, transpose_rhs=transpose_rhs, partial_sum=partial_sum) ** 2) + return jnp.sum(mock_gmm(lhs, rhs, partial_sum=partial_sum) ** 2) with mock.patch.object(megablox_ops.gmm_v2, "gmm_v2", side_effect=mock_gmm) as gmm_mock: with mock.patch.object(megablox_ops.tgmm_v2, "tgmm_v2", side_effect=mock_tgmm): actual = jax.jit(jax.value_and_grad(loss, argnums=(0, 1, 2)))(lhs, rhs, partial_sum) expected = jax.jit(jax.value_and_grad(reference_loss, argnums=(0, 1, 2)))(lhs, rhs, partial_sum) chex.assert_trees_all_close(actual, expected, atol=1e-4, rtol=1e-5) - self.assertEqual(gmm_mock.call_args.kwargs["transpose_rhs"], bool(use_dlhs_transpose_rhs) and not transpose_rhs) + self.assertEqual(gmm_mock.call_args.kwargs["transpose_rhs"], bool(use_dlhs_transpose_rhs)) class GmmTest(parameterized.TestCase):