Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions src/maxtext/configs/base.yml
Original file line number Diff line number Diff line change
Expand Up @@ -565,6 +565,10 @@ param_scan_axis: 1
# never saved and are re-gathered for the backward pass. See docs/guides/optimization/fsdp_prefetch.md for the
# recommended XLA flags.
prefetch_fsdp_weights: false
# With prefetch_fsdp_weights, defer each layer's weight-gradient reduce-scatters to the next layer's backward step
# and run them on the SparseCore dedicated to FSDP collectives. Hides them at the cost of extra HBM. Homogeneous
# blocks only for now (ignored for inhomogeneous blocks prefetched per sub-layer).
prefetch_fsdp_defer_grad_reduce_scatter: false

# The attention parameter dictates the specific algorithm/methodology used to compute the attention scores
# The attention_type parameter determines the variants of attention, e.g. global or local_sliding
Expand Down
1 change: 1 addition & 0 deletions src/maxtext/configs/post_train/rl.yml
Original file line number Diff line number Diff line change
Expand Up @@ -366,5 +366,6 @@ logical_axis_rules: [
['engram_dim', ['tensor']],
['dense_layers', []],
['moe_layers', []],
['local_layers', []],
['mhc', []],
]
12 changes: 12 additions & 0 deletions src/maxtext/configs/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -2127,6 +2127,18 @@ class LayoutAndSharding(BaseModel):
" FSDP mesh axis spans more than one device. See docs/guides/optimization/fsdp_prefetch.md."
),
)
prefetch_fsdp_defer_grad_reduce_scatter: bool = Field(
False,
description=(
"With prefetch_fsdp_weights and shard_mode=explicit, issue each layer's weight-gradient"
" reduce-scatters at the start of the next layer's backward step, tagged for XLA's FSDP scheduler,"
" which runs them on the SparseCore dedicated to FSDP collectives"
" (--xla_tpu_explicit_fsdp_dedicated_sparse_core_id) and overlaps them with that layer's backward."
" Keeps one extra layer of unreduced weight gradients live (more HBM). When False, they run untagged"
" on the other SparseCore and complete within their own layer's backward step. Not yet supported"
" (ignored) for inhomogeneous blocks prefetched per sub-layer."
),
)


class DcnParallelism(BaseModel):
Expand Down
39 changes: 26 additions & 13 deletions src/maxtext/integration/vllm/maxtext_vllm_rollout.py
Original file line number Diff line number Diff line change
Expand Up @@ -238,12 +238,15 @@ def _find_scanned_layer_idx(key_tuple, container_names=("layers", "scanned_block


def validate_direct_sync_layer_coverage(source, target) -> int:
"""Fail if an unrolled source would leave MaxText target layers untouched.
"""Fail if a direct-sync source would leave MaxText target layers untouched.

Tunix intentionally intersects direct-sync trees. For heterogeneous Qwen
scans, a schema error can therefore skip every transformer layer without an
exception. This check runs on the initial full-parameter load and requires
every unscanned target-layer parameter path to exist in the source.
every unscanned target-layer parameter path to be reachable from the source:
either directly (sources MaxText already unrolled, e.g. Gemma and Qwen) or
through the scanned-layer lookup of Tunix's `transfer_state_directly`, which
unrolls the homogeneous scans MaxText leaves stacked (e.g. Llama).
"""

def to_pure_params(state):
Expand All @@ -268,21 +271,31 @@ def unwrap(state, wrapper):
source_flat = flatten_dict(source)
target_flat = flatten_dict(target)

def is_unscanned_layer_path(path):
return any(isinstance(part, str) and re.fullmatch(r"layers_\d+", part) for part in path)
def unscanned_layer_index(path):
"""Position of the first `layers_N` component of `path`, or -1."""
for i, part in enumerate(path):
if isinstance(part, str) and re.fullmatch(r"layers_\d+", part):
return i
return -1

source_layer_keys = {key for key in source_flat if is_unscanned_layer_path(key)}
target_layer_keys = {key for key in target_flat if is_unscanned_layer_path(key)}
source_keys = set(source_flat)
target_layer_keys = {key for key in target_flat if unscanned_layer_index(key) != -1}

def source_covers(target_key):
if target_key in source_layer_keys:
return True
def has_split_wi(key):
# Tunix fuses split training weights into the inference-only prefused
# parameter before transfer. Treat the pair as coverage for target `wi`.
if target_key and target_key[-1] == "wi":
prefix = target_key[:-1]
return prefix + ("wi_0",) in source_layer_keys and prefix + ("wi_1",) in source_layer_keys
return False
return key[-1] == "wi" and key[:-1] + ("wi_0",) in source_keys and key[:-1] + ("wi_1",) in source_keys

def source_covers(target_key):
if target_key in source_keys or has_split_wi(target_key):
return True
# Mirror the fallback of Tunix's `transfer_state_directly`: a target
# `layers_N` is slice N of a scanned `layers` container (standard MaxText)
# or of an implicit stack without one. Only the former fuses split MoE.
idx = unscanned_layer_index(target_key)
scanned_key = target_key[:idx] + ("layers",) + target_key[idx + 1 :]
implicit_key = target_key[:idx] + target_key[idx + 1 :]
return scanned_key in source_keys or implicit_key in source_keys or has_split_wi(scanned_key)

missing = {key for key in target_layer_keys if not source_covers(key)}
if not target_layer_keys or missing:
Expand Down
Loading
Loading