From d7ed9930d15e169ea927bbe2dd878d7b49dc1a18 Mon Sep 17 00:00:00 2001 From: Yuchen Yao Date: Tue, 17 Nov 2026 00:01:28 +0000 Subject: [PATCH] Add selective pinned-host offload for large m3 embedding tables Qwen3.8's PLE n-gram embedding table (~51.2B params, ~102 GB bf16) plus its AdamW state consumes roughly half of per-chip HBM state while each step reads only B*S*ngram rows. Existing m3 offload (`parameter_memory_host_offload` / `optimizer_memory_host_offload`) is all-or-nothing. - Add generic path-pattern selective pinned-host offload in `src/maxtext/m3/infra/offload.py` (`selective_host_offload`, `host_offload_param_patterns`, `selective_host_offload_optimizer_mode`, `selective_host_offload_num_chunks`), off by default and HLO-identical when disabled, with Qwen3.8 model configs targeting the PLE n-gram embedding. - Keep selected embedding parameters and their AdamW state in `pinned_host` while leaving all other weights and optimizer state on device; gather only step-active rows on forward and produce row-sparse gradients (`SparseHostGrad`) without materializing a dense HBM gradient on backward. - Preserve exact dense AdamW semantics by default via `host_compute` (with `streamed_chunks` and documented opt-in `lazy_sparse` modes), isolate all host-compute ops inside `jax.shard_map` so single-slice TPU compilations emit zero MegaScale host collectives, and fuse post-update host param norm capture into the optimizer pass to avoid extra HBM spills. - Wire selective offload across both `m3/train/loop.py` and `model_creation_utils.create_model` (`train.py`, `train_compile.py`, `MaxTextTrainingEngine`, `AbstractMaxTextEngine`) while preserving param paths and `scan -> remat -> quantize -> named scopes -> offload -> optimizer` order. - Add `SelectiveHostOffloadTest` in `tests/unit/m3/offload_test.py` covering validation, default-off HLO identity, sharded 2D sparse gather/grad-norm/ clipping/optimizer modes, scanned and unscanned Qwen3.8-mini 3-step parity and `pinned_host` memory_kind invariants, dual-driver and transform composition, and TPU7x-8 AOT HBM savings with zero MegaScale collectives. --- src/maxtext/configs/base.yml | 7 + .../models/qwen3.8-flash-next-mini.yml | 3 + .../configs/models/qwen3.8-flash-next.yml | 3 + src/maxtext/configs/types.py | 40 + src/maxtext/m3/infra/offload.py | 1913 ++++++++++++++++- .../modeling_qwen3_8_flash_next.py | 10 + src/maxtext/m3/train/loop.py | 12 +- src/maxtext/trainers/pre_train/train.py | 81 +- .../trainers/pre_train/train_compile.py | 7 +- src/maxtext/training_engine/maxtext_engine.py | 119 +- .../training_engine/maxtext_engine_compile.py | 25 +- src/maxtext/utils/maxtext_utils.py | 4 + src/maxtext/utils/model_creation_utils.py | 4 + src/maxtext/utils/train_utils.py | 7 +- tests/unit/m3/offload_test.py | 653 ++++++ tests/unit/train_nnx_test.py | 2 + 16 files changed, 2817 insertions(+), 73 deletions(-) diff --git a/src/maxtext/configs/base.yml b/src/maxtext/configs/base.yml index 4645959d83..c22e7c68fe 100644 --- a/src/maxtext/configs/base.yml +++ b/src/maxtext/configs/base.yml @@ -619,6 +619,13 @@ engram: 'remat' optimizer_memory_host_offload: false parameter_memory_host_offload: false +# Selective host offload saves HBM for regex-matched embedding tables + AdamW state at the cost of +# synchronous host-CPU optimizer scan time (~1.5-1.9 s/step on v5p-8 at 2.5M x 160 rows/chip under +# default 'host_compute'; 'streamed_chunks' is ~150x slower; 'lazy_sparse' skips untouched-row decay). +selective_host_offload: false +host_offload_param_patterns: [] +selective_host_offload_optimizer_mode: host_compute +selective_host_offload_num_chunks: 16 # Whether to use jax.lax.scan over layers (stacked/unstacked checkpoint). We recommend setting this to false when using pipeline parallelism, instead scanning the PP iterations. When resuming from a checkpoint, this flag is auto-determined from metadata. scan_layers: true param_scan_axis: 1 diff --git a/src/maxtext/configs/models/qwen3.8-flash-next-mini.yml b/src/maxtext/configs/models/qwen3.8-flash-next-mini.yml index 81fbde492b..8aee01b44b 100644 --- a/src/maxtext/configs/models/qwen3.8-flash-next-mini.yml +++ b/src/maxtext/configs/models/qwen3.8-flash-next-mini.yml @@ -75,6 +75,9 @@ make_ngram_vocab_size_divisible_by: 4 # Mini-twin (HF: make_ngram_vocab_size_di split_ngram_parts: 4 # Mini-twin (HF: split_ngram_parts=128) ple_seed: 1234 # HF: seed=1234 ple_eos_token_id: 3 # In-range EOS token (< vocab_size=1024) +# Pattern used only when selective_host_offload=true (default false); host_compute adds ~1.5-1.9 s/step +# synchronous host-CPU AdamW scan on v5p-8 at 2.5M x 160 rows/chip (~24 -> 632 ms/step on v5p-8 mini). +host_offload_param_patterns: ["(?:^|/)ple_embedding/ngram_embedding/embedding$"] # Multi-Token Prediction (MTP) Parameters mtp_num_layers: 1 diff --git a/src/maxtext/configs/models/qwen3.8-flash-next.yml b/src/maxtext/configs/models/qwen3.8-flash-next.yml index 7e597027d2..0388a92d52 100644 --- a/src/maxtext/configs/models/qwen3.8-flash-next.yml +++ b/src/maxtext/configs/models/qwen3.8-flash-next.yml @@ -75,6 +75,9 @@ make_ngram_vocab_size_divisible_by: 128 # HF: make_ngram_vocab_size_divisible_b split_ngram_parts: 128 # HF config.json: split_ngram_parts=128 (class default in configuration_qwen4_exp.py is 512) ple_seed: 1234 # HF: seed=1234 ple_eos_token_id: 248044 # HF: eos_token_id=248044 +# Pattern used only when selective_host_offload=true (default false); host_compute adds ~1.5-1.9 s/step +# synchronous host-CPU AdamW scan on v5p-8 at 2.5M x 160 rows/chip (~24 -> 632 ms/step on v5p-8 mini). +host_offload_param_patterns: ["(?:^|/)ple_embedding/ngram_embedding/embedding$"] # Multi-Token Prediction (MTP) Parameters mtp_num_layers: 1 diff --git a/src/maxtext/configs/types.py b/src/maxtext/configs/types.py index 3101d9eea1..6eb0d3ecc7 100644 --- a/src/maxtext/configs/types.py +++ b/src/maxtext/configs/types.py @@ -2689,6 +2689,41 @@ class RematAndOffload(BaseModel): optimizer_memory_host_offload: bool = Field(False, description="Offload optimizer state to host memory.") parameter_memory_host_offload: bool = Field(False, description="Offload parameters to host memory.") + selective_host_offload: bool = Field( + False, + description=( + "Offload only parameters matching `host_offload_param_patterns` and their matching optimizer " + "state to `pinned_host` memory, leaving all other parameters and optimizer state on device. " + "Saves HBM at the cost of synchronous host-CPU optimizer latency (e.g., ~1.5-1.9 s/step on " + "v5p-8 for a 2.5M x 160 table/chip under default 'host_compute'; ~24 -> 632 ms/step on v5p-8 mini)." + ), + ) + host_offload_param_patterns: list[str] = Field( + default_factory=list, + description=( + "Regex patterns matching '/'-joined parameter paths to offload to `pinned_host` memory when " + "`selective_host_offload=True` (for example, ['(?:^|/)ple_embedding/ngram_embedding/embedding$'])." + ), + ) + selective_host_offload_optimizer_mode: Literal["host_compute", "streamed_chunks", "lazy_sparse"] = Field( + "host_compute", + description=( + "Optimizer update mode for selectively host-offloaded embedding tables: " + "'host_compute' (default) runs exact dense AdamW on host CPU with zero full-table HBM transfer, " + "but scans all offloaded rows and moments in host DRAM synchronously each step (~1.5-1.9 s/step " + "measured on v5p-8 at 2.5M x 160 rows/chip; ~24 -> 632 ms/step on v5p-8 mini); " + "'streamed_chunks' streams `selective_host_offload_num_chunks` row chunks through device compute " + "for exact dense AdamW with zero host temp DRAM but is ~150x slower on live TPU (~284 s/step at " + "2.5M x 160 on v5p-8, diagnostic use only); " + "'lazy_sparse' (opt-in approximation) updates only touched rows, skipping weight and moment decay " + "on untouched rows, and fails closed with dense host gradients (such as MaxTextTrainingEngine)." + ), + ) + selective_host_offload_num_chunks: int = Field( + 16, + ge=1, + description="Number of row chunks used when `selective_host_offload_optimizer_mode` is 'streamed_chunks'.", + ) class Tokenizer(BaseModel): @@ -5469,6 +5504,11 @@ def validate_save_fsdp_gathered_weights(self) -> "MaxTextConfig": raise ValueError( "save_fsdp_gathered_weights keeps gathered kernels in HBM, which defeats parameter_memory_host_offload." ) + if self.selective_host_offload: + if self.parameter_memory_host_offload: + raise ValueError("selective_host_offload=True cannot be combined with parameter_memory_host_offload=True.") + if not self.host_offload_param_patterns: + raise ValueError("selective_host_offload=True requires non-empty host_offload_param_patterns.") return self @model_validator(mode="after") diff --git a/src/maxtext/m3/infra/offload.py b/src/maxtext/m3/infra/offload.py index be93d79e61..1a75d1536d 100644 --- a/src/maxtext/m3/infra/offload.py +++ b/src/maxtext/m3/infra/offload.py @@ -19,9 +19,21 @@ inside `update`, preserving identical `nnx.state(model, nnx.Param)` paths and step-for-step numerical trajectories when compiled with buffer donation (`@nnx.jit(donate_argnums=(0, 1))`). `maybe_offload_parameters` optionally moves -`nnx.Param` state to `"pinned_host"` and gathers parameters to `"device"` for -forward and backward execution via a dynamic subclass mixin without adding -wrapper segments to module paths. +`nnx.Param` state to `"pinned_host"` — either all parameters +(`parameter_memory_host_offload=True`) or only parameters whose `"/"`-joined +paths match `host_offload_param_patterns` (`selective_host_offload=True`), +leaving all other parameters and optimizer state on device. + +For selectively offloaded `nnx.Embed` tables, `HostOffloadedEmbed` replaces the +module path-transparently with a static `nnx.Embed` subclass that gathers only +the needed rows per step from `pinned_host` to device via +`jax.experimental.compute_on("device_host")` (shard-locally inside +`jax.shard_map` + ICI `psum` when row-sharded). Under `value_and_grad` / +`pure_value_and_grad`, backward produces a row-sparse `SparseHostGrad` without +materializing a dense full-table gradient in HBM. Exact dense AdamW updates +(`host_compute` or `streamed_chunks`) remain the default; `lazy_sparse` is an +opt-in approximation that updates only rows touched in the current step and +skips weight/moment decay on untouched rows. Inside traced regions, `to_memory` lowers transfers via `jax.device_put(x, jax.memory.Space.*)` so XLA schedules host-device DMA copies @@ -39,20 +51,64 @@ capturing XLA HLO temporary workspace held in `bytes_reserved`. """ +from collections.abc import Callable, Sequence +import contextlib +import functools +import re +import threading from typing import Any from flax import nnx import jax +from jax.experimental import compute_on +import jax.numpy as jnp +from jax.sharding import PartitionSpec as P import numpy as np import optax +from maxtext.m3.infra import named_scopes + _GIB = float(2**30) _HOST_KINDS = frozenset({"pinned_host", "unpinned_host"}) _SPACES = { "pinned_host": jax.memory.Space.Host, "device": jax.memory.Space.Device, } +_OPTIMIZER_MODES = frozenset({"host_compute", "streamed_chunks", "lazy_sparse"}) +_SELECTIVE_HOST_PATH_KEY = "_m3_selective_host_path" _OFFLOADED_MODEL_CLASSES: dict[type[nnx.Module], type[nnx.Module]] = {} +_PREGATHERED_STATE = threading.local() +_HOST_SQNORM_CAPTURE = threading.local() +_ACTIVE_PARAM_SHARDINGS = threading.local() + + +@contextlib.contextmanager +def active_param_shardings(param_shardings: Any): + """Provides physical parameter shardings during `Auto`-mode `jit` traces (e.g. `MaxTextTrainingEngine`).""" + prev = getattr(_ACTIVE_PARAM_SHARDINGS, "map", None) + if param_shardings is None: + yield + return + spec_map: dict[tuple[Any, ...], Any] = {} + for path, leaf in jax.tree_util.tree_leaves_with_path( + param_shardings, is_leaf=lambda x: isinstance(x, (nnx.Variable, jax.sharding.Sharding, P)) + ): + if isinstance(leaf, nnx.Variable): + leaf = leaf.get_raw_value() + spec = getattr(leaf, "spec", leaf if isinstance(leaf, P) else None) + if spec is not None: + spec_map[tuple(_key_value(k) for k in path)] = spec + _ACTIVE_PARAM_SHARDINGS.map = spec_map + try: + yield + finally: + _ACTIVE_PARAM_SHARDINGS.map = prev + + +def _call_shard_map(fn: Any, *, in_specs: Any, out_specs: Any, args: tuple[Any, ...]) -> Any: + """Runs `jax.shard_map` under `jax.jit` so eager and `jax.checkpoint` callers can use `Space.Host` transfers.""" + smap = jax.shard_map(fn, in_specs=in_specs, out_specs=out_specs, check_vma=False) + return jax.jit(smap)(*args) def _apply_memory_kind_hook(var: nnx.Variable, value: Any) -> Any: @@ -86,13 +142,17 @@ def _is_host_leaf(x: Any) -> bool: aval = getattr(val, "aval", None) return getattr(aval, "memory_space", None) == jax.memory.Space.Host sharding = getattr(val, "sharding", None) - return getattr(sharding, "memory_kind", None) in _HOST_KINDS + if getattr(sharding, "memory_kind", None) in _HOST_KINDS: + return True + if isinstance(x, nnx.Variable) and x.get_metadata("memory_kind", default=None) in _HOST_KINDS: + return True + return False def _has_host_leaves(tree: Any) -> bool: """Returns True if any array or variable leaf in `tree` is in host memory.""" - leaves = jax.tree.leaves(tree, is_leaf=lambda x: isinstance(x, nnx.Variable)) - return any(_is_host_leaf(leaf) for leaf in leaves) + leaves = jax.tree.leaves(tree, is_leaf=lambda x: isinstance(x, (nnx.Variable, SparseHostGrad))) + return any(isinstance(leaf, SparseHostGrad) or _is_host_leaf(leaf) for leaf in leaves) def _validate_kind(kind: str) -> Any: @@ -151,6 +211,433 @@ def _move_variables_in_place(tree: Any, kind: str) -> None: leaf.set_raw_value(_move_raw_value(leaf.get_raw_value(), kind, target_space)) +def _key_value(key: Any) -> Any: + """Extracts the raw Python key from a JAX PyTree key entry.""" + if hasattr(key, "key"): + return key.key + if hasattr(key, "name"): + return key.name + if hasattr(key, "idx"): + return key.idx + return key + + +def _format_param_path(path: Sequence[Any]) -> str: + return "/".join(str(k) for k in path) + + +def is_selective_host_offload_enabled(config: Any) -> bool: + """Returns True when selective path-pattern host offload is enabled in `config`.""" + return bool(config.selective_host_offload) if hasattr(config, "selective_host_offload") else False + + +def get_host_offload_param_patterns(config: Any) -> tuple[str, ...]: + """Returns normalized regex patterns selecting parameters for host offload.""" + raw = (config.host_offload_param_patterns if hasattr(config, "host_offload_param_patterns") else None) or () + if isinstance(raw, str): + return (raw,) if raw else () + return tuple(str(p) for p in raw if str(p)) + + +def _get_optimizer_mode(config: Any) -> str: + """Returns validated selective host-offload optimizer mode from `config`.""" + mode = ( + str(config.selective_host_offload_optimizer_mode) + if hasattr(config, "selective_host_offload_optimizer_mode") + else "host_compute" + ) + if mode not in _OPTIMIZER_MODES: + raise ValueError( + f"Unsupported selective_host_offload_optimizer_mode {mode!r}; expected one of {sorted(_OPTIMIZER_MODES)}." + ) + return mode + + +def _get_num_chunks(config: Any) -> int: + chunks = int(config.selective_host_offload_num_chunks) if hasattr(config, "selective_host_offload_num_chunks") else 16 + if chunks < 1: + raise ValueError(f"selective_host_offload_num_chunks must be >= 1, got {chunks}.") + return chunks + + +def _match_param_paths(params_state: Any, patterns: Sequence[str]) -> frozenset[tuple[Any, ...]]: + compiled = [re.compile(p) for p in patterns] + matched: list[tuple[Any, ...]] = [] + for path, _ in nnx.to_flat_state(params_state): + path_str = _format_param_path(path) + if any(rx.search(path_str) for rx in compiled): + matched.append(tuple(path)) + return frozenset(matched) + + +def selected_host_param_paths(model_or_state: Any) -> frozenset[tuple[Any, ...]]: + """Returns canonical paths of `nnx.Param` variables marked for selective host offload.""" + state = nnx.state(model_or_state, nnx.Param) if isinstance(model_or_state, nnx.Module) else model_or_state + paths: list[tuple[Any, ...]] = [] + for leaf in jax.tree.leaves(state, is_leaf=lambda x: isinstance(x, nnx.Variable)): + if isinstance(leaf, nnx.Variable): + marked = leaf.get_metadata(_SELECTIVE_HOST_PATH_KEY, default=None) + if marked is not None: + paths.append(tuple(marked)) + return frozenset(paths) + + +def has_selective_host_offload(model_or_state: Any) -> bool: + """Returns True when `model_or_state` has selectively host-offloaded parameters.""" + return bool(selected_host_param_paths(model_or_state)) + + +def _active_mesh() -> Any | None: + mesh = jax.sharding.get_abstract_mesh() + if mesh is None or mesh.empty or mesh.size <= 1: + return None + return mesh + + +def _is_explicit_mesh(mesh: Any | None) -> bool: + return mesh is not None and all(t == jax.sharding.AxisType.Explicit for t in mesh.axis_types) + + +def _normalize_axis_entry(entry: Any, mesh: Any) -> tuple[str, ...]: + if entry is None: + return () + names = (entry,) if isinstance(entry, str) else tuple(entry) + return tuple(ax for ax in names if ax in mesh.shape and mesh.shape[ax] > 1) + + +def _resolve_2d_axes( + var_or_arr: Any, + mesh: Any | None, + sharding_hint: Any = None, +) -> tuple[tuple[str, ...], tuple[str, ...]]: + """Returns active mesh `(row_axes, col_axes)` for a 2D table variable or array.""" + if mesh is None: + return (), () + spec = None + if sharding_hint is not None: + hint = sharding_hint.get_raw_value() if isinstance(sharding_hint, nnx.Variable) else sharding_hint + spec = getattr(hint, "spec", hint if isinstance(hint, P) else None) + if isinstance(var_or_arr, nnx.Variable): + raw = var_or_arr.get_raw_value() + if spec is None: + active_map = getattr(_ACTIVE_PARAM_SHARDINGS, "map", None) + sel_path = var_or_arr.get_metadata(_SELECTIVE_HOST_PATH_KEY, default=None) + if active_map and sel_path is not None and sel_path in active_map: + spec = active_map[sel_path] + if spec is None: + spec = var_or_arr.get_metadata("out_sharding", default=None) + else: + raw = var_or_arr + if spec is None: + sh = getattr(raw, "sharding", None) + if sh is None and isinstance(raw, jax.core.Tracer): + sh = getattr(jax.typeof(raw), "sharding", None) + spec = getattr(sh, "spec", None) + if spec is None: + return (), () + row_axes = _normalize_axis_entry(spec[0], mesh) if len(spec) >= 1 else () + col_axes = _normalize_axis_entry(spec[1], mesh) if len(spec) >= 2 else () + return row_axes, col_axes + + +def _pspec_dim(axes: tuple[str, ...]) -> Any: + if not axes: + return None + return axes[0] if len(axes) == 1 else axes + + +def _maybe_reshard(x: jax.Array, spec: P, mesh: Any | None) -> jax.Array: + if _is_explicit_mesh(mesh): + return jax.sharding.reshard(x, spec) + return x + + +def _can_move_vars(vars_list: Sequence[nnx.Variable]) -> bool: + raw_leaves = [v.get_raw_value() for v in vars_list] + return bool(raw_leaves) and not any( + isinstance(v, jax.core.Tracer) or (isinstance(v, jax.ShapeDtypeStruct) and getattr(v, "sharding", None) is None) + for v in raw_leaves + ) + + +# --------------------------------------------------------------------------- +# Row-sparse host gather, host scatter-add, and SparseHostGrad container +# --------------------------------------------------------------------------- + + +@compute_on.compute_on(compute_type="device_host", out_memory_spaces=jax.memory.Space.Device) +def _replicated_host_gather(table_h: jax.Array, ids_h: jax.Array) -> jax.Array: + return table_h[ids_h] + + +@compute_on.compute_on(compute_type="device_host", out_memory_spaces=jax.memory.Space.Host) +def _replicated_host_scatter_add(table_like_h: jax.Array, ids_h: jax.Array, d_out_h: jax.Array) -> jax.Array: + zeros_h = jax.device_put(jnp.zeros(table_like_h.shape, dtype=table_like_h.dtype), jax.memory.Space.Host) + return zeros_h.at[ids_h].add(d_out_h.astype(table_like_h.dtype)) + + +@compute_on.compute_on(compute_type="device_host", out_memory_spaces=jax.memory.Space.Device) +def _local_host_gather(local_table_h: jax.Array, ids_h: jax.Array, start_h: jax.Array) -> jax.Array: + n_local = local_table_h.shape[0] + rel = ids_h - start_h + in_range = (rel >= 0) & (rel < n_local) + zero_i = jax.device_put(jnp.int32(0), jax.memory.Space.Host) + safe = jnp.where(in_range, rel, zero_i) + zero_v = jax.device_put(jnp.zeros((), dtype=local_table_h.dtype), jax.memory.Space.Host) + return jnp.where(in_range[..., None], local_table_h[safe], zero_v) + + +@compute_on.compute_on(compute_type="device_host", out_memory_spaces=jax.memory.Space.Host) +def _local_host_scatter_add( + local_table_like_h: jax.Array, ids_h: jax.Array, d_out_h: jax.Array, start_h: jax.Array +) -> jax.Array: + """Scatters `d_out_h` into a zero-initialized local shard on host.""" + n_local = local_table_like_h.shape[0] + rel = ids_h - start_h + in_range = (rel >= 0) & (rel < n_local) + zero_i = jax.device_put(jnp.int32(0), jax.memory.Space.Host) + safe = jnp.where(in_range, rel, zero_i) + zero_v = jax.device_put(jnp.zeros((), dtype=local_table_like_h.dtype), jax.memory.Space.Host) + vals = jnp.where(in_range[..., None], d_out_h.astype(local_table_like_h.dtype), zero_v) + zeros_h = jax.device_put(jnp.zeros(local_table_like_h.shape, dtype=local_table_like_h.dtype), jax.memory.Space.Host) + return zeros_h.at[safe].add(vals) + + +def _host_embed_gather( + table: jax.Array, + ids: jax.Array, + *, + dtype: Any, + row_axes: tuple[str, ...], + col_axes: tuple[str, ...], +) -> jax.Array: + """Gathers only rows indexed by `ids` from host-resident `table` onto device.""" + mesh = _active_mesh() + if mesh is not None: + # Always isolate host compute in shard_map so collectives run on device ICI, not MegaScale host. + r_spec, c_spec = _pspec_dim(row_axes), _pspec_dim(col_axes) + rep_ids_spec = P(*([None] * ids.ndim)) + ids_in = _maybe_reshard(ids, rep_ids_spec, mesh) + + def _shard_gather(local_table: jax.Array, local_ids: jax.Array) -> jax.Array: + ids_h = jax.device_put(local_ids, jax.memory.Space.Host) + if row_axes: + start = jax.lax.axis_index(row_axes).astype(jnp.int32) * jnp.int32(local_table.shape[0]) + part = _local_host_gather(local_table, ids_h, jax.device_put(start, jax.memory.Space.Host)) + return jax.lax.psum(part, row_axes).astype(dtype) + return _replicated_host_gather(local_table, ids_h).astype(dtype) + + return _call_shard_map( + _shard_gather, + in_specs=(P(r_spec, c_spec), rep_ids_spec), + out_specs=P(*([None] * ids.ndim), c_spec), + args=(table, ids_in), + ) + + gathered = _replicated_host_gather(table, jax.device_put(ids, jax.memory.Space.Host)) + return gathered.astype(dtype) + + +def _host_embed_scatter_add( + table_like: jax.Array, + ids: jax.Array, + d_out: jax.Array, + *, + row_axes: tuple[str, ...], + col_axes: tuple[str, ...], +) -> jax.Array: + """Scatters `d_out` into a host-resident gradient matching `table_like`.""" + mesh = _active_mesh() + if mesh is not None: + r_spec, c_spec = _pspec_dim(row_axes), _pspec_dim(col_axes) + rep_ids_spec = P(*([None] * ids.ndim)) + out_spec = P(*([None] * ids.ndim), c_spec) + ids_in = _maybe_reshard(ids, rep_ids_spec, mesh) + d_out_in = _maybe_reshard(d_out, out_spec, mesh) + + def _shard_scatter(local_tbl: jax.Array, local_ids: jax.Array, local_g: jax.Array) -> jax.Array: + ids_h = jax.device_put(local_ids, jax.memory.Space.Host) + g_h = jax.device_put(local_g, jax.memory.Space.Host) + if row_axes: + start = jax.lax.axis_index(row_axes).astype(jnp.int32) * jnp.int32(local_tbl.shape[0]) + return _local_host_scatter_add(local_tbl, ids_h, g_h, jax.device_put(start, jax.memory.Space.Host)) + return _replicated_host_scatter_add(local_tbl, ids_h, g_h) + + return _call_shard_map( + _shard_scatter, + in_specs=(P(r_spec, c_spec), rep_ids_spec, out_spec), + out_specs=P(r_spec, c_spec), + args=(table_like, ids_in, d_out_in), + ) + + return _replicated_host_scatter_add( + table_like, + jax.device_put(ids, jax.memory.Space.Host), + jax.device_put(d_out, jax.memory.Space.Host), + ) + + +@functools.partial(jax.custom_vjp, nondiff_argnums=(2, 3, 4)) +def _host_embed_lookup_vjp( + table: jax.Array, + ids: jax.Array, + dtype: Any, + row_axes: tuple[str, ...], + col_axes: tuple[str, ...], +) -> jax.Array: + return _host_embed_gather(table, ids, dtype=dtype, row_axes=row_axes, col_axes=col_axes) + + +def _host_embed_lookup_fwd( + table: jax.Array, + ids: jax.Array, + dtype: Any, + row_axes: tuple[str, ...], + col_axes: tuple[str, ...], +) -> tuple[jax.Array, tuple[jax.Array, jax.Array]]: + out = _host_embed_gather(table, ids, dtype=dtype, row_axes=row_axes, col_axes=col_axes) + return out, (table, ids) + + +def _host_embed_lookup_bwd( + dtype: Any, + row_axes: tuple[str, ...], + col_axes: tuple[str, ...], + res: tuple[jax.Array, jax.Array], + g: jax.Array, +) -> tuple[jax.Array, None]: + """VJP backward scattering device cotangent directly into a host gradient table.""" + del dtype + table, ids = res + d_table = _host_embed_scatter_add(table, ids, g, row_axes=row_axes, col_axes=col_axes) + return (d_table, None) + + +_host_embed_lookup_vjp.defvjp(_host_embed_lookup_fwd, _host_embed_lookup_bwd) + + +@jax.tree_util.register_pytree_node_class +class SparseHostGrad: + """Row-sparse gradient for a host-offloaded embedding table. + + Stores only the accessed row indices `ids` and device cotangent `d_gathered` + without materializing a dense `(num_embeddings, features)` gradient in HBM. + """ + + def __init__( + self, + ids: jax.Array, + d_gathered: jax.Array, + *, + table_shape: tuple[int, int], + table_dtype: Any, + row_axes: tuple[str, ...] = (), + col_axes: tuple[str, ...] = (), + grad_dtype: Any = None, + ): + self.ids = ids + self.d_gathered = d_gathered + self.table_shape = tuple(int(d) for d in table_shape) + self.table_dtype = jnp.dtype(table_dtype) + self.row_axes = tuple(row_axes) + self.col_axes = tuple(col_axes) + self.grad_dtype = jnp.dtype(grad_dtype) if grad_dtype is not None else self.table_dtype + + def with_d_gathered(self, new_d_gathered: jax.Array, *, grad_dtype: Any = None) -> "SparseHostGrad": + return SparseHostGrad( + self.ids, + new_d_gathered, + table_shape=self.table_shape, + table_dtype=self.table_dtype, + row_axes=self.row_axes, + col_axes=self.col_axes, + grad_dtype=grad_dtype if grad_dtype is not None else self.grad_dtype, + ) + + def tree_flatten(self): + children = (self.ids, self.d_gathered) + aux = (self.table_shape, self.table_dtype, self.row_axes, self.col_axes, self.grad_dtype) + return children, aux + + @classmethod + def tree_unflatten(cls, aux, children): + ids, d_gathered = children + table_shape, table_dtype, row_axes, col_axes, grad_dtype = aux + return cls( + ids, + d_gathered, + table_shape=table_shape, + table_dtype=table_dtype, + row_axes=row_axes, + col_axes=col_axes, + grad_dtype=grad_dtype, + ) + + +def materialize_sparse_grad(grad: SparseHostGrad) -> jax.Array: + """Scatters a `SparseHostGrad` into a dense `(num_embeddings, features)` array.""" + flat_ids = grad.ids.reshape(-1) + flat_dg = grad.d_gathered.reshape(-1, grad.table_shape[1]).astype(grad.grad_dtype) + return jnp.zeros(grad.table_shape, dtype=grad.grad_dtype).at[flat_ids].add(flat_dg) + + +@contextlib.contextmanager +def _pregathered_context(mapping: dict[tuple[Any, ...], tuple[jax.Array, jax.Array]]): + prev = getattr(_PREGATHERED_STATE, "mapping", None) + _PREGATHERED_STATE.mapping = mapping + try: + yield + finally: + _PREGATHERED_STATE.mapping = prev + + +class HostOffloadedEmbed(nnx.Embed): + """Path-transparent `nnx.Embed` subclass performing row-sparse host gathers.""" + + def __init__(self, embed: nnx.Embed, *, scope: str | None = None): # pylint: disable=super-init-not-called + self.num_embeddings = embed.num_embeddings + self.features = embed.features + self.dtype = embed.dtype + self.param_dtype = embed.param_dtype + self.promote_dtype = embed.promote_dtype + self.embedding = embed.embedding + self._m3_embed_scope = scope + + @classmethod + def from_embed(cls, embed: nnx.Embed, *, scope: str | None = None) -> "HostOffloadedEmbed": + return cls(embed, scope=scope) + + def __call__(self, inputs: jax.Array, out_sharding: Any = None) -> jax.Array: + if not jnp.issubdtype(inputs.dtype, jnp.integer): + raise ValueError("Input type must be an integer or unsigned integer.") + scope = getattr(self, "_m3_embed_scope", None) + scope_ctx = jax.named_scope(scope) if scope else contextlib.nullcontext() + with scope_ctx: + param_path = self.embedding.get_metadata(_SELECTIVE_HOST_PATH_KEY, default=None) + active_map = getattr(_PREGATHERED_STATE, "mapping", None) + if param_path is not None and active_map is not None and param_path in active_map: + pre_ids, pre_gathered = active_map[param_path] + if inputs.shape != pre_ids.shape: + raise ValueError( + f"HostOffloadedEmbed pre-gather shape mismatch for {param_path}: " + f"received inputs.shape={inputs.shape}, expected {pre_ids.shape}." + ) + out = pre_gathered + if out_sharding is not None: + out = jax.lax.with_sharding_constraint(out, out_sharding) + return out + if _is_host_leaf(self.embedding): + table = self.embedding.get_raw_value() + target_dtype = jnp.dtype(self.dtype) if self.dtype is not None else table.dtype + mesh = _active_mesh() + row_axes, col_axes = _resolve_2d_axes(self.embedding, mesh) + out = _host_embed_lookup_vjp(table, inputs, target_dtype, row_axes, col_axes) + if out_sharding is not None: + out = jax.lax.with_sharding_constraint(out, out_sharding) + return out + return super().__call__(inputs, out_sharding=out_sharding) + + class _HostOffloadedParametersMeta(type): """Metaclass recognizing single-inheritance parameter-offloaded model subclasses.""" @@ -175,12 +662,18 @@ def _offloaded_model_class(cls: type[nnx.Module]) -> type[nnx.Module]: parent_vocab_tiling = getattr(cls, "logits_from_hidden_states_for_vocab_tiling", None) def _with_device_params(self: Any, fn: Any, *args: Any, **kwargs: Any) -> Any: + # Never pull selectively offloaded `HostOffloadedEmbed` tables into HBM here. + embed_var_ids = { + id(node.embedding) + for _, node in nnx.iter_graph(self) + if isinstance(node, HostOffloadedEmbed) and isinstance(node.embedding, nnx.Variable) + } param_vars = [ v for v in jax.tree.leaves(nnx.state(self, nnx.Param), is_leaf=lambda x: isinstance(x, nnx.Variable)) - if isinstance(v, nnx.Variable) + if isinstance(v, nnx.Variable) and id(v) not in embed_var_ids and _is_host_leaf(v) ] - if not any(_is_host_leaf(v) for v in param_vars): + if not param_vars: return fn(self, *args, **kwargs) saved_raw = [v.get_raw_value() for v in param_vars] for v in param_vars: @@ -218,31 +711,1221 @@ def _vocab_tiling(self: Any, *args: Any, **kwargs: Any) -> Any: return subcls +def _replace_child(parent: Any, key: Any, new_child: Any) -> None: + if isinstance(parent, (list, nnx.List)): + parent[int(key)] = new_child + elif isinstance(parent, dict): + parent[key] = new_child + else: + setattr(parent, str(key), new_child) + + +def _apply_selective_parameter_offload(model: nnx.Module, config: Any) -> tuple[nnx.Module, bool]: + """Offloads pattern-matched parameters to `pinned_host` and wraps matched `nnx.Embed` modules.""" + patterns = get_host_offload_param_patterns(config) + if not patterns: + raise ValueError("selective_host_offload=True requires non-empty host_offload_param_patterns.") + params = nnx.state(model, nnx.Param) + selected_paths = _match_param_paths(params, patterns) + if not selected_paths: + raise ValueError( + f"selective_host_offload=True matched 0 parameters for host_offload_param_patterns={list(patterns)!r}." + ) + + selected_vars: list[nnx.Variable] = [] + for path, var in nnx.to_flat_state(params): + path_tuple = tuple(path) + if path_tuple in selected_paths and isinstance(var, nnx.Variable): + var.set_metadata(_SELECTIVE_HOST_PATH_KEY, path_tuple) + selected_vars.append(var) + + if _can_move_vars(selected_vars): + _move_variables_in_place(selected_vars, "pinned_host") + else: + for v in selected_vars: + _attach_memory_kind_metadata(v, "pinned_host") + + # Replace matching `nnx.Embed` modules path-transparently with `HostOffloadedEmbed`. + nodes = dict(nnx.iter_graph(model)) + embed_paths_covered: set[tuple[Any, ...]] = set() + for mod_path, node in list(nodes.items()): + if not mod_path or not isinstance(node, nnx.Embed): + continue + emb_param_path = (*mod_path, "embedding") + if emb_param_path not in selected_paths: + continue + embed_paths_covered.add(emb_param_path) + if not isinstance(node, HostOffloadedEmbed): + parent = nodes.get(mod_path[:-1]) + if parent is None: + raise ValueError(f"Could not resolve parent module for selected embedding at {mod_path}.") + scope = named_scopes.scope_of(node) + _replace_child(parent, mod_path[-1], HostOffloadedEmbed.from_embed(node, scope=scope)) + + non_embed_selected = selected_paths - embed_paths_covered + return model, bool(non_embed_selected) + + def maybe_offload_parameters(model: nnx.Module, config: Any) -> nnx.Module: - """Moves `nnx.Param` state to `pinned_host` when `config.parameter_memory_host_offload` is set.""" - if not getattr(config, "parameter_memory_host_offload", False): + """Moves all or pattern-selected `nnx.Param` state to `pinned_host` per `config`.""" + full_offload = bool(config.parameter_memory_host_offload) if hasattr(config, "parameter_memory_host_offload") else False + selective_offload = is_selective_host_offload_enabled(config) + if full_offload and selective_offload: + raise ValueError("selective_host_offload=True cannot be combined with parameter_memory_host_offload=True.") + if selective_offload: + model, needs_full_wrap = _apply_selective_parameter_offload(model, config) + if not needs_full_wrap: + return model + elif not full_offload: return model - params = nnx.state(model, nnx.Param) - param_leaves = jax.tree.leaves(params, is_leaf=lambda x: isinstance(x, nnx.Variable)) - raw_leaves = [v.get_raw_value() if isinstance(v, nnx.Variable) else v for v in param_leaves] - # Inside `nnx.eval_shape`, attach the dispatch subclass without moving abstract leaves prematurely. - should_move = bool(raw_leaves) and not any( - isinstance(v, jax.core.Tracer) or (isinstance(v, jax.ShapeDtypeStruct) and getattr(v, "sharding", None) is None) - for v in raw_leaves - ) - if should_move: - _move_variables_in_place(params, "pinned_host") else: - for v in param_leaves: - if isinstance(v, nnx.Variable): + params = nnx.state(model, nnx.Param) + param_leaves = [ + v for v in jax.tree.leaves(params, is_leaf=lambda x: isinstance(x, nnx.Variable)) if isinstance(v, nnx.Variable) + ] + if _can_move_vars(param_leaves): + _move_variables_in_place(params, "pinned_host") + else: + for v in param_leaves: _attach_memory_kind_metadata(v, "pinned_host") if not isinstance(model, HostOffloadedParameters): object.__setattr__(model, "__class__", _offloaded_model_class(type(model))) return model +def has_host_leaves(tree: Any) -> bool: + """Returns True if any array or variable leaf in `tree` resides in host memory.""" + return _has_host_leaves(tree) + + +def apply_selective_memory_kind_to_shardings(abs_var_state: Any, sharding_state: Any) -> Any: + """Propagates `pinned_host` metadata from traced abstract vars onto `NamedSharding` trees.""" + + def _is_leaf(x: Any) -> bool: + return isinstance(x, (nnx.Variable, jax.sharding.Sharding, jax.ShapeDtypeStruct)) + + def _apply(a_leaf: Any, s_leaf: Any) -> Any: + if s_leaf is None: + return s_leaf + kind = a_leaf.get_metadata("memory_kind", default=None) if isinstance(a_leaf, nnx.Variable) else None + if kind not in _HOST_KINDS: + raw = a_leaf.get_raw_value() if isinstance(a_leaf, nnx.Variable) else a_leaf + sh = getattr(raw, "sharding", None) + if getattr(sh, "memory_kind", None) in _HOST_KINDS: + kind = sh.memory_kind + if kind not in _HOST_KINDS: + return s_leaf + + def _to_host_sharding(sh: Any) -> Any: + if isinstance(sh, jax.sharding.NamedSharding) and sh.memory_kind != kind: + return sh.with_memory_kind(kind) + if isinstance(sh, jax.ShapeDtypeStruct): + inner = getattr(sh, "sharding", None) + if isinstance(inner, jax.sharding.NamedSharding) and inner.memory_kind != kind: + return jax.ShapeDtypeStruct(sh.shape, sh.dtype, sharding=inner.with_memory_kind(kind)) + return sh + + def _is_sh_leaf(x: Any) -> bool: + return isinstance(x, (jax.sharding.Sharding, jax.ShapeDtypeStruct)) + + if isinstance(s_leaf, nnx.Variable): + updated = jax.tree.map(_to_host_sharding, s_leaf.get_raw_value(), is_leaf=_is_sh_leaf) + return s_leaf.replace(updated) + return jax.tree.map(_to_host_sharding, s_leaf, is_leaf=_is_sh_leaf) + + return jax.tree.map(_apply, abs_var_state, sharding_state, is_leaf=_is_leaf) + + +# --------------------------------------------------------------------------- +# Sparse value_and_grad (row-sparse backward with zero dense HBM gradient) +# --------------------------------------------------------------------------- + + +def _collect_host_embeds(model: nnx.Module) -> list[tuple[tuple[Any, ...], HostOffloadedEmbed, Any]]: + """Returns `(param_path, embed_module, parent_module)` for each `HostOffloadedEmbed`.""" + nodes = dict(nnx.iter_graph(model)) + specs: list[tuple[tuple[Any, ...], HostOffloadedEmbed, Any]] = [] + for mod_path, node in nodes.items(): + if isinstance(node, HostOffloadedEmbed) and isinstance(node.embedding, nnx.Variable): + param_path = node.embedding.get_metadata(_SELECTIVE_HOST_PATH_KEY, default=None) + if param_path is None: + param_path = (*mod_path, "embedding") + parent = nodes.get(mod_path[:-1]) if mod_path else None + specs.append((tuple(param_path), node, parent)) + return specs + + +def _extract_tokens_and_segments( + args: tuple[Any, ...], + kwargs: dict[str, Any], + input_tokens: jax.Array | None, + segment_ids: jax.Array | None, +) -> tuple[jax.Array | None, jax.Array | None]: + """Extracts token and optional segment ID arrays from training-step batch arguments.""" + if input_tokens is not None: + return input_tokens, segment_ids + for candidate in (*args, *kwargs.values()): + if isinstance(candidate, dict): + toks = candidate.get("inputs") + if toks is None: + toks = candidate.get("decoder_input_tokens") + if toks is not None: + segs = segment_ids + if segs is None: + segs = candidate.get("inputs_segmentation") + if segs is None: + segs = candidate.get("decoder_segment_ids") + if segs is None: + segs = candidate.get("segment_ids") + return toks, segs + return None, segment_ids + + +def _prepare_host_embed_gathers( + model: nnx.Module, + input_tokens: jax.Array, + segment_ids: jax.Array | None, +) -> tuple[dict[tuple[Any, ...], jax.Array], dict[tuple[Any, ...], dict[str, Any]]]: + """Pre-gathers rows for every `HostOffloadedEmbed` in `model`.""" + mesh = _active_mesh() + gathered_map: dict[tuple[Any, ...], jax.Array] = {} + meta_map: dict[tuple[Any, ...], dict[str, Any]] = {} + for param_path, embed_mod, parent_mod in _collect_host_embeds(model): + ids_fn = getattr(parent_mod, "embed_ids_from_tokens", None) if parent_mod is not None else None + if callable(ids_fn): + ids = ids_fn(input_tokens, segment_ids=segment_ids) + else: + ids = input_tokens + table = embed_mod.embedding.get_raw_value() + target_dtype = jnp.dtype(embed_mod.dtype) if embed_mod.dtype is not None else table.dtype + row_axes, col_axes = _resolve_2d_axes(embed_mod.embedding, mesh) + gathered = _host_embed_gather(table, ids, dtype=target_dtype, row_axes=row_axes, col_axes=col_axes) + gathered_map[param_path] = gathered + meta_map[param_path] = { + "ids": ids, + "table_shape": table.shape, + "table_dtype": table.dtype, + "row_axes": row_axes, + "col_axes": col_axes, + } + return gathered_map, meta_map + + +def _partition_state_by_paths( + state: nnx.State, excluded_paths: frozenset[tuple[Any, ...]] +) -> tuple[nnx.State, nnx.State]: + """Splits `state` into `(kept_state, excluded_state)` by canonical leaf path.""" + kept_items = [] + excl_items = [] + for path, val in nnx.to_flat_state(state): + if tuple(path) in excluded_paths: + excl_items.append((path, val)) + else: + kept_items.append((path, val)) + return nnx.from_flat_state(kept_items), nnx.from_flat_state(excl_items) + + +def _assemble_grads_with_sparse( + dev_grads: nnx.State, + d_gathered_map: dict[tuple[Any, ...], jax.Array], + meta_map: dict[tuple[Any, ...], dict[str, Any]], + *, + grad_dtype: Any = None, +) -> nnx.State: + """Merges device gradients with `SparseHostGrad` leaves into a full parameter-shaped `nnx.State`.""" + items = list(nnx.to_flat_state(dev_grads)) + for param_path, dg in d_gathered_map.items(): + meta = meta_map[param_path] + t_dtype = meta["table_dtype"] + eff_grad_dtype = ( + jnp.dtype(grad_dtype) if grad_dtype is not None and jnp.dtype(t_dtype) == jnp.float32 else jnp.dtype(t_dtype) + ) + items.append( + ( + param_path, + SparseHostGrad( + meta["ids"], + dg, + table_shape=meta["table_shape"], + table_dtype=t_dtype, + row_axes=meta["row_axes"], + col_axes=meta["col_axes"], + grad_dtype=eff_grad_dtype, + ), + ) + ) + return nnx.from_flat_state(items) + + +def value_and_grad( + loss_fn: Callable[..., Any], + model: nnx.Module, + *args: Any, + has_aux: bool = False, + input_tokens: jax.Array | None = None, + segment_ids: jax.Array | None = None, + **kwargs: Any, +) -> tuple[Any, Any]: + """Computes `(value, grads)` with row-sparse gradients for `HostOffloadedEmbed` tables.""" + if not has_selective_host_offload(model) or not _collect_host_embeds(model): + return nnx.value_and_grad(loss_fn, has_aux=has_aux)(model, *args, **kwargs) + tokens, segs = _extract_tokens_and_segments(args, kwargs, input_tokens, segment_ids) + if tokens is None: + raise ValueError( + "m3_offload.value_and_grad requires token IDs (via `input_tokens` or a batch dict with " + "'inputs'/'input_tokens') when HostOffloadedEmbed modules are present; refusing dense HBM fallback." + ) + + gathered_map, meta_map = _prepare_host_embed_gathers(model, tokens, segs) + if not gathered_map: + raise ValueError( + "m3_offload.value_and_grad found HostOffloadedEmbed modules but pre-gather produced no active gathers." + ) + + host_embed_paths = frozenset(gathered_map.keys()) + graphdef, params, rest = nnx.split(model, nnx.Param, ...) + dev_params, host_params = _partition_state_by_paths(params, host_embed_paths) + + def _diff_fn(dev_p: nnx.State, g_map: dict[tuple[Any, ...], jax.Array]) -> Any: + full_p = nnx.merge_state(dev_p, host_params) if jax.tree.leaves(host_params) else dev_p + local_model = nnx.merge(graphdef, full_p, rest, copy=True) + ctx_map = {p: (meta_map[p]["ids"], g_map[p]) for p in g_map} + with _pregathered_context(ctx_map): + return loss_fn(local_model, *args, **kwargs) + + val_out, (dev_grads, d_gathered_map) = jax.value_and_grad(_diff_fn, argnums=(0, 1), has_aux=has_aux)( + dev_params, gathered_map + ) + grads = _assemble_grads_with_sparse(dev_grads, d_gathered_map, meta_map) + return val_out, grads + + +def pure_value_and_grad( + diff_wrapper: Callable[..., Any], + model_graphdef: Any, + curr_params: nnx.State, + *, + custom_params: nnx.State, + rest: nnx.State, + config: Any, + data: dict[str, Any], + has_aux: bool = True, +) -> tuple[Any, tuple[nnx.State, nnx.State]]: + """Pure-state `value_and_grad` for `train.py` with row-sparse host-embed gradients.""" + probe_model = nnx.merge(model_graphdef, curr_params, custom_params, rest, copy=True) + if not _collect_host_embeds(probe_model): + return jax.value_and_grad(diff_wrapper, argnums=(0, 1), has_aux=has_aux)( + curr_params, custom_params, rest, config, data + ) + tokens, segs = _extract_tokens_and_segments((data,), {}, None, None) + if tokens is None: + raise ValueError( + "m3_offload.pure_value_and_grad requires token IDs in `data` ('inputs'/'input_tokens') " + "when HostOffloadedEmbed modules are present; refusing dense HBM fallback." + ) + gathered_map, meta_map = _prepare_host_embed_gathers(probe_model, tokens, segs) + if not gathered_map: + raise ValueError( + "m3_offload.pure_value_and_grad found HostOffloadedEmbed modules but pre-gather produced no active gathers." + ) + + host_embed_paths = frozenset(gathered_map.keys()) + dev_params, host_params = _partition_state_by_paths(curr_params, host_embed_paths) + + def _wrapped( + dev_p: nnx.State, + cust_p: nnx.State, + g_map: dict[tuple[Any, ...], jax.Array], + ) -> Any: + full_p = nnx.merge_state(dev_p, host_params) if jax.tree.leaves(host_params) else dev_p + ctx_map = {p: (meta_map[p]["ids"], g_map[p]) for p in g_map} + with _pregathered_context(ctx_map): + return diff_wrapper(full_p, cust_p, rest, config, data) + + val_out, (dev_grads, custom_grads, d_gathered_map) = jax.value_and_grad(_wrapped, argnums=(0, 1, 2), has_aux=has_aux)( + dev_params, custom_params, gathered_map + ) + raw_grads = _assemble_grads_with_sparse( + dev_grads, + d_gathered_map, + meta_map, + grad_dtype=config.grad_dtype if hasattr(config, "grad_dtype") else None, + ) + return val_out, (raw_grads, custom_grads) + + +def engine_value_and_grad( + diff_wrapper: Callable[..., Any], + model_graphdef: Any, + params: nnx.State, + rest: nnx.State, + batch: Any, + *, + param_shardings: Any = None, +) -> tuple[Any, nnx.State]: + """Engine `fwd_bwd` differentiation avoiding `` cotangents across scan/cond boundaries.""" + with active_param_shardings(param_shardings): + full_model = nnx.merge(model_graphdef, params, rest, copy=True) + if not _collect_host_embeds(full_model): + return jax.value_and_grad(diff_wrapper, argnums=0, has_aux=True)(params, rest, batch) + tokens, segs = _extract_tokens_and_segments((batch,), {}, None, None) + if tokens is None: + raise ValueError( + "m3_offload.engine_value_and_grad requires token IDs in `batch` ('inputs'/'input_tokens') " + "when HostOffloadedEmbed modules are present; refusing dense HBM fallback." + ) + gathered_map, meta_map = _prepare_host_embed_gathers(full_model, tokens, segs) + if not gathered_map: + raise ValueError( + "m3_offload.engine_value_and_grad found HostOffloadedEmbed modules but pre-gather produced no active gathers." + ) + + host_embed_paths = frozenset(gathered_map.keys()) + dev_params, host_params = _partition_state_by_paths(params, host_embed_paths) + host_param_map = {tuple(p): v for p, v in nnx.to_flat_state(host_params)} + + def _wrapped(dev_p: nnx.State, g_map: dict[tuple[Any, ...], jax.Array]) -> Any: + full_p = nnx.merge_state(dev_p, host_params) if jax.tree.leaves(host_params) else dev_p + ctx_map = {p: (meta_map[p]["ids"], g_map[p]) for p in g_map} + with _pregathered_context(ctx_map): + return diff_wrapper(full_p, rest, batch) + + val_out, (dev_grads, d_gathered_map) = jax.value_and_grad(_wrapped, argnums=(0, 1), has_aux=True)( + dev_params, gathered_map + ) + grad_items = list(nnx.to_flat_state(dev_grads)) + for param_path, dg in d_gathered_map.items(): + meta = meta_map[param_path] + p_leaf = host_param_map[param_path] + table = p_leaf.get_raw_value() if isinstance(p_leaf, nnx.Variable) else p_leaf + d_table_h = _host_embed_scatter_add(table, meta["ids"], dg, row_axes=meta["row_axes"], col_axes=meta["col_axes"]) + grad_leaf = p_leaf.replace(d_table_h) if isinstance(p_leaf, nnx.Variable) else d_table_h + grad_items.append((param_path, grad_leaf)) + return val_out, nnx.from_flat_state(grad_items) + + +# --------------------------------------------------------------------------- +# Exact global L2 norm, clipping, and host-safe gradient ops +# --------------------------------------------------------------------------- + + +def _sparse_grad_sqnorm(grad: SparseHostGrad) -> jax.Array: + """Exact squared L2 norm `||scatter_add(0, ids, d_gathered)||_2^2` in O(M*D) memory.""" + mesh = _active_mesh() + + def _local_sqnorm(ids_arr: jax.Array, dg_arr: jax.Array, start: jax.Array, n_local: int) -> jax.Array: + flat_ids = ids_arr.reshape(-1) + flat_dg = dg_arr.reshape(-1, dg_arr.shape[-1]).astype(jnp.float32) + m = flat_ids.shape[0] + rel = flat_ids - start + in_range = (rel >= 0) & (rel < n_local) + # Send out-of-shard entries to a unique sentinel so they stay masked out cleanly. + sentinel = jnp.int32(n_local) + key = jnp.where(in_range, rel, sentinel) + vals = jnp.where(in_range[:, None], flat_dg, jnp.zeros((), dtype=jnp.float32)) + order = jnp.argsort(key) + sorted_key = key[order] + sorted_vals = vals[order] + is_new = jnp.concatenate([jnp.array([True]), sorted_key[1:] != sorted_key[:-1]]) + seg_ids = jnp.cumsum(is_new.astype(jnp.int32)) - 1 + seg_sums = jax.ops.segment_sum(sorted_vals, seg_ids, num_segments=m) + # Match exact grad_dtype rounding before squaring. + seg_sums = seg_sums.astype(grad.grad_dtype).astype(jnp.float32) + return jnp.sum(seg_sums * seg_sums) + + if mesh is not None: + c_spec = _pspec_dim(grad.col_axes) + rep_ids_spec = P(*([None] * grad.ids.ndim)) + dg_spec = P(*([None] * grad.ids.ndim), c_spec) + ids_in = _maybe_reshard(grad.ids, rep_ids_spec, mesh) + dg_in = _maybe_reshard(grad.d_gathered, dg_spec, mesh) + n_shards = 1 + for ax in grad.row_axes: + n_shards *= mesh.shape[ax] + n_local = grad.table_shape[0] // n_shards + sum_axes = (*grad.row_axes, *grad.col_axes) + + def _body(local_ids: jax.Array, local_dg: jax.Array) -> jax.Array: + start = jax.lax.axis_index(grad.row_axes).astype(jnp.int32) * jnp.int32(n_local) if grad.row_axes else jnp.int32(0) + local_sq = _local_sqnorm(local_ids, local_dg, start, n_local) + return jax.lax.psum(local_sq, sum_axes) if sum_axes else local_sq + + return _call_shard_map(_body, in_specs=(rep_ids_spec, dg_spec), out_specs=P(), args=(ids_in, dg_in)) + + return _local_sqnorm(grad.ids, grad.d_gathered, jnp.int32(0), grad.table_shape[0]) + + +@compute_on.compute_on(compute_type="device_host", out_memory_spaces=jax.memory.Space.Device) +def _host_array_sqnorm_raw(x_h: jax.Array) -> jax.Array: + x32 = x_h.astype(jnp.float32) + return jnp.sum(x32 * x32) + + +def _host_array_sqnorm(x: jax.Array, var_or_arr: Any = None, *, sharding_hint: Any = None) -> jax.Array: + """Computes squared L2 norm of a host-resident array on host without moving it to HBM.""" + mesh = _active_mesh() + row_axes, col_axes = _resolve_2d_axes(var_or_arr if var_or_arr is not None else x, mesh, sharding_hint=sharding_hint) + if mesh is not None and x.ndim == 2: + r_spec, c_spec = _pspec_dim(row_axes), _pspec_dim(col_axes) + sum_axes = (*row_axes, *col_axes) + + def _body(local_x: jax.Array) -> jax.Array: + local_sq = _host_array_sqnorm_raw(local_x) + return jax.lax.psum(local_sq, sum_axes) if sum_axes else local_sq + + return _call_shard_map(_body, in_specs=(P(r_spec, c_spec),), out_specs=P(), args=(x,)) + return _host_array_sqnorm_raw(x) + + +def _apply_host_elementwise( + host_fn: Any, + ref_leaf: Any, + host_arrays: tuple[jax.Array, ...], + device_scalars: tuple[jax.Array, ...] = (), + *, + sharding_hint: Any = None, +) -> jax.Array: + """Isolates an elementwise host-compute op inside `shard_map` when a mesh is active.""" + mesh = _active_mesh() + arr0 = host_arrays[0] + if mesh is not None and arr0.ndim == 2: + row_axes, col_axes = _resolve_2d_axes(ref_leaf, mesh, sharding_hint=sharding_hint) + tbl_spec = P(_pspec_dim(row_axes), _pspec_dim(col_axes)) + in_specs = (tbl_spec,) * len(host_arrays) + (P(),) * len(device_scalars) + n_arr = len(host_arrays) + + def _body(*local_args: jax.Array) -> jax.Array: + loc_arrs = local_args[:n_arr] + loc_scalars = tuple(jax.device_put(s, jax.memory.Space.Host) for s in local_args[n_arr:]) + return host_fn(*loc_arrs, *loc_scalars) + + return _call_shard_map(_body, in_specs=in_specs, out_specs=tbl_spec, args=(*host_arrays, *device_scalars)) + scalars_h = tuple(jax.device_put(s, jax.memory.Space.Host) for s in device_scalars) + return host_fn(*host_arrays, *scalars_h) + + +@compute_on.compute_on(compute_type="device_host", out_memory_spaces=jax.memory.Space.Host) +def _host_cast_array(x_h: jax.Array, dtype_holder_h: jax.Array) -> jax.Array: + return x_h.astype(dtype_holder_h.dtype) + + +@compute_on.compute_on(compute_type="device_host", out_memory_spaces=jax.memory.Space.Host) +def _host_add_arrays(a_h: jax.Array, b_h: jax.Array) -> jax.Array: + return a_h + b_h.astype(a_h.dtype) + + +@compute_on.compute_on(compute_type="device_host", out_memory_spaces=jax.memory.Space.Host) +def _host_scale_array(x_h: jax.Array, scale_h: jax.Array) -> jax.Array: + return x_h * scale_h.astype(x_h.dtype) + + +@compute_on.compute_on(compute_type="device_host", out_memory_spaces=jax.memory.Space.Host) +def _host_clip_array(x_h: jax.Array, trigger_h: jax.Array, inv_norm_h: jax.Array) -> jax.Array: + scaled = x_h * inv_norm_h.astype(x_h.dtype) + return jnp.where(trigger_h, x_h, scaled) + + +@compute_on.compute_on(compute_type="device_host", out_memory_spaces=jax.memory.Space.Host) +def _host_where_arrays(true_h: jax.Array, false_h: jax.Array, cond_h: jax.Array) -> jax.Array: + return jnp.where(cond_h, true_h, false_h) + + +@compute_on.compute_on(compute_type="device_host", out_memory_spaces=jax.memory.Space.Host) +def _host_zero_if(arr_h: jax.Array, cond_h: jax.Array) -> jax.Array: + return jnp.where(cond_h, jax.device_put(jnp.zeros((), dtype=arr_h.dtype), jax.memory.Space.Host), arr_h) + + +def _is_tree_leaf(x: Any) -> bool: + return isinstance(x, (nnx.Variable, SparseHostGrad, jax.sharding.Sharding, P)) + + +def select_tree_if(cond: jax.Array, true_tree: Any, false_tree: Any, *, shardings: Any = None) -> Any: + """Elementwise `jnp.where(cond, true_tree, false_tree)` keeping host leaves on host.""" + + def _sel(t_leaf: Any, f_leaf: Any, sh_leaf: Any = None) -> Any: + if isinstance(f_leaf, SparseHostGrad): + t_dg = t_leaf.d_gathered if isinstance(t_leaf, SparseHostGrad) else jnp.zeros_like(f_leaf.d_gathered) + return f_leaf.with_d_gathered(jnp.where(cond, t_dg, f_leaf.d_gathered)) + t_val = t_leaf.get_raw_value() if isinstance(t_leaf, nnx.Variable) else t_leaf + f_val = f_leaf.get_raw_value() if isinstance(f_leaf, nnx.Variable) else f_leaf + if not hasattr(f_val, "shape") or not hasattr(f_val, "dtype"): + return f_leaf + if _is_host_leaf(t_leaf) or _is_host_leaf(f_leaf): + t_h = t_val if _is_host_leaf(t_leaf) else jax.device_put(t_val, jax.memory.Space.Host) + f_h = f_val if _is_host_leaf(f_leaf) else jax.device_put(f_val, jax.memory.Space.Host) + out = _apply_host_elementwise(_host_where_arrays, f_leaf, (t_h, f_h), (cond,), sharding_hint=sh_leaf) + return f_leaf.replace(out) if isinstance(f_leaf, nnx.Variable) else out + out = jnp.where(cond, t_val, f_val) + return f_leaf.replace(out) if isinstance(f_leaf, nnx.Variable) else out + + if shardings is not None: + return jax.tree.map(_sel, true_tree, false_tree, shardings, is_leaf=_is_tree_leaf) + return jax.tree.map(_sel, true_tree, false_tree, is_leaf=_is_tree_leaf) + + +def zero_tree_if(cond: jax.Array, tree: Any, *, shardings: Any = None) -> Any: + """Zeros leaves when `cond` is True without copying host leaves onto HBM.""" + + def _zero(leaf: Any, sh_leaf: Any = None) -> Any: + if isinstance(leaf, SparseHostGrad): + return leaf.with_d_gathered(jnp.where(cond, jnp.zeros_like(leaf.d_gathered), leaf.d_gathered)) + val = leaf.get_raw_value() if isinstance(leaf, nnx.Variable) else leaf + if not hasattr(val, "shape") or not hasattr(val, "dtype"): + return leaf + if _is_host_leaf(leaf): + out = _apply_host_elementwise(_host_zero_if, leaf, (val,), (cond,), sharding_hint=sh_leaf) + return leaf.replace(out) if isinstance(leaf, nnx.Variable) else out + out = jnp.where(cond, jnp.zeros_like(val), val) + return leaf.replace(out) if isinstance(leaf, nnx.Variable) else out + + if shardings is not None: + return jax.tree.map(_zero, tree, shardings, is_leaf=_is_tree_leaf) + return jax.tree.map(_zero, tree, is_leaf=_is_tree_leaf) + + +def cast_grad_leaf(leaf: Any, grad_dtype: Any, *, sharding: Any = None) -> Any: + """Casts a gradient leaf (`device`, `pinned_host`, or `SparseHostGrad`) without moving host arrays to HBM.""" + target = jnp.dtype(grad_dtype) + if isinstance(leaf, SparseHostGrad): + if leaf.table_dtype == jnp.float32 and leaf.grad_dtype != target: + return leaf.with_d_gathered(leaf.d_gathered, grad_dtype=target) + return leaf + val = leaf.get_raw_value() if isinstance(leaf, nnx.Variable) else leaf + if not hasattr(val, "dtype") or val.dtype == target: + return leaf + if _is_host_leaf(leaf): + casted = _apply_host_elementwise( + _host_cast_array, leaf, (val,), (jnp.zeros((), dtype=target),), sharding_hint=sharding + ) + return leaf.replace(casted) if isinstance(leaf, nnx.Variable) else casted + casted = val.astype(target) + return leaf.replace(casted) if isinstance(leaf, nnx.Variable) else casted + + +def cast_grad_tree(grads: Any, grad_dtype: Any) -> Any: + """Casts float32 gradient leaves to `grad_dtype`, keeping host leaves on host.""" + target = jnp.dtype(grad_dtype) + + def _cast(leaf: Any) -> Any: + if isinstance(leaf, SparseHostGrad): + return cast_grad_leaf(leaf, target) + val = leaf.get_raw_value() if isinstance(leaf, nnx.Variable) else leaf + if hasattr(val, "dtype") and val.dtype == jnp.float32 and target != jnp.float32: + return cast_grad_leaf(leaf, target) + return leaf + + return jax.tree.map(_cast, grads, is_leaf=lambda x: isinstance(x, (nnx.Variable, SparseHostGrad))) + + +def accumulate_grad_leaves(acc: Any, g: Any, sharding: Any = None) -> Any: + """Adds gradient leaf `g` into `acc` on host when both reside in host memory.""" + acc_val = acc.get_raw_value() if isinstance(acc, nnx.Variable) else acc + g_val = g.get_raw_value() if isinstance(g, nnx.Variable) else g + if _is_host_leaf(acc) or _is_host_leaf(g): + acc_h = jax.device_put(acc_val, jax.memory.Space.Host) if not _is_host_leaf(acc) else acc_val + g_h = jax.device_put(g_val, jax.memory.Space.Host) if not _is_host_leaf(g) else g_val + out = _apply_host_elementwise(_host_add_arrays, acc, (acc_h, g_h), sharding_hint=sharding) + return acc.replace(out) if isinstance(acc, nnx.Variable) else out + out = acc_val + g_val.astype(acc_val.dtype) + return acc.replace(out) if isinstance(acc, nnx.Variable) else out + + +def scale_grad_leaf(leaf: Any, factor: jax.Array, *, sharding: Any = None) -> Any: + """Multiplies a gradient leaf by scalar `factor`, preserving host placement.""" + if isinstance(leaf, SparseHostGrad): + return leaf.with_d_gathered(leaf.d_gathered * factor.astype(leaf.d_gathered.dtype)) + val = leaf.get_raw_value() if isinstance(leaf, nnx.Variable) else leaf + if _is_host_leaf(leaf): + out = _apply_host_elementwise(_host_scale_array, leaf, (val,), (factor,), sharding_hint=sharding) + return leaf.replace(out) if isinstance(leaf, nnx.Variable) else out + out = val * factor.astype(val.dtype) + return leaf.replace(out) if isinstance(leaf, nnx.Variable) else out + + +def l2norm_pytree(tree: Any, *, shardings: Any = None) -> jax.Array: + """Exact global L2 norm across device arrays, `SparseHostGrad`, and `pinned_host` leaves.""" + leaves = jax.tree.leaves(tree, is_leaf=_is_tree_leaf) + sh_leaves = jax.tree.leaves(shardings, is_leaf=_is_tree_leaf) if shardings is not None else [None] * len(leaves) + sq_terms: list[jax.Array] = [] + for leaf, sh_leaf in zip(leaves, sh_leaves): + if leaf is None: + continue + if isinstance(leaf, SparseHostGrad): + sq_terms.append(_sparse_grad_sqnorm(leaf)) + continue + val = leaf.get_raw_value() if isinstance(leaf, nnx.Variable) else leaf + if not hasattr(val, "dtype") or not hasattr(val, "shape"): + continue + if _is_host_leaf(leaf): + sq_terms.append(_host_array_sqnorm(val, leaf, sharding_hint=sh_leaf)) + else: + v32 = val.astype(jnp.float32) + sq_terms.append(jnp.sum(v32 * v32)) + if not sq_terms: + return jnp.array(0.0, dtype=jnp.float32) + return jnp.sqrt(functools.reduce(jnp.add, sq_terms)) + + +def clip_grads_and_norms(grads: Any, max_norm: float, *, shardings: Any = None) -> tuple[Any, jax.Array, jax.Array]: + """Clips a mixed gradient tree in one pass, returning `(clipped, raw_norm, clipped_norm)`.""" + g_norm = l2norm_pytree(grads, shardings=shardings) + if max_norm <= 0: + return grads, g_norm, g_norm + max_norm_arr = jnp.asarray(max_norm, dtype=jnp.float32) + trigger = jnp.squeeze(g_norm < max_norm_arr) + safe_norm = jnp.where(trigger, jnp.ones_like(g_norm), g_norm) + scale_f32 = max_norm_arr / safe_norm + + def _clip(leaf: Any, sh_leaf: Any = None) -> Any: + if leaf is None: + return None + if isinstance(leaf, SparseHostGrad): + # Clip in `grad_dtype` (e.g. fp32 when `weight_dtype=float32, dtype=bfloat16`) to avoid bf16 pre-rounding. + dg = leaf.d_gathered.astype(leaf.grad_dtype) + clipped_dg = jax.lax.select(trigger, dg, (dg / safe_norm.astype(dg.dtype)) * max_norm_arr.astype(dg.dtype)) + return leaf.with_d_gathered(clipped_dg) + val = leaf.get_raw_value() if isinstance(leaf, nnx.Variable) else leaf + if not hasattr(val, "dtype") or not hasattr(val, "shape"): + return leaf + if _is_host_leaf(leaf): + clipped = _apply_host_elementwise(_host_clip_array, leaf, (val,), (trigger, scale_f32), sharding_hint=sh_leaf) + return leaf.replace(clipped) if isinstance(leaf, nnx.Variable) else clipped + clipped = jax.lax.select(trigger, val, (val / safe_norm.astype(val.dtype)) * max_norm_arr.astype(val.dtype)) + return leaf.replace(clipped) if isinstance(leaf, nnx.Variable) else clipped + + if shardings is not None: + clipped = jax.tree.map(_clip, grads, shardings, is_leaf=_is_tree_leaf) + else: + clipped = jax.tree.map(_clip, grads, is_leaf=_is_tree_leaf) + clipped_norm = jnp.where(trigger, g_norm, jnp.where(jnp.isfinite(g_norm), max_norm_arr, g_norm)) + return clipped, g_norm, clipped_norm + + +def clip_grads_by_global_norm(grads: Any, max_norm: float, *, shardings: Any = None) -> Any: + """Clips a mixed device/`SparseHostGrad`/`pinned_host` gradient tree by global L2 norm.""" + clipped, _, _ = clip_grads_and_norms(grads, max_norm, shardings=shardings) + return clipped + + +@contextlib.contextmanager +def capture_host_param_sqnorms(): + """Captures post-update squared L2 norms of host-offloaded params inside the optimizer host pass.""" + prev = getattr(_HOST_SQNORM_CAPTURE, "sqnorms", None) + captured: dict[tuple[Any, ...], jax.Array] = {} + _HOST_SQNORM_CAPTURE.sqnorms = captured + try: + yield captured + finally: + _HOST_SQNORM_CAPTURE.sqnorms = prev + + +def l2norm_pytree_with_overrides(tree: Any, sqnorm_overrides: dict[tuple[Any, ...], jax.Array]) -> jax.Array: + """Exact global L2 norm substituting pre-computed squared norms for captured host parameters.""" + if not sqnorm_overrides: + return l2norm_pytree(tree) + sq_terms: list[jax.Array] = [] + for path, leaf in nnx.to_flat_state(tree): + path_tuple = tuple(path) + if path_tuple in sqnorm_overrides: + sq_terms.append(sqnorm_overrides[path_tuple]) + continue + if leaf is None: + continue + if isinstance(leaf, SparseHostGrad): + sq_terms.append(_sparse_grad_sqnorm(leaf)) + continue + val = leaf.get_raw_value() if isinstance(leaf, nnx.Variable) else leaf + if not hasattr(val, "dtype") or not hasattr(val, "shape"): + continue + if _is_host_leaf(leaf): + sq_terms.append(_host_array_sqnorm(val, leaf)) + else: + v32 = val.astype(jnp.float32) + sq_terms.append(jnp.sum(v32 * v32)) + if not sq_terms: + return jnp.array(0.0, dtype=jnp.float32) + return jnp.sqrt(functools.reduce(jnp.add, sq_terms)) + + +# --------------------------------------------------------------------------- +# Selective optimizer state slicing & host-compute / streamed-chunks / lazy-sparse updates +# --------------------------------------------------------------------------- + + +def _nest_path(path: tuple[Any, ...], value: Any) -> dict[Any, Any]: + node: Any = value + for key in reversed(path): + node = {key: node} + return node + + +def _unnest_path(path: tuple[Any, ...], tree: Any) -> Any: + node = tree + for key in path: + node = node[key] + return node + + +def _filter_mapping_by_paths( + mapping: dict[Any, Any], + keep_fn: Callable[[tuple[Any, ...]], bool], + prefix: tuple[Any, ...] = (), +) -> dict[Any, Any]: + """Filters a nested parameter/state dict keeping only leaves where `keep_fn(path)` is True.""" + out: dict[Any, Any] = {} + for k, v in mapping.items(): + sub_path = (*prefix, k) + if isinstance(v, (dict, nnx.State)): + sub_dict = dict(v) if isinstance(v, nnx.State) else v + filtered = _filter_mapping_by_paths(sub_dict, keep_fn, sub_path) + if filtered: + out[k] = filtered + elif keep_fn(sub_path): + out[k] = v + return out + + +def _flat_dict_paths(mapping: dict[Any, Any], prefix=()) -> set[tuple[Any, ...]]: + paths: set[tuple[Any, ...]] = set() + for k, v in mapping.items(): + sub_path = (*prefix, k) + if isinstance(v, (dict, nnx.State)): + sub_dict = dict(v) if isinstance(v, nnx.State) else v + paths |= _flat_dict_paths(sub_dict, sub_path) + else: + paths.add(sub_path) + return paths + + +def _slice_opt_state( + opt_state: Any, + all_param_paths: frozenset[tuple[Any, ...]], + keep_fn: Callable[[tuple[Any, ...]], bool], +) -> Any: + """Filters parameter-shaped subtrees inside `opt_state` while keeping scalars intact.""" + if isinstance(opt_state, (dict, nnx.State)): + as_dict = dict(opt_state) if isinstance(opt_state, nnx.State) else opt_state + leaf_paths = _flat_dict_paths(as_dict) + if leaf_paths and leaf_paths.issubset(all_param_paths): + return _filter_mapping_by_paths(as_dict, keep_fn) + return {k: _slice_opt_state(v, all_param_paths, keep_fn) for k, v in as_dict.items()} + if isinstance(opt_state, tuple) and hasattr(opt_state, "_fields"): + return type(opt_state)( + *[_slice_opt_state(getattr(opt_state, f), all_param_paths, keep_fn) for f in opt_state._fields] + ) + if isinstance(opt_state, tuple): + return tuple(_slice_opt_state(v, all_param_paths, keep_fn) for v in opt_state) + if isinstance(opt_state, list): + return [_slice_opt_state(v, all_param_paths, keep_fn) for v in opt_state] + return opt_state + + +def _lookup_path_in_tree(tree: Any, key_tuple: tuple[Any, ...]) -> Any: + """Traverses a nested opt_state PyTree along `key_tuple`.""" + node = tree + for k in key_tuple: + if isinstance(node, (dict, nnx.State)): + node = node[k] + elif isinstance(node, (tuple, list)) and isinstance(k, int): + node = node[k] + elif hasattr(node, str(k)): + node = getattr(node, str(k)) + else: + node = node[k] + return node + + +def _is_matching_opt_leaf( + key_tuple: tuple[Any, ...], selected_paths: frozenset[tuple[Any, ...]] +) -> tuple[Any, ...] | None: + for p_path in selected_paths: + n = len(p_path) + if len(key_tuple) >= n and key_tuple[-n:] == p_path: + return p_path + return None + + +def _run_tx_on_single_leaf( + tx: optax.GradientTransformation, + p_path: tuple[Any, ...], + grad_arr: jax.Array, + opt_state_sub: Any, + param_arr: jax.Array, + **kwargs: Any, +) -> tuple[jax.Array, Any]: + """Runs `tx.update` + `apply_updates` on a single parameter wrapped at its exact path.""" + g_nested = _nest_path(p_path, grad_arr) + p_nested = _nest_path(p_path, param_arr) + updates_nested, new_st = tx.update(g_nested, opt_state_sub, p_nested, **kwargs) + upd_arr = _unnest_path(p_path, updates_nested) + new_p = optax.apply_updates(param_arr, upd_arr) + return new_p, new_st + + +def _selective_host_compute_update( + param: jax.Array, + grad: Any, + opt_state_sub: Any, + *, + tx: optax.GradientTransformation, + p_path: tuple[Any, ...], + row_axes: tuple[str, ...], + col_axes: tuple[str, ...], + kwargs_arrays: dict[str, Any], +) -> tuple[jax.Array, Any]: + """Runs exact dense optimizer update on host CPU with zero HBM full-table traffic.""" + mesh = _active_mesh() + is_sparse = isinstance(grad, SparseHostGrad) + + capture_map = getattr(_HOST_SQNORM_CAPTURE, "sqnorms", None) + capture_sqnorm = capture_map is not None + + @compute_on.compute_on(compute_type="device_host", out_memory_spaces=jax.memory.Space.Host) + def host_sparse_fn(p_loc, st_loc_h, sparse_h, start_h, kw_h): + ids_h, dg_h = sparse_h + n_local = p_loc.shape[0] + g_dtype = grad.grad_dtype + if row_axes: + rel = ids_h - start_h + in_range = (rel >= 0) & (rel < n_local) + zero_i = jax.device_put(jnp.int32(0), jax.memory.Space.Host) + safe = jnp.where(in_range, rel, zero_i) + zero_v = jax.device_put(jnp.zeros((), dtype=g_dtype), jax.memory.Space.Host) + vals = jnp.where(in_range[..., None], dg_h.astype(g_dtype), zero_v) + z_buf = jax.device_put(jnp.zeros(p_loc.shape, dtype=g_dtype), jax.memory.Space.Host) + g_loc = z_buf.at[safe].add(vals) + else: + z_buf = jax.device_put(jnp.zeros(p_loc.shape, dtype=g_dtype), jax.memory.Space.Host) + g_loc = z_buf.at[ids_h].add(dg_h.astype(g_dtype)) + p_new, st_new = _run_tx_on_single_leaf(tx, p_path, g_loc, st_loc_h, p_loc, **kw_h) + if capture_sqnorm: + p32 = p_new.astype(jnp.float32) + return p_new, st_new, jnp.sum(p32 * p32) + return p_new, st_new + + @compute_on.compute_on(compute_type="device_host", out_memory_spaces=jax.memory.Space.Host) + def host_dense_fn(p_loc, st_loc_h, g_loc, kw_h): + p_new, st_new = _run_tx_on_single_leaf(tx, p_path, g_loc, st_loc_h, p_loc, **kw_h) + if capture_sqnorm: + p32 = p_new.astype(jnp.float32) + return p_new, st_new, jnp.sum(p32 * p32) + return p_new, st_new + + def _move_scalars(tree: Any, space: Any) -> Any: + return jax.tree.map(lambda x: jax.device_put(x, space) if hasattr(x, "ndim") and x.ndim == 0 else x, tree) + + if mesh is not None: + r_spec, c_spec = _pspec_dim(row_axes), _pspec_dim(col_axes) + tbl_spec = P(r_spec, c_spec) + st_spec = jax.tree.map(lambda x: tbl_spec if hasattr(x, "ndim") and x.ndim == 2 else P(), opt_state_sub) + kw_spec = jax.tree.map(lambda _: P(), kwargs_arrays) + sum_axes = (*row_axes, *col_axes) + out_specs = (tbl_spec, st_spec, P()) if capture_sqnorm else (tbl_spec, st_spec) + if is_sparse: + rep_ids_spec = P(*([None] * grad.ids.ndim)) + dg_spec = P(*([None] * grad.ids.ndim), c_spec) + ids_in = _maybe_reshard(grad.ids, rep_ids_spec, mesh) + dg_in = _maybe_reshard(grad.d_gathered, dg_spec, mesh) + + def _smap_sparse(p_loc, st_loc, ids_loc, dg_loc, kw_loc): + start = jax.lax.axis_index(row_axes).astype(jnp.int32) * jnp.int32(p_loc.shape[0]) if row_axes else jnp.int32(0) + res = host_sparse_fn( + p_loc, + _move_scalars(st_loc, jax.memory.Space.Host), + (jax.device_put(ids_loc, jax.memory.Space.Host), jax.device_put(dg_loc, jax.memory.Space.Host)), + jax.device_put(start, jax.memory.Space.Host), + _move_scalars(kw_loc, jax.memory.Space.Host), + ) + p_new, st_new_h = res[0], res[1] + if capture_sqnorm: + sq_dev = jax.device_put(res[2], jax.memory.Space.Device) + sq_tot = jax.lax.psum(sq_dev, sum_axes) if sum_axes else sq_dev + return p_new, _move_scalars(st_new_h, jax.memory.Space.Device), sq_tot + return p_new, _move_scalars(st_new_h, jax.memory.Space.Device) + + out = _call_shard_map( + _smap_sparse, + in_specs=(tbl_spec, st_spec, rep_ids_spec, dg_spec, kw_spec), + out_specs=out_specs, + args=(param, opt_state_sub, ids_in, dg_in, kwargs_arrays), + ) + if capture_sqnorm: + capture_map[p_path] = out[2] + return out[0], out[1] + return out + + def _smap_dense(p_loc, st_loc, g_loc, kw_loc): + res = host_dense_fn( + p_loc, + _move_scalars(st_loc, jax.memory.Space.Host), + g_loc, + _move_scalars(kw_loc, jax.memory.Space.Host), + ) + p_new, st_new_h = res[0], res[1] + if capture_sqnorm: + sq_dev = jax.device_put(res[2], jax.memory.Space.Device) + sq_tot = jax.lax.psum(sq_dev, sum_axes) if sum_axes else sq_dev + return p_new, _move_scalars(st_new_h, jax.memory.Space.Device), sq_tot + return p_new, _move_scalars(st_new_h, jax.memory.Space.Device) + + out = _call_shard_map( + _smap_dense, + in_specs=(tbl_spec, st_spec, tbl_spec, kw_spec), + out_specs=out_specs, + args=(param, opt_state_sub, grad, kwargs_arrays), + ) + if capture_sqnorm: + capture_map[p_path] = out[2] + return out[0], out[1] + return out + + st_h = _move_scalars(opt_state_sub, jax.memory.Space.Host) + kw_h = _move_scalars(kwargs_arrays, jax.memory.Space.Host) + if is_sparse: + res = host_sparse_fn( + param, + st_h, + (jax.device_put(grad.ids, jax.memory.Space.Host), jax.device_put(grad.d_gathered, jax.memory.Space.Host)), + jax.device_put(jnp.int32(0), jax.memory.Space.Host), + kw_h, + ) + else: + res = host_dense_fn(param, st_h, grad, kw_h) + p_new, st_new_h = res[0], res[1] + if capture_sqnorm: + capture_map[p_path] = jax.device_put(res[2], jax.memory.Space.Device) + return p_new, _move_scalars(st_new_h, jax.memory.Space.Device) + + +def _selective_streamed_chunks_update( + param: jax.Array, + grad: Any, + opt_state_sub: Any, + *, + tx: optax.GradientTransformation, + p_path: tuple[Any, ...], + num_chunks: int, + row_axes: tuple[str, ...], + col_axes: tuple[str, ...], + kwargs_arrays: dict[str, Any], +) -> tuple[jax.Array, Any]: + """Runs exact dense optimizer update by streaming `num_chunks` row slices through device.""" + mesh = _active_mesh() + is_sparse = isinstance(grad, SparseHostGrad) + + def _run_local_stream(p_loc, st_loc, grad_arg, shard_start, kw_loc): + n_local, width = p_loc.shape + k = min(num_chunks, n_local) + cs = (n_local + k - 1) // k + k_eff = (n_local + cs - 1) // cs + + st_leaves, st_treedef = jax.tree_util.tree_flatten(st_loc) + arr_indices = [ + idx for idx, lf in enumerate(st_leaves) if hasattr(lf, "shape") and tuple(lf.shape) == (n_local, width) + ] + host_arrs_init = (p_loc, *[st_leaves[idx] for idx in arr_indices]) + + @compute_on.compute_on( + compute_type="device_host", + out_memory_spaces=(jax.memory.Space.Device,) * len(host_arrs_init), + ) + def _read_chunk(arrs_h, off_h): + return tuple(jax.lax.dynamic_slice(a, (off_h, 0), (cs, width)) for a in arrs_h) + + @compute_on.compute_on( + compute_type="device_host", + out_memory_spaces=(jax.memory.Space.Host,) * len(host_arrs_init), + ) + def _write_chunk(arrs_h, chunks_h, off_h): + return tuple(jax.lax.dynamic_update_slice(a, c, (off_h, 0)) for a, c in zip(arrs_h, chunks_h)) + + @compute_on.compute_on(compute_type="device_host", out_memory_spaces=jax.memory.Space.Device) + def _read_grad_chunk(g_h, off_h): + return jax.lax.dynamic_slice(g_h, (off_h, 0), (cs, width)) + + def _scan_body(carry, i): + arrs_h = carry + raw_off = i * jnp.int32(cs) + slice_off = jnp.minimum(raw_off, jnp.int32(n_local - cs)) + off_h = jax.device_put(slice_off, jax.memory.Space.Host) + chunks_dev = _read_chunk(arrs_h, off_h) + p_c = chunks_dev[0] + st_arr_c = chunks_dev[1:] + if is_sparse: + ids_loc, dg_loc = grad_arg + g_dtype = grad.grad_dtype + rel = ids_loc - (shard_start + slice_off) + in_chunk = (rel >= 0) & (rel < cs) + safe = jnp.where(in_chunk, rel, jnp.int32(0)) + vals = jnp.where(in_chunk[..., None], dg_loc.astype(g_dtype), jnp.zeros((), dtype=g_dtype)) + g_c = jnp.zeros((cs, width), dtype=g_dtype).at[safe].add(vals) + else: + g_c = _read_grad_chunk(grad_arg, off_h) + + cur_leaves = list(st_leaves) + for pos, idx in enumerate(arr_indices): + cur_leaves[idx] = st_arr_c[pos] + st_c = jax.tree_util.tree_unflatten(st_treedef, cur_leaves) + p_new_c, st_new_c = _run_tx_on_single_leaf(tx, p_path, g_c, st_c, p_c, **kw_loc) + new_leaves, _ = jax.tree_util.tree_flatten(st_new_c) + new_arr_c = (p_new_c, *[new_leaves[idx] for idx in arr_indices]) + + # Mask overlapping rows on the final chunk so every row updates exactly once. + active = ((slice_off + jnp.arange(cs, dtype=jnp.int32)) >= raw_off)[:, None] + masked_chunks = tuple(jnp.where(active, new_c, old_c) for new_c, old_c in zip(new_arr_c, chunks_dev)) + chunks_h = tuple(jax.device_put(c, jax.memory.Space.Host) for c in masked_chunks) + arrs_h = _write_chunk(arrs_h, chunks_h, off_h) + scalar_out = tuple(lf for idx, lf in enumerate(new_leaves) if idx not in arr_indices) + return arrs_h, scalar_out + + final_arrs_h, scalar_hist = jax.lax.scan(_scan_body, host_arrs_init, jnp.arange(k_eff, dtype=jnp.int32)) + final_scalars = [s[-1] for s in scalar_hist] + out_leaves = list(st_leaves) + arr_set = set(arr_indices) + sc_iter = iter(final_scalars) + arr_iter = iter(final_arrs_h[1:]) + for idx, _ in enumerate(out_leaves): + out_leaves[idx] = next(arr_iter) if idx in arr_set else next(sc_iter) + return final_arrs_h[0], jax.tree_util.tree_unflatten(st_treedef, out_leaves) + + if mesh is not None: + r_spec, c_spec = _pspec_dim(row_axes), _pspec_dim(col_axes) + tbl_spec = P(r_spec, c_spec) + st_spec = jax.tree.map(lambda x: tbl_spec if hasattr(x, "ndim") and x.ndim == 2 else P(), opt_state_sub) + kw_spec = jax.tree.map(lambda _: P(), kwargs_arrays) + if is_sparse: + rep_ids_spec = P(*([None] * grad.ids.ndim)) + dg_spec = P(*([None] * grad.ids.ndim), c_spec) + ids_in = _maybe_reshard(grad.ids, rep_ids_spec, mesh) + dg_in = _maybe_reshard(grad.d_gathered, dg_spec, mesh) + + def _smap_sparse(p_loc, st_loc, ids_loc, dg_loc, kw_loc): + start = jax.lax.axis_index(row_axes).astype(jnp.int32) * jnp.int32(p_loc.shape[0]) if row_axes else jnp.int32(0) + return _run_local_stream(p_loc, st_loc, (ids_loc, dg_loc), start, kw_loc) + + return _call_shard_map( + _smap_sparse, + in_specs=(tbl_spec, st_spec, rep_ids_spec, dg_spec, kw_spec), + out_specs=(tbl_spec, st_spec), + args=(param, opt_state_sub, ids_in, dg_in, kwargs_arrays), + ) + + def _smap_dense(p_loc, st_loc, g_loc, kw_loc): + start = jax.lax.axis_index(row_axes).astype(jnp.int32) * jnp.int32(p_loc.shape[0]) if row_axes else jnp.int32(0) + return _run_local_stream(p_loc, st_loc, g_loc, start, kw_loc) + + return _call_shard_map( + _smap_dense, + in_specs=(tbl_spec, st_spec, tbl_spec, kw_spec), + out_specs=(tbl_spec, st_spec), + args=(param, opt_state_sub, grad, kwargs_arrays), + ) + + grad_arg = (grad.ids, grad.d_gathered) if is_sparse else grad + return _run_local_stream(param, opt_state_sub, grad_arg, jnp.int32(0), kwargs_arrays) + + +def _selective_lazy_sparse_update( + param: jax.Array, + grad: SparseHostGrad, + opt_state_sub: Any, + *, + tx: optax.GradientTransformation, + p_path: tuple[Any, ...], + row_axes: tuple[str, ...], + col_axes: tuple[str, ...], + kwargs_arrays: dict[str, Any], +) -> tuple[jax.Array, Any]: + """Opt-in sparse-row AdamW approximation updating only touched rows (skips decay on untouched rows).""" + mesh = _active_mesh() + + def _run_local_sparse(p_loc, st_loc, sparse_loc, shard_start, kw_loc): + ids_loc, dg_loc = sparse_loc + n_local, width = p_loc.shape + g_dtype = grad.grad_dtype + flat_ids = ids_loc.reshape(-1) + flat_dg = dg_loc.reshape(-1, width).astype(g_dtype) + m = flat_ids.shape[0] + rel = flat_ids - shard_start + in_range = (rel >= 0) & (rel < n_local) + sentinel = jnp.int32(n_local) + key = jnp.where(in_range, rel, sentinel) + vals = jnp.where(in_range[:, None], flat_dg, jnp.zeros((), dtype=g_dtype)) + order = jnp.argsort(key) + sorted_key = key[order] + sorted_vals = vals[order] + is_new = jnp.concatenate([jnp.array([True]), sorted_key[1:] != sorted_key[:-1]]) + seg_ids = jnp.cumsum(is_new.astype(jnp.int32)) - 1 + unique_grads = jax.ops.segment_sum(sorted_vals, seg_ids, num_segments=m) + unique_ids = jnp.full((m,), sentinel, dtype=jnp.int32).at[seg_ids].min(sorted_key) + valid_row = unique_ids < sentinel + safe_ids = jnp.where(valid_row, unique_ids, jnp.int32(0)) + + st_leaves, st_treedef = jax.tree_util.tree_flatten(st_loc) + arr_indices = [ + idx for idx, lf in enumerate(st_leaves) if hasattr(lf, "shape") and tuple(lf.shape) == (n_local, width) + ] + host_arrs_init = (p_loc, *[st_leaves[idx] for idx in arr_indices]) + + @compute_on.compute_on( + compute_type="device_host", + out_memory_spaces=(jax.memory.Space.Device,) * len(host_arrs_init), + ) + def _gather_rows(arrs_h, idx_h): + return tuple(a[idx_h] for a in arrs_h) + + @compute_on.compute_on( + compute_type="device_host", + out_memory_spaces=(jax.memory.Space.Host,) * len(host_arrs_init), + ) + def _scatter_rows(arrs_h, rows_h, write_idx_h): + # Out-of-bounds index `n_local` with `mode="drop"` avoids clobbering row 0. + return tuple(a.at[write_idx_h].set(r, mode="drop") for a, r in zip(arrs_h, rows_h)) + + rows_dev = _gather_rows(host_arrs_init, jax.device_put(safe_ids, jax.memory.Space.Host)) + cur_leaves = list(st_leaves) + for pos, idx in enumerate(arr_indices): + cur_leaves[idx] = rows_dev[pos + 1] + st_rows = jax.tree_util.tree_unflatten(st_treedef, cur_leaves) + p_rows_new, st_rows_new = _run_tx_on_single_leaf(tx, p_path, unique_grads, st_rows, rows_dev[0], **kw_loc) + new_leaves, _ = jax.tree_util.tree_flatten(st_rows_new) + new_rows_h = tuple( + jax.device_put(r, jax.memory.Space.Host) for r in (p_rows_new, *[new_leaves[idx] for idx in arr_indices]) + ) + write_idx = jnp.where(valid_row, unique_ids, sentinel) + updated_arrs_h = _scatter_rows(host_arrs_init, new_rows_h, jax.device_put(write_idx, jax.memory.Space.Host)) + + out_leaves = list(new_leaves) + for pos, idx in enumerate(arr_indices): + out_leaves[idx] = updated_arrs_h[pos + 1] + return updated_arrs_h[0], jax.tree_util.tree_unflatten(st_treedef, out_leaves) + + if mesh is not None: + r_spec, c_spec = _pspec_dim(row_axes), _pspec_dim(col_axes) + tbl_spec = P(r_spec, c_spec) + st_spec = jax.tree.map(lambda x: tbl_spec if hasattr(x, "ndim") and x.ndim == 2 else P(), opt_state_sub) + rep_ids_spec = P(*([None] * grad.ids.ndim)) + dg_spec = P(*([None] * grad.ids.ndim), c_spec) + kw_spec = jax.tree.map(lambda _: P(), kwargs_arrays) + ids_in = _maybe_reshard(grad.ids, rep_ids_spec, mesh) + dg_in = _maybe_reshard(grad.d_gathered, dg_spec, mesh) + + def _smap(p_loc, st_loc, ids_loc, dg_loc, kw_loc): + start = jax.lax.axis_index(row_axes).astype(jnp.int32) * jnp.int32(p_loc.shape[0]) if row_axes else jnp.int32(0) + return _run_local_sparse(p_loc, st_loc, (ids_loc, dg_loc), start, kw_loc) + + return _call_shard_map( + _smap, + in_specs=(tbl_spec, st_spec, rep_ids_spec, dg_spec, kw_spec), + out_specs=(tbl_spec, st_spec), + args=(param, opt_state_sub, ids_in, dg_in, kwargs_arrays), + ) + + return _run_local_sparse(param, opt_state_sub, (grad.ids, grad.d_gathered), jnp.int32(0), kwargs_arrays) + + class HostOffloadedOptimizer(nnx.Optimizer): - """An `nnx.Optimizer` parking `opt_state` (and host-offloaded params) in `pinned_host` between steps.""" + """An `nnx.Optimizer` parking all or selected `opt_state`/params in `pinned_host` between steps.""" def __init__( self, @@ -251,8 +1934,57 @@ def __init__( wrt: Any = nnx.Param, *, offload_opt_state: bool = True, + selected_param_paths: frozenset[tuple[Any, ...]] = frozenset(), + optimizer_mode: str = "host_compute", + num_chunks: int = 16, + clip_max_norm: float = 0.0, ): + if optimizer_mode not in _OPTIMIZER_MODES: + raise ValueError(f"Unsupported optimizer_mode {optimizer_mode!r}; expected one of {sorted(_OPTIMIZER_MODES)}.") self._offload_opt_state = offload_opt_state + self._selected_param_paths = frozenset(tuple(p) for p in selected_param_paths) + self._optimizer_mode = optimizer_mode + self._num_chunks = int(num_chunks) + self._clip_max_norm = float(clip_max_norm) + + if self._selected_param_paths: + # Selective mode: leave selected host params in `pinned_host` during `tx.init` so + # `zeros_like` allocates their moments directly in host memory without an HBM spike. + super().__init__(model, tx, wrt=wrt) + selected_opt_vars: list[nnx.Variable] = [] + other_opt_vars: list[nnx.Variable] = [] + for path, leaf in jax.tree_util.tree_leaves_with_path( + self.opt_state, is_leaf=lambda x: isinstance(x, nnx.Variable) + ): + if not isinstance(leaf, nnx.Variable): + continue + raw = leaf.get_raw_value() + if not hasattr(raw, "ndim") or raw.ndim == 0: + continue + key_tuple = tuple(_key_value(k) for k in path) + if _is_matching_opt_leaf(key_tuple, self._selected_param_paths) is not None: + selected_opt_vars.append(leaf) + else: + other_opt_vars.append(leaf) + if _can_move_vars(selected_opt_vars): + _move_variables_in_place(selected_opt_vars, "pinned_host") + else: + for v in selected_opt_vars: + _attach_memory_kind_metadata(v, "pinned_host") + raw = v.get_raw_value() + if isinstance(raw, jax.core.Tracer) and not _is_host_leaf(v): + v.set_raw_value(jax.device_put(raw, jax.memory.Space.Host)) + if self._offload_opt_state and other_opt_vars: + if _can_move_vars(other_opt_vars): + _move_variables_in_place(other_opt_vars, "pinned_host") + else: + for v in other_opt_vars: + _attach_memory_kind_metadata(v, "pinned_host") + raw = v.get_raw_value() + if isinstance(raw, jax.core.Tracer) and not _is_host_leaf(v): + v.set_raw_value(jax.device_put(raw, jax.memory.Space.Host)) + return + param_vars = [ v for v in jax.tree.leaves(nnx.state(model, wrt), is_leaf=lambda x: isinstance(x, nnx.Variable)) @@ -272,8 +2004,111 @@ def __init__( if self._offload_opt_state: _move_variables_in_place(self.opt_state, "pinned_host") - def update(self, model: nnx.Module, grads: Any, **kwargs: Any) -> None: # pylint: disable=arguments-differ - """Performs an optimizer step, gathering host-offloaded state to device.""" + def _update_selective(self, model: nnx.Module, grads: Any, **kwargs: Any) -> None: + """Updates device and selectively host-offloaded parameters in place.""" + if self._clip_max_norm > 0.0: + grads = clip_grads_by_global_norm(grads, self._clip_max_norm) + + params_state = nnx.state(model, self.wrt, graph=self.graph) + param_var_map = {tuple(p): v for p, v in nnx.to_flat_state(params_state)} + all_param_paths = frozenset(param_var_map.keys()) + selected = self._selected_param_paths & all_param_paths + + grads_state = nnx.state(grads, self.wrt, graph=self.graph) if isinstance(grads, nnx.Module) else grads + grad_val_map = { + tuple(p): (v.get_raw_value() if isinstance(v, nnx.Variable) else v) for p, v in nnx.to_flat_state(grads_state) + } + opt_state_pure = nnx.as_pure(self.opt_state) + kwargs_arrays = nnx.as_pure(kwargs) + + # 1. Update non-selected parameters on device. + dev_paths = all_param_paths - selected + updated_dev_opt_state: Any = None + if dev_paths: + dev_params_pure = _filter_mapping_by_paths(nnx.as_pure(params_state), lambda p: p in dev_paths) + dev_grads_pure = _filter_mapping_by_paths(nnx.as_pure(grads_state), lambda p: p in dev_paths) + dev_opt_in = _slice_opt_state(opt_state_pure, all_param_paths, lambda p: p in dev_paths) + if self._offload_opt_state: + dev_opt_in = to_memory(dev_opt_in, "device") + dev_updates, updated_dev_opt_state = self.tx.update(dev_grads_pure, dev_opt_in, dev_params_pure, **kwargs_arrays) + new_dev_params = optax.apply_updates(dev_params_pure, dev_updates) + for p_path in dev_paths: + param_var_map[p_path].set_raw_value(_unnest_path(p_path, new_dev_params)) + if self._offload_opt_state: + updated_dev_opt_state = to_memory(updated_dev_opt_state, "pinned_host") + + # 2. Update each selectively host-offloaded parameter without materializing dense HBM gradients. + mesh = _active_mesh() + updated_host_opt_states: dict[tuple[Any, ...], Any] = {} + for p_path in sorted(selected): + p_var = param_var_map[p_path] + p_val = p_var.get_raw_value() + g_val = grad_val_map[p_path] + st_sub = _slice_opt_state(opt_state_pure, all_param_paths, lambda p, target=p_path: p == target) + row_axes, col_axes = _resolve_2d_axes(p_var, mesh) + + if self._optimizer_mode == "streamed_chunks": + p_new, st_new = _selective_streamed_chunks_update( + p_val, + g_val, + st_sub, + tx=self.tx, + p_path=p_path, + num_chunks=self._num_chunks, + row_axes=row_axes, + col_axes=col_axes, + kwargs_arrays=kwargs_arrays, + ) + elif self._optimizer_mode == "lazy_sparse": + if not isinstance(g_val, SparseHostGrad): + raise ValueError( + f"selective_host_offload_optimizer_mode='lazy_sparse' requires SparseHostGrad for {p_path}, " + f"got dense {type(g_val).__name__}. Use 'host_compute' or 'streamed_chunks' with dense host " + "gradients (such as MaxTextTrainingEngine)." + ) + p_new, st_new = _selective_lazy_sparse_update( + p_val, + g_val, + st_sub, + tx=self.tx, + p_path=p_path, + row_axes=row_axes, + col_axes=col_axes, + kwargs_arrays=kwargs_arrays, + ) + else: + p_new, st_new = _selective_host_compute_update( + p_val, + g_val, + st_sub, + tx=self.tx, + p_path=p_path, + row_axes=row_axes, + col_axes=col_axes, + kwargs_arrays=kwargs_arrays, + ) + p_var.set_raw_value(p_new) + updated_host_opt_states[p_path] = st_new + + # 3. Write updated optimizer state back into existing Variables in place. + fallback_scalar_state = ( + updated_dev_opt_state if updated_dev_opt_state is not None else next(iter(updated_host_opt_states.values())) + ) + for path, leaf in jax.tree_util.tree_leaves_with_path(self.opt_state, is_leaf=lambda x: isinstance(x, nnx.Variable)): + if not isinstance(leaf, nnx.Variable): + continue + key_tuple = tuple(_key_value(k) for k in path) + matched_p = _is_matching_opt_leaf(key_tuple, selected) + if matched_p is not None: + leaf.set_raw_value(_lookup_path_in_tree(updated_host_opt_states[matched_p], key_tuple)) + else: + leaf.set_raw_value(_lookup_path_in_tree(fallback_scalar_state, key_tuple)) + self.step[...] += 1 + + def update(self, model: nnx.Module, grads: Any, **kwargs: Any) -> Any: # pylint: disable=arguments-differ + """Performs an optimizer step, handling full or selective pinned-host offload.""" + if self._selected_param_paths: + return self._update_selective(model, grads, **kwargs) if self._offload_opt_state: _move_variables_in_place(self.opt_state, "device") params = nnx.state(model, self.wrt) @@ -287,6 +2122,7 @@ def update(self, model: nnx.Module, grads: Any, **kwargs: Any) -> None: # pylin _move_variables_in_place(nnx.state(model, self.wrt), "pinned_host") if self._offload_opt_state: _move_variables_in_place(self.opt_state, "pinned_host") + return None def make_optimizer( @@ -294,9 +2130,34 @@ def make_optimizer( tx: optax.GradientTransformation, config: Any, wrt: Any = nnx.Param, + *, + clip_max_norm: float = 0.0, + dense_host_grads: bool = False, ) -> nnx.Optimizer: - """Builds the optimizer, using `HostOffloadedOptimizer` when optimizer or parameter offload is active.""" + """Builds the optimizer, using `HostOffloadedOptimizer` when any host offload mode is active.""" offload_opt = bool(getattr(config, "optimizer_memory_host_offload", False)) + selective = is_selective_host_offload_enabled(config) or has_selective_host_offload(model) + if selective: + mode = _get_optimizer_mode(config) + if dense_host_grads and mode == "lazy_sparse": + raise ValueError( + "selective_host_offload_optimizer_mode='lazy_sparse' is incompatible with dense pinned_host " + "gradients (MaxTextTrainingEngine); use 'host_compute' or 'streamed_chunks'." + ) + selected_paths = selected_host_param_paths(model) + if not selected_paths: + patterns = get_host_offload_param_patterns(config) + selected_paths = _match_param_paths(nnx.state(model, wrt), patterns) + return HostOffloadedOptimizer( + model, + tx, + wrt=wrt, + offload_opt_state=offload_opt, + selected_param_paths=selected_paths, + optimizer_mode=mode, + num_chunks=_get_num_chunks(config), + clip_max_norm=clip_max_norm, + ) offload_params = bool(getattr(config, "parameter_memory_host_offload", False)) or _has_host_leaves( nnx.state(model, wrt) ) diff --git a/src/maxtext/m3/models/qwen3_8_flash_next/modeling_qwen3_8_flash_next.py b/src/maxtext/m3/models/qwen3_8_flash_next/modeling_qwen3_8_flash_next.py index db5f651d74..70ef2e8805 100644 --- a/src/maxtext/m3/models/qwen3_8_flash_next/modeling_qwen3_8_flash_next.py +++ b/src/maxtext/m3/models/qwen3_8_flash_next/modeling_qwen3_8_flash_next.py @@ -1467,6 +1467,16 @@ def compute_ngram_ids(self, input_ids: jax.Array) -> jax.Array: return jnp.concatenate(blocks, axis=-1)[:, self.ngram_size - 1 :, :] + def embed_ids_from_tokens(self, tokens: jax.Array, segment_ids: jax.Array | None = None) -> jax.Array: + """Computes n-gram embedding row indices from token IDs and optional segment IDs.""" + if segment_ids is not None: + tokens = jnp.where(segment_ids != 0, tokens, self.eos_token_id) + ngram_ids = self.compute_ngram_ids(tokens) + mesh = jax.sharding.get_abstract_mesh() + if not mesh.empty and mesh.size > 1: + ngram_ids = jax.lax.with_sharding_constraint(ngram_ids, jax.sharding.PartitionSpec()) + return ngram_ids + def __call__(self, input_ids: jax.Array) -> jax.Array: ngram_ids = self.compute_ngram_ids(input_ids) mesh = jax.sharding.get_abstract_mesh() diff --git a/src/maxtext/m3/train/loop.py b/src/maxtext/m3/train/loop.py index 17b11515c5..c5689b3d95 100644 --- a/src/maxtext/m3/train/loop.py +++ b/src/maxtext/m3/train/loop.py @@ -137,7 +137,12 @@ def setup_model(config, mesh: Mesh, sharding: Sharding | None = None) -> tuple[n model = quantization.maybe_quantize(model, config, *quantization.example_inputs(config)) model = add_named_scopes(model) model = m3_offload.maybe_offload_parameters(model, config) - optimizer = m3_offload.make_optimizer(model, make_optimizer(config), config) + if m3_offload.is_selective_host_offload_enabled(config): + # Global-norm clipping must span device and row-sparse host gradients together. + tx = optimizers.get_optimizer(config, config.learning_rate) + optimizer = m3_offload.make_optimizer(model, tx, config, clip_max_norm=float(config.gradient_clipping_threshold)) + else: + optimizer = m3_offload.make_optimizer(model, make_optimizer(config), config) return model, optimizer @@ -218,7 +223,10 @@ def loss_fn(model: nnx.Module, batch: dict[str, jax.Array]) -> jax.Array: @nnx.jit(donate_argnums=(0, 1)) def train_step(model: nnx.Module, optimizer: nnx.Optimizer, batch: dict[str, jax.Array]) -> jax.Array: """One forward/backward/update. Mutates `model` and `optimizer` in place.""" - loss, grads = nnx.value_and_grad(loss_fn)(model, batch) + if m3_offload.has_selective_host_offload(model): + loss, grads = m3_offload.value_and_grad(loss_fn, model, batch) + else: + loss, grads = nnx.value_and_grad(loss_fn)(model, batch) with jax.named_scope("optimizer"): optimizer.update(model, grads) return loss diff --git a/src/maxtext/trainers/pre_train/train.py b/src/maxtext/trainers/pre_train/train.py index 1eb18ba497..87121e5616 100644 --- a/src/maxtext/trainers/pre_train/train.py +++ b/src/maxtext/trainers/pre_train/train.py @@ -92,6 +92,7 @@ from maxtext.utils import train_utils from maxtext.utils import mllog_utils from maxtext.utils import gpu_collective_overlap_pass +from maxtext.m3.infra import offload as m3_offload from maxtext.utils.gradient_accumulation import gradient_accumulation_loss_and_grad from maxtext.utils.vocabulary_tiling import vocab_tiling_nnx_loss @@ -574,8 +575,15 @@ def train_step(model, config, state_mesh_shardings, params_shardings, state, dat state = nnx.merge(model, state) # reconstruct TrainStateNNX loss_model, loss_params, loss_rng = state.model, None, None + selective_offload = m3_offload.has_selective_host_offload(state.model) + # --- Gradient computation --- if config.gradient_accumulation_steps > 1: + if selective_offload: + raise ValueError( + "selective_host_offload=True with gradient_accumulation_steps > 1 is not supported in " + "train.py fused train_step; use MaxTextTrainingEngine for multi-microbatch accumulation." + ) loss, aux, raw_grads = gradient_accumulation_loss_and_grad( loss_fn, config, @@ -643,16 +651,33 @@ def diff_wrapper(curr_params, custom_params, rest, config, data): ) return loss, (aux, non_param_rest) - grad_func = jax.value_and_grad(diff_wrapper, argnums=(0, 1), has_aux=True) - # Potentially mark model's loops for double buffering on GPUs. - with max_utils.model_loop_unroll_context(config): - (loss, (aux, non_param_rest)), (raw_grads, custom_grads) = grad_func(curr_params, custom_params, rest, config, data) + if selective_offload: + with max_utils.model_loop_unroll_context(config): + (loss, (aux, non_param_rest)), (raw_grads, custom_grads) = m3_offload.pure_value_and_grad( + diff_wrapper, + model_graphdef, + curr_params, + custom_params=custom_params, + rest=rest, + config=config, + data=data, + ) + else: + grad_func = jax.value_and_grad(diff_wrapper, argnums=(0, 1), has_aux=True) + # Potentially mark model's loops for double buffering on GPUs. + with max_utils.model_loop_unroll_context(config): + (loss, (aux, non_param_rest)), (raw_grads, custom_grads) = grad_func( + curr_params, custom_params, rest, config, data + ) nnx.update(state.model, nnx.State.merge(custom_grads, non_param_rest)) - raw_grads = jax.tree_util.tree_map( - lambda x: x.astype(config.grad_dtype) if x.dtype == jnp.float32 else x, - raw_grads, - ) + if selective_offload: + raw_grads = m3_offload.cast_grad_tree(raw_grads, config.grad_dtype) + else: + raw_grads = jax.tree_util.tree_map( + lambda x: x.astype(config.grad_dtype) if x.dtype == jnp.float32 else x, + raw_grads, + ) if config.parameter_memory_host_offload: raw_grads = jax.device_put( raw_grads, @@ -675,11 +700,17 @@ def diff_wrapper(curr_params, custom_params, rest, config, data): # log_step_diagnostics: the updated router biases and this step's bias updates (see step_diagnostics). diag_bias_values, diag_bias_updates = [], [] - if config.gradient_clipping_threshold > 0: + pre_raw_grad_norm = None + pre_grad_norm = None + if selective_offload: + grads, pre_raw_grad_norm, pre_grad_norm = m3_offload.clip_grads_and_norms( + raw_grads, config.gradient_clipping_threshold + ) + elif config.gradient_clipping_threshold > 0: grads = maxtext_utils.apply_gradient_clipping(raw_grads, None, config.gradient_clipping_threshold) else: grads = raw_grads - if config.optimizer_memory_host_offload: + if config.optimizer_memory_host_offload and not selective_offload: # state.optimizer is an NNX Optimizer module; state_mesh_shardings.optimizer # is an NNX State. Use nnx.state() to get a compatible State for device_put. device_opt_shardings = jax.tree_util.tree_map_with_path( @@ -690,7 +721,14 @@ def diff_wrapper(curr_params, custom_params, rest, config, data): opt_state = nnx.state(state.optimizer) new_opt_state = jax.device_put(opt_state, device_opt_shardings) nnx.update(state.optimizer, new_opt_state) - if config.skip_step_on_spikes: + host_param_sqnorms: dict[tuple, jax.Array] = {} + if selective_offload: + with m3_offload.capture_host_param_sqnorms() as host_param_sqnorms: + if config.skip_step_on_spikes: + state.apply_gradients(grads, loss=loss, grad_norm=pre_grad_norm) + else: + state.apply_gradients(grads) + elif config.skip_step_on_spikes: # The skip-step optimizer is a GradientTransformationExtraArgs that reads # loss/grad_norm to decide whether to zero the update on a spike. nnx # Optimizer.update forwards these kwargs to tx.update. @@ -795,18 +833,27 @@ def diff_wrapper(curr_params, custom_params, rest, config, data): if global_max_logit is not None: scalar_metrics["learning/max_logits"] = global_max_logit + def _param_norm(params_tree): + if selective_offload: + return m3_offload.l2norm_pytree_with_overrides(params_tree, host_param_sqnorms) + return max_utils.l2norm_pytree(params_tree) + if not config.optimizer_memory_host_offload: - scalar_metrics["learning/grad_norm"] = max_utils.l2norm_pytree(grads) - scalar_metrics["learning/raw_grad_norm"] = max_utils.l2norm_pytree(raw_grads) + scalar_metrics["learning/grad_norm"] = pre_grad_norm if selective_offload else max_utils.l2norm_pytree(grads) + scalar_metrics["learning/raw_grad_norm"] = ( + pre_raw_grad_norm if selective_offload else max_utils.l2norm_pytree(raw_grads) + ) model_params = nnx.state(new_state.model, nnx.Param) - scalar_metrics["learning/param_norm"] = max_utils.l2norm_pytree(model_params) + scalar_metrics["learning/param_norm"] = _param_norm(model_params) if getattr(config, "log_step_diagnostics", False): scalar_metrics.update(step_diagnostics(raw_grads, has_moe_overflow, diag_bias_values, diag_bias_updates)) if "learning/raw_grad_norm" not in scalar_metrics: # optimizer_memory_host_offload skips the norms above - scalar_metrics["learning/raw_grad_norm"] = max_utils.l2norm_pytree(raw_grads) - scalar_metrics["learning/grad_norm"] = max_utils.l2norm_pytree(grads) - scalar_metrics["learning/param_norm"] = max_utils.l2norm_pytree(nnx.state(new_state.model, nnx.Param)) + scalar_metrics["learning/raw_grad_norm"] = ( + pre_raw_grad_norm if selective_offload else max_utils.l2norm_pytree(raw_grads) + ) + scalar_metrics["learning/grad_norm"] = pre_grad_norm if selective_offload else max_utils.l2norm_pytree(grads) + scalar_metrics["learning/param_norm"] = _param_norm(nnx.state(new_state.model, nnx.Param)) # Surface skip-step rejections as a TB metric. The skip-step optimizer stores # is_skipped in its opt_state; read it back off the optimizer just updated in place. diff --git a/src/maxtext/trainers/pre_train/train_compile.py b/src/maxtext/trainers/pre_train/train_compile.py index bc5408858e..18bf1ed98c 100644 --- a/src/maxtext/trainers/pre_train/train_compile.py +++ b/src/maxtext/trainers/pre_train/train_compile.py @@ -141,7 +141,12 @@ def create_train_state_fn(): if getattr(getattr(config, "lora", None), "enable_lora", False) else nnx.Param ) - optimizer = nnx.Optimizer(nnx_model, tx, wrt=wrt) + if config.selective_host_offload: + from maxtext.m3.infra import offload as m3_offload # pylint: disable=import-outside-toplevel + + optimizer = m3_offload.make_optimizer(nnx_model, tx, config, wrt=wrt) + else: + optimizer = nnx.Optimizer(nnx_model, tx, wrt=wrt) return train_state_nnx.TrainStateNNX(nnx_model, optimizer) init_state_fn = create_train_state_fn diff --git a/src/maxtext/training_engine/maxtext_engine.py b/src/maxtext/training_engine/maxtext_engine.py index f30f26ef46..227de68534 100644 --- a/src/maxtext/training_engine/maxtext_engine.py +++ b/src/maxtext/training_engine/maxtext_engine.py @@ -43,6 +43,7 @@ resolve_prefuse_moe_weights, resolve_rollout_tp, ) +from maxtext.m3.infra import offload as m3_offload from maxtext.optimizers import optimizers from maxtext.trainers.pre_train import train as maxtext_train from maxtext.training_engine import abstract_engine @@ -838,6 +839,8 @@ def __init__( # Set by `_compile_for_batch` under `optimizer_memory_host_offload`: the device shardings # `_update_kernel` moves the pinned-host optimizer state onto. `None` without offload. self._optimizer_device_shardings: Any = None + self._selective_grad_shardings: Any = None + self._selective_state_shardings: Any = None # Micro step count within the current training step. Resets on `update()`. self._micro_step_count = 0 # Every micro step this run has ever folded in across optimizer steps. @@ -925,6 +928,8 @@ def _build_optimizer(self, tx: Any) -> Any: A subclass that cannot allocate moments overrides this, and may install `self._state` and rebind `self._model` on the way: `__init__` does not touch either again. """ + if m3_offload.is_selective_host_offload_enabled(self._config): + return m3_offload.make_optimizer(self._model, tx, self._config, wrt=nnx.Param, dense_host_grads=True) return nnx.Optimizer(self._model, tx, wrt=nnx.Param) def _checkpoint_dir(self) -> str: @@ -947,6 +952,8 @@ def _reset_compiled_state(self, *, reset_model: bool) -> None: self._compiled_eval = None self._compiled_eval_signature = None self._optimizer_device_shardings = None + self._selective_grad_shardings = None + self._selective_state_shardings = None self._compiled_fwd_only = None self._compiled_fwd_only_signature = None if reset_model: @@ -1365,14 +1372,18 @@ def _reshard_model_params(self, state_pure: Any, params_shardings: Any) -> Any: params_pure = jax.tree.map(jax.sharding.reshard, params_pure, params_shardings) return self._with_model_state(state_pure, nnx.merge_state(params_pure, rest_pure)) - def _to_grad_dtype(self, x: Any) -> Any: + def _to_grad_dtype(self, x: Any, leaf_sharding: Any = None) -> Any: """Returns one gradient leaf, or its `jax.ShapeDtypeStruct`, in `grad_dtype`.""" if not hasattr(x, "dtype") or not jnp.issubdtype(x.dtype, jnp.floating): return x target = jnp.dtype(self._config.grad_dtype) if x.dtype == target: return x - return x.update(dtype=target) if isinstance(x, jax.ShapeDtypeStruct) else x.astype(target) + if isinstance(x, jax.ShapeDtypeStruct): + return x.update(dtype=target) + if m3_offload.is_selective_host_offload_enabled(self._config): + return m3_offload.cast_grad_leaf(x, target, sharding=leaf_sharding) + return x.astype(target) def _freeze_mask_for(self, tree: Any) -> Any: """Returns which leaves of `tree` `trainable_parameters_mask` freezes, or None if it freezes none. @@ -1453,24 +1464,45 @@ def diff_wrapper(p, r, b): # would put an all-reduce back into every micro-batch. params = jax.tree.map(jax.sharding.reshard, params, self._reduced_params_shardings) - grad_func = jax.value_and_grad(diff_wrapper, argnums=0, has_aux=True) # Every non-raising branch of `diff_wrapper` builds a LossOutput, so `loss_out` is always # one. The value returned by `value_and_grad` is the unreduced sum that was # differentiated, which `loss_out.primary_loss` already carries, so it is discarded here. - (_, (loss_out, new_rest)), micro_grads = grad_func(params, rest, batch) + if m3_offload.is_selective_host_offload_enabled(self._config): + (_, (loss_out, new_rest)), micro_grads = m3_offload.engine_value_and_grad( + diff_wrapper, + self._model_graphdef, + params, + rest, + batch, + param_shardings=self._selective_grad_shardings, + ) + else: + grad_func = jax.value_and_grad(diff_wrapper, argnums=0, has_aux=True) + (_, (loss_out, new_rest)), micro_grads = grad_func(params, rest, batch) # Summed in `grad_dtype` by `_accumulate_kernel`. A float32 `grad_dtype` holds a float32 # accumulator tree across kernel calls, and a second one while `fwd_bwd` runs, since these # gradients are returned in that dtype. if self._uncast_grad_mask is None: - micro_grads = jax.tree.map(self._to_grad_dtype, micro_grads) + if self._selective_grad_shardings is not None and m3_offload.is_selective_host_offload_enabled(self._config): + micro_grads = jax.tree.map(self._to_grad_dtype, micro_grads, self._selective_grad_shardings) + else: + micro_grads = jax.tree.map(self._to_grad_dtype, micro_grads) else: # Leaves whose gradients are summed across devices by an all-reduce alone are cast as they # join the running sum instead: `_start_running_sum` for the first micro-batch of an update, # `_accumulate_kernel` for the others. See `_all_reduced_grad_mask`. - micro_grads = jax.tree.map( - lambda g, uncast: g if uncast else self._to_grad_dtype(g), micro_grads, self._uncast_grad_mask - ) + if self._selective_grad_shardings is not None and m3_offload.is_selective_host_offload_enabled(self._config): + micro_grads = jax.tree.map( + lambda g, uncast, sh: g if uncast else self._to_grad_dtype(g, sh), + micro_grads, + self._uncast_grad_mask, + self._selective_grad_shardings, + ) + else: + micro_grads = jax.tree.map( + lambda g, uncast: g if uncast else self._to_grad_dtype(g), micro_grads, self._uncast_grad_mask + ) # Accumulated UNREDUCED, with no `1/denominator` applied: `_update_kernel` divides once # by the total, so the optimizer sees `sum(grads)/sum(denom)` rather than a mean of @@ -1480,7 +1512,7 @@ def diff_wrapper(p, r, b): return loss_out.primary_loss, loss_out.aux_metrics, new_rest, micro_grads, denominator @staticmethod - def _accumulate_kernel(acc_grads, acc_denom, grads, denominator): + def _accumulate_kernel(acc_grads, acc_denom, grads, denominator, grad_shardings=None): """Adds one micro-batch's gradients and loss denominator to the running sums. Each micro-batch runs the same forward/backward program without receiving the running sum. @@ -1503,6 +1535,13 @@ def _accumulate_kernel(acc_grads, acc_denom, grads, denominator): Returns: `(acc_grads + grads, acc_denom + denominator)`. """ + if m3_offload.has_host_leaves(acc_grads) or m3_offload.has_host_leaves(grads): + if grad_shardings is not None: + return ( + jax.tree.map(m3_offload.accumulate_grad_leaves, acc_grads, grads, grad_shardings), + acc_denom + denominator, + ) + return jax.tree.map(m3_offload.accumulate_grad_leaves, acc_grads, grads), acc_denom + denominator return jax.tree.map(lambda acc, g: acc + g.astype(acc.dtype), acc_grads, grads), acc_denom + denominator def _update_kernel( @@ -1565,31 +1604,61 @@ def normalize(g): wide = jnp.promote_types(g.dtype, jnp.float32) return jnp.where(has_weights, (g.astype(wide) / safe_denominator).astype(g.dtype), jnp.zeros_like(g)) - grads = jax.tree.map(normalize, accumulated_grads) + selective_offload = m3_offload.is_selective_host_offload_enabled(self._config) + active_grad_sh = ( + self._zero1_params_shardings if self._zero1_params_shardings is not None else self._selective_grad_shardings + ) + if selective_offload: + inv_denom = jnp.where(has_weights, 1.0 / safe_denominator, jnp.float32(0.0)) + if active_grad_sh is not None: + grads = jax.tree.map( + lambda g, sh: m3_offload.scale_grad_leaf(g, inv_denom, sharding=sh), accumulated_grads, active_grad_sh + ) + else: + grads = jax.tree.map(lambda g: m3_offload.scale_grad_leaf(g, inv_denom), accumulated_grads) + else: + grads = jax.tree.map(normalize, accumulated_grads) # Compute global grad_norm in float32 before clipping so a sum of squares over bf16 # does not overflow. Frozen parameters already have exact zero gradients from # `_stop_frozen_gradients` in `_fwd_bwd_kernel`, so they do not contribute to `grad_norm`. - grad_norm = max_utils.l2norm_pytree(jax.tree.map(lambda g: g.astype(jnp.float32), grads)) + if selective_offload: + grad_norm = m3_offload.l2norm_pytree(grads, shardings=active_grad_sh) + else: + grad_norm = max_utils.l2norm_pytree(jax.tree.map(lambda g: g.astype(jnp.float32), grads)) if self._config.skip_step_on_nan: should_skip = jnp.logical_not(jnp.isfinite(grad_norm)) # Zeroed, so a non-finite gradient cannot reach the optimizer's arithmetic either. - grads = jax.tree.map(lambda g: jnp.where(should_skip, jnp.zeros_like(g), g), grads) + if selective_offload: + grads = m3_offload.zero_tree_if(should_skip, grads, shardings=active_grad_sh) + else: + grads = jax.tree.map(lambda g: jnp.where(should_skip, jnp.zeros_like(g), g), grads) if self._config.gradient_clipping_threshold > 0: - grads = maxtext_utils.apply_gradient_clipping(grads, None, self._config.gradient_clipping_threshold) + if selective_offload: + grads = m3_offload.clip_grads_by_global_norm( + grads, self._config.gradient_clipping_threshold, shardings=active_grad_sh + ) + else: + grads = maxtext_utils.apply_gradient_clipping(grads, None, self._config.gradient_clipping_threshold) local_state = nnx.merge(self._state_graphdef, state_pure, copy=True) if isinstance(local_state, train_state_nnx.TrainStateNNX): - if self._config.skip_step_on_spikes: - local_state.apply_gradients(grads, loss=mean_loss, grad_norm=grad_norm) - else: - local_state.apply_gradients(grads) + with m3_offload.active_param_shardings(active_grad_sh if selective_offload else None): + if self._config.skip_step_on_spikes: + local_state.apply_gradients(grads, loss=mean_loss, grad_norm=grad_norm) + else: + local_state.apply_gradients(grads) _, new_state_pure = nnx.split(local_state) if self._config.skip_step_on_spikes and _OPTIMIZER_STATE_KEY in new_state_pure: opt_state = nnx.to_pure_dict(new_state_pure[_OPTIMIZER_STATE_KEY])["opt_state"] if isinstance(opt_state, dict) and "is_skipped" in opt_state: is_skipped_val = opt_state["is_skipped"].astype(jnp.float32) if self._config.skip_step_on_nan: - new_state_pure = jax.tree.map(lambda new, old: jnp.where(should_skip, old, new), new_state_pure, state_pure) + if selective_offload: + new_state_pure = m3_offload.select_tree_if( + should_skip, state_pure, new_state_pure, shardings=self._selective_state_shardings + ) + else: + new_state_pure = jax.tree.map(lambda new, old: jnp.where(should_skip, old, new), new_state_pure, state_pure) is_skipped_val = jnp.where(should_skip, 1.0, is_skipped_val) if self._zero1_params_shardings is not None: @@ -1835,7 +1904,7 @@ def fwd_bwd(params, rest, dynamic): return self._fwd_bwd_kernel(params, rest, batch) def accumulate(acc_grads, acc_denom, grads, denominator): - return self._accumulate_kernel(acc_grads, acc_denom, grads, denominator) + return self._accumulate_kernel(acc_grads, acc_denom, grads, denominator, self._selective_grad_shardings) def update(state_pure, accumulated_grads, accumulated_denominator, mean_loss): # `_optimizer_device_shardings` is read at trace time, after it is set below. @@ -1868,6 +1937,8 @@ def update(state_pure, accumulated_grads, accumulated_denominator, mean_loss): self._plain_grad_shardings = None else: self._plain_grad_shardings = params_shardings + self._selective_grad_shardings = grad_shardings + self._selective_state_shardings = state_mesh_shardings self._uncast_grad_mask = _all_reduced_grad_mask(self._config, self._mesh, grad_shardings, params_pure) self._cast_uncast_grads = None if self._uncast_grad_mask is not None: @@ -1879,7 +1950,7 @@ def update(state_pure, accumulated_grads, accumulated_denominator, mean_loss): if uncast ] self._cast_uncast_grads = jax.jit( - lambda leaves: [self._to_grad_dtype(g) for g in leaves], + lambda leaves: [self._to_grad_dtype(g, sh) for g, sh in zip(leaves, uncast_shardings)], in_shardings=(uncast_shardings,), out_shardings=uncast_shardings, ) @@ -1894,7 +1965,11 @@ def update(state_pure, accumulated_grads, accumulated_denominator, mean_loss): # state onto the same shardings in device memory, and `update_out_shardings` returns the new # state to the host inside the same executable. self._optimizer_device_shardings = None - if self._optimizer_offload_enabled() and _OPTIMIZER_STATE_KEY in state_mesh_shardings: + if ( + self._optimizer_offload_enabled() + and not m3_offload.is_selective_host_offload_enabled(self._config) + and _OPTIMIZER_STATE_KEY in state_mesh_shardings + ): self._optimizer_device_shardings = jax.tree.map( lambda s: s.with_memory_kind(_DEVICE_MEMORY_KIND), state_mesh_shardings[_OPTIMIZER_STATE_KEY], @@ -1922,6 +1997,8 @@ def update(state_pure, accumulated_grads, accumulated_denominator, mean_loss): self._zero1_params_shardings = None self._gathered_params_shardings = None self._optimizer_device_shardings = None + self._selective_grad_shardings = None + self._selective_state_shardings = None # 1. JIT Compile Micro FWD/BWD Pass, and the add that accumulates it. # diff --git a/src/maxtext/training_engine/maxtext_engine_compile.py b/src/maxtext/training_engine/maxtext_engine_compile.py index db339d222e..d54a02bf8e 100644 --- a/src/maxtext/training_engine/maxtext_engine_compile.py +++ b/src/maxtext/training_engine/maxtext_engine_compile.py @@ -63,6 +63,7 @@ from maxtext.common import train_state_nnx from maxtext.configs import maxtext_engine_types from maxtext.configs import pyconfig +from maxtext.m3.infra import offload as m3_offload from maxtext.trainers.pre_train import train_compile as pre_train_compile from maxtext.training_engine import grpo_engine_compile from maxtext.training_engine import maxtext_engine @@ -111,8 +112,15 @@ def _rehome_aval(aval: Any, mesh: jax.sharding.Mesh) -> Any: """ if not hasattr(aval, "shape") or not hasattr(aval, "dtype"): return aval - spec = getattr(getattr(aval, "sharding", None), "spec", None) - target = jax.sharding.NamedSharding(mesh, spec) if spec is not None else None + sharding = getattr(aval, "sharding", None) + spec = getattr(sharding, "spec", None) + if spec is not None: + target = jax.sharding.NamedSharding(mesh, spec) + memory_kind = getattr(sharding, "memory_kind", None) + if memory_kind is not None and target.memory_kind != memory_kind: + target = target.with_memory_kind(memory_kind) + else: + target = None return jax.ShapeDtypeStruct(aval.shape, aval.dtype, sharding=target) @@ -201,10 +209,14 @@ def _trace_train_state(self, tx: Any) -> Any: def build(model_state): model = nnx.merge(model_graphdef, model_state) - return train_state_nnx.TrainStateNNX(model, nnx.Optimizer(model, tx, wrt=nnx.Param)) + if m3_offload.is_selective_host_offload_enabled(self._config): + opt = m3_offload.make_optimizer(model, tx, self._config, wrt=nnx.Param, dense_host_grads=True) + else: + opt = nnx.Optimizer(model, tx, wrt=nnx.Param) + return train_state_nnx.TrainStateNNX(model, opt) propagation_mesh = _propagation_mesh(self._mesh) - state_graphdef, _ = nnx.split(nnx.eval_shape(build, model_pure)) + state_graphdef, abs_state_pure = nnx.split(nnx.eval_shape(build, model_pure)) with self._sharding_ctx(): # Displaces the real mesh for this trace only: the graph trace above needs none, and a # stand-in set around it collides with the config's own AbstractMesh under `shard_mode=auto`. @@ -213,7 +225,10 @@ def build(model_state): lambda model_state: nnx.split(build(model_state))[1], jax.tree.map(lambda aval: _rehome_aval(aval, propagation_mesh), model_pure), ) - return nnx.merge(state_graphdef, jax.tree.map(lambda aval: _rehome_aval(aval, self._mesh), state_pure)) + rehomed = jax.tree.map(lambda aval: _rehome_aval(aval, self._mesh), state_pure) + if m3_offload.is_selective_host_offload_enabled(self._config): + rehomed = m3_offload.apply_selective_memory_kind_to_shardings(abs_state_pure, rehomed) + return nnx.merge(state_graphdef, rehomed) def train_state_avals(self) -> Any: """Returns a copy of the pure train state as avals on their compiled shardings. diff --git a/src/maxtext/utils/maxtext_utils.py b/src/maxtext/utils/maxtext_utils.py index 22b580b3f2..9ed3c4aa46 100644 --- a/src/maxtext/utils/maxtext_utils.py +++ b/src/maxtext/utils/maxtext_utils.py @@ -2116,6 +2116,10 @@ def _make_abstract_leaf(leaf_a, leaf_s): is_leaf=lambda x: isinstance(x, NamedSharding), ) nnx.update(state_mesh_shardings, state_params) + if is_training and config.selective_host_offload: + from maxtext.m3.infra import offload as m3_offload # pylint: disable=import-outside-toplevel + + state_mesh_shardings = m3_offload.apply_selective_memory_kind_to_shardings(abs_var_state, state_mesh_shardings) abstract_sharded_state = maxtext_utils_nnx.set_named_sharding_nnx(abstract_state, state_mesh_shardings) state_mesh_annotations = maxtext_utils_nnx.get_partition_spec_nnx(state_mesh_shardings) diff --git a/src/maxtext/utils/model_creation_utils.py b/src/maxtext/utils/model_creation_utils.py index 81c70db5aa..ff00673060 100644 --- a/src/maxtext/utils/model_creation_utils.py +++ b/src/maxtext/utils/model_creation_utils.py @@ -823,6 +823,10 @@ def create_nnx_abstract_model( named_sharding_state = sharding.nnx_construct_named_sharding(abs_var_state, mesh) if getattr(config, "use_m3_model", False) and mesh is not None and getattr(mesh, "size", 1) > 1: named_sharding_state = _apply_m3_default_named_sharding(abs_var_state, named_sharding_state, mesh) + if hasattr(config, "selective_host_offload") and config.selective_host_offload: + from maxtext.m3.infra import offload as m3_offload # pylint: disable=import-outside-toplevel + + named_sharding_state = m3_offload.apply_selective_memory_kind_to_shardings(abs_var_state, named_sharding_state) abstract_state = jax.tree.map( lambda a, s: jax.ShapeDtypeStruct(a.shape, a.dtype, sharding=s), abs_var_state, diff --git a/src/maxtext/utils/train_utils.py b/src/maxtext/utils/train_utils.py index 86941521be..55db07e242 100644 --- a/src/maxtext/utils/train_utils.py +++ b/src/maxtext/utils/train_utils.py @@ -424,7 +424,12 @@ def create_train_state_fn(): if getattr(getattr(config, "lora", None), "enable_lora", False) else nnx.Param ) - optimizer = nnx.Optimizer(model, tx, wrt=wrt) + if config.selective_host_offload: + from maxtext.m3.infra import offload as m3_offload # pylint: disable=import-outside-toplevel + + optimizer = m3_offload.make_optimizer(model, tx, config, wrt=wrt) + else: + optimizer = nnx.Optimizer(model, tx, wrt=wrt) return train_state_nnx.TrainStateNNX(model, optimizer) init_state_fn = create_train_state_fn diff --git a/tests/unit/m3/offload_test.py b/tests/unit/m3/offload_test.py index 3cc6a8d85b..623311318f 100644 --- a/tests/unit/m3/offload_test.py +++ b/tests/unit/m3/offload_test.py @@ -14,6 +14,7 @@ """Unit tests for `maxtext.m3.infra.offload` and runtime TPU HBM telemetry.""" +import re from unittest import mock from absl.testing import absltest @@ -558,5 +559,657 @@ def test_create_model_preserves_qwix_quantization_with_parameter_offload(self): self.assertEqual(out.shape, (2, 16, 256)) +def _qwen38_mini_config(output_dir: str, **overrides): + """Creates a compact `qwen3.8-flash-next-mini` M3 configuration for selective offload tests.""" + args = { + "model_name": "qwen3.8-flash-next-mini", + "override_model_config": "true", + "use_m3_model": "true", + "per_device_batch_size": "1", + "max_target_length": "16", + "steps": "4", + "learning_rate": "1e-3", + "gradient_clipping_threshold": "1.0", + "adam_weight_decay": "0.1", + "dtype": "float32", + "weight_dtype": "float32", + "matmul_precision": "highest", + "enable_checkpointing": "false", + "base_output_directory": output_dir, + "run_name": "m3_selective_offload_test", + } + num_devices = len(jax.devices()) + if num_devices >= 4: + args["ici_fsdp_parallelism"] = "2" + args["ici_expert_parallelism"] = str(num_devices // 2) + else: + args["ici_fsdp_parallelism"] = str(num_devices) + args.update({k: str(v) for k, v in overrides.items()}) + return pyconfig.initialize(["", get_test_config_path()] + [f"{k}={v}" for k, v in args.items()]) + + +class SelectiveHostOffloadTest(parameterized.TestCase): + """Tests pattern-selected pinned-host offload for large embedding tables (e.g. Qwen3.8 PLE).""" + + def test_validation_fails_closed(self): + out_dir = self.create_tempdir().full_path + with self.assertRaisesRegex(ValueError, "host_offload_param_patterns"): + _qwen38_mini_config(out_dir, selective_host_offload="true", host_offload_param_patterns="[]") + + with self.assertRaisesRegex(ValueError, "parameter_memory_host_offload"): + _qwen38_mini_config( + out_dir, + selective_host_offload="true", + parameter_memory_host_offload="true", + ) + + cfg_unmatched = _qwen38_mini_config( + out_dir, + selective_host_offload="true", + host_offload_param_patterns='["nonexistent_param_path_xyz$"]', + ) + mesh = mesh_from_config(cfg_unmatched) + with jax.set_mesh(mesh): + with self.assertRaisesRegex(ValueError, "matched 0 parameters"): + loop.setup_model(cfg_unmatched, mesh) + + with self.assertRaises(ValueError): + _qwen38_mini_config( + out_dir, + selective_host_offload="true", + selective_host_offload_optimizer_mode="unknown_mode", + ) + + def test_selective_offload_fails_closed_instead_of_silent_fallback(self): + """Verifies missing tokens, pre-gather shape mismatch, and dense lazy_sparse raise ValueError.""" + import importlib.util # pylint: disable=import-outside-toplevel + import optax # pylint: disable=import-outside-toplevel + + out_dir = self.create_tempdir().full_path + cfg = _qwen38_mini_config(out_dir, scan_layers="true", selective_host_offload="true") + mesh = mesh_from_config(cfg) + sharding = sharding_from_config(cfg) + + with jax.set_mesh(mesh): + model, _ = loop.setup_model(cfg, mesh, sharding) + bad_batch = {"targets": jnp.zeros((4, 16), dtype=jnp.int32)} + + # 1. value_and_grad / pure_value_and_grad / engine_value_and_grad refuse dense fallback when tokens=None. + with self.assertRaisesRegex(ValueError, "requires token IDs"): + m3_offload.value_and_grad(loop.loss_fn, model, bad_batch) + + graphdef, params, rest = nnx.split(model, nnx.Param, ...) + with self.assertRaisesRegex(ValueError, "requires token IDs"): + m3_offload.pure_value_and_grad( + lambda p, cp, r, c, d: (jnp.array(0.0), None), + graphdef, + params, + custom_params=nnx.State({}), + rest=rest, + config=cfg, + data=bad_batch, + ) + + with self.assertRaisesRegex(ValueError, "requires token IDs"): + m3_offload.engine_value_and_grad( + lambda p, r, b: (jnp.array(0.0), (None, r)), + graphdef, + params, + rest, + bad_batch, + ) + + # 2. HostOffloadedEmbed refuses pre-gather shape mismatch instead of bypassing pre_gathered. + ple_embeds = m3_offload._collect_host_embeds(model) # pylint: disable=protected-access + self.assertEqual(len(ple_embeds), 1) + ple_path, ple_embed, _ = ple_embeds[0] + dummy_pre_ids = jnp.zeros((2, 4), dtype=jnp.int32) + dummy_pre_vals = jnp.zeros((2, 4, ple_embed.features), dtype=jnp.float32) + with m3_offload._pregathered_context({ple_path: (dummy_pre_ids, dummy_pre_vals)}): # pylint: disable=protected-access + with self.assertRaisesRegex(ValueError, "pre-gather shape mismatch"): + ple_embed(jnp.zeros((2, 5), dtype=jnp.int32)) + + # 3. lazy_sparse rejects dense host gradients both at config/make_optimizer time and in update(). + cfg_ls = _qwen38_mini_config( + out_dir, + scan_layers="true", + selective_host_offload="true", + selective_host_offload_optimizer_mode="lazy_sparse", + ) + with self.assertRaisesRegex(ValueError, "incompatible with dense pinned_host gradients"): + m3_offload.make_optimizer(model, optax.adamw(1e-3), cfg_ls, dense_host_grads=True) + + opt_ls = m3_offload.make_optimizer(model, optax.adamw(1e-3), cfg_ls) + dense_grads = jax.tree.map( + lambda v: ( + v.replace( + m3_offload.to_memory(jnp.zeros_like(v.get_raw_value()), "pinned_host") + if m3_offload._is_host_leaf(v) # pylint: disable=protected-access + else jnp.zeros_like(v.get_raw_value()) + ) + if isinstance(v, nnx.Variable) + else jnp.zeros_like(v) + ), + nnx.state(model, nnx.Param), + is_leaf=lambda x: isinstance(x, nnx.Variable), + ) + with self.assertRaisesRegex(ValueError, "requires SparseHostGrad"): + opt_ls.update(model, dense_grads) + + # 4. AbstractMaxTextEngine fails closed at construction time for lazy_sparse (when tunix is installed). + if importlib.util.find_spec("tunix") is not None: + from maxtext.trainers.pre_train import train_compile # pylint: disable=import-outside-toplevel + from maxtext.training_engine import maxtext_engine_compile # pylint: disable=import-outside-toplevel + + cfg_eng_ls = _qwen38_mini_config( + out_dir, + compile_topology="tpu7x-8", + compile_topology_num_slices="1", + ici_fsdp_parallelism="4", + ici_expert_parallelism="2", + scan_layers="true", + selective_host_offload="true", + selective_host_offload_optimizer_mode="lazy_sparse", + ) + tpu_mesh = train_compile.get_topology_mesh(cfg_eng_ls) + with self.assertRaisesRegex(ValueError, "incompatible with dense pinned_host gradients"): + maxtext_engine_compile.AbstractMaxTextEngine(cfg_eng_ls, tpu_mesh) + + def test_sharded_2d_embed_gather_sqnorm_clip_and_optimizer_modes(self): + """Tests 2-axis row-sharded P(('fsdp', 'ep'), None) table with non-divisible n_local and duplicate IDs.""" + devices = np.array(jax.devices()) + if len(devices) >= 4: + mesh = Mesh(devices[:4].reshape(2, 2), ("fsdp", "ep")) + row_spec = ("fsdp", "ep") + n_local = 37 # Odd & non-divisible by num_chunks=5 (mirrors 320,001,920 / 128 = 2,500,015). + vocab_size = n_local * 4 + else: + mesh = Mesh(devices.reshape(-1, 1), ("fsdp", "ep")) + row_spec = ("fsdp",) + n_local = 37 + vocab_size = n_local * len(devices) + width = 16 + + class _ToyEmbedModel(nnx.Module): + """Minimal embedding + linear test module.""" + + def __init__(self, rngs: nnx.Rngs): + self.ple_embedding = nnx.Embed( + num_embeddings=vocab_size, + features=width, + dtype=jnp.float32, + param_dtype=jnp.float32, + embedding_init=nnx.initializers.normal(0.1), + rngs=rngs, + ) + self.proj = nnx.Linear(width, width, use_bias=False, dtype=jnp.float32, param_dtype=jnp.float32, rngs=rngs) + + def __call__(self, inputs: jax.Array) -> jax.Array: + return self.proj(self.ple_embedding(inputs)) + + class _ToyCfg: + parameter_memory_host_offload = False + optimizer_memory_host_offload = False + selective_host_offload = True + host_offload_param_patterns = ("(?:^|/)ple_embedding/embedding$",) + selective_host_offload_optimizer_mode = "host_compute" + selective_host_offload_num_chunks = 5 + + def _build(selective: bool, opt_mode: str = "host_compute"): + model = _ToyEmbedModel(nnx.Rngs(0)) + sh_tbl = NamedSharding(mesh, P(row_spec, None)) + model.ple_embedding.embedding.set_metadata("out_sharding", P(row_spec, None)) + model.ple_embedding.embedding.set_raw_value(jax.device_put(model.ple_embedding.embedding.get_raw_value(), sh_tbl)) + model.proj.kernel.set_raw_value( + jax.device_put(model.proj.kernel.get_raw_value(), NamedSharding(mesh, P(None, None))) + ) + import optax # pylint: disable=import-outside-toplevel + + base_tx = optax.adamw(learning_rate=1e-2, weight_decay=0.1) + if not selective: + tx = optax.chain(optax.clip_by_global_norm(0.5), base_tx) + return model, nnx.Optimizer(model, tx, wrt=nnx.Param) + cfg = _ToyCfg() + cfg.selective_host_offload_optimizer_mode = opt_mode + model = m3_offload.maybe_offload_parameters(model, cfg) + opt = m3_offload.make_optimizer(model, base_tx, cfg, clip_max_norm=0.5) + return model, opt + + def _loss(m, batch): + return jnp.mean(m(batch["inputs"]) ** 2) + + @nnx.jit(donate_argnums=(0, 1)) + def _step(m, opt, batch): + if m3_offload.has_selective_host_offload(m): + loss, grads = m3_offload.value_and_grad(_loss, m, batch) + else: + loss, grads = nnx.value_and_grad(_loss)(m, batch) + opt.update(m, grads) + return loss + + with jax.set_mesh(mesh): + m_ref, opt_ref = _build(False) + m_hc, opt_hc = _build(True, "host_compute") + m_sc, opt_sc = _build(True, "streamed_chunks") + m_ls, opt_ls = _build(True, "lazy_sparse") + + # Construct batches with explicit duplicate row IDs across shards and untouched rows. + batches = [] + for s in range(3): + raw_ids = jax.random.randint(jax.random.key(50 + s), (4, 8), 0, vocab_size, dtype=jnp.int32) + # Force duplicate hits on row 3 and row n_local + 2 across shards. + raw_ids = raw_ids.at[:, :2].set(jnp.array([3, min(n_local + 2, vocab_size - 1)], dtype=jnp.int32)) + batches.append({"inputs": jax.device_put(raw_ids, NamedSharding(mesh, P(None, None)))}) + + # Step-0 exact gradient and global-norm parity (including duplicate-ID accumulation). + l_ref0, g_ref0 = nnx.value_and_grad(_loss)(m_ref, batches[0]) + l_hc0, g_hc0 = m3_offload.value_and_grad(_loss, m_hc, batches[0]) + np.testing.assert_allclose(float(l_hc0), float(l_ref0), rtol=1e-6, atol=1e-6) + norm_ref = float(m3_offload.l2norm_pytree(g_ref0)) + norm_hc = float(m3_offload.l2norm_pytree(g_hc0)) + np.testing.assert_allclose(norm_hc, norm_ref, rtol=1e-5, atol=1e-6) + + # Run 3 optimizer steps across all modes. + losses_ref, losses_hc, losses_sc, losses_ls = [], [], [], [] + for b in batches: + losses_ref.append(float(_step(m_ref, opt_ref, b))) + losses_hc.append(float(_step(m_hc, opt_hc, b))) + losses_sc.append(float(_step(m_sc, opt_sc, b))) + losses_ls.append(float(_step(m_ls, opt_ls, b))) + + np.testing.assert_allclose(losses_hc, losses_ref, rtol=1e-5, atol=1e-5) + np.testing.assert_allclose(losses_sc, losses_ref, rtol=1e-5, atol=1e-5) + p_ref = _param_numpy_map(m_ref) + p_hc = _param_numpy_map(m_hc) + p_sc = _param_numpy_map(m_sc) + p_ls = _param_numpy_map(m_ls) + for path in p_ref: + np.testing.assert_allclose(p_hc[path], p_ref[path], rtol=1e-5, atol=1e-5) + np.testing.assert_allclose(p_sc[path], p_ref[path], rtol=1e-5, atol=1e-5) + + # Verify opt-in lazy_sparse intentionally skips weight decay on untouched rows (documenting deviation). + emb_path = ("ple_embedding", "embedding") + touched = set(int(x) for b in batches for x in np.asarray(b["inputs"]).reshape(-1)) + untouched = [i for i in range(vocab_size) if i not in touched] + self.assertNotEmpty(untouched) + # Dense AdamW decayed untouched rows; lazy_sparse left them untouched, so they must diverge. + untouched_diff = float(np.max(np.abs(p_ls[emb_path][untouched] - p_ref[emb_path][untouched]))) + self.assertGreater(untouched_diff, 1e-4) + + @parameterized.named_parameters( + ("scanned", True), + ("unscanned", False), + ) + def test_qwen38_mini_loop_parity_and_memory_kind(self, scan_layers: bool): + out_dir = self.create_tempdir().full_path + cfg_off = _qwen38_mini_config( + out_dir, + scan_layers=str(scan_layers).lower(), + selective_host_offload="false", + ) + cfg_on = _qwen38_mini_config( + out_dir, + scan_layers=str(scan_layers).lower(), + selective_host_offload="true", + ) + mesh = mesh_from_config(cfg_off) + sharding = sharding_from_config(cfg_off) + + with jax.set_mesh(mesh): + # Verify eval_shape preserves pinned_host ONLY on selected PLE table + its moments. + abs_model, abs_opt = nnx.eval_shape(lambda: loop.setup_model(cfg_on, mesh, sharding)) + abs_ple_paths = m3_offload.selected_host_param_paths(abs_model) + self.assertEqual(len(abs_ple_paths), 1) + for path, var in nnx.to_flat_state(nnx.state(abs_model, nnx.Param)): + mk = var.get_raw_value().sharding.memory_kind + if tuple(path) in abs_ple_paths: + self.assertEqual(mk, "pinned_host") + else: + self.assertIn(mk, (None, "device")) + abs_host_opt = 0 + abs_dev_opt = 0 + for path, leaf in jax.tree_util.tree_leaves_with_path( + abs_opt.opt_state, is_leaf=lambda x: isinstance(x, nnx.Variable) + ): + raw = leaf.get_raw_value() + if not hasattr(raw, "ndim") or raw.ndim == 0: + continue + key_t = tuple(m3_offload._key_value(k) for k in path) # pylint: disable=protected-access + is_sel = m3_offload._is_matching_opt_leaf(key_t, abs_ple_paths) is not None # pylint: disable=protected-access + if is_sel: + self.assertEqual(raw.sharding.memory_kind, "pinned_host") + else: + self.assertIn(raw.sharding.memory_kind, (None, "device")) + abs_host_opt += int(is_sel) + abs_dev_opt += int(not is_sel) + self.assertEqual(abs_host_opt, 2) + self.assertGreater(abs_dev_opt, 10) + + m_off, opt_off = loop.setup_model(cfg_off, mesh, sharding) + m_on, opt_on = loop.setup_model(cfg_on, mesh, sharding) + self.assertEqual(_param_paths(m_on), _param_paths(m_off)) + + # Step-0 loss and full gradient parity (materializing SparseHostGrad for PLE). + batch0 = loop.make_synthetic_batch(cfg_off, mesh, sharding, jax.random.key(0)) + l0_off, g0_off = nnx.value_and_grad(loop.loss_fn)(m_off, batch0) + l0_on, g0_on = m3_offload.value_and_grad(loop.loss_fn, m_on, batch0) + np.testing.assert_allclose(float(l0_on), float(l0_off), rtol=1e-6, atol=1e-6) + g_off_map = {tuple(p): v.get_raw_value() for p, v in nnx.to_flat_state(g0_off)} + g_on_map = { + tuple(p): (v.get_raw_value() if isinstance(v, nnx.Variable) else v) for p, v in nnx.to_flat_state(g0_on) + } + for path, go in g_off_map.items(): + gn = g_on_map[path] + if isinstance(gn, m3_offload.SparseHostGrad): + gn = m3_offload.materialize_sparse_grad(gn) + np.testing.assert_allclose(np.asarray(gn), np.asarray(go), rtol=1e-5, atol=1e-5) + + # 3-step exact AdamW trajectory parity (step-0 elementwise <= 1e-6; 3-step Frobenius <= 1e-5). + losses_off, losses_on = [], [] + for step in range(3): + batch = loop.make_synthetic_batch(cfg_off, mesh, sharding, jax.random.key(10 + step)) + losses_off.append(float(loop.train_step(m_off, opt_off, batch))) + losses_on.append(float(loop.train_step(m_on, opt_on, batch))) + if step == 0: + p_off0 = _param_numpy_map(m_off) + p_on0 = _param_numpy_map(m_on) + for path in p_off0: + np.testing.assert_allclose(p_on0[path], p_off0[path], rtol=1e-6, atol=1e-6) + np.testing.assert_allclose(losses_on, losses_off, rtol=1e-5, atol=1e-5) + p_off = _param_numpy_map(m_off) + p_on = _param_numpy_map(m_on) + for path in p_off: + # Damp denominator at 1.0 for zero-centered `1+scale` norms (`||scale||_2 ~ 0.01`, max_d < 1e-6). + rel_frob = float(np.linalg.norm(p_on[path] - p_off[path]) / max(float(np.linalg.norm(p_off[path])), 1.0)) + self.assertLess(rel_frob, 1e-5, f"Frobenius drift too large on {path}: {rel_frob:.2e}") + np.testing.assert_allclose(p_on[path], p_off[path], rtol=1e-4, atol=1e-4) + + # Post-step memory_kind invariants: ONLY PLE weight + its mu/nu remain in pinned_host. + ple_paths = m3_offload.selected_host_param_paths(m_on) + for path, var in nnx.to_flat_state(nnx.state(m_on, nnx.Param)): + mk = var.get_raw_value().sharding.memory_kind + if tuple(path) in ple_paths: + self.assertEqual(mk, "pinned_host") + else: + self.assertIn(mk, (None, "device")) + post_host_opt = 0 + for path, leaf in jax.tree_util.tree_leaves_with_path( + opt_on.opt_state, is_leaf=lambda x: isinstance(x, nnx.Variable) + ): + raw = leaf.get_raw_value() + if not hasattr(raw, "ndim") or raw.ndim == 0: + continue + key_t = tuple(m3_offload._key_value(k) for k in path) # pylint: disable=protected-access + is_sel = m3_offload._is_matching_opt_leaf(key_t, ple_paths) is not None # pylint: disable=protected-access + if is_sel: + self.assertEqual(raw.sharding.memory_kind, "pinned_host") + post_host_opt += 1 + else: + self.assertIn(raw.sharding.memory_kind, (None, "device")) + self.assertEqual(post_host_opt, 2) + + # Negative control: perturbing the offloaded PLE table must break loss parity. + ple_path = next(iter(ple_paths)) + ple_var = dict(nnx.to_flat_state(nnx.state(m_on, nnx.Param)))[ple_path] + ple_var.set_raw_value(m3_offload.to_memory(ple_var.get_raw_value() + 1.0, "pinned_host")) + perturbed_loss = float(loop.loss_fn(m_on, batch0)) + ref_loss = float(loop.loss_fn(m_off, batch0)) + self.assertGreater(abs(perturbed_loss - ref_loss), 1e-3) + + def test_default_off_hlo_byte_identical(self): + """Verifies Qwen3.8 default-off (`selective_host_offload=False`) emits byte-identical HLO.""" + out_dir = self.create_tempdir().full_path + cfg_default = _qwen38_mini_config(out_dir, scan_layers="true", selective_host_offload="false") + cfg_empty_pat = _qwen38_mini_config( + out_dir, + scan_layers="true", + selective_host_offload="false", + host_offload_param_patterns="[]", + ) + self.assertNotEmpty(cfg_default.host_offload_param_patterns) + mesh = mesh_from_config(cfg_default) + sharding = sharding_from_config(cfg_default) + + def _strip_debug_metadata(hlo_text: str) -> str: + # Strip XLA source-location intern tables (FileNames/FunctionNames/FileLocations/StackFrames) + # and inline `metadata={...}` annotations while keeping every HLO instruction, shape, layout, + # sharding, and fusion byte-identical. + body = re.split(r"\n(?:FileNames|FunctionNames|FileLocations|StackFrames)\b", hlo_text, maxsplit=1)[0] + return re.sub(r",\s*metadata=\{[^}]*\}", "", body) + + def _compile_hlo(cfg): + with jax.set_mesh(mesh): + m, opt = loop.setup_model(cfg, mesh, sharding) + batch = loop.make_synthetic_batch(cfg, mesh, sharding, jax.random.key(0)) + return _strip_debug_metadata(loop.train_step.lower(m, opt, batch).compile().as_text()) + + self.assertEqual(_compile_hlo(cfg_default), _compile_hlo(cfg_empty_pat)) + + def test_dual_drivers_and_transform_composition(self): + """Verifies selective offload across `train.py`, `MaxTextTrainingEngine`, and scan+remat+quantize.""" + out_dir = self.create_tempdir().full_path + + # 1. Legacy `train.py` driver (`train_utils.setup_train_loop` + `legacy_train.train_step`). + def _run_legacy(selective: bool): + cfg = _qwen38_mini_config( + out_dir, + scan_layers="true", + dataset_type="synthetic", + selective_host_offload=str(selective).lower(), + ) + mesh = mesh_from_config(cfg) + with jax.set_mesh(mesh): + ( + _, + _, + state_mesh_shardings, + _, + mesh, + _, + data_iter, + _, + _, + _, + train_state, + ) = train_utils.setup_train_loop(cfg, None, mesh=mesh) + jit_model, state = nnx.split(train_state) + p_step, _ = train_utils.jit_train_and_eval_step( + cfg, jit_model, mesh, state, state_mesh_shardings, legacy_train.train_step + ) + losses, gnorms, pnorms = [], [], [] + for _ in range(2): + batch = next(data_iter) + state, metrics = p_step(state, batch) + losses.append(float(metrics["scalar"]["learning/loss"])) + gnorms.append(float(metrics["scalar"]["learning/grad_norm"])) + pnorms.append(float(metrics["scalar"]["learning/param_norm"])) + return state, losses, gnorms, pnorms + + _, l_off, gn_off, pn_off = _run_legacy(False) + st_on, l_on, gn_on, pn_on = _run_legacy(True) + np.testing.assert_allclose(l_on, l_off, rtol=1e-5, atol=1e-5) + np.testing.assert_allclose(gn_on, gn_off, rtol=1e-4, atol=1e-4) + np.testing.assert_allclose(pn_on, pn_off, rtol=1e-4, atol=1e-4) + host_m = [ + x for x in jax.tree.leaves(st_on.model) if hasattr(x, "sharding") and x.sharding.memory_kind == "pinned_host" + ] + dev_m = [x for x in jax.tree.leaves(st_on.model) if hasattr(x, "sharding") and x.sharding.memory_kind == "device"] + host_o = [ + x + for x in jax.tree.leaves(st_on.optimizer) + if hasattr(x, "sharding") and getattr(x, "ndim", 0) > 0 and x.sharding.memory_kind == "pinned_host" + ] + dev_o = [ + x + for x in jax.tree.leaves(st_on.optimizer) + if hasattr(x, "sharding") and getattr(x, "ndim", 0) > 0 and x.sharding.memory_kind == "device" + ] + self.assertLen(host_m, 1) + self.assertGreater(len(dev_m), 10) + self.assertLen(host_o, 2) + self.assertGreater(len(dev_o), 10) + + # 2. Transform composition: scan + remat + Qwix int8 quantize + named scopes + selective offload. + cfg_comp = _qwen38_mini_config( + out_dir, + scan_layers="true", + remat_policy="full", + quantization="int8", + use_qwix_quantization="true", + selective_host_offload="true", + ) + cfg_comp_off = _qwen38_mini_config( + out_dir, + scan_layers="true", + remat_policy="full", + quantization="int8", + use_qwix_quantization="true", + selective_host_offload="false", + ) + mesh = mesh_from_config(cfg_comp) + with jax.set_mesh(mesh): + m_comp_off = model_creation_utils.create_model(cfg_comp_off, mesh, model_mode=MODEL_MODE_TRAIN, rngs=nnx.Rngs(0)) + m_comp_on = model_creation_utils.create_model(cfg_comp, mesh, model_mode=MODEL_MODE_TRAIN, rngs=nnx.Rngs(0)) + self.assertEqual(_param_paths(m_comp_on), _param_paths(m_comp_off)) + ple_paths = m3_offload.selected_host_param_paths(m_comp_on) + self.assertLen(ple_paths, 1) + toks = jnp.ones((cfg_comp.global_batch_size_to_train_on, 16), dtype=jnp.int32) + pos = jnp.broadcast_to(jnp.arange(16, dtype=jnp.int32), toks.shape) + self.assertGreater(m3_quant.quantized_matmuls(m_comp_on, jnp.int8, toks, pos), 0) + + # 3. MaxTextTrainingEngine (when tunix is installed). + import importlib.util # pylint: disable=import-outside-toplevel + + if importlib.util.find_spec("tunix") is not None: + from maxtext.training_engine import maxtext_engine # pylint: disable=import-outside-toplevel + + def _run_engine(selective: bool): + cfg_e = _qwen38_mini_config(out_dir, scan_layers="true", selective_host_offload=str(selective).lower()) + eng = maxtext_engine.MaxTextTrainingEngine(cfg_e, mesh=mesh) + bs = cfg_e.global_batch_size_to_train_on + seq = cfg_e.max_target_length + losses = [] + for step in range(2): + toks = jax.random.randint(jax.random.key(200 + step), (bs, seq), 0, cfg_e.vocab_size, dtype=jnp.int32) + pos = jnp.broadcast_to(jnp.arange(seq, dtype=jnp.int32), (bs, seq)) + seg = jnp.ones((bs, seq), dtype=jnp.int32) + batch = { + "inputs": toks, + "targets": jnp.roll(toks, -1, axis=-1), + "inputs_position": pos, + "targets_position": pos, + "inputs_segmentation": seg, + "targets_segmentation": seg, + } + if step == 0: + eng.compile(batch) + eng.fwd_bwd(batch) + eng.update() + m = eng.get_metrics() + lk = next(iter(m.weighted_metrics)) + losses.append( + float(jnp.sum(m.weighted_metrics[lk].unreduced_sum) / jnp.sum(m.weighted_metrics[lk].denominator)) + ) + jax.block_until_ready(eng._read_state_pure()) # pylint: disable=protected-access + return losses + + np.testing.assert_allclose(_run_engine(True), _run_engine(False), rtol=1e-5, atol=1e-5) + + def test_tpu7x_aot_selective_ple_offload_reduces_hbm_without_dense_grad(self): + """Compiles Qwen3.8 mini with an enlarged PLE table on virtual `tpu7x-8` and checks HBM drop.""" + import os # pylint: disable=import-outside-toplevel + + os.environ.setdefault("ALLOW_MULTIPLE_LIBTPU_LOAD", "1") + out_dir = self.create_tempdir().full_path + th = accelerator_to_spec_map.get_system_characteristics("tpu7x-8") + topo = get_topology_desc( + platform=th.platform, + topology_name=th.topology_name, + chip_config_name=th.chip_config_name, + chips_per_host_bounds=th.chips_per_host_bounds, + num_slices=1, + wrap=th.wrap, + ) + + def _compile(selective: bool): + cfg = _qwen38_mini_config( + out_dir, + compile_topology="tpu7x-8", + ici_fsdp_parallelism="4", + ici_expert_parallelism="2", + scan_layers="true", + dtype="bfloat16", + weight_dtype="bfloat16", + # Enlarge PLE table (~51.2 MB bf16 + ~102.4 MB bf16 AdamW mu/nu = ~153.6 MB) so HBM drop is unmistakable. + ngram_vocab_size_base="100000", + selective_host_offload=str(selective).lower(), + ) + tpu_mesh = mesh_from_config(cfg, devices=topo.devices) + sharding = sharding_from_config(cfg) + with jax.set_mesh(tpu_mesh): + data_spec = sharding("tokens", ("batch", "length"), TensorType.ACTIVATION) + data_sharding = NamedSharding(tpu_mesh, data_spec) + batch_shape = (int(cfg.per_device_batch_size) * len(topo.devices), int(cfg.max_target_length)) + batch = { + "inputs": jax.ShapeDtypeStruct(batch_shape, jnp.int32, sharding=data_sharding), + "targets": jax.ShapeDtypeStruct(batch_shape, jnp.int32, sharding=data_sharding), + "positions": jax.ShapeDtypeStruct(batch_shape, jnp.int32, sharding=data_sharding), + } + abs_model, abs_opt = nnx.eval_shape(lambda: loop.setup_model(cfg, tpu_mesh)) + compiled = loop.train_step.lower(abs_model, abs_opt, batch).compile() + rows = memory_report.memory_report({"update": compiled}, {}) + return compiled.memory_analysis(), rows[0], compiled.as_text() + + mem_off, row_off, _ = _compile(False) + _, row_on, hlo_on = _compile(True) + + self.assertEqual(mem_off.host_argument_size_in_bytes, 0) + self.assertNotIn("MegaScaleCollective", hlo_on) + # PLE table + AdamW mu/nu (~153.6 MB in bf16) must move from HBM argument to S(5) host_argument. + arg_drop = row_off.argument - row_on.argument + self.assertGreater(arg_drop, 140 * 1024 * 1024) + self.assertEqual(row_on.host_argument, arg_drop) + self.assertEqual(row_on.host_alias, row_on.host_argument) + # Host-compute uses bounded S(5) host scratch (<= 1x offloaded table+moments), never HBM. + self.assertLessEqual(row_on.host_temp, row_on.host_argument) + # Device resident peak must drop substantially without materializing a dense PLE gradient in HBM. + self.assertLess(row_on.device_total, row_off.device_total - 100 * 1024 * 1024) + + # Also verify AbstractMaxTextEngine AOT (when tunix is installed): zero MegaScale & large HBM drop. + import importlib.util # pylint: disable=import-outside-toplevel + + if importlib.util.find_spec("tunix") is not None: + from maxtext.trainers.pre_train import train_compile # pylint: disable=import-outside-toplevel + from maxtext.training_engine import maxtext_engine_compile # pylint: disable=import-outside-toplevel + + def _compile_engine(selective: bool): + cfg = _qwen38_mini_config( + out_dir, + compile_topology="tpu7x-8", + compile_topology_num_slices="1", + ici_fsdp_parallelism="4", + ici_expert_parallelism="2", + scan_layers="true", + remat_policy="full", + dtype="bfloat16", + weight_dtype="bfloat16", + ngram_vocab_size_base="100000", + selective_host_offload=str(selective).lower(), + ) + tpu_mesh = train_compile.get_topology_mesh(cfg) + eng = maxtext_engine_compile.AbstractMaxTextEngine(cfg, tpu_mesh) + compiled = eng.compile_kernels(maxtext_engine_compile.get_shaped_micro_batch(cfg)) + rows = memory_report.memory_report(compiled, eng.train_state_avals()) + peak = memory_report.device_peak(rows) + ms_count = sum(c.as_text().count("MegaScaleCollective") for c in compiled.values()) + return peak, ms_count + + peak_off, ms_off = _compile_engine(False) + peak_on, ms_on = _compile_engine(True) + self.assertEqual(ms_off, 0) + self.assertEqual(ms_on, 0) + self.assertLess(peak_on.device_total, peak_off.device_total - 100 * 1024 * 1024) + + if __name__ == "__main__": absltest.main() diff --git a/tests/unit/train_nnx_test.py b/tests/unit/train_nnx_test.py index b5db1d4c2c..b13e1df839 100644 --- a/tests/unit/train_nnx_test.py +++ b/tests/unit/train_nnx_test.py @@ -72,6 +72,8 @@ class _Cfg: shard_optimizer_over_data: bool = False optimizer_memory_host_offload: bool = False parameter_memory_host_offload: bool = False + selective_host_offload: bool = False + host_offload_param_patterns: tuple[str, ...] = () gradient_clipping_threshold: float = 0.0 grad_dtype: jnp.dtype = jnp.float32 record_internal_nn_metrics: bool = False