Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions src/maxtext/configs/base.yml
Original file line number Diff line number Diff line change
Expand Up @@ -619,6 +619,13 @@ engram: 'remat'

optimizer_memory_host_offload: false
parameter_memory_host_offload: false
# Selective host offload saves HBM for regex-matched embedding tables + AdamW state at the cost of
# synchronous host-CPU optimizer scan time (~1.5-1.9 s/step on v5p-8 at 2.5M x 160 rows/chip under
# default 'host_compute'; 'streamed_chunks' is ~150x slower; 'lazy_sparse' skips untouched-row decay).
selective_host_offload: false
host_offload_param_patterns: []
selective_host_offload_optimizer_mode: host_compute
selective_host_offload_num_chunks: 16
# Whether to use jax.lax.scan over layers (stacked/unstacked checkpoint). We recommend setting this to false when using pipeline parallelism, instead scanning the PP iterations. When resuming from a checkpoint, this flag is auto-determined from metadata.
scan_layers: true
param_scan_axis: 1
Expand Down
3 changes: 3 additions & 0 deletions src/maxtext/configs/models/qwen3.8-flash-next-mini.yml
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,9 @@ make_ngram_vocab_size_divisible_by: 4 # Mini-twin (HF: make_ngram_vocab_size_di
split_ngram_parts: 4 # Mini-twin (HF: split_ngram_parts=128)
ple_seed: 1234 # HF: seed=1234
ple_eos_token_id: 3 # In-range EOS token (< vocab_size=1024)
# Pattern used only when selective_host_offload=true (default false); host_compute adds ~1.5-1.9 s/step
# synchronous host-CPU AdamW scan on v5p-8 at 2.5M x 160 rows/chip (~24 -> 632 ms/step on v5p-8 mini).
host_offload_param_patterns: ["(?:^|/)ple_embedding/ngram_embedding/embedding$"]

# Multi-Token Prediction (MTP) Parameters
mtp_num_layers: 1
Expand Down
3 changes: 3 additions & 0 deletions src/maxtext/configs/models/qwen3.8-flash-next.yml
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,9 @@ make_ngram_vocab_size_divisible_by: 128 # HF: make_ngram_vocab_size_divisible_b
split_ngram_parts: 128 # HF config.json: split_ngram_parts=128 (class default in configuration_qwen4_exp.py is 512)
ple_seed: 1234 # HF: seed=1234
ple_eos_token_id: 248044 # HF: eos_token_id=248044
# Pattern used only when selective_host_offload=true (default false); host_compute adds ~1.5-1.9 s/step
# synchronous host-CPU AdamW scan on v5p-8 at 2.5M x 160 rows/chip (~24 -> 632 ms/step on v5p-8 mini).
host_offload_param_patterns: ["(?:^|/)ple_embedding/ngram_embedding/embedding$"]

# Multi-Token Prediction (MTP) Parameters
mtp_num_layers: 1
Expand Down
40 changes: 40 additions & 0 deletions src/maxtext/configs/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -2689,6 +2689,41 @@ class RematAndOffload(BaseModel):

optimizer_memory_host_offload: bool = Field(False, description="Offload optimizer state to host memory.")
parameter_memory_host_offload: bool = Field(False, description="Offload parameters to host memory.")
selective_host_offload: bool = Field(
False,
description=(
"Offload only parameters matching `host_offload_param_patterns` and their matching optimizer "
"state to `pinned_host` memory, leaving all other parameters and optimizer state on device. "
"Saves HBM at the cost of synchronous host-CPU optimizer latency (e.g., ~1.5-1.9 s/step on "
"v5p-8 for a 2.5M x 160 table/chip under default 'host_compute'; ~24 -> 632 ms/step on v5p-8 mini)."
),
)
host_offload_param_patterns: list[str] = Field(
default_factory=list,
description=(
"Regex patterns matching '/'-joined parameter paths to offload to `pinned_host` memory when "
"`selective_host_offload=True` (for example, ['(?:^|/)ple_embedding/ngram_embedding/embedding$'])."
),
)
selective_host_offload_optimizer_mode: Literal["host_compute", "streamed_chunks", "lazy_sparse"] = Field(
"host_compute",
description=(
"Optimizer update mode for selectively host-offloaded embedding tables: "
"'host_compute' (default) runs exact dense AdamW on host CPU with zero full-table HBM transfer, "
"but scans all offloaded rows and moments in host DRAM synchronously each step (~1.5-1.9 s/step "
"measured on v5p-8 at 2.5M x 160 rows/chip; ~24 -> 632 ms/step on v5p-8 mini); "
"'streamed_chunks' streams `selective_host_offload_num_chunks` row chunks through device compute "
"for exact dense AdamW with zero host temp DRAM but is ~150x slower on live TPU (~284 s/step at "
"2.5M x 160 on v5p-8, diagnostic use only); "
"'lazy_sparse' (opt-in approximation) updates only touched rows, skipping weight and moment decay "
"on untouched rows, and fails closed with dense host gradients (such as MaxTextTrainingEngine)."
),
)
selective_host_offload_num_chunks: int = Field(
16,
ge=1,
description="Number of row chunks used when `selective_host_offload_optimizer_mode` is 'streamed_chunks'.",
)


class Tokenizer(BaseModel):
Expand Down Expand Up @@ -5469,6 +5504,11 @@ def validate_save_fsdp_gathered_weights(self) -> "MaxTextConfig":
raise ValueError(
"save_fsdp_gathered_weights keeps gathered kernels in HBM, which defeats parameter_memory_host_offload."
)
if self.selective_host_offload:
if self.parameter_memory_host_offload:
raise ValueError("selective_host_offload=True cannot be combined with parameter_memory_host_offload=True.")
if not self.host_offload_param_patterns:
raise ValueError("selective_host_offload=True requires non-empty host_offload_param_patterns.")
Comment on lines +5507 to +5511

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟠 Fixes at the wrong layer: this config validation for selective_host_offload is placed inside validate_save_fsdp_gathered_weights. Create a dedicated @model_validator(mode="after") named validate_selective_host_offload to keep unrelated concerns separate.

return self

@model_validator(mode="after")
Expand Down
Loading
Loading