Repository navigation
Update Qwen3.5 scan-remat pattern by re-using Qwen3-Next's nested block scan #5009
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
99ccb1b
7b84837
f2488b3
981a5e9
0b5cc81
cc9a8fe
f452a91
e2fe51e
38974ea
ec3af18
697a9e9
f7ab0ff
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Large diffs are not rendered by default.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -31,7 +31,7 @@ | |
| import re | ||
| import time | ||
| import traceback | ||
| from typing import Any, Optional, Tuple | ||
| from typing import Any, NamedTuple, Optional, Tuple | ||
|
|
||
| from flax import nnx | ||
| from flax.traverse_util import flatten_dict, unflatten_dict | ||
|
|
@@ -300,25 +300,47 @@ def source_covers(target_key): | |
| return len(target_layer_keys) | ||
|
|
||
|
|
||
| def _find_qwen_scanned_layer_idx(key_tuple): | ||
| """Finds a Qwen scanned block path like `layers.layer_0` or `layers.moe_block`.""" | ||
| _QWEN_NESTED_LOCAL = "local_layers" | ||
| _QWEN_NESTED_GLOBAL = "global_layer" | ||
|
|
||
|
|
||
| class _QwenScannedRef(NamedTuple): | ||
| """Where a scanned Qwen parameter sits, and how it maps onto cycle slots.""" | ||
|
|
||
| container_idx: int | ||
| slot_idx: Optional[int] | ||
| consumed: int | ||
| nested: bool | ||
| trailing: bool | ||
|
|
||
|
|
||
| _NO_QWEN_SCAN = _QwenScannedRef(-1, -1, 0, False, False) | ||
|
|
||
|
|
||
| def _find_qwen_scanned_layer_idx(key_tuple) -> _QwenScannedRef: | ||
| """Finds a Qwen scanned block path like `layers.local_layers` or `layers.layer_0`.""" | ||
| for i in range(len(key_tuple) - 1): | ||
| if key_tuple[i] != "layers" or not isinstance(key_tuple[i + 1], str): | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This only matches
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done |
||
| continue | ||
| if key_tuple[i + 1].startswith("layers_"): | ||
| name = key_tuple[i + 1] | ||
| if name.startswith("layers_"): | ||
| continue | ||
| match = re.fullmatch(r"layer_(\d+)", key_tuple[i + 1]) | ||
| if name == _QWEN_NESTED_LOCAL: | ||
| return _QwenScannedRef(i, None, 2, True, False) | ||
| if name == _QWEN_NESTED_GLOBAL: | ||
| return _QwenScannedRef(i, None, 2, False, True) | ||
| match = re.fullmatch(r"layer_(\d+)", name) | ||
| if match: | ||
| return i, int(match.group(1)), 2 | ||
| return i, 0, 1 | ||
| return -1, -1, 0 | ||
| return _QwenScannedRef(i, int(match.group(1)), 2, False, False) | ||
| return _QwenScannedRef(i, 0, 1, False, False) | ||
| return _NO_QWEN_SCAN | ||
|
|
||
|
|
||
| def unroll_qwen_scanned_weights(weights, scan_axis: int = 1, pattern_length: Optional[int] = None): | ||
| """Unroll Qwen's heterogeneous or homogeneous scanned blocks for an unscanned MaxText target. | ||
|
|
||
| Qwen 3 Next/3.5 training stores a repeating layer cycle as | ||
| `decoder.layers.layer_{slot}`, with repetitions stacked on `scan_axis`. | ||
| Qwen 3 Next/3.5 training nests two scans: `decoder.layers.local_layers` holds the cycle's linear-attention | ||
| layers with the slot on `scan_axis + 1`, and `decoder.layers.global_layer` holds the trailing full-attention layer. | ||
| Qwen 3 base training stores homogeneous layers as `decoder.layers.*` stacked on `scan_axis`. | ||
| The inference model stores every layer as a direct decoder attribute named | ||
| `layers_{global_index}`. Tunix's generic direct-sync mapper cannot bridge | ||
|
|
@@ -345,19 +367,48 @@ def unroll_qwen_scanned_weights(weights, scan_axis: int = 1, pattern_length: Opt | |
| scanned_keys = [] | ||
| slot_indices = set() | ||
| scan_lengths = set() | ||
| local_widths = set() | ||
| has_trailing = False | ||
| for key, value in flat_w.items(): | ||
| container_idx, slot_idx, consumed = _find_qwen_scanned_layer_idx(key) | ||
| if container_idx == -1 or "dropout" in key or "rngs" in key: | ||
| if "layers_remainder" in key and "dropout" not in key and "rngs" not in key: | ||
| # Like MaxTextToMaxTextConverter and raiden_unscan, which reject such a layer count up front. | ||
| raise ValueError( | ||
| f"Qwen scanned parameter {'.'.join(map(str, key))} is in `layers_remainder`: unrolling the layers past " | ||
| "the last whole cycle is not supported, so num_decoder_layers must be a multiple of the cycle." | ||
| ) | ||
| ref = _find_qwen_scanned_layer_idx(key) | ||
| if ref.container_idx == -1 or "dropout" in key or "rngs" in key: | ||
| continue | ||
| if not hasattr(value, "shape") or len(value.shape) <= scan_axis: | ||
| min_ndim = scan_axis + 2 if ref.nested else scan_axis + 1 | ||
| if not hasattr(value, "shape") or len(value.shape) < min_ndim: | ||
| raise ValueError(f"Qwen scanned parameter {'.'.join(map(str, key))} has no scan axis {scan_axis}: {value!r}") | ||
| scanned_keys.append((key, value, container_idx, slot_idx, consumed)) | ||
| slot_indices.add(slot_idx) | ||
| scanned_keys.append((key, value, ref)) | ||
| scan_lengths.add(value.shape[scan_axis]) | ||
| if ref.nested: | ||
| local_widths.add(value.shape[scan_axis + 1]) | ||
| slot_indices.update(range(value.shape[scan_axis + 1])) | ||
| elif ref.trailing: | ||
| has_trailing = True | ||
| else: | ||
| slot_indices.add(ref.slot_idx) | ||
|
|
||
| if not scanned_keys: | ||
| return weights | ||
|
|
||
| if len(local_widths) > 1: | ||
| raise ValueError(f"Qwen nested `local_layers` parameters disagree on cycle width: {sorted(local_widths)}") | ||
| if pattern_length is None: | ||
| trailing_slot = (max(slot_indices) + 1) if slot_indices else 0 | ||
| else: | ||
| if local_widths and local_widths != {pattern_length - 1}: | ||
| raise ValueError( | ||
| f"Qwen nested `local_layers` hold {local_widths.pop()} cycle slots, but pattern_length {pattern_length} " | ||
| f"leaves {pattern_length - 1} before the global layer" | ||
| ) | ||
| trailing_slot = pattern_length - 1 | ||
| if has_trailing: | ||
| slot_indices.add(trailing_slot) | ||
|
|
||
| if pattern_length is None: | ||
| expected_slots = set(range(max(slot_indices) + 1)) | ||
| if slot_indices != expected_slots: | ||
|
|
@@ -371,16 +422,25 @@ def unroll_qwen_scanned_weights(weights, scan_axis: int = 1, pattern_length: Opt | |
| raise ValueError(f"Qwen scanned parameters disagree on scan length: {sorted(scan_lengths)}") | ||
|
|
||
| scan_length = scan_lengths.pop() | ||
| scanned_key_paths = {key for key, _, _, _, _ in scanned_keys} | ||
| scanned_key_paths = {key for key, _, _ in scanned_keys} | ||
| new_flat_w = {key: value for key, value in flat_w.items() if key not in scanned_key_paths} | ||
|
|
||
| for key, value, container_idx, slot_idx, consumed in scanned_keys: | ||
| prefix = key[:container_idx] | ||
| suffix = key[container_idx + consumed :] | ||
| for key, value, ref in scanned_keys: | ||
| prefix = key[: ref.container_idx] | ||
| suffix = key[ref.container_idx + ref.consumed :] | ||
| if ref.nested: | ||
| slots = tuple(range(value.shape[scan_axis + 1])) | ||
| else: | ||
| slots = (trailing_slot if ref.trailing else ref.slot_idx,) | ||
| for repetition in range(scan_length): | ||
| global_idx = repetition * pattern_length + slot_idx | ||
| new_key = prefix + (f"layers_{global_idx}",) + suffix | ||
| new_flat_w[new_key] = jnp.take(value, repetition, axis=scan_axis) | ||
| block = jax.lax.index_in_dim(value, repetition, axis=scan_axis, keepdims=False) | ||
| for local_idx, slot_idx in enumerate(slots): | ||
| global_idx = repetition * pattern_length + slot_idx | ||
| new_key = prefix + (f"layers_{global_idx}",) + suffix | ||
| # Taking the block collapsed `scan_axis`, so the nested slot axis has shifted down into it. | ||
| new_flat_w[new_key] = ( | ||
| jax.lax.index_in_dim(block, local_idx, axis=scan_axis, keepdims=False) if ref.nested else block | ||
| ) | ||
|
|
||
| logging.info( | ||
| "MaxTextVllmSampler: unrolled %d Qwen tensor components across %d layers for direct MaxText weight sync.", | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
nit: With
param_scan_axis=0and FP8 experts, this returns(0, 0, 1), andmoveaxisfails on the repeated axis. Assertparam_scan_axis >= 1, or handle0.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It now raises ValueError