diff --git a/src/maxtext/configs/types.py b/src/maxtext/configs/types.py index 3101d9eea1..4cbe943ceb 100644 --- a/src/maxtext/configs/types.py +++ b/src/maxtext/configs/types.py @@ -2137,8 +2137,9 @@ class Qwen3_8FlashNext(BaseModel): qsa_attention_kernel: Literal["dense", "flash"] = Field( "dense", description=( - "Qwen3.8 QSA attention kernel: 'dense' materializes the score matrix, 'flash' dispatches through" - " Tokamax Splash Attention (requires max_target_length <= index_select_budget)." + "Qwen3.8 QSA attention kernel: 'dense' materializes the full score matrix; 'flash' dispatches through" + " Tokamax Splash Attention when s <= index_select_budget and tiled block-sparse FlashAttention with a" + " 32-bit packed block bitmap when s > index_select_budget." ), ) use_ple_layer: bool = Field(False, description="Whether to use Per-Layer Embedding (PLE).") diff --git a/src/maxtext/m3/models/__init__.py b/src/maxtext/m3/models/__init__.py index 6cd070c101..8b7232f49f 100644 --- a/src/maxtext/m3/models/__init__.py +++ b/src/maxtext/m3/models/__init__.py @@ -77,8 +77,8 @@ def create_granitemoe_model(config, mesh=None, *, rngs, **kwargs): "granitemoe": _FOUR_AXIS_MESH, "qwen3-0.6b": modeling_qwen3.m3_sharding.DEFAULT_SHARDING_AXES, "qwen3-vl-2b": _FOUR_AXIS_MESH, - "qwen3.8-flash-next": ("data", "fsdp", "fsdp_transpose", "tensor", "tensor_sequence", "expert"), - "qwen3.8-flash-next-mini": ("data", "fsdp", "fsdp_transpose", "tensor", "tensor_sequence", "expert"), + "qwen3.8-flash-next": modeling_qwen3.m3_sharding.DEFAULT_SHARDING_AXES, + "qwen3.8-flash-next-mini": modeling_qwen3.m3_sharding.DEFAULT_SHARDING_AXES, } MODEL_EXPLICIT_SHARDING_SUPPORT: frozenset[str] = frozenset({"qwen3-0.6b"}) diff --git a/src/maxtext/m3/models/qwen3_8_flash_next/modeling_qwen3_8_flash_next.py b/src/maxtext/m3/models/qwen3_8_flash_next/modeling_qwen3_8_flash_next.py index db5f651d74..10e7840508 100644 --- a/src/maxtext/m3/models/qwen3_8_flash_next/modeling_qwen3_8_flash_next.py +++ b/src/maxtext/m3/models/qwen3_8_flash_next/modeling_qwen3_8_flash_next.py @@ -49,7 +49,7 @@ from maxtext.kernels.tokamax_splash_attention import splash_attention_mask as tokamax_splash_mask from maxtext.m3.core import kernels as m3_kernels from maxtext.m3.core import moe as m3_moe -from maxtext.m3.core.sharding import DefaultSharding, Sharding, param_metadata +from maxtext.m3.core.sharding import DefaultSharding, Sharding, TensorType, param_metadata # ================================================================================================= # Normalization layers and 4-stream gated residual (hyper-connection) @@ -125,15 +125,38 @@ def l2norm(x: jax.Array, dim: int = -1, eps: float = 1e-6) -> jax.Array: return (x_f32 * jax.lax.rsqrt(jnp.sum(jnp.square(x_f32), axis=dim, keepdims=True) + eps)).astype(x.dtype) -_BATCH_MESH_AXES = ("data", "stage", "fsdp", "fsdp_transpose", "expert") +def _mesh_axis_names(entry: Any) -> tuple[str, ...]: + """Mesh axis names carried by one `PartitionSpec` entry.""" + if entry is None: + return () + return (entry,) if isinstance(entry, str) else tuple(entry) + + +def get_batch_mesh_axes(sharding: Sharding | None = None) -> tuple[str, ...]: + """Returns the active mesh axes `sharding` maps the `batch` axis onto when a multi-device mesh is set.""" + mesh = jax.sharding.get_abstract_mesh() + if mesh.empty or mesh.size <= 1: + return () + spec = (sharding or DefaultSharding())("batch", ("batch",), TensorType.ACTIVATION) + return tuple(ax for ax in _mesh_axis_names(spec[0]) if mesh.shape.get(ax, 1) > 1) + + +def get_length_mesh_axes(sharding: Sharding | None = None) -> tuple[str, ...]: + """Returns the active mesh axes `sharding` maps the `length` axis onto when a multi-device mesh is set.""" + mesh = jax.sharding.get_abstract_mesh() + if mesh.empty or mesh.size <= 1: + return () + spec = (sharding or DefaultSharding())("length", ("length",), TensorType.ACTIVATION) + return tuple(ax for ax in _mesh_axis_names(spec[0]) if mesh.shape.get(ax, 1) > 1) -def get_batch_mesh_axes() -> tuple[str, ...]: - """Returns active batch mesh axis names when a multi-device mesh is set.""" +def get_heads_mesh_axes(sharding: Sharding | None = None) -> tuple[str, ...]: + """Returns the active mesh axes `sharding` maps the `heads` axis onto when a multi-device mesh is set.""" mesh = jax.sharding.get_abstract_mesh() if mesh.empty or mesh.size <= 1: return () - return tuple(ax for ax in _BATCH_MESH_AXES if mesh.shape.get(ax, 1) > 1) + spec = (sharding or DefaultSharding())("q", ("heads",), TensorType.ACTIVATION) + return tuple(ax for ax in _mesh_axis_names(spec[0]) if mesh.shape.get(ax, 1) > 1) def shard_batch(x: jax.Array) -> jax.Array: @@ -589,6 +612,17 @@ def __call__(self, x: jax.Array, position_ids: jax.Array) -> tuple[jax.Array, ja return jnp.cos(emb).astype(x.dtype), jnp.sin(emb).astype(x.dtype) +def _compute_qsa_doc_start(segment_ids: jax.Array) -> jax.Array: + """Computes first token index of each token's document segment in O(B * S log S) without O(S^2) masks.""" + b, s = segment_ids.shape + pos = jnp.broadcast_to(jnp.arange(s, dtype=jnp.int32)[None, :], (b, s)) + sorted_seg, sorted_pos = jax.lax.sort([segment_ids.astype(jnp.int32), pos], dimension=-1, is_stable=True) + is_first = jnp.concatenate([jnp.ones((b, 1), dtype=bool), sorted_seg[:, 1:] != sorted_seg[:, :-1]], axis=-1) + first_sorted_idx = jax.lax.associative_scan(jnp.maximum, jnp.where(is_first, pos, 0), axis=-1) + sorted_doc_start = jnp.where(sorted_seg != 0, jnp.take_along_axis(sorted_pos, first_sorted_idx, axis=-1), 0) + return jnp.zeros((b, s), dtype=jnp.int32).at[jnp.arange(b, dtype=jnp.int32)[:, None], sorted_pos].set(sorted_doc_start) + + class Qwen3_8QSAIndexer(nnx.Module): """MQA micro-block indexer selecting top-k complete blocks for QSA.""" @@ -668,8 +702,7 @@ def __call__( if attention_mask is not None: segment_ids = jnp.where(attention_mask != 0, segment_ids, 0) - same_doc = (segment_ids[:, :, None] == segment_ids[:, None, :]) & (segment_ids[:, :, None] != 0) - doc_start = jnp.argmax(same_doc, axis=-1) + doc_start = jnp.zeros((b, s), dtype=jnp.int32) if unpacked else _compute_qsa_doc_start(segment_ids) num_blocks = max(1, s // c) local_pos, end_t = jnp.arange(s)[None, :] - doc_start, jnp.clip(jnp.arange(s) + c - 1, 0, s - 1) is_block_start = ( @@ -714,6 +747,7 @@ def __call__( if not return_token_mask: return None, selected_block_indices + same_doc = (segment_ids[:, :, None] == segment_ids[:, None, :]) & (segment_ids[:, :, None] != 0) if self.mask_mode == "reshape" and s <= self.token_budget: ar_k, ar_q = jnp.arange(s)[None, None, :], jnp.arange(s)[None, :, None] token_mask = (same_doc & (ar_k <= ar_q))[:, None, :, :] @@ -747,6 +781,29 @@ def __call__( return token_mask, selected_block_indices +def pack_qsa_block_mask_32bit(selected_block_indices: jax.Array, num_words: int) -> jax.Array: + """Packs document-local selected_block_indices (B, S, topk) into a carry-free 32-bit uint32 bitmap (B, S, num_words).""" + max_block = num_words * 32 + + def _pack_token(row: jax.Array) -> jax.Array: + valid = (row >= 0) & (row < max_block) + safe_idx = jnp.where(valid, row, max_block) + sorted_idx = jnp.sort(safe_idx, axis=-1) + dedup_valid = (sorted_idx < max_block) & jnp.concatenate( + [jnp.ones((1,), dtype=bool), sorted_idx[1:] != sorted_idx[:-1]], + axis=-1, + ) + word_idx = jnp.where(dedup_valid, sorted_idx >> 5, 0) + bit_mask = jnp.where( + dedup_valid, + jnp.uint32(1) << (sorted_idx & 31).astype(jnp.uint32), + jnp.uint32(0), + ) + return jnp.zeros((num_words,), dtype=jnp.uint32).at[word_idx].add(bit_mask) + + return jax.vmap(jax.vmap(_pack_token))(selected_block_indices) + + class Qwen3_8SparseAttention(nnx.Module): """Qwen Sparse Attention (QSA) with RMSNorm, gated output, and micro-block indexer.""" @@ -767,13 +824,13 @@ def __init__( eps: float = 1e-6, dtype: Any = jnp.bfloat16, weight_dtype: Any = jnp.bfloat16, - sharding: Any | None = None, + sharding: Sharding | None = None, *, indexer_mask_mode: str = "gather", attention_kernel: str = "dense", rngs: nnx.Rngs, ): - del sharding + self.sharding = sharding or DefaultSharding() if num_query_heads % num_kv_heads != 0: raise ValueError(f"num_query_heads ({num_query_heads}) must be divisible by num_kv_heads ({num_kv_heads}).") if attention_kernel not in ("dense", "flash"): @@ -872,7 +929,7 @@ def _run_batch( )(q_b, k_b, v_b, sq_b, skv_b) mesh = jax.sharding.get_abstract_mesh() - batch_axes = get_batch_mesh_axes() + batch_axes = get_batch_mesh_axes(self.sharding) batch_shards = math.prod(mesh.shape[ax] for ax in batch_axes) if batch_axes else 1 if batch_axes and b % batch_shards == 0: qkv_spec = jax.sharding.PartitionSpec(batch_axes, None, None, None) @@ -890,6 +947,578 @@ def _run_batch( out = jnp.swapaxes(out[:, :, :s, :], 1, 2) return out.reshape(b, s, self.num_query_heads * self.head_dim) + def flash_block_sparse_attention( + self, + query_states: jax.Array, + key_states: jax.Array, + value_states: jax.Array, + selected_block_indices: jax.Array, + *, + attention_mask: jax.Array | None = None, + segment_ids: jax.Array | None = None, + block_q: int | None = None, + block_kv: int | None = None, + sharding: Sharding | None = None, + ) -> jax.Array: + """Runs tiled online-softmax FlashAttention with a packed 32-bit block bitmap and LSE custom_vjp.""" + b, s, h_q, d = query_states.shape + c = self.indexer.compress_ratio + num_blocks = max(1, s // c) + scale = 1.0 / math.sqrt(float(d)) + + if block_q is None: + b_q = 512 if s >= 1024 else max(c, ((max(8, s // 4) + c - 1) // c) * c) + else: + b_q = block_q + if block_kv is None: + b_kv = (1024 if s >= 2048 else 512) if s >= 1024 else b_q + else: + b_kv = block_kv + + tile_lcm = math.lcm(b_q, b_kv) + pad_len = (tile_lcm - s % tile_lcm) % tile_lcm + s_pad = s + pad_len + + unpacked = segment_ids is None and attention_mask is None + if unpacked: + seg = jnp.ones((b, s), dtype=jnp.int32) + local_pos = jnp.broadcast_to(jnp.arange(s, dtype=jnp.int32)[None, :], (b, s)) + else: + seg = jnp.ones((b, s), dtype=jnp.int32) if segment_ids is None else segment_ids.astype(jnp.int32) + if attention_mask is not None: + seg = jnp.where(attention_mask != 0, seg, 0) + doc_start = _compute_qsa_doc_start(seg) + local_pos = jnp.arange(s, dtype=jnp.int32)[None, :] - doc_start + + fast_unpacked = unpacked and (pad_len == 0) and (s % c == 0) and (b_kv % (32 * c) == 0) + num_words = max(1, (s_pad // c) // 32) if fast_unpacked else max(1, (num_blocks + 31) // 32) + packed_mask = pack_qsa_block_mask_32bit(selected_block_indices, num_words) + + q, k, v = query_states, key_states, value_states + if pad_len > 0: + q = jnp.pad(q, ((0, 0), (0, pad_len), (0, 0), (0, 0))) + k = jnp.pad(k, ((0, 0), (0, pad_len), (0, 0), (0, 0))) + v = jnp.pad(v, ((0, 0), (0, pad_len), (0, 0), (0, 0))) + packed_mask = jnp.pad(packed_mask, ((0, 0), (0, pad_len), (0, 0))) + seg = jnp.pad(seg, ((0, 0), (0, pad_len)), constant_values=0) + local_pos = jnp.pad(local_pos, ((0, 0), (0, pad_len)), constant_values=0) + + mesh = jax.sharding.get_abstract_mesh() + sharding_obj = sharding or getattr(self, "sharding", None) or DefaultSharding() + use_sharding = ( + not mesh.empty and mesh.size > 1 and not any(t == jax.sharding.AxisType.Explicit for t in mesh.axis_types) + ) + batch_axes = get_batch_mesh_axes(sharding_obj) if use_sharding else () + batch_shards = math.prod(mesh.shape[ax] for ax in batch_axes) if batch_axes else 1 + b_spec: tuple[str, ...] | None = batch_axes if (batch_axes and b % batch_shards == 0) else None + + head_axes = get_heads_mesh_axes(sharding_obj) if use_sharding else () + head_shards = math.prod(mesh.shape[ax] for ax in head_axes) if head_axes else 1 + h_kv_total = k.shape[2] + tp_spec: tuple[str, ...] | None = None + if head_axes and h_q % head_shards == 0: + if h_kv_total % head_shards == 0: + tp_spec = head_axes + elif head_shards % h_kv_total == 0: + rep = head_shards // h_kv_total + k = jnp.repeat(k, rep, axis=2) + v = jnp.repeat(v, rep, axis=2) + tp_spec = head_axes + + length_axes = get_length_mesh_axes(sharding_obj) if use_sharding else () + q_seq_shards = math.prod(mesh.shape[ax] for ax in length_axes) if length_axes else 1 + use_q_seq_shard_map = bool(length_axes) and s_pad % (q_seq_shards * b_q) == 0 + length_spec: tuple[str, ...] | None = length_axes if use_q_seq_shard_map else None + + concrete_mesh: jax.sharding.Mesh | None = None + if use_sharding: + try: + cand_mesh = jax.sharding.get_mesh() + if isinstance(cand_mesh, jax.sharding.Mesh) and not cand_mesh.empty: + concrete_mesh = cand_mesh + except ValueError: + concrete_mesh = None + if use_sharding and concrete_mesh is None: + raw_q = getattr(q, "primal", q) + if isinstance(getattr(raw_q, "sharding", None), jax.sharding.SingleDeviceSharding): + use_sharding = False + b_spec, tp_spec, length_spec = None, None, None + use_q_seq_shard_map = False + shmap_mesh = concrete_mesh if concrete_mesh is not None else mesh + + def _apply_spec(x: jax.Array, spec: jax.sharding.PartitionSpec) -> jax.Array: + if concrete_mesh is not None and isinstance(getattr(x, "sharding", None), jax.sharding.SingleDeviceSharding): + return jax.device_put(x, jax.sharding.NamedSharding(concrete_mesh, spec)) + return jax.lax.with_sharding_constraint(x, spec) + + def _constrain_outer(x: jax.Array, *axes: tuple[str, ...] | None) -> jax.Array: + if not use_sharding or (b_spec is None and tp_spec is None and length_spec is None): + return x + return _apply_spec(x, jax.sharding.PartitionSpec(*axes)) + + def _constrain(x: jax.Array, *axes: tuple[str, ...] | None) -> jax.Array: + if use_q_seq_shard_map or not use_sharding or (b_spec is None and tp_spec is None): + return x + return _apply_spec(x, jax.sharding.PartitionSpec(*axes)) + + q = _constrain_outer(q, b_spec, length_spec, tp_spec, None) + k = _constrain_outer(k, b_spec, None, tp_spec, None) + v = _constrain_outer(v, b_spec, None, tp_spec, None) + packed_mask = _constrain_outer(packed_mask, b_spec, length_spec, None) + seg_q = _constrain_outer(seg, b_spec, length_spec) + local_pos_q = _constrain_outer(local_pos, b_spec, length_spec) + seg_k = _constrain_outer(seg, b_spec, None) + local_pos_k = _constrain_outer(local_pos, b_spec, None) + + words_per_kv_tile = max(1, (b_kv // c) // 32) + bit_shifts_32 = jnp.arange(32, dtype=jnp.uint32) + + def _q_tile_offset(num_q_tiles: int) -> jax.Array: + if not use_q_seq_shard_map: + return jnp.int32(0) + q_shard_idx = jnp.int32(0) + for ax in length_axes: + q_shard_idx = q_shard_idx * int(mesh.shape[ax]) + jax.lax.axis_index(ax) + return q_shard_idx * jnp.int32(num_q_tiles) + + def _tile_valid_mask( + mask_t: jax.Array, + seg_q_t: jax.Array, + local_q_t: jax.Array, + seg_k_t: jax.Array, + local_k_t: jax.Array, + q_tile_idx: jax.Array, + kv_tile_idx: jax.Array, + ) -> jax.Array: + q_idx = q_tile_idx * b_q + jnp.arange(b_q, dtype=jnp.int32) + k_idx = kv_tile_idx * b_kv + jnp.arange(b_kv, dtype=jnp.int32) + causal = k_idx[None, None, :] <= q_idx[None, :, None] + if fast_unpacked: + w_start = kv_tile_idx * words_per_kv_tile + words = jax.lax.dynamic_slice_in_dim(mask_t, w_start, words_per_kv_tile, axis=-1) + bits = ((words[:, :, :, None] >> bit_shifts_32) & jnp.uint32(1)) != 0 + blk_bool = bits.reshape(mask_t.shape[0], b_q, b_kv // c) + is_selected = jnp.repeat(blk_bool, c, axis=-1) + in_tail = k_idx[None, None, :] >= (((q_idx + 1) // c) * c)[None, :, None] + return (is_selected | in_tail) & causal + + lblk_k = jnp.clip(local_k_t // c, 0, num_blocks - 1) + word_idx = lblk_k >> 5 + bit_shift = (lblk_k & 31).astype(jnp.uint32) + gathered_words = jnp.take_along_axis(mask_t, word_idx[:, None, :], axis=-1) + is_selected = ((gathered_words >> bit_shift[:, None, :]) & jnp.uint32(1)) != 0 + in_tail = local_k_t[:, None, :] >= (((local_q_t + 1) // c) * c)[:, :, None] + same_doc = (seg_q_t[:, :, None] == seg_k_t[:, None, :]) & (seg_q_t[:, :, None] != 0) + return same_doc & causal & (is_selected | in_tail) + + def _fwd_impl( + q_in: jax.Array, + k_in: jax.Array, + v_in: jax.Array, + pmask: jax.Array, + seg_q_in: jax.Array, + lpos_q_in: jax.Array, + seg_k_in: jax.Array, + lpos_k_in: jax.Array, + ) -> tuple[jax.Array, jax.Array]: + b_loc, s_q_loc, h_q_loc, _ = q_in.shape + _, s_kv, h_kv, _ = k_in.shape + g = h_q_loc // h_kv + num_q_tiles = s_q_loc // b_q + num_kv_tiles = s_kv // b_kv + + # Tie loop-invariant index and segment inputs to dynamic activations so outer scan/remat + # partial evaluation (_scan_known_hoisting / _cond_partial_eval) never treats tile indices + # as statically known and never stacks 6D/7D [num_q_tiles, num_kv_tiles, ...] branch residuals. + _unk_zero = jax.lax.optimization_barrier( + jnp.sum(q_in[:1, :1, :1, :1].astype(jnp.int32) * 0) + + jnp.sum(k_in[:1, :1, :1, :1].astype(jnp.int32) * 0) + + jnp.sum(v_in[:1, :1, :1, :1].astype(jnp.int32) * 0) + ) + pmask = pmask + _unk_zero.astype(jnp.uint32) + seg_q_in = seg_q_in + _unk_zero + lpos_q_in = lpos_q_in + _unk_zero + seg_k_in = seg_k_in + _unk_zero + lpos_k_in = lpos_k_in + _unk_zero + q_tile_indices = _q_tile_offset(num_q_tiles) + jnp.arange(num_q_tiles, dtype=jnp.int32) + _unk_zero + kv_tile_indices = jnp.arange(num_kv_tiles, dtype=jnp.int32) + _unk_zero + + q_tiled = _constrain( + q_in.reshape(b_loc, num_q_tiles, b_q, h_kv, g, d).transpose(1, 0, 2, 3, 4, 5), + None, + b_spec, + None, + tp_spec, + None, + None, + ) + k_tiled = _constrain( + k_in.reshape(b_loc, num_kv_tiles, b_kv, h_kv, d).transpose(1, 0, 2, 3, 4), + None, + b_spec, + None, + tp_spec, + None, + ) + v_tiled = _constrain( + v_in.reshape(b_loc, num_kv_tiles, b_kv, h_kv, d).transpose(1, 0, 2, 3, 4), + None, + b_spec, + None, + tp_spec, + None, + ) + mask_tiled = _constrain( + pmask.reshape(b_loc, num_q_tiles, b_q, num_words).transpose(1, 0, 2, 3), + None, + b_spec, + None, + None, + ) + seg_q_tiled = _constrain(seg_q_in.reshape(b_loc, num_q_tiles, b_q).transpose(1, 0, 2), None, b_spec, None) + local_q_tiled = _constrain(lpos_q_in.reshape(b_loc, num_q_tiles, b_q).transpose(1, 0, 2), None, b_spec, None) + seg_k_tiled = _constrain(seg_k_in.reshape(b_loc, num_kv_tiles, b_kv).transpose(1, 0, 2), None, b_spec, None) + local_k_tiled = _constrain(lpos_k_in.reshape(b_loc, num_kv_tiles, b_kv).transpose(1, 0, 2), None, b_spec, None) + + def process_q_tile(carry: None, q_inputs: tuple[jax.Array, ...]) -> tuple[None, tuple[jax.Array, jax.Array]]: + del carry + q_t, mask_t, seg_q_t, local_q_t, q_tile_idx = q_inputs + m_init = _constrain(jnp.full((b_loc, b_q, h_kv, g), -1e30, dtype=jnp.float32), b_spec, None, tp_spec, None) + l_init = _constrain(jnp.zeros((b_loc, b_q, h_kv, g), dtype=jnp.float32), b_spec, None, tp_spec, None) + acc_init = _constrain(jnp.zeros((b_loc, b_q, h_kv, g, d), dtype=jnp.float32), b_spec, None, tp_spec, None, None) + + def process_kv_tile( + kv_carry: tuple[jax.Array, jax.Array, jax.Array], + kv_inputs: tuple[jax.Array, ...], + ) -> tuple[tuple[jax.Array, jax.Array, jax.Array], None]: + m_prev, l_prev, acc_prev = kv_carry + k_t, v_t, seg_k_t, local_k_t, kv_tile_idx = kv_inputs + q_end = (q_tile_idx + 1) * b_q - 1 + k_start = kv_tile_idx * b_kv + + def compute_tile(_: None) -> tuple[jax.Array, jax.Array, jax.Array]: + valid_mask = _tile_valid_mask(mask_t, seg_q_t, local_q_t, seg_k_t, local_k_t, q_tile_idx, kv_tile_idx) + scores = ( + jnp.einsum( + "bqhgd,bkhd->bqhgk", + q_t, + k_t, + preferred_element_type=jnp.float32, + ) + * scale + ) + scores = jnp.where(valid_mask[:, :, None, None, :], scores, -1e30) + m_curr = jnp.max(scores, axis=-1) + m_new = jnp.maximum(m_prev, m_curr) + alpha = jnp.exp(m_prev - m_new) + exp_scores = jnp.exp(jnp.where(valid_mask[:, :, None, None, :], scores - m_new[..., None], -1e30)) + p = jnp.where(valid_mask[:, :, None, None, :], exp_scores, 0.0) + l_new = l_prev * alpha + jnp.sum(p, axis=-1) + acc_new = acc_prev * alpha[..., None] + jnp.einsum( + "bqhgk,bkhd->bqhgd", + p.astype(v_t.dtype), + v_t, + preferred_element_type=jnp.float32, + ) + return (m_new, l_new, acc_new) + + return ( + jax.lax.cond(k_start <= q_end, compute_tile, lambda _: (m_prev, l_prev, acc_prev), operand=None), + None, + ) + + (m_fin, l_fin, acc_fin), _ = jax.lax.scan( + process_kv_tile, + (m_init, l_init, acc_init), + (k_tiled, v_tiled, seg_k_tiled, local_k_tiled, kv_tile_indices), + ) + l_safe = jnp.where(l_fin > 0.0, l_fin, 1.0) + out_t = (acc_fin / l_safe[..., None]).astype(q_in.dtype) + lse_t = jnp.where(l_fin > 0.0, m_fin + jnp.log(l_safe), 0.0) + return None, (out_t, lse_t) + + _, (out_tiled, lse_tiled) = jax.lax.scan( + process_q_tile, + None, + ( + q_tiled, + mask_tiled, + seg_q_tiled, + local_q_tiled, + q_tile_indices, + ), + ) + out = _constrain( + out_tiled.transpose(1, 0, 2, 3, 4, 5).reshape(b_loc, s_q_loc, h_q_loc, d), + b_spec, + None, + tp_spec, + None, + ) + lse = _constrain(lse_tiled.transpose(1, 0, 2, 3, 4), b_spec, None, None, tp_spec, None) + return out, lse + + @jax.custom_vjp + def _kernel( + q_in: jax.Array, + k_in: jax.Array, + v_in: jax.Array, + pmask: jax.Array, + seg_q_in: jax.Array, + lpos_q_in: jax.Array, + seg_k_in: jax.Array, + lpos_k_in: jax.Array, + ) -> jax.Array: + out, _ = _fwd_impl(q_in, k_in, v_in, pmask, seg_q_in, lpos_q_in, seg_k_in, lpos_k_in) + return out + + def _fwd( + q_in: jax.Array, + k_in: jax.Array, + v_in: jax.Array, + pmask: jax.Array, + seg_q_in: jax.Array, + lpos_q_in: jax.Array, + seg_k_in: jax.Array, + lpos_k_in: jax.Array, + ) -> tuple[jax.Array, tuple[jax.Array, ...]]: + out, lse = _fwd_impl(q_in, k_in, v_in, pmask, seg_q_in, lpos_q_in, seg_k_in, lpos_k_in) + return out, (q_in, k_in, v_in, pmask, seg_q_in, lpos_q_in, seg_k_in, lpos_k_in, out, lse) + + def _bwd( + res: tuple[jax.Array, ...], + do: jax.Array, + ) -> tuple[jax.Array, jax.Array, jax.Array, None, None, None, None, None]: + q_in, k_in, v_in, pmask, seg_q_in, lpos_q_in, seg_k_in, lpos_k_in, out, lse = res + b_loc, s_q_loc, h_q_loc, _ = q_in.shape + _, s_kv, h_kv, _ = k_in.shape + g = h_q_loc // h_kv + num_q_tiles = s_q_loc // b_q + num_kv_tiles = s_kv // b_kv + + _unk_zero = jax.lax.optimization_barrier( + jnp.sum(do[:1, :1, :1, :1].astype(jnp.int32) * 0) + + jnp.sum(q_in[:1, :1, :1, :1].astype(jnp.int32) * 0) + + jnp.sum(k_in[:1, :1, :1, :1].astype(jnp.int32) * 0) + + jnp.sum(v_in[:1, :1, :1, :1].astype(jnp.int32) * 0) + ) + pmask = pmask + _unk_zero.astype(jnp.uint32) + seg_q_in = seg_q_in + _unk_zero + lpos_q_in = lpos_q_in + _unk_zero + seg_k_in = seg_k_in + _unk_zero + lpos_k_in = lpos_k_in + _unk_zero + q_tile_indices = _q_tile_offset(num_q_tiles) + jnp.arange(num_q_tiles, dtype=jnp.int32) + _unk_zero + kv_tile_indices = jnp.arange(num_kv_tiles, dtype=jnp.int32) + _unk_zero + + q_tiled = _constrain( + q_in.reshape(b_loc, num_q_tiles, b_q, h_kv, g, d).transpose(1, 0, 2, 3, 4, 5), + None, + b_spec, + None, + tp_spec, + None, + None, + ) + k_tiled = _constrain( + k_in.reshape(b_loc, num_kv_tiles, b_kv, h_kv, d).transpose(1, 0, 2, 3, 4), + None, + b_spec, + None, + tp_spec, + None, + ) + v_tiled = _constrain( + v_in.reshape(b_loc, num_kv_tiles, b_kv, h_kv, d).transpose(1, 0, 2, 3, 4), + None, + b_spec, + None, + tp_spec, + None, + ) + do_tiled = _constrain( + do.reshape(b_loc, num_q_tiles, b_q, h_kv, g, d).transpose(1, 0, 2, 3, 4, 5), + None, + b_spec, + None, + tp_spec, + None, + None, + ) + out_tiled = _constrain( + out.reshape(b_loc, num_q_tiles, b_q, h_kv, g, d).transpose(1, 0, 2, 3, 4, 5), + None, + b_spec, + None, + tp_spec, + None, + None, + ) + lse_tiled = _constrain(lse.transpose(1, 0, 2, 3, 4), None, b_spec, None, tp_spec, None) + mask_tiled = _constrain( + pmask.reshape(b_loc, num_q_tiles, b_q, num_words).transpose(1, 0, 2, 3), + None, + b_spec, + None, + None, + ) + seg_q_tiled = _constrain(seg_q_in.reshape(b_loc, num_q_tiles, b_q).transpose(1, 0, 2), None, b_spec, None) + local_q_tiled = _constrain(lpos_q_in.reshape(b_loc, num_q_tiles, b_q).transpose(1, 0, 2), None, b_spec, None) + seg_k_tiled = _constrain(seg_k_in.reshape(b_loc, num_kv_tiles, b_kv).transpose(1, 0, 2), None, b_spec, None) + local_k_tiled = _constrain(lpos_k_in.reshape(b_loc, num_kv_tiles, b_kv).transpose(1, 0, 2), None, b_spec, None) + + delta_tiled = _constrain( + jnp.sum(out_tiled.astype(jnp.float32) * do_tiled.astype(jnp.float32), axis=-1), + None, + b_spec, + None, + tp_spec, + None, + ) + dk_init = _constrain(jnp.zeros_like(k_tiled, dtype=jnp.float32), None, b_spec, None, tp_spec, None) + dv_init = _constrain(jnp.zeros_like(v_tiled, dtype=jnp.float32), None, b_spec, None, tp_spec, None) + + def bwd_q_tile( + dk_dv_carry: tuple[jax.Array, jax.Array], + q_inputs: tuple[jax.Array, ...], + ) -> tuple[tuple[jax.Array, jax.Array], jax.Array]: + dk_acc, dv_acc = dk_dv_carry + q_t, do_t, lse_t, delta_t, mask_t, seg_q_t, local_q_t, q_tile_idx = q_inputs + dq_init = _constrain(jnp.zeros((b_loc, b_q, h_kv, g, d), dtype=jnp.float32), b_spec, None, tp_spec, None, None) + + def bwd_kv_tile( + dq_carry: jax.Array, + kv_inputs: tuple[jax.Array, ...], + ) -> tuple[jax.Array, tuple[jax.Array, jax.Array]]: + k_t, v_t, dk_t, dv_t, seg_k_t, local_k_t, kv_tile_idx = kv_inputs + q_end = (q_tile_idx + 1) * b_q - 1 + k_start = kv_tile_idx * b_kv + + def compute_bwd(_: None) -> tuple[jax.Array, tuple[jax.Array, jax.Array]]: + valid_mask = _tile_valid_mask(mask_t, seg_q_t, local_q_t, seg_k_t, local_k_t, q_tile_idx, kv_tile_idx) + scores = ( + jnp.einsum( + "bqhgd,bkhd->bqhgk", + q_t, + k_t, + preferred_element_type=jnp.float32, + ) + * scale + ) + scores = jnp.where(valid_mask[:, :, None, None, :], scores, -1e30) + exp_scores = jnp.exp(jnp.where(valid_mask[:, :, None, None, :], scores - lse_t[..., None], -1e30)) + p = jnp.where(valid_mask[:, :, None, None, :], exp_scores, 0.0) + dv_new = dv_t + jnp.einsum( + "bqhgk,bqhgd->bkhd", + p.astype(do_t.dtype), + do_t, + preferred_element_type=jnp.float32, + ) + dp = jnp.einsum( + "bqhgd,bkhd->bqhgk", + do_t, + v_t, + preferred_element_type=jnp.float32, + ) + ds = p * (dp - delta_t[..., None]) * scale + ds_cast = ds.astype(q_t.dtype) + dq_new = dq_carry + jnp.einsum( + "bqhgk,bkhd->bqhgd", + ds_cast, + k_t, + preferred_element_type=jnp.float32, + ) + dk_new = dk_t + jnp.einsum( + "bqhgk,bqhgd->bkhd", + ds_cast, + q_t, + preferred_element_type=jnp.float32, + ) + return dq_new, (dk_new, dv_new) + + return jax.lax.cond( + k_start <= q_end, + compute_bwd, + lambda _: (dq_carry, (dk_t, dv_t)), + operand=None, + ) + + dq_t, (dk_next, dv_next) = jax.lax.scan( + bwd_kv_tile, + dq_init, + ( + k_tiled, + v_tiled, + dk_acc, + dv_acc, + seg_k_tiled, + local_k_tiled, + kv_tile_indices, + ), + ) + return (dk_next, dv_next), dq_t.astype(q_in.dtype) + + (dk_fin, dv_fin), dq_tiled = jax.lax.scan( + bwd_q_tile, + (dk_init, dv_init), + ( + q_tiled, + do_tiled, + lse_tiled, + delta_tiled, + mask_tiled, + seg_q_tiled, + local_q_tiled, + q_tile_indices, + ), + ) + dq = _constrain( + dq_tiled.transpose(1, 0, 2, 3, 4, 5).reshape(b_loc, s_q_loc, h_q_loc, d), + b_spec, + None, + tp_spec, + None, + ) + dk = _constrain( + dk_fin.transpose(1, 0, 2, 3, 4).reshape(b_loc, s_kv, h_kv, d).astype(k_in.dtype), + b_spec, + None, + tp_spec, + None, + ) + dv = _constrain( + dv_fin.transpose(1, 0, 2, 3, 4).reshape(b_loc, s_kv, h_kv, d).astype(v_in.dtype), + b_spec, + None, + tp_spec, + None, + ) + return dq, dk, dv, None, None, None, None, None + + _kernel.defvjp(_fwd, _bwd) + if use_q_seq_shard_map: + pspec = jax.sharding.PartitionSpec + out_pad = jax.shard_map( + _kernel, + mesh=shmap_mesh, + in_specs=( + pspec(b_spec, length_spec, tp_spec, None), + pspec(b_spec, None, tp_spec, None), + pspec(b_spec, None, tp_spec, None), + pspec(b_spec, length_spec, None), + pspec(b_spec, length_spec), + pspec(b_spec, length_spec), + pspec(b_spec, None), + pspec(b_spec, None), + ), + out_specs=pspec(b_spec, length_spec, tp_spec, None), + check_vma=False, + )(q, k, v, packed_mask, seg_q, local_pos_q, seg_k, local_pos_k) + else: + out_pad = _kernel(q, k, v, packed_mask, seg_q, local_pos_q, seg_k, local_pos_k) + if pad_len > 0: + out_pad = out_pad[:, :s, :, :] + return out_pad.reshape(b, s, h_q * d) + def __call__( self, hidden_states: jax.Array, @@ -912,10 +1541,6 @@ def __call__( Attention output tensor, or tuple of (output, selected_block_indices) if return_selected_blocks is True. """ b, s, _ = hidden_states.shape - if self.attention_kernel == "flash" and s > self.indexer.token_budget: - raise ValueError( - f"qsa_attention_kernel='flash' requires seq_len <= token_budget ({self.indexer.token_budget}), got seq_len={s}." - ) if position_ids is None: position_ids = jnp.broadcast_to(jnp.arange(s)[None, :], (b, s)) @@ -945,9 +1570,20 @@ def __call__( query_states, key_states = res if self.attention_kernel == "flash": - attn_output = self._flash_attention( - query_states, key_states, value_states, attention_mask=attention_mask, segment_ids=segment_ids - ) + if s <= self.indexer.token_budget: + attn_output = self._flash_attention( + query_states, key_states, value_states, attention_mask=attention_mask, segment_ids=segment_ids + ) + else: + attn_output = self.flash_block_sparse_attention( + query_states, + key_states, + value_states, + selected_block_indices, + attention_mask=attention_mask, + segment_ids=segment_ids, + sharding=self.sharding, + ) else: assert selected_token_mask is not None if (rep := self.num_query_heads // self.num_kv_heads) > 1: @@ -1656,6 +2292,7 @@ def __init__( compress_ratio=int(c.index_select_block_size), indexer_mask_mode=str(c.qsa_indexer_mask), attention_kernel=str(c.qsa_attention_kernel), + sharding=sharding or DefaultSharding(), ) self.mlp_hyper_connection = Qwen3_8GatedResidual(use_combine=True, **hc_kw) sharding_obj = sharding or DefaultSharding() diff --git a/tests/unit/m3/qwen3_8_flash_next/modeling_qwen3_8_flash_next_test.py b/tests/unit/m3/qwen3_8_flash_next/modeling_qwen3_8_flash_next_test.py index 21d1c17e90..02d37eb969 100644 --- a/tests/unit/m3/qwen3_8_flash_next/modeling_qwen3_8_flash_next_test.py +++ b/tests/unit/m3/qwen3_8_flash_next/modeling_qwen3_8_flash_next_test.py @@ -25,6 +25,7 @@ * MTP draft block and integrated loss """ +import functools import types from typing import Any import unittest @@ -1015,15 +1016,10 @@ def test_indexer_reshape_packed_multi_doc_parity(self): np.testing.assert_array_equal(np.array(b_r), np.array(b_g)) def test_flash_attention_kernel_parity_and_validation(self): - """Verifies attention_kernel='flash' matches 'dense' when s <= token_budget and rejects s > token_budget.""" + """Verifies attention_kernel='flash' matches 'dense' for both s <= token_budget and s > token_budget.""" with self.assertRaises(ValueError): modeling_qwen3_8_flash_next.Qwen3_8SparseAttention(attention_kernel="invalid", rngs=nnx.Rngs(0)) - args_16 = (64, 4, 2, 32, 0.25, 10000.0, (2, 1, 1), 2, 1, 32, 16, 4, 1e-6, jnp.float32, jnp.float32) - flash_16 = modeling_qwen3_8_flash_next.Qwen3_8SparseAttention(*args_16, attention_kernel="flash", rngs=nnx.Rngs(0)) - with self.assertRaises(ValueError): - flash_16(jnp.zeros((1, 32, 64), dtype=jnp.float32)) - b, s, emb_dim = 2, 32, 64 args_64 = (emb_dim, 4, 2, 32, 0.25, 10000.0, (2, 1, 1), 2, 1, 32, 64, 4, 1e-6, jnp.float32, jnp.float32) dense_attn = modeling_qwen3_8_flash_next.Qwen3_8SparseAttention(*args_64, attention_kernel="dense", rngs=nnx.Rngs(11)) @@ -1079,6 +1075,525 @@ def loss_fn( _assert_allclose_to_scale(dx_f, dx_d) _assert_allclose_to_scale(grads_f.q_proj.kernel[...], grads_d.q_proj.kernel[...]) + # s > token_budget: dispatches to Variant D (_flash_block_sparse_attention with 32-bit packed block bitmap). + s_long, budget_short = 48, 16 + pos_long = jnp.tile(jnp.arange(s_long, dtype=jnp.int32), (b, 1)) + for dt in (jnp.float32, jnp.bfloat16): + args_long = (emb_dim, 4, 2, 32, 0.25, 10000.0, (2, 1, 1), 2, 1, 32, budget_short, 4, 1e-6, dt, jnp.float32) + dense_long = modeling_qwen3_8_flash_next.Qwen3_8SparseAttention( + *args_long, attention_kernel="dense", rngs=nnx.Rngs(23) + ) + flash_long = modeling_qwen3_8_flash_next.Qwen3_8SparseAttention( + *args_long, indexer_mask_mode="reshape", attention_kernel="flash", rngs=nnx.Rngs(23) + ) + nnx.update(flash_long, nnx.state(dense_long)) + + x_long = jnp.array(rng.standard_normal((b, s_long, emb_dim), dtype=np.float32), dtype=dt) + mask_long = jnp.ones((b, s_long), dtype=jnp.int32).at[0, 40:].set(0).at[1, 44:].set(0) + seg_long = jnp.ones((b, s_long), dtype=jnp.int32).at[0, 18:36].set(2).at[0, 36:].set(0).at[1, 22:].set(2) + + for case_name, mask_in, seg_in in ( + ("unpacked", None, None), + ("mask_2d", mask_long, None), + ("packed_seg", None, seg_long), + ("both_mask_and_seg", mask_long, seg_long), + ): + with self.subTest(mode="block_sparse", dtype=str(dt), case_name=case_name): + valid_l = jnp.ones((b, s_long), dtype=bool) + if mask_in is not None: + valid_l &= mask_in != 0 + if seg_in is not None: + valid_l &= seg_in != 0 + out_d, blk_d = dense_long( + x_long, position_ids=pos_long, attention_mask=mask_in, segment_ids=seg_in, return_selected_blocks=True + ) + out_f, blk_f = flash_long( + x_long, position_ids=pos_long, attention_mask=mask_in, segment_ids=seg_in, return_selected_blocks=True + ) + np.testing.assert_array_equal(np.array(blk_f), np.array(blk_d)) + self.assertTrue(np.all(np.isfinite(np.array(out_f, dtype=np.float32)))) + if dt == jnp.float32: + np.testing.assert_allclose( + np.array(out_f)[np.array(valid_l)], np.array(out_d)[np.array(valid_l)], rtol=1e-4, atol=1e-4 + ) + else: + _assert_allclose_to_scale( + jnp.where(valid_l[:, :, None], out_f, 0.0), + jnp.where(valid_l[:, :, None], out_d, 0.0), + rtol=2e-2, + atol_scale=2e-2, + ) + + def loss_long( + mod: modeling_qwen3_8_flash_next.Qwen3_8SparseAttention, + inp: jax.Array, + mask_in: jax.Array | None = mask_in, + seg_in: jax.Array | None = seg_in, + valid_l: jax.Array = valid_l, + ) -> jax.Array: + out = mod(inp, position_ids=pos_long, attention_mask=mask_in, segment_ids=seg_in) + assert isinstance(out, jax.Array) + w_out = jnp.arange(1, emb_dim + 1, dtype=jnp.float32) / float(emb_dim) + return jnp.sum(jnp.where(valid_l[:, :, None], out.astype(jnp.float32), 0.0) * w_out) + + _, (grads_d, dx_d) = nnx.value_and_grad(loss_long, argnums=(0, 1))(dense_long, x_long) + _, (grads_f, dx_f) = nnx.value_and_grad(loss_long, argnums=(0, 1))(flash_long, x_long) + self.assertTrue(np.all(np.isfinite(np.array(dx_f, dtype=np.float32)))) + tol = 1e-4 if dt == jnp.float32 else 2.5e-2 + _assert_allclose_to_scale(dx_f, dx_d, rtol=tol, atol_scale=tol) + _assert_allclose_to_scale(grads_f.q_proj.kernel[...], grads_d.q_proj.kernel[...], rtol=tol, atol_scale=tol) + _assert_allclose_to_scale(grads_f.k_proj.kernel[...], grads_d.k_proj.kernel[...], rtol=tol, atol_scale=tol) + _assert_allclose_to_scale(grads_f.v_proj.kernel[...], grads_d.v_proj.kernel[...], rtol=tol, atol_scale=tol) + + def test_qsa_flash_block_sparse_tail_packing_sharding_and_negative_controls(self): + """Verifies 32-bit bitmap packing, tail blocks, packed==unpacked isolation, 4-device sharding, and negative controls.""" + rng = np.random.default_rng(41) + + # 1. Direct verification of carry-free 32-bit bitmap packing with duplicate indices and -1 slots across 3 words. + b, s, topk, num_blocks, num_words = 2, 16, 12, 80, 3 + raw_idx = rng.integers(-2, num_blocks, size=(b, s, topk), dtype=np.int32) + raw_idx[:, :, -2:] = raw_idx[:, :, :2] # Inject duplicate block indices per query row. + sel_idx = jnp.array(raw_idx, dtype=jnp.int32) + packed = modeling_qwen3_8_flash_next.pack_qsa_block_mask_32bit(sel_idx, num_words) + self.assertEqual(packed.shape, (b, s, num_words)) + self.assertEqual(packed.dtype, jnp.uint32) + bits = ((packed[:, :, :, None] >> jnp.arange(32, dtype=jnp.uint32)) & jnp.uint32(1)) != 0 + unpacked_bool = bits.reshape(b, s, num_words * 32)[:, :, :num_blocks] + m_local = jnp.arange(num_blocks, dtype=jnp.int32)[None, None, :] + ref_bool = jnp.any((sel_idx[:, :, :, None] == m_local) & (sel_idx[:, :, :, None] >= 0), axis=2) + np.testing.assert_array_equal(np.array(unpacked_bool), np.array(ref_bool)) + + # 2. Non-multiple-of-c tail block (s=70, c=4, budget=20) and fast_unpacked multi-word slice (s=256, b_kv=128). + emb_dim = 64 + for s_case, budget_case, bq_case, bkv_case in ((70, 20, 20, 20), (256, 64, 64, 128)): + args_c = (emb_dim, 4, 2, 32, 0.25, 10000.0, (2, 1, 1), 2, 1, 32, budget_case, 4, 1e-6, jnp.float32, jnp.float32) + dense_c = modeling_qwen3_8_flash_next.Qwen3_8SparseAttention(*args_c, attention_kernel="dense", rngs=nnx.Rngs(31)) + flash_c = modeling_qwen3_8_flash_next.Qwen3_8SparseAttention(*args_c, attention_kernel="flash", rngs=nnx.Rngs(31)) + nnx.update(flash_c, nnx.state(dense_c)) + x_c = jnp.array(rng.standard_normal((b, s_case, emb_dim), dtype=np.float32)) + pos_c = jnp.tile(jnp.arange(s_case, dtype=jnp.int32), (b, 1)) + out_d, blk_d = dense_c(x_c, position_ids=pos_c, return_selected_blocks=True) + out_f, blk_f = flash_c(x_c, position_ids=pos_c, return_selected_blocks=True) + np.testing.assert_array_equal(np.array(blk_f), np.array(blk_d)) + np.testing.assert_allclose(np.array(out_f), np.array(out_d), rtol=1e-4, atol=1e-4) + + # Negative control 1: corrupting selected_block_indices must change _flash_block_sparse_attention output by > 0.05. + cos_c, sin_c = flash_c.rotary_emb(x_c, pos_c) + q_gate = flash_c.q_proj(x_c).reshape(b, s_case, 4, 64) + q_st = flash_c.q_norm(q_gate[..., :32]) + k_st = flash_c.k_norm(flash_c.k_proj(x_c).reshape(b, s_case, 2, 32)) + v_st = flash_c.v_proj(x_c).reshape(b, s_case, 2, 32) + q_st, k_st = modeling_qwen3_8_flash_next.apply_rotary_pos_emb(q_st, k_st, cos=cos_c, sin=sin_c, unsqueeze_dim=2) + ctx_good = flash_c.flash_block_sparse_attention(q_st, k_st, v_st, blk_f, block_q=bq_case, block_kv=bkv_case) + blk_corrupt = jnp.full_like(blk_f, -1) + ctx_corrupt = flash_c.flash_block_sparse_attention( + q_st, k_st, v_st, blk_corrupt, block_q=bq_case, block_kv=bkv_case + ) + self.assertGreater(float(jnp.max(jnp.abs(ctx_good - ctx_corrupt))), 0.05) + + # 3. Packed sequence isolation: packed 2-doc output & gradients == concatenation of per-doc unpacked runs (s > budget). + len1, len2 = 28, 24 + args_p = (emb_dim, 4, 2, 32, 0.25, 10000.0, (2, 1, 1), 2, 1, 32, 16, 4, 1e-6, jnp.float32, jnp.float32) + flash_p = modeling_qwen3_8_flash_next.Qwen3_8SparseAttention(*args_p, attention_kernel="flash", rngs=nnx.Rngs(53)) + x1 = jnp.array(rng.standard_normal((1, len1, emb_dim), dtype=np.float32)) + x2 = jnp.array(rng.standard_normal((1, len2, emb_dim), dtype=np.float32)) + x_pack = jnp.concatenate([x1, x2], axis=1) + pos1 = jnp.arange(len1, dtype=jnp.int32)[None, :] + pos2 = jnp.arange(len2, dtype=jnp.int32)[None, :] + pos_pack = jnp.concatenate([pos1, pos2], axis=1) + seg_pack = jnp.concatenate([jnp.ones((1, len1), dtype=jnp.int32), jnp.full((1, len2), 2, dtype=jnp.int32)], axis=1) + + w_probe = jnp.arange(1, emb_dim + 1, dtype=jnp.float32) / float(emb_dim) + + def packed_loss( + mod: modeling_qwen3_8_flash_next.Qwen3_8SparseAttention, inp: jax.Array + ) -> tuple[jax.Array, jax.Array]: + out = mod(inp, position_ids=pos_pack, segment_ids=seg_pack) + assert isinstance(out, jax.Array) + return jnp.sum(out * w_probe), out + + def unpacked_loss( + mod: modeling_qwen3_8_flash_next.Qwen3_8SparseAttention, inp1: jax.Array, inp2: jax.Array + ) -> tuple[jax.Array, jax.Array]: + out1 = mod(inp1, position_ids=pos1) + out2 = mod(inp2, position_ids=pos2) + assert isinstance(out1, jax.Array) and isinstance(out2, jax.Array) + out_cat = jnp.concatenate([out1, out2], axis=1) + return jnp.sum(out_cat * w_probe), out_cat + + (_, out_pack), (g_pack, dx_pack) = nnx.value_and_grad(packed_loss, argnums=(0, 1), has_aux=True)(flash_p, x_pack) + (_, out_cat), (g_cat, dx1, dx2) = nnx.value_and_grad(unpacked_loss, argnums=(0, 1, 2), has_aux=True)(flash_p, x1, x2) + dx_cat = jnp.concatenate([dx1, dx2], axis=1) + np.testing.assert_allclose(np.array(out_pack), np.array(out_cat), rtol=1e-4, atol=1e-4) + np.testing.assert_allclose(np.array(dx_pack), np.array(dx_cat), rtol=1e-4, atol=1e-4) + _assert_allclose_to_scale(g_pack.q_proj.kernel[...], g_cat.q_proj.kernel[...]) + + # Negative control 2: omitting segment_ids on the packed sequence leaks doc-1 keys into doc-2 and diverges > 0.05. + out_leaked = flash_p(x_pack, position_ids=pos_pack, segment_ids=None) + assert isinstance(out_leaked, jax.Array) + self.assertGreater(float(jnp.max(jnp.abs(out_leaked[:, len1:, :] - out_cat[:, len1:, :]))), 0.05) + + # 4. Qwen3_8QSAIndexer(return_token_mask=False) bit-exact selected_block_indices parity vs return_token_mask=True + # across unpacked, 2D-padded, and multi-doc packed segment_ids, with zero O(B * S^2) bool[B,S,S] allocation. + b_ix, s_ix = 2, 48 + x_ix = jnp.array(rng.standard_normal((b_ix, s_ix, emb_dim), dtype=np.float32)) + pos_ix = jnp.tile(jnp.arange(s_ix, dtype=jnp.int32), (b_ix, 1)) + cos_ix, sin_ix = flash_p.rotary_emb(x_ix, pos_ix) + mask_ix = jnp.ones((b_ix, s_ix), dtype=jnp.int32).at[0, 38:].set(0).at[1, :6].set(0) + seg_ix = jnp.ones((b_ix, s_ix), dtype=jnp.int32).at[:, 20:36].set(2).at[:, 36:].set(3) + for m_in, s_in in ((None, None), (mask_ix, None), (None, seg_ix), (mask_ix, seg_ix)): + tmask_t, blk_t = flash_p.indexer( + x_ix, (cos_ix, sin_ix), attention_mask=m_in, segment_ids=s_in, return_token_mask=True + ) + tmask_f, blk_f = flash_p.indexer( + x_ix, (cos_ix, sin_ix), attention_mask=m_in, segment_ids=s_in, return_token_mask=False + ) + self.assertIsNotNone(tmask_t) + self.assertIsNone(tmask_f) + np.testing.assert_array_equal(np.array(blk_f), np.array(blk_t)) + ix_jaxpr = str( + jax.make_jaxpr( + lambda x_i, m_i, s_i: flash_p.indexer( + x_i, (cos_ix, sin_ix), attention_mask=m_i, segment_ids=s_i, return_token_mask=False + )[1] + )(x_ix, mask_ix, seg_ix) + ) + self.assertNotIn("bool[2,48,48]", ix_jaxpr) + + # 5. Outer scan + custom remat policy (save_only_these_names("query_proj")) must NOT leak 6D/7D stacked + # [num_q_tiles, num_kv_tiles, ...] branch residuals or dense [B, H, S, S] attention tensors. + b_rm, s_rm = 2, 48 + pos_rm = jnp.tile(jnp.arange(s_rm, dtype=jnp.int32), (b_rm, 1)) + seg_rm = jnp.ones((b_rm, s_rm), dtype=jnp.int32).at[:, 24:].set(2) + args_rm = (emb_dim, 4, 2, 32, 0.25, 10000.0, (2, 1, 1), 2, 1, 32, 16, 4, 1e-6, jnp.float32, jnp.float32) + flash_rm = modeling_qwen3_8_flash_next.Qwen3_8SparseAttention(*args_rm, attention_kernel="flash", rngs=nnx.Rngs(73)) + gd_rm, params_rm, rest_rm = nnx.split(flash_rm, nnx.Param, ...) + stacked_params = jax.tree.map(lambda p: jnp.stack([p, p], axis=0), params_rm) + x_rm = jnp.array(rng.standard_normal((b_rm, s_rm, emb_dim), dtype=np.float32)) + + for pol in ( + jax.checkpoint_policies.save_only_these_names("query_proj"), + jax.checkpoint_policies.nothing_saveable, + jax.checkpoint_policies.everything_saveable, + ): + + def scanned_loss( + h_in: jax.Array, + sp_in: Any, + pol_fn: Any = pol, + ) -> jax.Array: + def body_fn(carry: jax.Array, p_slice: Any) -> tuple[jax.Array, None]: + @functools.partial(jax.checkpoint, policy=pol_fn, prevent_cse=True) + def _step(c_in: jax.Array, ps: Any) -> jax.Array: + mod = nnx.merge(gd_rm, ps, rest_rm) + out = mod(c_in, position_ids=pos_rm, segment_ids=seg_rm) + assert isinstance(out, jax.Array) + return c_in + out + + return _step(carry, p_slice), None + + h_out, _ = jax.lax.scan(body_fn, h_in, sp_in) + return jnp.sum(h_out) + + jaxpr_str = str(jax.make_jaxpr(jax.grad(scanned_loss, argnums=(0, 1)))(x_rm, stacked_params)) + # With s_rm=48, b_q=b_kv=16 -> num_q_tiles=num_kv_tiles=3 (or 12 -> 4). Neither 7D [T, T, B, b_q, h_kv, g, b_kv] + # nor 6D [T, B, b_q, h_kv, g, b_kv] nor dense [B, H, S, S] may appear in the backward jaxpr. + for forbidden in ( + "f32[3,3,", + "f32[4,4,", + "f32[3,2,12,2,2,12]", + "f32[4,2,16,2,2,16]", + "f32[2,4,48,48]", + "bool[2,48,48]", + ): + self.assertNotIn(forbidden, jaxpr_str) + + def test_qsa_flash_block_sparse_4device_sharded_parity(self): + """Verifies 4-device CPU sharded parity (data=2, tensor=2) and TP head sharding for s > token_budget.""" + if len(jax.devices()) < 4: + self.skipTest("Requires >= 4 JAX devices") + rng = np.random.default_rng(61) + emb_dim = 64 + w_probe = jnp.arange(1, emb_dim + 1, dtype=jnp.float32) / float(emb_dim) + mesh = jax.sharding.Mesh(np.array(jax.devices()[:4]).reshape(2, 2), ("data", "tensor")) + P = jax.sharding.PartitionSpec + + def ns(spec: jax.sharding.PartitionSpec) -> jax.sharding.NamedSharding: + return jax.sharding.NamedSharding(mesh, spec) + + b_sh, s_sh = 2, 48 + args_sh = (emb_dim, 4, 2, 32, 0.25, 10000.0, (2, 1, 1), 2, 1, 32, 16, 4, 1e-6, jnp.float32, jnp.float32) + dense_sh = modeling_qwen3_8_flash_next.Qwen3_8SparseAttention(*args_sh, attention_kernel="dense", rngs=nnx.Rngs(61)) + flash_sh = modeling_qwen3_8_flash_next.Qwen3_8SparseAttention(*args_sh, attention_kernel="flash", rngs=nnx.Rngs(61)) + nnx.update(flash_sh, nnx.state(dense_sh)) + x_sh = jax.device_put( + jnp.array(rng.standard_normal((b_sh, s_sh, emb_dim), dtype=np.float32)), ns(P("data", None, None)) + ) + pos_sh = jax.device_put(jnp.tile(jnp.arange(s_sh, dtype=jnp.int32), (b_sh, 1)), ns(P("data", None))) + seg_sh = jax.device_put(jnp.ones((b_sh, s_sh), dtype=jnp.int32).at[:, 24:].set(2), ns(P("data", None))) + with jax.set_mesh(mesh): + out_sh_d = dense_sh(x_sh, position_ids=pos_sh, segment_ids=seg_sh) + out_sh_f = flash_sh(x_sh, position_ids=pos_sh, segment_ids=seg_sh) + np.testing.assert_allclose(np.array(out_sh_f), np.array(out_sh_d), rtol=1e-4, atol=1e-4) + + def sh_loss( + mod: modeling_qwen3_8_flash_next.Qwen3_8SparseAttention, + inp: jax.Array, + ) -> jax.Array: + out = mod(inp, position_ids=pos_sh, segment_ids=seg_sh) + assert isinstance(out, jax.Array) + return jnp.sum(out * w_probe) + + sh_grad_fn = nnx.jit(nnx.value_and_grad(sh_loss, argnums=(0, 1))) + _, (g_sh_d, dx_sh_d) = sh_grad_fn(dense_sh, x_sh) + _, (g_sh_f, dx_sh_f) = sh_grad_fn(flash_sh, x_sh) + np.testing.assert_allclose(np.array(dx_sh_f), np.array(dx_sh_d), rtol=1e-4, atol=1e-4) + _assert_allclose_to_scale(g_sh_f.q_proj.kernel[...], g_sh_d.q_proj.kernel[...]) + + # Verify TP head sharding (h_kv=2 -> 1 per TP rank on tensor=2) inside flash_block_sparse_attention HLO. + q_sh = jax.device_put(jnp.ones((b_sh, s_sh, 4, 32), dtype=jnp.float32), ns(P("data", None, "tensor", None))) + k_sh = jax.device_put(jnp.ones((b_sh, s_sh, 2, 32), dtype=jnp.float32), ns(P("data", None, "tensor", None))) + v_sh = jax.device_put(jnp.ones((b_sh, s_sh, 2, 32), dtype=jnp.float32), ns(P("data", None, "tensor", None))) + blk_sh = jax.device_put(jnp.zeros((b_sh, s_sh, 4), dtype=jnp.int32), ns(P("data", None, None))) + hlo_tp = ( + jax.jit( + lambda q_i, k_i, v_i, b_i: jnp.sum( + jax.grad( + lambda qq, kk, vv: jnp.sum( + flash_sh.flash_block_sparse_attention(qq, kk, vv, b_i, block_q=16, block_kv=16) + ), + argnums=(0, 1, 2), + )(q_i, k_i, v_i)[0] + ) + ) + .lower(q_sh, k_sh, v_sh, blk_sh) + .compile() + .as_text() + ) + self.assertIn("f32[1,16,1,2,16]", hlo_tp) + self.assertNotIn("f32[1,16,2,2,16]", hlo_tp) + + def test_qsa_flash_block_sparse_sequence_sharding_and_custom_strategy(self): + """Verifies flash_block_sparse_attention preserves length_axes sharding and respects custom Sharding.""" + if len(jax.devices()) < 4: + self.skipTest("Requires 4 devices.") + rng = np.random.default_rng(1309) + b, s, h_q, d = 2, 64, 4, 32 + q_np = rng.standard_normal((b, s, h_q, d), dtype=np.float32) + k_np = rng.standard_normal((b, s, 2, d), dtype=np.float32) + v_np = rng.standard_normal((b, s, 2, d), dtype=np.float32) + blk_np = rng.integers(0, s // 4, size=(b, s, 4), dtype=np.int32) + seg_np = np.ones((b, s), dtype=np.int32) + seg_np[:, 40:] = 2 + w_out = jnp.array(rng.standard_normal((b, s, h_q * d), dtype=np.float32)) + + mod_default = modeling_qwen3_8_flash_next.Qwen3_8SparseAttention( + emb_dim=64, + num_query_heads=h_q, + num_kv_heads=2, + head_dim=d, + mrope_section=(2, 1, 1), + index_n_heads=2, + index_kv_heads=1, + index_head_dim=16, + token_budget=16, + compress_ratio=4, + dtype=jnp.float32, + weight_dtype=jnp.float32, + attention_kernel="flash", + rngs=nnx.Rngs(params=0), + ) + + def unsharded_loss(q_i, k_i, v_i, seg_i): + out = mod_default.flash_block_sparse_attention( + q_i, k_i, v_i, jnp.asarray(blk_np), segment_ids=seg_i, block_q=16, block_kv=16 + ) + return jnp.sum(out * w_out) + + q_ref, k_ref, v_ref = jnp.asarray(q_np), jnp.asarray(k_np), jnp.asarray(v_np) + seg_ref = jnp.asarray(seg_np) + loss_ref, (dq_ref, dk_ref, dv_ref) = jax.jit(jax.value_and_grad(unsharded_loss, argnums=(0, 1, 2)))( + q_ref, k_ref, v_ref, seg_ref + ) + + # 1. Context + Tensor (2, 2) mesh: query & packed bitmap stay sharded on `context` (0 query all-gathers) + # and also handles GQA head repetition when head_shards (2) > num_kv_heads (1). + P = jax.sharding.PartitionSpec + mesh_ct = jax.sharding.Mesh(np.array(jax.devices()[:4]).reshape(2, 2), ("context", "tensor")) + + def ns_ct(spec): + return jax.sharding.NamedSharding(mesh_ct, spec) + + with jax.set_mesh(mesh_ct): + q_ct = jax.device_put(q_ref, ns_ct(P(None, "context", "tensor", None))) + k_ct = jax.device_put(k_ref, ns_ct(P(None, None, "tensor", None))) + v_ct = jax.device_put(v_ref, ns_ct(P(None, None, "tensor", None))) + blk_ct = jax.device_put(jnp.asarray(blk_np), ns_ct(P(None, "context", None))) + seg_ct = jax.device_put(seg_ref, ns_ct(P(None, "context"))) + + w_ct = jax.device_put(w_out, ns_ct(P(None, "context", None))) + loss_ct, (dq_ct, dk_ct, dv_ct) = jax.jit( + jax.value_and_grad( + lambda qq, kk, vv: jnp.sum( + mod_default.flash_block_sparse_attention( + qq, kk, vv, blk_ct, segment_ids=seg_ct, block_q=16, block_kv=16 + ) + * w_ct + ), + argnums=(0, 1, 2), + ) + )(q_ct, k_ct, v_ct) + np.testing.assert_allclose(float(loss_ct), float(loss_ref), rtol=1e-4, atol=1e-4) + np.testing.assert_allclose(np.asarray(dq_ct), np.asarray(dq_ref), rtol=1e-4, atol=1e-4) + np.testing.assert_allclose(np.asarray(dk_ct), np.asarray(dk_ref), rtol=1e-4, atol=1e-4) + np.testing.assert_allclose(np.asarray(dv_ct), np.asarray(dv_ref), rtol=1e-4, atol=1e-4) + + # Unpacked forward HLO with pre-replicated KV has zero all-gathers (positive control: forcing None on Q emits one). + fwd_hlo = ( + jax.jit( + lambda qq, kk, vv, bb: mod_default.flash_block_sparse_attention(qq, kk, vv, bb, block_q=16, block_kv=16) + ) + .lower(q_ct, k_ct, v_ct, blk_ct) + .compile() + .as_text() + ) + pos_hlo = ( + jax.jit(lambda qq: jnp.sum(jax.lax.with_sharding_constraint(qq, P(None, None, "tensor", None)))) + .lower(q_ct) + .compile() + .as_text() + ) + self.assertIn("all-gather(", pos_hlo) + self.assertNotIn("all-gather(", fwd_hlo) + # Each context shard only holds 2 local Q tiles (32 tokens = 2 * 16), never all 4 Q tiles. + self.assertIn("f32[2,2,16,1,2,32]", fwd_hlo) + self.assertNotIn("f32[4,2,16,1,2,32]", fwd_hlo) + + # Eager call on SingleDeviceSharding inputs inside `with jax.set_mesh` also succeeds cleanly. + out_eager = mod_default.flash_block_sparse_attention( + q_ref, k_ref, v_ref, jnp.asarray(blk_np), block_q=16, block_kv=16 + ) + self.assertEqual(out_eager.shape, (b, s, h_q * d)) + + # GQA head repetition when head_shards (2) > num_kv_heads (1), compared against unsharded reference. + k1_ref, v1_ref = k_ref[:, :, :1, :], v_ref[:, :, :1, :] + l1_ref, (dq1_ref, dk1_ref, dv1_ref) = jax.jit( + jax.value_and_grad( + lambda qq, kk, vv: jnp.sum( + mod_default.flash_block_sparse_attention(qq, kk, vv, jnp.asarray(blk_np), block_q=16, block_kv=16) * w_out + ), + argnums=(0, 1, 2), + ) + )(q_ref, k1_ref, v1_ref) + with jax.set_mesh(mesh_ct): + k1_ct = jax.device_put(k1_ref, ns_ct(P(None, None, None, None))) + v1_ct = jax.device_put(v1_ref, ns_ct(P(None, None, None, None))) + l1_ct, (dq1_ct, dk1_ct, dv1_ct) = jax.jit( + jax.value_and_grad( + lambda qq, kk, vv: jnp.sum( + mod_default.flash_block_sparse_attention(qq, kk, vv, blk_ct, block_q=16, block_kv=16) * w_ct + ), + argnums=(0, 1, 2), + ) + )(q_ct, k1_ct, v1_ct) + np.testing.assert_allclose(float(l1_ct), float(l1_ref), rtol=1e-4, atol=1e-4) + np.testing.assert_allclose(np.asarray(dq1_ct), np.asarray(dq1_ref), rtol=1e-4, atol=1e-4) + np.testing.assert_allclose(np.asarray(dk1_ct), np.asarray(dk1_ref), rtol=1e-4, atol=1e-4) + np.testing.assert_allclose(np.asarray(dv1_ct), np.asarray(dv1_ref), rtol=1e-4, atol=1e-4) + + # 2. Custom EP=CP Sharding (`batch -> fsdp`, `length -> expert`) + `tensor_sequence` head sharding. + class _EpCpSharding(modeling_qwen3_8_flash_next.DefaultSharding): + """Custom sharding mapping activation batch off `expert` and length onto `expert`.""" + + def map_axis(self, axis, tensor_name, tensor_type): + if tensor_type == modeling_qwen3_8_flash_next.TensorType.ACTIVATION: + if axis == "batch": + return ("data", "fsdp", "fsdp_transpose") + if axis == "length": + return ("context", "expert") + return super().map_axis(axis, tensor_name, tensor_type) + + ep_cp = _EpCpSharding() + mod_epcp = modeling_qwen3_8_flash_next.Qwen3_8SparseAttention( + emb_dim=64, + num_query_heads=h_q, + num_kv_heads=2, + head_dim=d, + mrope_section=(2, 1, 1), + index_n_heads=2, + index_kv_heads=1, + index_head_dim=16, + token_budget=16, + compress_ratio=4, + dtype=jnp.float32, + weight_dtype=jnp.float32, + sharding=ep_cp, + attention_kernel="flash", + rngs=nnx.Rngs(params=0), + ) + mesh_ep = jax.sharding.Mesh(np.array(jax.devices()[:4]).reshape(2, 2), ("fsdp", "expert")) + + def ns_ep(spec): + return jax.sharding.NamedSharding(mesh_ep, spec) + + with jax.set_mesh(mesh_ep): + q_ep = jax.device_put(q_ref, ns_ep(P("fsdp", "expert", None, None))) + k_ep = jax.device_put(k_ref, ns_ep(P("fsdp", None, None, None))) + v_ep = jax.device_put(v_ref, ns_ep(P("fsdp", None, None, None))) + blk_ep = jax.device_put(jnp.asarray(blk_np), ns_ep(P("fsdp", "expert", None))) + w_ep = jax.device_put(w_out, ns_ep(P("fsdp", "expert", None))) + l_ep, (dq_ep, dk_ep, dv_ep) = jax.jit( + jax.value_and_grad( + lambda qq, kk, vv: jnp.sum( + mod_epcp.flash_block_sparse_attention(qq, kk, vv, blk_ep, block_q=16, block_kv=16) * w_ep + ), + argnums=(0, 1, 2), + ) + )(q_ep, k_ep, v_ep) + loss_unpacked_ref, (dq_u_ref, dk_u_ref, dv_u_ref) = jax.jit( + jax.value_and_grad( + lambda qq, kk, vv: jnp.sum( + mod_epcp.flash_block_sparse_attention(qq, kk, vv, jnp.asarray(blk_np), block_q=16, block_kv=16) * w_out + ), + argnums=(0, 1, 2), + ) + )(q_ref, k_ref, v_ref) + with jax.set_mesh(mesh_ep): + np.testing.assert_allclose(float(l_ep), float(loss_unpacked_ref), rtol=1e-4, atol=1e-4) + np.testing.assert_allclose(np.asarray(dq_ep), np.asarray(dq_u_ref), rtol=1e-4, atol=1e-4) + np.testing.assert_allclose(np.asarray(dk_ep), np.asarray(dk_u_ref), rtol=1e-4, atol=1e-4) + np.testing.assert_allclose(np.asarray(dv_ep), np.asarray(dv_u_ref), rtol=1e-4, atol=1e-4) + ep_hlo = ( + jax.jit(lambda qq, kk, vv, bb: mod_epcp.flash_block_sparse_attention(qq, kk, vv, bb, block_q=16, block_kv=16)) + .lower(q_ep, k_ep, v_ep, blk_ep) + .compile() + .as_text() + ) + self.assertNotIn("all-gather(", ep_hlo) + self.assertIn("f32[2,1,16,2,2,32]", ep_hlo) + + # 3. `tensor_sequence` head sharding via `get_heads_mesh_axes`. + mesh_tsp = jax.sharding.Mesh(np.array(jax.devices()[:4]).reshape(2, 2), ("fsdp", "tensor_sequence")) + + def ns_tsp(spec): + return jax.sharding.NamedSharding(mesh_tsp, spec) + + with jax.set_mesh(mesh_tsp): + q_tsp = jax.device_put(q_ref, ns_tsp(P("fsdp", None, "tensor_sequence", None))) + k_tsp = jax.device_put(k_ref, ns_tsp(P("fsdp", None, "tensor_sequence", None))) + v_tsp = jax.device_put(v_ref, ns_tsp(P("fsdp", None, "tensor_sequence", None))) + blk_tsp = jax.device_put(jnp.asarray(blk_np), ns_tsp(P("fsdp", None, None))) + tsp_hlo = ( + jax.jit( + lambda qq, kk, vv, bb: mod_default.flash_block_sparse_attention(qq, kk, vv, bb, block_q=16, block_kv=16) + ) + .lower(q_tsp, k_tsp, v_tsp, blk_tsp) + .compile() + .as_text() + ) + self.assertIn("f32[4,1,16,1,2,32]", tsp_hlo) + self.assertNotIn("f32[4,1,16,2,2,32]", tsp_hlo) + # ================================================================================================= # Sparse MoE block