From 5fdba82e94985b8f2bcfaacc6aed0d59225bc06d Mon Sep 17 00:00:00 2001 From: Jitendra Jalwaniya Date: Tue, 13 Oct 2026 17:15:15 +0000 Subject: [PATCH 1/2] wan: add Sol-Attn sparse self-attention (arXiv:2607.24027) Training-free sparse attention with query-dependent threshold routing (tau = mu + beta * sigma over block-proxy scores) and a zeroth-order pooled-key approximation of unselected blocks, both folded into one online softmax. - kernels/sol_attention: model-agnostic routing (pure JAX), Pallas TPU kernel, and a dense reference oracle. The kernel keeps K/V in HBM, walks only routed blocks through a DMA ring, uses a [keys, queries] score layout, runs the ragged tail block as an epilogue, and fuses selected blocks into ~1024-key tiles. - models/sol_attention: config resolution, dense warm-up dispatch, and a sharded entry point (Ulysses all-to-all or head-sharded) reusable by other DiTs. - models/sparse_schedule: step/layer warm-up schedule shared with SVG; svg_attention.is_svg_active is now an alias with unchanged messages. - Wire into FlaxWanAttention, the Wan pipeline, AOT cache metadata, and base_wan_14b.yml (use_sol_attention, sol_beta, sol_block_q/kv, sol_active_*). Disabled runs keep their transformer config and AOT cache identity. Wan 2.1 14B T2V, 720p, 81 frames, 50 steps on v6e-8 (ulysses_custom, data=2 x context=4): 231.3s -> 160.6s (1.44x) at the default 1024x256 blocks, beta=1.0; PSNR 20.20 dB / SSIM 0.702 vs. dense. Self-attention layer: 83.7 ms -> 36.6 ms. --- docs/sol_attention.md | 119 ++++++ src/maxdiffusion/aot_cache.py | 48 +++ src/maxdiffusion/configs/base_wan_14b.yml | 19 + src/maxdiffusion/generate_wan.py | 1 + .../kernels/sol_attention/__init__.py | 20 + .../kernels/sol_attention/reference.py | 117 ++++++ .../kernels/sol_attention/routing.py | 245 ++++++++++++ .../sol_attention/sol_attention_kernel.py | 378 ++++++++++++++++++ src/maxdiffusion/models/attention_flax.py | 23 ++ src/maxdiffusion/models/sol_attention.py | 279 +++++++++++++ src/maxdiffusion/models/sparse_schedule.py | 119 ++++++ src/maxdiffusion/models/svg_attention.py | 91 +---- .../pipelines/wan/wan_pipeline.py | 2 + src/maxdiffusion/tests/sol_attention_test.py | 324 +++++++++++++++ .../tests/wan/sol_config_propagation_test.py | 110 +++++ 15 files changed, 1810 insertions(+), 85 deletions(-) create mode 100644 docs/sol_attention.md create mode 100644 src/maxdiffusion/kernels/sol_attention/__init__.py create mode 100644 src/maxdiffusion/kernels/sol_attention/reference.py create mode 100644 src/maxdiffusion/kernels/sol_attention/routing.py create mode 100644 src/maxdiffusion/kernels/sol_attention/sol_attention_kernel.py create mode 100644 src/maxdiffusion/models/sol_attention.py create mode 100644 src/maxdiffusion/models/sparse_schedule.py create mode 100644 src/maxdiffusion/tests/sol_attention_test.py create mode 100644 src/maxdiffusion/tests/wan/sol_config_propagation_test.py diff --git a/docs/sol_attention.md b/docs/sol_attention.md new file mode 100644 index 000000000..476a91f85 --- /dev/null +++ b/docs/sol_attention.md @@ -0,0 +1,119 @@ +# Sol-Attn: sparse self-attention for video/image DiTs + +[Sol-Attn](https://arxiv.org/abs/2607.24027) is a training-free sparse +attention method. For every query block it picks the key/value blocks whose +block-proxy score `mean(Q_i) · mean(K_j)` exceeds a query-dependent threshold +`tau_i = mu_i + beta * sigma_i`, computes those blocks exactly, and replaces +every other block with its zeroth-order approximation (one pooled key with +weight `log(block size)`, value `mean(V_j)`). Both branches share one +online-softmax state, so skipped blocks still contribute to the softmax +numerator and denominator. + +The expected exact-block density is about `1 - Phi(beta)`: `beta = 1.0` keeps +~16% of the blocks (~84% sparsity), close to the paper's Wan 2.1 setting. + +## Using it with Wan 2.1 14B + +```bash +python src/maxdiffusion/generate_wan.py src/maxdiffusion/configs/base_wan_14b.yml \ + attention=ulysses_custom ici_data_parallelism=2 ici_context_parallelism=4 \ + use_sol_attention=True sol_beta=1.0 sol_block_q=1024 sol_block_kv=256 +``` + +| Key | Default | Meaning | +|---|---|---| +| `use_sol_attention` | `False` | Enable Sol-Attn for self-attention (cross-attention stays dense). Mutually exclusive with `use_svg_attention`. | +| `sol_beta` | `1.0` | Standardized threshold; higher is sparser. | +| `sol_block_q` | `1024` | Query routing block and kernel query tile, in tokens (multiple of 128). | +| `sol_block_kv` | `256` | Key/value block for exact compute and pooling (multiple of 128). | +| `sol_active_{start,end}_step` | `10, 50` | Sparse steps `[start, end)`; earlier steps are a dense warm-up (paper: first 20%). | +| `sol_active_{start,end}_layer` | `1, 40` | Sparse layers `[start, end)`; layer 0 stays dense. | +| `sol_dense_layer_fraction`, `sol_dense_timestep_fraction` | `0.0` | Fraction-based warm-up, used when no explicit interval is set. | +| `sol_log_density` | `False` | Print the measured exact-block density per call (debug). | + +Sol-Attn works with any attention backend whose self-attention either shards +heads (`flash`) or the sequence over one mesh axis (`ulysses*`): in the latter +case an all-to-all gives each device full sequences for a subset of heads. + +## Results on TPU v6e-8 + +Wan 2.1 14B T2V, 720p, 81 frames, 50 steps, `attention=ulysses_custom`, +`ici_data_parallelism=2 ici_context_parallelism=4`, one prompt, seed 12345. +The dense warm-up follows the paper (steps `[0, 10)` and layer 0 dense). PSNR +and SSIM compare each video with the dense video, as in Table 1 of the paper +(the paper reports PSNR 20.59 / SSIM 0.68 at 85% sparsity on H100). + +| Config | Exact density | Generation time | Speedup | PSNR | SSIM | +|---|---|---|---|---|---| +| dense | 100% | 231.3 s | 1.00x | – | – | +| `sol_block_q=1024 sol_block_kv=256 sol_beta=1.0` (default) | ~17% | 160.6 s | 1.44x | 20.20 dB | 0.702 | +| `sol_block_q=1024 sol_block_kv=256 sol_beta=0.8` | – | 170.5 s | 1.36x | 20.60 dB | 0.735 | +| `sol_block_q=512 sol_block_kv=256 sol_beta=1.0` | – | 175.0 s | 1.32x | 20.99 dB | 0.755 | +| `sol_block_q=2048 sol_block_kv=512 sol_beta=1.0` | ~16% | 150.4 s* | 1.54x | 18.55 dB | 0.633 | + +\* measured with `sol_log_density=True`, which adds a host callback per layer. + +Per self-attention layer (all 8 chips, synthetic video-like inputs at the +same shapes), dense `ulysses_custom` takes 83.7 ms and Sol-Attn 36.6 ms at +`1024x256` (2.3x) and 23.4 ms at `2048x512` (3.6x). About 13 ms of each +layer is shared overhead (Ulysses all-to-alls and layout copies), so +end-to-end gains are bounded by Amdahl's law and the 20% dense warm-up. + +## Code layout + +``` +kernels/sol_attention/ + routing.py pooling, thresholds (Eq. 3-5), index compaction (pure JAX) + sol_attention_kernel.py Pallas TPU kernel + `sol_attention(q, k, v, ...)` entry point + reference.py dense O(L^2) oracle of Eqs. (9)-(10), for tests +models/ + sol_attention.py config resolution, warm-up dispatch, sharded entry point + sparse_schedule.py step/layer warm-up schedule shared with SVG +``` + +`kernels/sol_attention` has no model dependencies: `sol_attention(q, k, v, +scale=..., beta=...)` takes `[..., L, d]` tensors and is usable on its own. + +## TPU adaptation + +The paper's GPU kernel fuses routing into the attention kernel to avoid +materializing the `N x N` block-proxy map. On TPU the design differs in a few +places, each driven by measurements on v6e: + +* **Routing in a small XLA pre-pass.** With TPU-sized blocks the proxy map is + tiny (e.g. 148 x 148 per head at 512-token blocks), so `mu_i` and `sigma_i` + are taken directly from the materialized row (identical to the moment + estimator of Eq. 5). Only compacted per-query-block index rows reach the + kernel; the selection mask for the approximate branch is rebuilt on chip + from them. +* **Exact branch via manual DMAs.** Keys/values stay in HBM; the kernel walks + the selected blocks through a ring of async copies, so only routed blocks are + read. The next grid step's index row is prefetched into SMEM. +* **Transposed score layout.** Scores are `[keys, queries]` so softmax + reductions run across sublanes, as in the dense splash kernel. +* **Branch-free hot loop.** The ragged tail block (sequence length not a + multiple of `sol_block_kv`) runs as a masked epilogue; a per-block branch + cost ~40%. +* **Larger blocks than the paper's 64 x 64.** MXU efficiency and per-block + overheads favor 512-2048 query tokens and 256-512 key tokens. +* **Grouped hot loop.** Selected blocks land in contiguous ring slots, so the + loop fuses several of them into one ~1024-key score tile (one softmax + rescale per tile), keeping routing at `sol_block_kv` granularity. This + alone is 10-25% faster for 128-256 token blocks. + +## Extending to other models + +1. In the attention module constructor, call + `sol_attention.init_sol_config(attention_config, num_layers, is_self_attention)` + and `setattr` the result on the module. +2. In `__call__`, wrap the attention op: + `sol_attention.apply_sol_or_dense(self, run_attention, layer_index, step_index, timestep)`. + `run_attention(sparse_config_override=...)` must forward the override to + `attention_flax._apply_attention(spatiotemporal_config=...)`. +3. In the pipeline, merge `sol_attention.attention_config_from_pyconfig(config, num_layers)` + into the transformer's `attention_config`, and add + `aot_cache.extract_sol_meta(config, pipeline)` to the AOT metadata. + +For MMDiT-style joint attention (text and video tokens in one sequence), pass +`always_exact` to the kernel to keep text blocks in the exact branch, as the +paper does for HunyuanVideo. diff --git a/src/maxdiffusion/aot_cache.py b/src/maxdiffusion/aot_cache.py index 1c27c2c92..39c14d349 100644 --- a/src/maxdiffusion/aot_cache.py +++ b/src/maxdiffusion/aot_cache.py @@ -143,6 +143,54 @@ def extract_svg_meta(config: Any, pipeline: Any = None) -> dict[str, Any]: return meta +def extract_sol_meta(config: Any, pipeline: Any = None) -> dict[str, Any]: + """Extracts all graph-affecting Sol-Attn settings for AOT cache identity.""" + sol_keys = ( + "use_sol_attention", + "sol_beta", + "sol_block_q", + "sol_block_kv", + "sol_active_start_step", + "sol_active_end_step", + "sol_active_start_layer", + "sol_active_end_layer", + "sol_dense_layer_fraction", + "sol_dense_timestep_fraction", + "sol_num_train_timesteps", + "sol_num_layers", + "sol_vmem_limit_bytes", + "sol_log_density", + ) + enabled = config is not None and bool( + config.get("use_sol_attention", False) if isinstance(config, dict) else getattr(config, "use_sol_attention", False) + ) + if not enabled: + # Dense runs keep their pre-Sol cache identity. + return {} + meta: dict[str, Any] = {} + if config is not None: + for k in sol_keys: + val = getattr(config, k, None) if not isinstance(config, dict) else config.get(k, None) + if val is not None: + meta[k] = str(val) + if pipeline is not None: + for attr in ("transformer", "high_noise_transformer", "low_noise_transformer"): + t = getattr(pipeline, attr, None) + if t is None: + continue + t_cfg = getattr(t, "config", None) + attn_cfg = getattr(t_cfg, "attention_config", None) or ( + t_cfg.get("attention_config") if isinstance(t_cfg, dict) else None + ) + if isinstance(attn_cfg, dict): + meta[f"{attr}_sol_config"] = json.dumps( + {k: v for k, v in attn_cfg.items() if str(k).startswith(("sol_", "use_sol_"))}, + sort_keys=True, + default=str, + ) + return meta + + def _metadata_fingerprint(meta: dict[str, Any]) -> str: """Returns the stable filename fingerprint for install-time metadata.""" serialized = json.dumps(meta, sort_keys=True, default=str) diff --git a/src/maxdiffusion/configs/base_wan_14b.yml b/src/maxdiffusion/configs/base_wan_14b.yml index 4c2b2b30c..5e49ac45c 100644 --- a/src/maxdiffusion/configs/base_wan_14b.yml +++ b/src/maxdiffusion/configs/base_wan_14b.yml @@ -101,6 +101,25 @@ svg_low_noise_density: -1.0 # compute blocks and the sparse kernel for small ones. Must stay a dict so the # command line can override it with JSON. svg_flash_block_sizes: {} +# Sol-Attn (arXiv:2607.24027): training-free sparse self-attention with +# query-dependent threshold routing and pooled-key approximation of skipped +# blocks. Mutually exclusive with use_svg_attention. Works with any attention +# backend whose self-attention shards either heads or (Ulysses) sequence. +use_sol_attention: False +# Standardized cutoff: expected exact density ~= 1 - Phi(sol_beta). +# 1.0 -> ~16% density (~84% sparsity), matching the paper's Wan 2.1 setup. +sol_beta: 1.0 +# Query routing block / kernel query tile and key/value block, in tokens. +# Smaller blocks route more precisely but run less efficiently on the MXU. +sol_block_q: 1024 +sol_block_kv: 256 +# Dense warm-up, [start, end) intervals. The paper keeps the first 20% of +# denoising steps and the first layer dense: steps [10, 50), layers [1, 40). +sol_active_start_step: 10 +sol_active_end_step: 50 +sol_active_start_layer: 1 +sol_active_end_layer: 40 +sol_log_density: False attention: 'flash' # Supported attention: dot_product, flash, tokamax_flash, cudnn_flash_te, ring, tokamax_ring, ulysses, ulysses_custom, ulysses_ring use_base2_exp: True diff --git a/src/maxdiffusion/generate_wan.py b/src/maxdiffusion/generate_wan.py index a8a184f48..c7fef2d91 100644 --- a/src/maxdiffusion/generate_wan.py +++ b/src/maxdiffusion/generate_wan.py @@ -315,6 +315,7 @@ def run(config, pipeline=None, filename_prefix="", commit_hash=None): "scan_layers": str(config.scan_layers), "jax": jax.__version__, **aot_cache.extract_svg_meta(config, pipeline), + **aot_cache.extract_sol_meta(config, pipeline), }, mesh=pipeline.mesh, ) diff --git a/src/maxdiffusion/kernels/sol_attention/__init__.py b/src/maxdiffusion/kernels/sol_attention/__init__.py new file mode 100644 index 000000000..7c590691d --- /dev/null +++ b/src/maxdiffusion/kernels/sol_attention/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Sol-Attn: training-free sparse attention with threshold routing and +approximate correction (arXiv:2607.24027).""" + +from maxdiffusion.kernels.sol_attention.routing import SolBlockSizes +from maxdiffusion.kernels.sol_attention.sol_attention_kernel import sol_attention + +__all__ = ["SolBlockSizes", "sol_attention"] diff --git a/src/maxdiffusion/kernels/sol_attention/reference.py b/src/maxdiffusion/kernels/sol_attention/reference.py new file mode 100644 index 000000000..8c62ad975 --- /dev/null +++ b/src/maxdiffusion/kernels/sol_attention/reference.py @@ -0,0 +1,117 @@ +""" +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Dense O(L^2)-memory reference for Sol-Attn. + +Materializes the full score matrix, so it is only meant for tests and small +sequence lengths. It evaluates Eqs. (9)-(10) of arXiv:2607.24027 literally: +one softmax over [exact scores of selected blocks | zeroth-order pooled scores +of unselected blocks]. +""" + +from __future__ import annotations + +from typing import Optional + +import jax +import jax.numpy as jnp + +from maxdiffusion.kernels.sol_attention import routing as sol_routing + + +def _flatten_heads(x: jax.Array) -> jax.Array: + return x.reshape(-1, *x.shape[-2:]) + + +def sol_attention_reference( + q: jax.Array, + k: jax.Array, + v: jax.Array, + *, + scale: float, + beta: float, + block_sizes: sol_routing.SolBlockSizes = sol_routing.SolBlockSizes(), + always_exact: Optional[jax.Array] = None, + approximate: bool = True, + selected: Optional[jax.Array] = None, +) -> jax.Array: + """Sol-Attn on [..., L, d] tensors (any leading batch/head dims). + + Args: + q, k, v: Query/key/value [..., L, d] self-attention tensors. + scale: Attention logit scale. + beta: Threshold cutoff, see `routing.threshold_route`. + block_sizes: Block granularity. + always_exact: Optional bool [Nk] of key blocks forced into the exact branch. + approximate: If False, unselected blocks are dropped instead of + approximated (standard keep-or-drop block-sparse attention). Used for the + approximate-correction ablation (Fig. 9 of the paper). + selected: Optional precomputed bool [prod(...), Nq, Nk] selection that + overrides threshold routing. + + Returns: + Output [..., L, d] in q.dtype. + """ + lead = q.shape[:-2] + seq_len, head_dim = q.shape[-2:] + padded_len = sol_routing.padded_length(seq_len, block_sizes) + qp, kp, vp = (sol_routing.pad_sequence(_flatten_heads(x), padded_len) for x in (q, k, v)) + routing = sol_routing.compute_routing( + qp, + kp, + vp, + seq_len=seq_len, + scale=scale, + beta=beta, + block_sizes=block_sizes, + always_exact=always_exact, + ) + if selected is not None: + routing = sol_routing.SolRouting( + selected=selected, + tau=routing.tau, + kv_counts=routing.kv_counts, + k_pooled=routing.k_pooled, + v_pooled=routing.v_pooled, + ) + bq, bkv = block_sizes.block_q, block_sizes.block_kv + + hp = jax.lax.Precision.HIGHEST + qf, kf, vf = (x.astype(jnp.float32) for x in (qp, kp, vp)) + scores = jnp.einsum("hqd,hkd->hqk", qf, kf, precision=hp) * scale + token_selected = jnp.repeat(jnp.repeat(routing.selected, bq, axis=1), bkv, axis=2) + key_valid = jnp.arange(padded_len) < seq_len + exact_logits = jnp.where(jnp.logical_and(token_selected, key_valid), scores, sol_routing.MASK_VALUE) + + if approximate: + approx_logits = jnp.einsum("hqd,hkd->hqk", qf, routing.k_pooled, precision=hp) * scale + approx_logits = approx_logits + jnp.repeat(sol_routing.approx_bias(routing), bq, axis=1) + logits = jnp.concatenate([exact_logits, approx_logits], axis=-1) + values = jnp.concatenate([vf, routing.v_pooled], axis=-2) + else: + logits, values = exact_logits, vf + + m = jnp.max(logits, axis=-1, keepdims=True) + p = jnp.where(logits > 0.5 * sol_routing.MASK_VALUE, jnp.exp(logits - m), 0.0) + out = jnp.einsum("hqk,hkd->hqd", p, values, precision=hp) / jnp.maximum(jnp.sum(p, axis=-1, keepdims=True), 1e-30) + return out[:, :seq_len].reshape(*lead, seq_len, head_dim).astype(q.dtype) + + +def dense_attention_reference(q: jax.Array, k: jax.Array, v: jax.Array, *, scale: float) -> jax.Array: + """Plain softmax attention on [..., L, d] tensors in float32.""" + hp = jax.lax.Precision.HIGHEST + qf, kf, vf = (x.astype(jnp.float32) for x in (q, k, v)) + p = jax.nn.softmax(jnp.einsum("...qd,...kd->...qk", qf, kf, precision=hp) * scale, axis=-1) + return jnp.einsum("...qk,...kd->...qd", p, vf, precision=hp).astype(q.dtype) diff --git a/src/maxdiffusion/kernels/sol_attention/routing.py b/src/maxdiffusion/kernels/sol_attention/routing.py new file mode 100644 index 000000000..26eb9b888 --- /dev/null +++ b/src/maxdiffusion/kernels/sol_attention/routing.py @@ -0,0 +1,245 @@ +""" +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Sol-Attn block pooling and query-dependent threshold routing. + +Implements Sec. 3.1 of "Sol-Attn: Accelerating Video Generation Inference via +On-the-Fly Attention Sparsification" (arXiv:2607.24027): + + s_ij = mean(Q_i) . mean(K_j) * scale (Eq. 3) + tau_i = mu_i + beta * sigma_i, S_i = {j : s_ij > tau_i} (Eq. 4) + +The functions here are pure JAX, model agnostic, and shared by the Pallas TPU +kernel and the dense reference so that both see bit-identical routing. + +TPU note: the paper fuses routing into the GPU kernel to avoid materializing +the N x N block-proxy map in HBM. With TPU-friendly blocks (>=128 tokens) the +map is tiny (e.g. 592 x 592 per head for Wan 2.1 14B 720p), so it is computed +here in a cheap XLA pre-pass. mu_i / sigma_i are taken directly from the +materialized row, which equals the exact moment estimator of Eq. (5). +""" + +from __future__ import annotations + +import dataclasses +import math +from typing import Optional + +import jax +import jax.numpy as jnp + +# Finite "minus infinity" keeps exp() and max() NaN-free on TPU. +MASK_VALUE = -0.7 * float(jnp.finfo(jnp.float32).max) + + +@dataclasses.dataclass(frozen=True) +class SolBlockSizes: + """Token granularity of Sol-Attn. + + Attributes: + block_q: Query tokens per routing block (and per kernel query tile). + block_kv: Key/value tokens per block. Selected blocks are attended + exactly; unselected blocks are approximated by their pooled key. + """ + + block_q: int = 1024 + block_kv: int = 256 + + def __post_init__(self): + for name in ("block_q", "block_kv"): + value = getattr(self, name) + if value <= 0 or value % 128: + raise ValueError(f"Sol-Attn {name} must be a positive multiple of 128, got {value}.") + + @property + def seq_multiple(self) -> int: + return math.lcm(self.block_q, self.block_kv) + + +def padded_length(seq_len: int, block_sizes: SolBlockSizes) -> int: + multiple = block_sizes.seq_multiple + return -(-seq_len // multiple) * multiple + + +def pad_sequence(x: jax.Array, padded_len: int) -> jax.Array: + """Zero-pads axis -2 (sequence) of `x` to `padded_len`.""" + pad = padded_len - x.shape[-2] + if pad == 0: + return x + widths = [(0, 0)] * x.ndim + widths[-2] = (0, pad) + return jnp.pad(x, widths) + + +def block_counts(seq_len: int, padded_len: int, block: int) -> jax.Array: + """Number of real (non-padding) tokens in each block, shape [padded_len // block].""" + starts = jnp.arange(padded_len // block, dtype=jnp.int32) * block + return jnp.clip(seq_len - starts, 0, block) + + +def pool_blocks(x: jax.Array, seq_len: int, block: int) -> jax.Array: + """Mean over the real tokens of each block. + + Args: + x: Zero-padded tensor [..., padded_len, d]. + seq_len: Number of real tokens. + block: Block size along the sequence. + + Returns: + float32 tensor [..., padded_len // block, d]. Fully padded blocks are 0. + """ + *lead, padded_len, d = x.shape + sums = x.astype(jnp.float32).reshape(*lead, padded_len // block, block, d).sum(axis=-2) + counts = block_counts(seq_len, padded_len, block).astype(jnp.float32) + return sums / jnp.maximum(counts, 1.0)[:, None] + + +def block_proxy_scores(q_pooled: jax.Array, k_pooled: jax.Array, scale: float) -> jax.Array: + """Pre-softmax block-proxy logits s_ij (Eq. 3), shape [..., Nq, Nk].""" + return ( + jnp.einsum( + "...qd,...kd->...qk", + q_pooled.astype(jnp.float32), + k_pooled.astype(jnp.float32), + precision=jax.lax.Precision.HIGHEST, + ) + * scale + ) + + +def threshold_route( + scores: jax.Array, + kv_counts: jax.Array, + beta: float, + always_exact: Optional[jax.Array] = None, +) -> tuple[jax.Array, jax.Array]: + """Query-dependent threshold routing (Eq. 4). + + Args: + scores: Block-proxy logits [..., Nq, Nk]. + kv_counts: Real tokens per key block [Nk]; blocks with 0 tokens are padding + and are neither selected nor approximated. + beta: Standardized cutoff. Under the Gaussian reference the selected + density is 1 - Phi(beta) (beta=1.0 -> ~16% density, ~84% sparsity). + always_exact: Optional bool [Nk] (or broadcastable to [..., Nq, Nk]) of key + blocks that always take the exact branch, e.g. text/sink tokens in + MMDiT-style joint attention. + + Returns: + (selected, tau): bool [..., Nq, Nk] selection mask and float32 [..., Nq] + thresholds. + """ + valid = kv_counts > 0 + num_valid = jnp.maximum(jnp.sum(valid), 1).astype(jnp.float32) + masked = jnp.where(valid, scores, 0.0) + mu = jnp.sum(masked, axis=-1) / num_valid + centered = jnp.where(valid, scores - mu[..., None], 0.0) + sigma = jnp.sqrt(jnp.sum(centered * centered, axis=-1) / num_valid) + tau = mu + beta * sigma + selected = jnp.logical_and(scores > tau[..., None], valid) + if always_exact is not None: + selected = jnp.logical_or(selected, jnp.logical_and(always_exact, valid)) + return selected, tau + + +def compact_selection(selected: jax.Array, row_width: int) -> jax.Array: + """Packs a selection mask into per-row index lists for the kernel. + + Row layout: [count, j_0, j_1, ..., j_{count-1}, ...], padded to + `row_width` (>= Nk + 1) int32 entries. Selected indices are ascending so the + kernel walks HBM in order. + """ + nk = selected.shape[-1] + if row_width < nk + 1: + raise ValueError(f"row_width={row_width} must be >= Nk + 1 = {nk + 1}.") + j = jnp.arange(nk, dtype=jnp.int32) + # Selected indices sort first; the modulo recovers the index for both groups. + order = jnp.sort(jnp.where(selected, j, nk + j), axis=-1) % nk + counts = jnp.sum(selected, axis=-1, dtype=jnp.int32) + rows = jnp.concatenate([counts[..., None], order], axis=-1) + widths = [(0, 0)] * (rows.ndim - 1) + [(0, row_width - nk - 1)] + return jnp.pad(rows, widths) + + +@jax.tree_util.register_dataclass +@dataclasses.dataclass(frozen=True) +class SolRouting: + """Everything the exact/approximate branches need, derived from Q and K. + + Shapes use H for the flattened batch*heads axis. + """ + + selected: jax.Array # bool [H, Nq, Nk] + tau: jax.Array # f32 [H, Nq] + kv_counts: jax.Array # int32 [Nk] + k_pooled: jax.Array # f32 [H, Nk, d] mean key per block + v_pooled: jax.Array # f32 [H, Nk, d] mean value per block + + def density(self) -> jax.Array: + """Fraction of (query block, real key block) pairs computed exactly.""" + valid = (self.kv_counts > 0).astype(jnp.float32) + return jnp.sum(self.selected * valid) / (jnp.sum(valid) * self.selected.shape[0] * self.selected.shape[1]) + + +def compute_routing( + q: jax.Array, + k: jax.Array, + v: jax.Array, + *, + seq_len: int, + scale: float, + beta: float, + block_sizes: SolBlockSizes, + always_exact: Optional[jax.Array] = None, +) -> SolRouting: + """Pools Q/K/V into blocks and routes key blocks per query block. + + Args: + q, k, v: Zero-padded [H, padded_len, d] tensors. + seq_len: Number of real tokens. + scale: Attention logit scale. + beta: Standardized cutoff (see `threshold_route`). + block_sizes: Block granularity. + always_exact: Optional bool [Nk] of key blocks forced into the exact branch. + """ + q_pooled = pool_blocks(q, seq_len, block_sizes.block_q) + k_pooled = pool_blocks(k, seq_len, block_sizes.block_kv) + v_pooled = pool_blocks(v, seq_len, block_sizes.block_kv) + kv_counts = block_counts(seq_len, k.shape[-2], block_sizes.block_kv) + scores = block_proxy_scores(q_pooled, k_pooled, scale) + selected, tau = threshold_route(scores, kv_counts, beta, always_exact) + return SolRouting( + selected=selected, + tau=tau, + kv_counts=kv_counts, + k_pooled=k_pooled, + v_pooled=v_pooled, + ) + + +def approx_bias(routing: SolRouting) -> jax.Array: + """Additive logit bias for the zeroth-order approximate branch. + + For an unselected block j with c_j real tokens, Eqs. (9)-(10) contribute + c_j * exp(q . Kbar_j) to the denominator and exp(q . Kbar_j) * sum(V_j) to the + numerator, i.e. one pooled key with logit q . Kbar_j + log(c_j) and value + mean(V_j). Selected and padding blocks get MASK_VALUE (no approx term). + + Returns: + float32 [H, Nq, Nk]. + """ + log_counts = jnp.log(jnp.maximum(routing.kv_counts, 1).astype(jnp.float32)) + keep = jnp.logical_and(jnp.logical_not(routing.selected), routing.kv_counts > 0) + return jnp.where(keep, log_counts, MASK_VALUE) diff --git a/src/maxdiffusion/kernels/sol_attention/sol_attention_kernel.py b/src/maxdiffusion/kernels/sol_attention/sol_attention_kernel.py new file mode 100644 index 000000000..655ba1d08 --- /dev/null +++ b/src/maxdiffusion/kernels/sol_attention/sol_attention_kernel.py @@ -0,0 +1,378 @@ +""" +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Pallas TPU kernel for Sol-Attn (arXiv:2607.24027, Algorithm 1). + +One grid program per (head, query block). Each program keeps a single +online-softmax state and runs two branches into it: + + 1. Approximate branch (outer loop of Algorithm 1): one GEMM of the query + block against all pooled keys. Unselected blocks contribute their + zeroth-order term (Eqs. 9-10) through a log-count bias; the selected + blocks (read from the SMEM routing row) are masked on chip so nothing is + double counted. + 2. Exact branch (inner loop): walks the selected key/value blocks of this + query block through a ring of manual HBM->VMEM DMAs, so only routed + blocks are ever read. + +Routing (pooling + thresholds) runs in a tiny XLA pre-pass, see `routing.py`; +only the compacted per-query-block index rows reach the kernel. +""" + +from __future__ import annotations + +import functools +from typing import Optional + +import jax +import jax.numpy as jnp +from jax import lax +from jax.experimental import pallas as pl +from jax.experimental.pallas import tpu as pltpu + +from maxdiffusion.kernels.sol_attention import routing as sol_routing + +NUM_LANES = 128 +NUM_SUBLANES = 8 +# The hot loop fuses selected blocks into score tiles of about this many keys; +# measured on v6e, 1024 beats 256/512 by 10-25% for 128-256 token blocks. +TARGET_TILE_KEYS = 1024 +NT_DIM_NUMBERS = (((1,), (1,)), ((), ())) +LOG2E = 1.4426950408889634 +_MASK_VALUE = sol_routing.MASK_VALUE +_HALF_MASK_VALUE = 0.5 * sol_routing.MASK_VALUE + + +def _round_up(x: int, multiple: int) -> int: + return -(-x // multiple) * multiple + + +def _resolve_pipelining(block_kv: int, group_size: Optional[int], num_buffers: Optional[int]) -> tuple[int, int]: + if group_size is None: + group_size = max(1, TARGET_TILE_KEYS // block_kv) + if num_buffers is None: + num_buffers = 2 * group_size + if group_size < 1 or num_buffers % group_size or num_buffers < 2 * group_size: + raise ValueError(f"num_buffers={num_buffers} must be a multiple of group_size={group_size} and >= 2 * group_size.") + return group_size, num_buffers + + +def _sol_kernel( + # Inputs. + q_ref, # VMEM [bq, d], pre-scaled by scale * log2(e) + k_pooled_ref, # VMEM [nk_pad, d] + v_pooled_ref, # VMEM [nk_pad, d] + log_count_ref, # VMEM [nk_pad, 1] f32: log2(tokens in block j), MASK for padding blocks + rows_hbm, # HBM [H, Nq, 8, row_cols] int32; each (8, row_cols) row is one tile-aligned DMA + k_hbm, # HBM [H, Lp, d] + v_hbm, # HBM [H, Lp, d] + # Outputs. + o_ref, # VMEM [d, bq] (transposed) + # Scratch. + row_smem, # SMEM [2, 8, row_cols] int32, double-buffered across grid steps + k_buf, # VMEM [num_buffers, bkv, d] + v_buf, # VMEM [num_buffers, bkv, d] + bias_scr, # VMEM [nk_pad, 1] f32 + o_scr, # VMEM [d, bq] f32 + row_sem, # DMA semaphores [2] + kv_sem, # DMA semaphores [2, num_buffers] + *, + seq_len: int, + block_kv: int, + num_buffers: int, + group_size: int, +): + """One (head, query block) program: approximate branch + exact sparse loop.""" + # Scores are laid out [keys, queries] so softmax reductions run over + # sublanes (cheap VPU ops) instead of lanes, as in the dense splash kernel. + h = pl.program_id(0) + i = pl.program_id(1) + num_q_blocks = pl.num_programs(1) + step = h * num_q_blocks + i + num_steps = pl.num_programs(0) * num_q_blocks + row_slot = lax.rem(step, 2) + row_cols = row_smem.shape[2] + + def row_copy(n, slot): + return pltpu.make_async_copy( + rows_hbm.at[n // num_q_blocks, lax.rem(n, num_q_blocks)], row_smem.at[slot], row_sem.at[slot] + ) + + # The routing row of step n+1 is prefetched during step n (grid runs in + # order), so only the very first step pays the DMA latency. + @pl.when(step == 0) + def fetch_first_row(): + row_copy(step, row_slot).start() + + row_copy(step, row_slot).wait() + + @pl.when(step + 1 < num_steps) + def prefetch_next_row(): + row_copy(step + 1, 1 - row_slot).start() + + def row_entry(n): + return row_smem[row_slot, n // row_cols, lax.rem(n, row_cols)] + + count = row_entry(0) + + def kv_copies(t): + """K/V block DMAs for the t-th selected block, into ring slot t % num_buffers.""" + slot = lax.rem(t, num_buffers) + start = pl.multiple_of(row_entry(t + 1) * block_kv, block_kv) + k_copy = pltpu.make_async_copy(k_hbm.at[h, pl.ds(start, block_kv), :], k_buf.at[slot], kv_sem.at[0, slot]) + v_copy = pltpu.make_async_copy(v_hbm.at[h, pl.ds(start, block_kv), :], v_buf.at[slot], kv_sem.at[1, slot]) + return k_copy, v_copy + + def start_kv_copies(t): + for c in kv_copies(t): + c.start() + + # Ring of num_buffers block slots, consumed group_size blocks at a time. + # Prime all but the last group; these DMAs overlap the approximate branch. + for t in range(num_buffers - group_size): + pl.when(t < count)(functools.partial(start_kv_copies, t)) + + # Routing mask on chip: selected blocks get no approximate term. + bias_scr[...] = log_count_ref[...] + + def mask_selected(t, carry): + bias_scr[pl.ds(row_entry(t + 1), 1), :] = jnp.full((1, 1), _MASK_VALUE, jnp.float32) + return carry + + lax.fori_loop(0, count, mask_selected, 0) + + q = q_ref[...] + sv_dims = (((0,), (0,)), ((), ())) + + # Approximate branch: one GEMM over all pooled keys, zeroth-order term of + # Eqs. (9)-(10) via the log2(count) bias. + s = lax.dot_general(k_pooled_ref[...], q, NT_DIM_NUMBERS, preferred_element_type=jnp.float32) + s = s + bias_scr[...] + m = jnp.max(s, axis=0, keepdims=True) + p = jnp.where(s > _HALF_MASK_VALUE, jnp.exp2(s - m), 0.0) + l = jnp.sum(p, axis=0, keepdims=True) + o_scr[...] = lax.dot_general(v_pooled_ref[...], p.astype(v_pooled_ref.dtype), sv_dims, preferred_element_type=jnp.float32) + + ragged = seq_len % block_kv != 0 + last_block = seq_len // block_kv + + def exact_blocks(t0, m_prev, l_prev, size: int, prefetch: bool, mask_tail: bool): + """Online-softmax update with selected blocks t0..t0+size-1. + + Their ring slots are contiguous, so `size` routed blocks form one + [size * bkv, bq] score tile: routing stays fine-grained while each loop + iteration amortizes the softmax rescale over a larger tile. + """ + slot = lax.rem(t0, num_buffers) + copies = [kv_copies(t0 + j) for j in range(size)] + if prefetch: + for j in range(size): + nxt = t0 + num_buffers - group_size + j + pl.when(nxt < count)(functools.partial(start_kv_copies, nxt)) + for k_copy, _ in copies: + k_copy.wait() + k = k_buf[pl.ds(slot, size)].reshape(size * block_kv, k_buf.shape[-1]) + s = lax.dot_general(k, q, NT_DIM_NUMBERS, preferred_element_type=jnp.float32) + if mask_tail: + valid = lax.broadcasted_iota(jnp.int32, s.shape, 0) < seq_len - last_block * block_kv + s = jnp.where(valid, s, _MASK_VALUE) + m_next = jnp.maximum(m_prev, jnp.max(s, axis=0, keepdims=True)) + alpha = jnp.exp2(m_prev - m_next) + p = jnp.exp2(s - m_next) + l_next = alpha * l_prev + jnp.sum(p, axis=0, keepdims=True) + for _, v_copy in copies: + v_copy.wait() + v = v_buf[pl.ds(slot, size)].reshape(size * block_kv, v_buf.shape[-1]) + o_scr[...] = alpha * o_scr[...] + lax.dot_general(v, p.astype(v.dtype), sv_dims, preferred_element_type=jnp.float32) + return m_next, l_next + + # Exact branch over routed blocks only, sharing the online-softmax state. + # Indices are ascending, so a selected ragged tail block is always last; it + # runs as a masked epilogue to keep the hot loop branch-free (a per-block + # branch costs ~40% here). Every block's DMA is issued by the prologue or + # by a hot-loop iteration, so the leftover/tail steps never prefetch. + if ragged: + has_tail = jnp.logical_and(count > 0, row_entry(count) == last_block) + num_main = count - has_tail.astype(jnp.int32) + else: + num_main = count + num_groups = num_main // group_size + m, l = lax.fori_loop( + 0, + num_groups, + lambda g, c: exact_blocks(g * group_size, *c, size=group_size, prefetch=True, mask_tail=False), + (m, l), + ) + if group_size > 1: + m, l = lax.fori_loop( + num_groups * group_size, + num_main, + lambda t, c: exact_blocks(t, *c, size=1, prefetch=False, mask_tail=False), + (m, l), + ) + if ragged: + m, l = lax.cond( + has_tail, + lambda c: exact_blocks(num_main, *c, size=1, prefetch=False, mask_tail=True), + lambda c: c, + (m, l), + ) + o_ref[...] = (o_scr[...] * pl.reciprocal(l, approx=False)).astype(o_ref.dtype) + + +def sol_attention_from_routing( + q: jax.Array, + k: jax.Array, + v: jax.Array, + routing: sol_routing.SolRouting, + *, + seq_len: int, + scale: float, + block_sizes: sol_routing.SolBlockSizes, + vmem_limit_bytes: Optional[int] = None, + num_buffers: Optional[int] = None, + group_size: Optional[int] = None, + interpret: bool = False, +) -> jax.Array: + """Runs the kernel on padded [H, Lp, d] tensors with precomputed routing. + + Returns the padded [H, Lp, d] output. + """ + group_size, num_buffers = _resolve_pipelining(block_sizes.block_kv, group_size, num_buffers) + num_heads, padded_len, head_dim = q.shape + bq, bkv = block_sizes.block_q, block_sizes.block_kv + nq, nk = padded_len // bq, padded_len // bkv + nk_pad = _round_up(nk, NUM_LANES) + # Row tables are (8, row_cols) tiles so HBM->SMEM DMAs are layout aligned. + row_cols = _round_up(-(-(nk + 1) // NUM_SUBLANES), NUM_LANES) + row_width = NUM_SUBLANES * row_cols + + def pad_blocks(x): + widths = [(0, 0)] * x.ndim + widths[-2] = (0, nk_pad - nk) + return jnp.pad(x, widths) + + # exp2 everywhere: fold log2(e) into the query scale and the count bias. + q = (q.astype(jnp.float32) * (scale * LOG2E)).astype(q.dtype) + k_pooled = pad_blocks(routing.k_pooled).astype(k.dtype) + v_pooled = pad_blocks(routing.v_pooled).astype(v.dtype) + counts = jnp.pad(routing.kv_counts, (0, nk_pad - nk)) + log_count = jnp.where(counts > 0, jnp.log2(jnp.maximum(counts, 1).astype(jnp.float32)), _MASK_VALUE)[:, None] + rows = sol_routing.compact_selection(routing.selected, row_width).reshape(num_heads, nq, NUM_SUBLANES, row_cols) + + kernel = functools.partial(_sol_kernel, seq_len=seq_len, block_kv=bkv, num_buffers=num_buffers, group_size=group_size) + out = pl.pallas_call( + kernel, + grid=(num_heads, nq), + in_specs=[ + pl.BlockSpec((None, bq, head_dim), lambda h, i: (h, i, 0)), + pl.BlockSpec((None, nk_pad, head_dim), lambda h, i: (h, 0, 0)), + pl.BlockSpec((None, nk_pad, head_dim), lambda h, i: (h, 0, 0)), + pl.BlockSpec((nk_pad, 1), lambda h, i: (0, 0)), + pl.BlockSpec(memory_space=pltpu.HBM), + pl.BlockSpec(memory_space=pltpu.HBM), + pl.BlockSpec(memory_space=pltpu.HBM), + ], + out_specs=pl.BlockSpec((None, head_dim, bq), lambda h, i: (h, 0, i)), + out_shape=jax.ShapeDtypeStruct((num_heads, head_dim, padded_len), q.dtype), + scratch_shapes=[ + pltpu.SMEM((2, NUM_SUBLANES, row_cols), jnp.int32), + pltpu.VMEM((num_buffers, bkv, head_dim), k.dtype), + pltpu.VMEM((num_buffers, bkv, head_dim), v.dtype), + pltpu.VMEM((nk_pad, 1), jnp.float32), + pltpu.VMEM((head_dim, bq), jnp.float32), + pltpu.SemaphoreType.DMA((2,)), + pltpu.SemaphoreType.DMA((2, num_buffers)), + ], + compiler_params=pltpu.CompilerParams( + # Sequential grid: each step prefetches the next step's routing row. + dimension_semantics=("arbitrary", "arbitrary"), + vmem_limit_bytes=vmem_limit_bytes, + ), + interpret=interpret, + name="sol_attention", + )(q, k_pooled, v_pooled, log_count, rows, k, v) + return jnp.swapaxes(out, 1, 2) + + +def sol_attention( + q: jax.Array, + k: jax.Array, + v: jax.Array, + *, + scale: float, + beta: float, + block_sizes: sol_routing.SolBlockSizes = sol_routing.SolBlockSizes(), + always_exact: Optional[jax.Array] = None, + vmem_limit_bytes: Optional[int] = None, + num_buffers: Optional[int] = None, + group_size: Optional[int] = None, + interpret: bool = False, + return_density: bool = False, +): + """Sol-Attn self-attention on [..., L, d] tensors (any leading dims). + + Args: + q, k, v: Query/key/value [..., L, d]; all leading dims are folded into one + head axis. Keys must cover the full sequence (no sequence sharding). + scale: Attention logit scale (usually head_dim ** -0.5). + beta: Standardized threshold cutoff. Expected density ~= 1 - Phi(beta). + block_sizes: Query/key block granularity. + always_exact: Optional bool [Nk] of key blocks always computed exactly. + vmem_limit_bytes: Optional Mosaic VMEM limit. + num_buffers: Exact-branch K/V DMA ring depth in blocks (default 2 * group_size). + group_size: Selected blocks fused into one score tile per loop step + (default: TARGET_TILE_KEYS // block_kv). + interpret: Run the Pallas kernel in interpret mode (CPU testing). + return_density: Also return the exact-branch block density (scalar). + + Returns: + Output [..., L, d] in q.dtype (and the density if requested). + """ + if q.shape != k.shape or q.shape != v.shape: + raise ValueError(f"Sol-Attn needs matching self-attention q/k/v, got {q.shape}, {k.shape}, {v.shape}.") + lead = q.shape[:-2] + seq_len, head_dim = q.shape[-2:] + padded_len = sol_routing.padded_length(seq_len, block_sizes) + qp, kp, vp = (sol_routing.pad_sequence(x.reshape(-1, seq_len, head_dim), padded_len) for x in (q, k, v)) + with jax.named_scope("sol_routing"): + routing = sol_routing.compute_routing( + qp, + kp, + vp, + seq_len=seq_len, + scale=scale, + beta=beta, + block_sizes=block_sizes, + always_exact=always_exact, + ) + with jax.named_scope("sol_kernel"): + out = sol_attention_from_routing( + qp, + kp, + vp, + routing, + seq_len=seq_len, + scale=scale, + block_sizes=block_sizes, + vmem_limit_bytes=vmem_limit_bytes, + num_buffers=num_buffers, + group_size=group_size, + interpret=interpret, + ) + out = out[:, :seq_len].reshape(*lead, seq_len, head_dim) + if return_density: + return out, routing.density() + return out diff --git a/src/maxdiffusion/models/attention_flax.py b/src/maxdiffusion/models/attention_flax.py index a6e341775..37ee26a51 100644 --- a/src/maxdiffusion/models/attention_flax.py +++ b/src/maxdiffusion/models/attention_flax.py @@ -38,6 +38,7 @@ from ..kernels import custom_svg_attention_dispatch from ..kernels import custom_svg_static_range_attention from . import quantizations +from . import sol_attention from . import svg_attention from . import svg_head_local from .modeling_flax_utils import get_activation @@ -2049,6 +2050,10 @@ def _apply_attention( "spatiotemporal_shape": spatiotemporal_shape, } + if spatiotemporal_config and spatiotemporal_config.get("use_sol_attention"): + with jax.named_scope("sol_attention"): + return sol_attention.sol_sharded_attention(query, key, value, context) + if spatiotemporal_config and spatiotemporal_config.get("use_svg_attention"): if effective_attention_kernel not in ( "ulysses_custom", @@ -2615,6 +2620,13 @@ def __init__( ) for name, value in svg_config.items(): setattr(self, name, value) + sol_config = sol_attention.init_sol_config( + attention_config, + default_num_layers=40, + is_self_attention=is_self_attention, + ) + for name, value in sol_config.items(): + setattr(self, name, value) self.is_self_attention = is_self_attention if attention_kernel in {"flash", "cudnn_flash_te"} and mesh is None: @@ -2879,6 +2891,8 @@ def __call__( raise ValueError("SVG attention supports deterministic inference only.") if spatiotemporal_shape is None: raise ValueError("SVG attention requires spatiotemporal_shape.") + if self.use_sol_attention and is_self_attention and not deterministic: + raise ValueError("Sol-Attn supports deterministic inference only.") if encoder_hidden_states is None: encoder_hidden_states = hidden_states @@ -2939,6 +2953,15 @@ def run_attention(**svg_kwargs): svg_step_index=svg_step_index, svg_timestep=svg_timestep, ) + elif self.use_sol_attention and is_self_attention: + # The svg_* indices are the generic sparse-attention schedule signals. + attn_output = sol_attention.apply_sol_or_dense( + self, + run_attention, + layer_index=svg_layer_index, + step_index=svg_step_index, + timestep=svg_timestep, + ) else: with jax.named_scope("apply_attention"): attn_output = run_attention() diff --git a/src/maxdiffusion/models/sol_attention.py b/src/maxdiffusion/models/sol_attention.py new file mode 100644 index 000000000..ac763caa8 --- /dev/null +++ b/src/maxdiffusion/models/sol_attention.py @@ -0,0 +1,279 @@ +""" +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Model-agnostic integration layer for Sol-Attn (arXiv:2607.24027). + +Sol-Attn is a training-free drop-in replacement for full self-attention: it +only needs per-head Q/K/V over the full token sequence. Integrating it into a +model takes three steps (see `FlaxWanAttention` for an example): + + 1. In the attention module's constructor, resolve the `sol_*` settings with + `init_sol_config(attention_config, num_layers, is_self_attention)` and + store them on the module. + 2. In `__call__`, wrap the attention op with `apply_sol_or_dense(...)`, which + evaluates the dense warm-up schedule (steps / layers) and passes a + `sparse_config_override` to the attention op on sparse steps. + 3. The attention dispatcher (`attention_flax._apply_attention`) routes that + override to `sol_sharded_attention`, which gathers full sequences per head + (Ulysses all-to-all if the sequence is sharded) and runs the TPU kernel. + +Nothing here depends on video-specific token layouts, so the same path works +for image DiTs. MMDiT-style joint attention can force text tokens into the +exact branch via the kernel's `always_exact` argument. +""" + +from __future__ import annotations + +import functools +from typing import Any, Callable, Mapping, Optional + +import flax.linen as nn +import jax +from jax.sharding import PartitionSpec as P + +from maxdiffusion.common_types import BATCH, HEAD, LENGTH +from maxdiffusion.kernels.sol_attention import routing as sol_routing +from maxdiffusion.kernels.sol_attention import sol_attention_kernel +from maxdiffusion.models import sparse_schedule + +# Attention-level Sol-Attn keys and their defaults. `sol_num_layers` is model +# specific and supplied by the caller. +SOL_ATTENTION_DEFAULTS = { + "use_sol_attention": False, + # Standardized threshold cutoff; expected exact density ~= 1 - Phi(beta). + # beta=1.0 -> ~16% density (~84% sparsity), close to the paper's Wan setup. + "sol_beta": 1.0, + # Query block (routing granularity and kernel query tile) and key/value + # block (exact-branch granularity). Multiples of 128. + "sol_block_q": 1024, + "sol_block_kv": 256, + # Dense warm-up schedule. Explicit [start, end) intervals win over fractions. + "sol_active_start_step": -1, + "sol_active_end_step": -1, + "sol_active_start_layer": -1, + "sol_active_end_layer": -1, + "sol_dense_layer_fraction": 0.0, + "sol_dense_timestep_fraction": 0.0, + "sol_num_train_timesteps": 1000, + # Optional Mosaic VMEM limit for the kernel (None = compiler default). + "sol_vmem_limit_bytes": None, + # Print the per-call exact-branch density (debug; adds a host callback). + "sol_log_density": False, +} + + +def attention_config_from_pyconfig(config: Any, num_layers: int) -> dict[str, Any]: + """Extracts the transformer `attention_config` entries from a pipeline config. + + Pipelines merge the result into the `attention_config` they hand to the + transformer. When Sol-Attn is disabled nothing is emitted, so dense runs keep + their exact transformer config (and AOT cache identity). + """ + if not bool(getattr(config, "use_sol_attention", False)): + return {} + resolved = {"use_sol_attention": True, "sol_num_layers": int(getattr(config, "sol_num_layers", num_layers) or num_layers)} + for name in SOL_ATTENTION_DEFAULTS: + if name != "use_sol_attention" and getattr(config, name, None) is not None: + resolved[name] = getattr(config, name) + return resolved + + +def init_sol_config( + attention_config: Optional[Mapping[str, Any]], + default_num_layers: int, + is_self_attention: bool = True, +) -> dict[str, Any]: + """Resolves attention-level Sol-Attn settings from `attention_config`. + + Returns a dict keyed by `SOL_ATTENTION_DEFAULTS` plus `sol_num_layers`. Sol + only runs on self-attention with `use_sol_attention` set; otherwise the + canonical defaults are returned so unused `sol_*` overrides do not change the + module's static graph (and thus the AOT cache key). + """ + attention_config = attention_config or {} + resolved = {**SOL_ATTENTION_DEFAULTS, "sol_num_layers": default_num_layers} + if not (is_self_attention and bool(attention_config.get("use_sol_attention", False))): + return resolved + if bool(attention_config.get("use_svg_attention", False)): + raise ValueError("use_sol_attention and use_svg_attention are mutually exclusive.") + for name in resolved: + if name in attention_config and attention_config[name] is not None: + resolved[name] = attention_config[name] + resolved["use_sol_attention"] = True + # Fail at construction time instead of at trace time. + sol_routing.SolBlockSizes(block_q=int(resolved["sol_block_q"]), block_kv=int(resolved["sol_block_kv"])) + if float(resolved["sol_beta"]) < 0.0: + raise ValueError(f"sol_beta must be >= 0, got {resolved['sol_beta']}.") + return resolved + + +def is_sol_active( + sol: Any, + layer_index: int | jax.Array | None, + step_index: int | jax.Array | None, + timestep: int | float | jax.Array | None, +) -> bool | jax.Array: + """Dense warm-up schedule for an object carrying `init_sol_config` attributes.""" + return sparse_schedule.is_sparse_active( + step_index=step_index, + layer_index=layer_index, + timestep=timestep, + start_step=sol.sol_active_start_step, + end_step=sol.sol_active_end_step, + start_layer=sol.sol_active_start_layer, + end_layer=sol.sol_active_end_layer, + dense_layer_fraction=sol.sol_dense_layer_fraction, + dense_timestep_fraction=sol.sol_dense_timestep_fraction, + num_train_timesteps=sol.sol_num_train_timesteps, + num_layers=sol.sol_num_layers, + name="Sol-Attn", + ) + + +def sparse_config(sol: Any, layer_index=None, step_index=None) -> dict[str, Any]: + """The `sparse_config_override` consumed by `sol_sharded_attention`.""" + return { + "use_sol_attention": True, + "beta": float(sol.sol_beta), + "block_q": int(sol.sol_block_q), + "block_kv": int(sol.sol_block_kv), + "vmem_limit_bytes": sol.sol_vmem_limit_bytes or None, + "log_density": bool(sol.sol_log_density), + "layer_index": layer_index, + "step_index": step_index, + } + + +def apply_sol_or_dense( + sol: Any, + run_attention: Callable[..., jax.Array], + layer_index: int | jax.Array | None, + step_index: int | jax.Array | None, + timestep: int | float | jax.Array | None, + named_scope: Callable[[str], Any] = jax.named_scope, +) -> jax.Array: + """Runs Sol-Attn on active steps/layers and dense attention otherwise. + + Args: + sol: Object exposing the attributes resolved by `init_sol_config` + (typically the attention module itself). + run_attention: Calls the attention op. Invoked with no arguments for the + dense path and with a `sparse_config_override` keyword for the sparse path. + layer_index: Transformer layer index (static int or traced array). + step_index: Denoising step index (static int or traced array). + timestep: Diffusion timestep, used by fraction-based schedules. + named_scope: Context manager factory used to label the dispatch. + """ + is_active = is_sol_active(sol, layer_index, step_index, timestep) + + def run_dense(_): + return run_attention() + + def run_sparse(_): + return run_attention(sparse_config_override=sparse_config(sol, layer_index, step_index)) + + with named_scope("apply_attention"): + if isinstance(is_active, bool): + return run_sparse(None) if is_active else run_dense(None) + return jax.lax.cond(is_active, run_sparse, run_dense, operand=None) + + +def _to_bhld(x: jax.Array, heads: int) -> jax.Array: + if x.ndim == 4: + return x + b, l, hd = x.shape + return x.reshape(b, l, heads, hd // heads).transpose(0, 2, 1, 3) + + +def _sequence_axis(spec_entry) -> Optional[str]: + if spec_entry is None: + return None + if isinstance(spec_entry, str): + return spec_entry + if isinstance(spec_entry, (tuple, list)) and len(spec_entry) == 1: + return spec_entry[0] + raise ValueError(f"Sol-Attn supports sequence sharding over a single mesh axis, got {spec_entry!r}.") + + +def sol_sharded_attention(query: jax.Array, key: jax.Array, value: jax.Array, context: Mapping[str, Any]) -> jax.Array: + """Sol-Attn entry point for `attention_flax._apply_attention`. + + Accepts [B, L, H*D] or [B, H, L, D] self-attention inputs and returns + [B, L, H*D]. If the sequence is sharded over a mesh axis, heads are + exchanged with an all-to-all so every device sees full sequences for a + subset of heads (Ulysses), which is what block routing needs. + """ + cfg = context["spatiotemporal_config"] + heads = context["heads"] + mesh = context["mesh"] + if context.get("attention_mask") is not None: + raise ValueError("Sol-Attn does not support attention masks.") + if context.get("is_causal", False): + raise ValueError("Sol-Attn supports bidirectional attention only.") + + q, k, v = (_to_bhld(x, heads) for x in (query, key, value)) + if q.shape != k.shape or q.shape != v.shape: + raise ValueError(f"Sol-Attn requires matched self-attention q/k/v, got {q.shape}, {k.shape}, {v.shape}.") + + kernel = functools.partial( + sol_attention_kernel.sol_attention, + scale=context["scale"], + beta=cfg["beta"], + block_sizes=sol_routing.SolBlockSizes(block_q=cfg["block_q"], block_kv=cfg["block_kv"]), + vmem_limit_bytes=cfg.get("vmem_limit_bytes"), + return_density=True, + ) + + if mesh is None: + out, density = kernel(q, k, v) + else: + qspec = nn.logical_to_mesh_axes(context["axis_names_q"]) + kvspec = nn.logical_to_mesh_axes(context["axis_names_kv"]) + if qspec != kvspec: + raise ValueError(f"Sol-Attn requires identical q/kv sharding, got {qspec} vs {kvspec}.") + if qspec[3] is not None: + raise ValueError("Sol-Attn does not support sharding the head dimension.") + seq_axis = _sequence_axis(qspec[2]) + mesh_axes = tuple(mesh.axis_names) + + @functools.partial( + jax.shard_map, + mesh=mesh, + in_specs=(qspec, kvspec, kvspec), + out_specs=(qspec, P()), + check_vma=False, + ) + def local(q, k, v): + if seq_axis is not None: + a2a = functools.partial(jax.lax.all_to_all, axis_name=seq_axis, tiled=True) + q, k, v = (a2a(x, split_axis=1, concat_axis=2) for x in (q, k, v)) + out, density = kernel(q, k, v) + if seq_axis is not None: + out = a2a(out, split_axis=2, concat_axis=1) + return out, jax.lax.pmean(density, mesh_axes) + + out, density = local(q, k, v) + + if cfg.get("log_density", False): + jax.debug.print( + "sol_attention density={d:.4f} layer={l} step={s}", + d=density, + l=cfg.get("layer_index") if cfg.get("layer_index") is not None else -1, + s=cfg.get("step_index") if cfg.get("step_index") is not None else -1, + ) + + b, h, l, d = out.shape + out = out.transpose(0, 2, 1, 3).reshape(b, l, h * d) + return nn.with_logical_constraint(out, (BATCH, LENGTH, HEAD)) diff --git a/src/maxdiffusion/models/sparse_schedule.py b/src/maxdiffusion/models/sparse_schedule.py new file mode 100644 index 000000000..82f298492 --- /dev/null +++ b/src/maxdiffusion/models/sparse_schedule.py @@ -0,0 +1,119 @@ +""" +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Step/layer warm-up schedules shared by training-free sparse attention methods +(Sparse VideoGen, Sol-Attn, ...). Most methods keep the first denoising steps +and the first transformer layers dense and sparsify the rest. +""" + +from __future__ import annotations + +import math +from numbers import Integral + +import jax +import jax.numpy as jnp + + +def is_sparse_active( + step_index: int | jax.Array | None = None, + layer_index: int | jax.Array | None = None, + timestep: int | float | jax.Array | None = None, + start_step: int = -1, + end_step: int = -1, + start_layer: int = -1, + end_layer: int = -1, + dense_layer_fraction: float = 0.0, + dense_timestep_fraction: float = 0.0, + num_train_timesteps: int = 1000, + num_layers: int = 40, + name: str = "sparse attention", +) -> bool | jax.Array: + """Evaluates whether sparse attention is active for the current step and layer. + + Explicit `[start, end)` intervals take precedence over the fraction-based + schedules. Static Python ints resolve to a Python bool at trace time (no + `lax.cond` is emitted); traced indices yield a boolean array. + """ + has_start_step = start_step >= 0 + has_end_step = end_step >= 0 + if has_start_step != has_end_step: + raise ValueError( + f"Incomplete explicit {name} step schedule: start_step={start_step}, end_step={end_step}. " + "Both bounds must be non-negative or both unset (< 0)." + ) + has_explicit_step = has_start_step and has_end_step + + has_start_layer = start_layer >= 0 + has_end_layer = end_layer >= 0 + if has_start_layer != has_end_layer: + raise ValueError( + f"Incomplete explicit {name} layer schedule: start_layer={start_layer}, end_layer={end_layer}. " + "Both bounds must be non-negative or both unset (< 0)." + ) + has_explicit_layer = has_start_layer and has_end_layer + + if has_explicit_step and step_index is None: + raise ValueError(f"Explicit {name} step schedule requires step_index.") + if has_explicit_layer and layer_index is None: + raise ValueError(f"Explicit {name} layer schedule requires layer_index.") + + # 1. Evaluate layer interval / fraction + if has_explicit_layer: + if isinstance(layer_index, Integral): + layer_active: bool | jax.Array = bool(start_layer <= layer_index < end_layer) + else: + layer_arr = jnp.asarray(layer_index) + layer_active = jnp.logical_and(layer_arr >= start_layer, layer_arr < end_layer) + else: + dense_layer_count = math.ceil(dense_layer_fraction * num_layers) + if dense_layer_count > 0 and layer_index is not None: + if isinstance(layer_index, Integral): + layer_active = bool(layer_index >= dense_layer_count) + else: + layer_active = jnp.asarray(layer_index) >= dense_layer_count + else: + layer_active = True + + if isinstance(layer_active, bool) and not layer_active: + return False + + # 2. Evaluate step interval / fraction + if has_explicit_step: + if isinstance(step_index, Integral): + step_active: bool | jax.Array = bool(start_step <= step_index < end_step) + else: + step_arr = jnp.asarray(step_index) + step_active = jnp.logical_and(step_arr >= start_step, step_arr < end_step) + else: + if dense_timestep_fraction > 0.0 and timestep is not None: + first_sparse_timestep = (1.0 - dense_timestep_fraction) * num_train_timesteps + if isinstance(timestep, (Integral, float)): + step_active = bool(timestep < first_sparse_timestep) + else: + step_active = jnp.max(jnp.asarray(timestep)) < first_sparse_timestep + else: + step_active = True + + if isinstance(step_active, bool) and not step_active: + return False + + if isinstance(layer_active, bool) and isinstance(step_active, bool): + return bool(layer_active and step_active) + if isinstance(layer_active, bool): + return step_active + if isinstance(step_active, bool): + return layer_active + return jnp.logical_and(step_active, layer_active) diff --git a/src/maxdiffusion/models/svg_attention.py b/src/maxdiffusion/models/svg_attention.py index 98614b6c9..a23e0ab01 100644 --- a/src/maxdiffusion/models/svg_attention.py +++ b/src/maxdiffusion/models/svg_attention.py @@ -18,13 +18,15 @@ from __future__ import annotations +import functools import math -from numbers import Integral from typing import Any, Callable, Mapping, Optional, Tuple import jax import jax.numpy as jnp +from maxdiffusion.models import sparse_schedule + # Attention-level SVG keys and their defaults. `svg_num_layers` is model # specific and supplied by the caller. Pipeline-level keys such as # `svg_high_noise_density` / `svg_low_noise_density` (Wan expert selection) @@ -320,90 +322,9 @@ def svg_placement_unpermute( return jnp.where(is_temporal[:, :, None, None], out_fm, output) -def is_svg_active( - step_index: int | jax.Array | None = None, - layer_index: int | jax.Array | None = None, - timestep: int | float | jax.Array | None = None, - start_step: int = -1, - end_step: int = -1, - start_layer: int = -1, - end_layer: int = -1, - dense_layer_fraction: float = 0.0, - dense_timestep_fraction: float = 0.0, - num_train_timesteps: int = 1000, - num_layers: int = 40, -) -> bool | jax.Array: - """Evaluate whether SVG attention is active for the current step and layer.""" - has_start_step = start_step >= 0 - has_end_step = end_step >= 0 - if has_start_step != has_end_step: - raise ValueError( - f"Incomplete explicit SVG step schedule: start_step={start_step}, end_step={end_step}. " - "Both bounds must be non-negative or both unset (< 0)." - ) - has_explicit_step = has_start_step and has_end_step - - has_start_layer = start_layer >= 0 - has_end_layer = end_layer >= 0 - if has_start_layer != has_end_layer: - raise ValueError( - f"Incomplete explicit SVG layer schedule: start_layer={start_layer}, end_layer={end_layer}. " - "Both bounds must be non-negative or both unset (< 0)." - ) - has_explicit_layer = has_start_layer and has_end_layer - - if has_explicit_step and step_index is None: - raise ValueError("Explicit SVG step schedule requires step_index.") - if has_explicit_layer and layer_index is None: - raise ValueError("Explicit SVG layer schedule requires layer_index.") - - # 1. Evaluate layer interval / fraction - if has_explicit_layer: - if isinstance(layer_index, Integral): - layer_active: bool | jax.Array = bool(start_layer <= layer_index < end_layer) - else: - layer_arr = jnp.asarray(layer_index) - layer_active = jnp.logical_and(layer_arr >= start_layer, layer_arr < end_layer) - else: - dense_layer_count = math.ceil(dense_layer_fraction * num_layers) - if dense_layer_count > 0 and layer_index is not None: - if isinstance(layer_index, Integral): - layer_active = bool(layer_index >= dense_layer_count) - else: - layer_active = jnp.asarray(layer_index) >= dense_layer_count - else: - layer_active = True - - if isinstance(layer_active, bool) and not layer_active: - return False - - # 2. Evaluate step interval / fraction - if has_explicit_step: - if isinstance(step_index, Integral): - step_active: bool | jax.Array = bool(start_step <= step_index < end_step) - else: - step_arr = jnp.asarray(step_index) - step_active = jnp.logical_and(step_arr >= start_step, step_arr < end_step) - else: - if dense_timestep_fraction > 0.0 and timestep is not None: - first_sparse_timestep = (1.0 - dense_timestep_fraction) * num_train_timesteps - if isinstance(timestep, (Integral, float)): - step_active = bool(timestep < first_sparse_timestep) - else: - step_active = jnp.max(jnp.asarray(timestep)) < first_sparse_timestep - else: - step_active = True - - if isinstance(step_active, bool) and not step_active: - return False - - if isinstance(layer_active, bool) and isinstance(step_active, bool): - return bool(layer_active and step_active) - if isinstance(layer_active, bool): - return step_active - if isinstance(step_active, bool): - return layer_active - return jnp.logical_and(step_active, layer_active) +# The schedule is shared with other sparse attention methods; keep the +# SVG-specific name (and error messages) for existing callers. +is_svg_active = functools.partial(sparse_schedule.is_sparse_active, name="SVG") def place_sequence_for_mask( diff --git a/src/maxdiffusion/pipelines/wan/wan_pipeline.py b/src/maxdiffusion/pipelines/wan/wan_pipeline.py index 4006e2fd1..c9e7d6782 100644 --- a/src/maxdiffusion/pipelines/wan/wan_pipeline.py +++ b/src/maxdiffusion/pipelines/wan/wan_pipeline.py @@ -33,6 +33,7 @@ from ... import max_logging from ... import max_utils from ...max_utils import get_flash_block_sizes, get_precision, device_put_replicated +from ...models import sol_attention from ...models.wan.wan_utils import load_wan_transformer, load_wan_vae from ...models.wan.transformers.transformer_wan import WanModel from ...models.wan.autoencoder_kl_wan import AutoencoderKLWan, AutoencoderKLWanCache @@ -393,6 +394,7 @@ def create_model(rngs: nnx.Rngs, wan_config: dict): "svg_high_noise_density": high_density, "svg_low_noise_density": low_density, "svg_flash_block_sizes": getattr(config, "svg_flash_block_sizes", None) or None, + **sol_attention.attention_config_from_pyconfig(config, wan_config.get("num_layers", 40)), } # 2. eval_shape - will not use flops or create weights on device diff --git a/src/maxdiffusion/tests/sol_attention_test.py b/src/maxdiffusion/tests/sol_attention_test.py new file mode 100644 index 000000000..cac3a247e --- /dev/null +++ b/src/maxdiffusion/tests/sol_attention_test.py @@ -0,0 +1,324 @@ +""" +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Tests for Sol-Attn (arXiv:2607.24027): routing math, the dense reference, the +Pallas TPU kernel, sharded execution, and model/pipeline config plumbing. +""" + +from types import SimpleNamespace + +import flax.linen as nn +import jax +import jax.numpy as jnp +import numpy as np +import pytest +from jax.sharding import Mesh, NamedSharding, PartitionSpec as P +from scipy.stats import norm + +from maxdiffusion import aot_cache +from maxdiffusion import common_types as ct +from maxdiffusion.kernels.sol_attention import reference +from maxdiffusion.kernels.sol_attention import routing +from maxdiffusion.kernels.sol_attention import sol_attention_kernel +from maxdiffusion.models import sol_attention + +ON_TPU = jax.devices()[0].platform == "tpu" +requires_tpu = pytest.mark.skipif(not ON_TPU, reason="Pallas TPU kernel requires a TPU.") + + +def _rel_l2(actual, expected): + actual = np.asarray(actual, np.float32) + expected = np.asarray(expected, np.float32) + return float(np.linalg.norm(actual - expected) / np.linalg.norm(expected)) + + +def _video_like_qkv(heads, seq_len, head_dim, *, seed=0, smooth=64, noise=0.5, dtype=jnp.float32): + """Tokens that are locally correlated (like video latents) so pooling matters.""" + k1, k2, k3, k4 = jax.random.split(jax.random.PRNGKey(seed), 4) + base = jax.random.normal(k1, (heads, seq_len // smooth + 1, head_dim)) + base = jnp.repeat(base, smooth, axis=1)[:, :seq_len] + q = base + noise * jax.random.normal(k2, (heads, seq_len, head_dim)) + k = base + noise * jax.random.normal(k3, (heads, seq_len, head_dim)) + v = jax.random.normal(k4, (heads, seq_len, head_dim)) + return q.astype(dtype), k.astype(dtype), v.astype(dtype) + + +# --- Routing ----------------------------------------------------------------- + + +def test_block_sizes_validation(): + with pytest.raises(ValueError, match="multiple of 128"): + routing.SolBlockSizes(block_q=96) + assert routing.SolBlockSizes(block_q=256, block_kv=384).seq_multiple == 768 + assert routing.padded_length(1000, routing.SolBlockSizes(256, 128)) == 1024 + + +def test_pool_blocks_ignores_padding(): + x = jnp.arange(5 * 2, dtype=jnp.float32).reshape(1, 5, 2) + # Pad 5 tokens to 8 with block 4: block 0 averages tokens 0..3, block 1 only token 4. + pooled = routing.pool_blocks(routing.pad_sequence(x, 8), seq_len=5, block=4) + np.testing.assert_allclose(pooled[0, 0], x[0, :4].mean(axis=0)) + np.testing.assert_allclose(pooled[0, 1], x[0, 4]) + np.testing.assert_array_equal(routing.block_counts(5, 8, 4), [4, 1]) + + +def test_threshold_matches_pooled_key_moments(): + """Row mean/std equal the pooled-key moment form of Eq. (5).""" + rng = np.random.default_rng(0) + q_pooled = rng.normal(size=(3, 7, 16)) + k_pooled = rng.normal(size=(3, 11, 16)) + scale = 0.25 + scores = routing.block_proxy_scores(jnp.asarray(q_pooled), jnp.asarray(k_pooled), scale) + _, tau = routing.threshold_route(scores, jnp.full((11,), 128), beta=1.3) + + qs = q_pooled * scale + mean_k = k_pooled.mean(axis=1) # [H, d] + second_k = np.einsum("hkd,hke->hde", k_pooled, k_pooled) / k_pooled.shape[1] # [H, d, d] + mu = np.einsum("hqd,hd->hq", qs, mean_k) + var = np.einsum("hqd,hde,hqe->hq", qs, second_k, qs) - mu**2 + np.testing.assert_allclose(tau, mu + 1.3 * np.sqrt(var), rtol=1e-5, atol=1e-5) + + +def test_threshold_density_tracks_gaussian_tail(): + """For Gaussian proxy logits the selected density is ~ 1 - Phi(beta) (Fig. 3).""" + scores = jax.random.normal(jax.random.PRNGKey(1), (4, 64, 2048)) + counts = jnp.full((2048,), 128) + for beta in (0.5, 1.0, 1.5): + selected, _ = routing.threshold_route(scores, counts, beta) + assert abs(float(selected.mean()) - (1.0 - norm.cdf(beta))) < 0.01 + + +def test_threshold_skips_padding_and_honors_always_exact(): + scores = jnp.array([[[5.0, 0.0, 0.0, 9.0]]]) + counts = jnp.array([128, 128, 128, 0]) # last block is pure padding + selected, _ = routing.threshold_route(scores, counts, beta=1.0) + np.testing.assert_array_equal(selected[0, 0], [True, False, False, False]) + always = jnp.array([False, True, False, True]) + selected, _ = routing.threshold_route(scores, counts, beta=1.0, always_exact=always) + np.testing.assert_array_equal(selected[0, 0], [True, True, False, False]) + + +def test_compact_selection_layout(): + selected = jnp.array([[[False, True, False, True, True], [False] * 5]]) + rows = routing.compact_selection(selected, row_width=8) + assert rows.shape == (1, 2, 8) + np.testing.assert_array_equal(rows[0, 0, :4], [3, 1, 3, 4]) + assert int(rows[0, 1, 0]) == 0 + + +# --- Dense reference --------------------------------------------------------- + + +def test_reference_selecting_everything_is_dense(): + q, k, v = _video_like_qkv(2, 300, 32) + bs = routing.SolBlockSizes(128, 128) + nq = nk = routing.padded_length(300, bs) // 128 + out = reference.sol_attention_reference( + q, k, v, scale=32**-0.5, beta=0.0, block_sizes=bs, selected=jnp.ones((2, nq, nk), bool) + ) + np.testing.assert_allclose(out, reference.dense_attention_reference(q, k, v, scale=32**-0.5), atol=1e-5) + + +def test_zeroth_order_term_is_exact_for_constant_key_blocks(): + """With constant keys per block, Eqs. (9)-(10) reproduce dense attention even + when every block is approximated, including the partial last block.""" + seq_len, block = 300, 128 + rng = np.random.default_rng(3) + block_keys = rng.normal(size=(2, 3, 16)).astype(np.float32) + k = jnp.asarray(np.repeat(block_keys, block, axis=1)[:, :seq_len]) + q = jnp.asarray(rng.normal(size=(2, seq_len, 16)).astype(np.float32)) + v = jnp.asarray(rng.normal(size=(2, seq_len, 16)).astype(np.float32)) + bs = routing.SolBlockSizes(128, 128) + out = reference.sol_attention_reference(q, k, v, scale=0.3, beta=0.0, block_sizes=bs, selected=jnp.zeros((2, 3, 3), bool)) + np.testing.assert_allclose(out, reference.dense_attention_reference(q, k, v, scale=0.3), atol=1e-5) + + +def test_approximate_correction_beats_keep_or_drop(): + """Same routing, with and without the pooled correction (Fig. 9).""" + q, k, v = _video_like_qkv(2, 1024, 64, seed=5, noise=0.3) + q = q * 3.0 # sharper attention, so dropping blocks actually hurts + dense = reference.dense_attention_reference(q, k, v, scale=64**-0.5) + bs = routing.SolBlockSizes(128, 128) + for beta in (0.5, 1.0, 1.5): + sol = reference.sol_attention_reference(q, k, v, scale=64**-0.5, beta=beta, block_sizes=bs) + drop = reference.sol_attention_reference(q, k, v, scale=64**-0.5, beta=beta, block_sizes=bs, approximate=False) + assert _rel_l2(sol, dense) < _rel_l2(drop, dense) + + +# --- Pallas kernel ----------------------------------------------------------- + + +@requires_tpu +@pytest.mark.parametrize( + "heads,seq_len,block_q,block_kv,beta", + [ + (2, 1000, 128, 128, 1.0), + (3, 3000, 256, 128, 1.0), + (2, 4100, 256, 256, 0.5), + (2, 2048, 512, 128, 1.5), + ], +) +def test_kernel_matches_reference(heads, seq_len, block_q, block_kv, beta): + q, k, v = _video_like_qkv(heads, seq_len, 128, dtype=jnp.bfloat16) + bs = routing.SolBlockSizes(block_q, block_kv) + scale = 128**-0.5 + padded = routing.padded_length(seq_len, bs) + qp, kp, vp = (routing.pad_sequence(x, padded) for x in (q, k, v)) + # Route once and feed both implementations so threshold ties cannot differ. + route = routing.compute_routing(qp, kp, vp, seq_len=seq_len, scale=scale, beta=beta, block_sizes=bs) + out = sol_attention_kernel.sol_attention_from_routing(qp, kp, vp, route, seq_len=seq_len, scale=scale, block_sizes=bs) + want = reference.sol_attention_reference(q, k, v, scale=scale, beta=beta, block_sizes=bs, selected=route.selected) + assert 0.0 < float(route.density()) < 1.0 + assert _rel_l2(out[:, :seq_len], want) < 1e-2 + + +@requires_tpu +@pytest.mark.parametrize("group_size,num_buffers", [(1, 2), (2, 4), (3, 6), (4, 12)]) +def test_kernel_pipelining_variants(group_size, num_buffers): + """Grouped hot loop, leftover blocks, and the ragged tail epilogue.""" + q, k, v = _video_like_qkv(2, 2500, 128, seed=3, dtype=jnp.bfloat16) + bs = routing.SolBlockSizes(256, 128) + padded = routing.padded_length(2500, bs) + qp, kp, vp = (routing.pad_sequence(x, padded) for x in (q, k, v)) + route = routing.compute_routing(qp, kp, vp, seq_len=2500, scale=0.09, beta=0.3, block_sizes=bs) + # Force the ragged tail block into every row so the epilogue always runs. + sel = route.selected.at[:, :, 2500 // 128].set(True) + route = routing.SolRouting(sel, route.tau, route.kv_counts, route.k_pooled, route.v_pooled) + out = sol_attention_kernel.sol_attention_from_routing( + qp, kp, vp, route, seq_len=2500, scale=0.09, block_sizes=bs, group_size=group_size, num_buffers=num_buffers + ) + want = reference.sol_attention_reference(q, k, v, scale=0.09, beta=0.3, block_sizes=bs, selected=sel) + assert _rel_l2(out[:, :2500], want) < 1e-2 + + +def test_invalid_pipelining_is_rejected(): + with pytest.raises(ValueError, match="multiple of group_size"): + sol_attention_kernel._resolve_pipelining(128, group_size=4, num_buffers=6) # pylint: disable=protected-access + assert sol_attention_kernel._resolve_pipelining(256, None, None) == (4, 8) # pylint: disable=protected-access + + +@requires_tpu +def test_kernel_handles_empty_and_full_selection(): + q, k, v = _video_like_qkv(1, 700, 128, dtype=jnp.bfloat16) + bs = routing.SolBlockSizes(128, 128) + padded = routing.padded_length(700, bs) + qp, kp, vp = (routing.pad_sequence(x, padded) for x in (q, k, v)) + route = routing.compute_routing(qp, kp, vp, seq_len=700, scale=0.1, beta=1.0, block_sizes=bs) + for fill in (False, True): + sel = jnp.full_like(route.selected, fill) + forced = routing.SolRouting(sel, route.tau, route.kv_counts, route.k_pooled, route.v_pooled) + out = sol_attention_kernel.sol_attention_from_routing(qp, kp, vp, forced, seq_len=700, scale=0.1, block_sizes=bs) + want = reference.sol_attention_reference(q, k, v, scale=0.1, beta=1.0, block_sizes=bs, selected=sel) + assert _rel_l2(out[:, :700], want) < 1e-2 + + +@requires_tpu +def test_sharded_ulysses_matches_single_device(): + devices = jax.devices() + if len(devices) < 2: + pytest.skip("Requires multiple TPU devices.") + mesh = Mesh(np.array(devices).reshape(1, -1), ("data", "context")) + cp = len(devices) + batch, heads, seq_len, head_dim = 2, 2 * cp, 128 * cp * 3, 128 + rng = jax.random.PRNGKey(7) + q, k, v = (jax.random.normal(r, (batch, heads, seq_len, head_dim), jnp.bfloat16) for r in jax.random.split(rng, 3)) + cfg = sol_attention.sparse_config( + SimpleNamespace( + sol_beta=1.0, + sol_block_q=256, + sol_block_kv=128, + sol_vmem_limit_bytes=None, + sol_log_density=False, + ) + ) + context = { + "spatiotemporal_config": cfg, + "heads": heads, + "mesh": mesh, + "scale": head_dim**-0.5, + "attention_mask": None, + "axis_names_q": (ct.BATCH, ct.SELF_ATTN_HEAD, ct.SELF_ATTN_Q_LENGTH, ct.D_KV), + "axis_names_kv": (ct.BATCH, ct.SELF_ATTN_HEAD, ct.SELF_ATTN_KV_LENGTH, ct.D_KV), + } + rules = tuple(tuple(r) for r in ct.ULYSSES_ATTENTION_AXIS_RULES) + ( + ("activation_batch", "data"), + ("activation_length", "context"), + ("activation_heads", None), + ) + spec = NamedSharding(mesh, P("data", None, "context", None)) + with mesh, nn.logical_axis_rules(rules): + sharded = jax.jit(lambda q, k, v: sol_attention.sol_sharded_attention(q, k, v, context))( + *(jax.device_put(x, spec) for x in (q, k, v)) + ) + local = sol_attention_kernel.sol_attention( + q, k, v, scale=head_dim**-0.5, beta=1.0, block_sizes=routing.SolBlockSizes(256, 128) + ) + local = local.transpose(0, 2, 1, 3).reshape(batch, seq_len, heads * head_dim) + np.testing.assert_allclose(np.asarray(sharded, np.float32), np.asarray(local, np.float32), atol=2e-2, rtol=2e-2) + + +# --- Config plumbing --------------------------------------------------------- + + +def test_init_sol_config_defaults_when_disabled(): + cfg = sol_attention.init_sol_config({"sol_beta": 2.0}, default_num_layers=30) + assert cfg["use_sol_attention"] is False + assert cfg["sol_beta"] == sol_attention.SOL_ATTENTION_DEFAULTS["sol_beta"] + assert cfg["sol_num_layers"] == 30 + # Cross-attention never uses Sol. + cfg = sol_attention.init_sol_config({"use_sol_attention": True}, 30, is_self_attention=False) + assert cfg["use_sol_attention"] is False + + +def test_init_sol_config_validation(): + cfg = sol_attention.init_sol_config({"use_sol_attention": True, "sol_beta": 1.5, "sol_block_q": 512}, 40) + assert cfg["use_sol_attention"] and cfg["sol_beta"] == 1.5 and cfg["sol_block_q"] == 512 + with pytest.raises(ValueError, match="mutually exclusive"): + sol_attention.init_sol_config({"use_sol_attention": True, "use_svg_attention": True}, 40) + with pytest.raises(ValueError, match="multiple of 128"): + sol_attention.init_sol_config({"use_sol_attention": True, "sol_block_kv": 64}, 40) + + +def test_apply_sol_or_dense_follows_warmup_schedule(): + sol = SimpleNamespace(**sol_attention.init_sol_config({"use_sol_attention": True}, 40)) + sol.sol_active_start_step, sol.sol_active_end_step = 10, 50 + sol.sol_active_start_layer, sol.sol_active_end_layer = 1, 40 + calls = [] + + def run_attention(**kwargs): + calls.append(kwargs.get("sparse_config_override")) + return jnp.zeros(()) + + sol_attention.apply_sol_or_dense(sol, run_attention, layer_index=0, step_index=20, timestep=None) + sol_attention.apply_sol_or_dense(sol, run_attention, layer_index=5, step_index=5, timestep=None) + sol_attention.apply_sol_or_dense(sol, run_attention, layer_index=5, step_index=20, timestep=None) + assert calls[0] is None and calls[1] is None + assert calls[2]["use_sol_attention"] and calls[2]["beta"] == 1.0 + with pytest.raises(ValueError, match="Incomplete explicit Sol-Attn step schedule"): + sol.sol_active_end_step = -1 + sol_attention.apply_sol_or_dense(sol, run_attention, layer_index=5, step_index=20, timestep=None) + + +def test_pipeline_and_aot_plumbing_is_inert_when_disabled(): + dense = SimpleNamespace(use_sol_attention=False, sol_beta=1.0) + assert not sol_attention.attention_config_from_pyconfig(dense, 40) + assert not aot_cache.extract_sol_meta(dense) + enabled = SimpleNamespace(use_sol_attention=True, sol_beta=0.8, sol_block_q=512, sol_active_start_step=10) + attn_cfg = sol_attention.attention_config_from_pyconfig(enabled, 40) + assert attn_cfg["use_sol_attention"] and attn_cfg["sol_beta"] == 0.8 and attn_cfg["sol_num_layers"] == 40 + assert sol_attention.init_sol_config(attn_cfg, 40)["sol_block_q"] == 512 + meta_a = aot_cache.extract_sol_meta(enabled) + meta_b = aot_cache.extract_sol_meta(SimpleNamespace(**{**vars(enabled), "sol_beta": 1.2})) + assert meta_a and meta_a != meta_b diff --git a/src/maxdiffusion/tests/wan/sol_config_propagation_test.py b/src/maxdiffusion/tests/wan/sol_config_propagation_test.py new file mode 100644 index 000000000..d3f729945 --- /dev/null +++ b/src/maxdiffusion/tests/wan/sol_config_propagation_test.py @@ -0,0 +1,110 @@ +""" +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Configuration propagation tests for Sol-Attn in the Wan transformer. +""" + +from types import SimpleNamespace +import unittest + +from flax import nnx +import jax +import numpy as np +from jax.sharding import Mesh + +from maxdiffusion.pipelines.wan.wan_pipeline import create_sharded_logical_transformer + +_WAN_CONFIG = { + "num_layers": 2, + "num_attention_heads": 2, + "attention_head_dim": 32, + "in_channels": 16, + "patch_size": (1, 2, 2), + "text_dim": 64, + "freq_dim": 64, + "ffn_dim": 64, + "eps": 1e-6, +} + +_COMMON = { + "precision": "DEFAULT", + "flash_block_sizes": {}, + "activations_dtype": "bfloat16", + "weights_dtype": "bfloat16", + "attention": "dot_product", + "remat_policy": "none", + "names_which_can_be_saved": [], + "names_which_can_be_offloaded": [], + "flash_min_seq_length": 0, + "dropout": 0.0, + "mask_padding_tokens": False, + "scan_layers": False, + "enable_jax_named_scopes": False, + "use_base2_exp": False, + "use_experimental_scheduler": False, + "logical_axis_rules": (), + "model_type": "T2V", + "model_name": "wan2.1", +} + + +def _build(**extra): + devices = np.array(jax.devices()[:1]).reshape((1, 1)) + return create_sharded_logical_transformer( + devices_array=devices, + mesh=Mesh(devices, ("data", "fsdp")), + rngs=nnx.Rngs(0), + config=SimpleNamespace(**_COMMON, **extra), + restored_checkpoint={"wan_config": dict(_WAN_CONFIG), "wan_state": {}}, + subfolder="transformer", + ) + + +class SolConfigPropagationTest(unittest.TestCase): + + def test_sol_settings_reach_self_attention_only(self): + model = _build( + use_sol_attention=True, + sol_beta=1.25, + sol_block_q=1024, + sol_block_kv=256, + sol_active_start_step=10, + sol_active_end_step=50, + sol_active_start_layer=1, + sol_active_end_layer=40, + ) + attn1 = model.blocks[0].attn1 + self.assertTrue(attn1.use_sol_attention) + self.assertFalse(attn1.use_svg_attention) + self.assertEqual(attn1.sol_beta, 1.25) + self.assertEqual((attn1.sol_block_q, attn1.sol_block_kv), (1024, 256)) + self.assertEqual((attn1.sol_active_start_step, attn1.sol_active_end_step), (10, 50)) + self.assertEqual((attn1.sol_active_start_layer, attn1.sol_active_end_layer), (1, 40)) + self.assertEqual(attn1.sol_num_layers, 2) + # Cross-attention stays dense, as in the paper. + self.assertFalse(model.blocks[0].attn2.use_sol_attention) + + def test_dense_config_is_untouched(self): + model = _build(use_sol_attention=False, sol_beta=3.0) + self.assertFalse(model.blocks[0].attn1.use_sol_attention) + self.assertNotIn("sol_beta", model.config.attention_config) + + def test_sol_and_svg_are_mutually_exclusive(self): + with self.assertRaisesRegex(ValueError, "mutually exclusive"): + _build(use_sol_attention=True, use_svg_attention=True, svg_spatial_density=0.25) + + +if __name__ == "__main__": + unittest.main() From 01c500c6165373e9fa3d2697ed71b47297e60960 Mon Sep 17 00:00:00 2001 From: Jitendra Jalwaniya Date: Tue, 20 Oct 2026 10:26:13 +0000 Subject: [PATCH 2/2] sol_attention: fused pooling, unpadded kernel, fixed softmax offset - kernels/sol_attention/pooling.py: one Pallas pass computes Q/K/V block means and key block radii straight from the unpadded [H, L, d] layout (XLA's reshape-and-mean cost ~0.4 ms per tensor). Routing: 2.3 -> 0.65 ms. - Kernel reads unpadded tensors: the ragged tail key block is fetched as the last block_kv rows and masked, the query scale is applied on chip, and the output is transposed on chip, removing the pad, scale, swapaxes, and slice copies around the kernel. - Fixed softmax offset m = max(m_est, ub - 80) per query, where m_est (the largest pooled score) rules out underflow and ub (pooled score plus ||q|| * key block radius) rules out f32 overflow. The hot loop has no max reduction or rescale (kernel 16.7 -> 13.7 ms per chip). Query tiles whose bounds are more than 120 apart (e.g. an outlier key) fall back to the online softmax. - models/sol_attention: skip the density all-reduce unless sol_log_density, and chunk heads by ulysses_attention_chunks so XLA can overlap one chunk's all-to-all with another's kernel (neutral on v6e-8). - Tests: fused pooling, unpadded path, outlier keys, chunked sharding. Wan 2.1 14B T2V 720p, 81 frames, 50 steps on v6e-8 (ulysses_custom, data=2 x context=4, async all-to-all flags): 229.6s dense -> 147.1s (1.56x, was 1.44x) at the default 1024x256 blocks; PSNR 20.18 dB / SSIM 0.704 vs. dense (unchanged). Self-attention layer: 82.5 -> 22.8 ms. --- docs/sol_attention.md | 73 +++-- .../kernels/sol_attention/pooling.py | 116 ++++++++ .../kernels/sol_attention/routing.py | 71 ++++- .../sol_attention/sol_attention_kernel.py | 258 ++++++++++++------ src/maxdiffusion/models/sol_attention.py | 53 +++- src/maxdiffusion/tests/sol_attention_test.py | 54 +++- 6 files changed, 494 insertions(+), 131 deletions(-) create mode 100644 src/maxdiffusion/kernels/sol_attention/pooling.py diff --git a/docs/sol_attention.md b/docs/sol_attention.md index 476a91f85..65f5e8c7f 100644 --- a/docs/sol_attention.md +++ b/docs/sol_attention.md @@ -38,32 +38,41 @@ case an all-to-all gives each device full sequences for a subset of heads. ## Results on TPU v6e-8 Wan 2.1 14B T2V, 720p, 81 frames, 50 steps, `attention=ulysses_custom`, -`ici_data_parallelism=2 ici_context_parallelism=4`, one prompt, seed 12345. -The dense warm-up follows the paper (steps `[0, 10)` and layer 0 dense). PSNR -and SSIM compare each video with the dense video, as in Table 1 of the paper -(the paper reports PSNR 20.59 / SSIM 0.68 at 85% sparsity on H100). - -| Config | Exact density | Generation time | Speedup | PSNR | SSIM | -|---|---|---|---|---|---| -| dense | 100% | 231.3 s | 1.00x | – | – | -| `sol_block_q=1024 sol_block_kv=256 sol_beta=1.0` (default) | ~17% | 160.6 s | 1.44x | 20.20 dB | 0.702 | -| `sol_block_q=1024 sol_block_kv=256 sol_beta=0.8` | – | 170.5 s | 1.36x | 20.60 dB | 0.735 | -| `sol_block_q=512 sol_block_kv=256 sol_beta=1.0` | – | 175.0 s | 1.32x | 20.99 dB | 0.755 | -| `sol_block_q=2048 sol_block_kv=512 sol_beta=1.0` | ~16% | 150.4 s* | 1.54x | 18.55 dB | 0.633 | - -\* measured with `sol_log_density=True`, which adds a host callback per layer. - -Per self-attention layer (all 8 chips, synthetic video-like inputs at the -same shapes), dense `ulysses_custom` takes 83.7 ms and Sol-Attn 36.6 ms at -`1024x256` (2.3x) and 23.4 ms at `2048x512` (3.6x). About 13 ms of each -layer is shared overhead (Ulysses all-to-alls and layout copies), so -end-to-end gains are bounded by Amdahl's law and the 20% dense warm-up. +`ici_data_parallelism=2 ici_context_parallelism=4`, one prompt, seed 12345, +with the async all-to-all flags below. The dense warm-up follows the paper +(steps `[0, 10)` and layer 0 dense). PSNR and SSIM compare each video with the +dense video, as in Table 1 of the paper (the paper reports PSNR 20.59 / SSIM +0.68 at 85% sparsity on H100). + +```bash +export LIBTPU_INIT_ARGS="--xla_tpu_enable_async_all_to_all=true --xla_tpu_overlap_compute_collective_tc=true \ + --xla_tpu_enable_async_collective_fusion=true --xla_tpu_enable_async_collective_fusion_multiple_steps=true" +``` + +| Config | Generation time | Speedup | PSNR | SSIM | +|---|---|---|---|---| +| dense | 229.6 s | 1.00x | – | – | +| `sol_block_q=1024 sol_block_kv=256` (default) | 147.1 s | 1.56x | 20.18 dB | 0.704 | +| `sol_block_q=2048 sol_block_kv=256` | 138.4 s | 1.66x | 18.49 dB | 0.633 | +| default + `ulysses_attention_chunks=2` | 146.9 s | 1.56x | 20.28 dB | 0.708 | +| default + `sol_active_start_step=5` (10% dense warm-up) | 136.6 s | 1.68x | 17.01 dB | 0.587 | + +Shortening the dense warm-up below the paper's 20% buys speed at a clear +fidelity cost. + +The measured exact-block density on Wan at `sol_beta=1.0` is about 16-17%. + +Per self-attention layer (all 8 chips, synthetic video-like inputs at the same +shapes), dense `ulysses_custom` takes 82.5 ms and Sol-Attn 22.8 ms at +`1024x256` (3.6x). Of the 22.8 ms, the kernel is ~14 ms and the Ulysses +exchange (all-to-alls plus XLA layout copies, the same as for dense) ~8 ms. ## Code layout ``` kernels/sol_attention/ routing.py pooling, thresholds (Eq. 3-5), index compaction (pure JAX) + pooling.py fused single-pass Pallas pooling of Q/K/V (+ key block radii) sol_attention_kernel.py Pallas TPU kernel + `sol_attention(q, k, v, ...)` entry point reference.py dense O(L^2) oracle of Eqs. (9)-(10), for tests models/ @@ -97,9 +106,27 @@ places, each driven by measurements on v6e: * **Larger blocks than the paper's 64 x 64.** MXU efficiency and per-block overheads favor 512-2048 query tokens and 256-512 key tokens. * **Grouped hot loop.** Selected blocks land in contiguous ring slots, so the - loop fuses several of them into one ~1024-key score tile (one softmax - rescale per tile), keeping routing at `sol_block_kv` granularity. This - alone is 10-25% faster for 128-256 token blocks. + loop fuses several of them into one ~1024-key score tile, keeping routing + at `sol_block_kv` granularity. This alone is 10-25% faster for 128-256 + token blocks. +* **Fixed softmax offset.** Instead of a running max, each query uses + `m = max(m_est, ub - 80)` (log2 units): `m_est` (the largest pooled score, + a lower bound of the true max) rules out underflow, and + `ub = max_j (q . mean(K_j) + ||q|| * radius_j)` (an upper bound of every + token score, with per-block key radii from the pooling kernel) rules out + f32 overflow. The hot loop then has no max reduction and no accumulator + rescale (~20% faster). Query tiles whose bounds are too far apart (e.g. an + outlier key inflating a block radius) fall back to the online softmax. +* **Fused pooling, no padding.** One Pallas pass computes the Q/K/V block + means and key radii straight from the unpadded `[H, L, d]` layout (XLA's + reshape-and-mean cost ~0.4 ms per tensor). The attention kernel reads + unpadded tensors too: the ragged tail key block is fetched as the last + `block_kv` rows and masked, and the output is transposed on chip, so no + pad/slice/transpose copies remain around the kernel. +* **Head-chunked exchange.** `ulysses_attention_chunks` splits heads into + chunks so XLA can overlap one chunk's all-to-all with another's kernel. + On v6e-8 this is neutral (the exchange is dominated by layout copies), but + it may help on slower interconnects. ## Extending to other models diff --git a/src/maxdiffusion/kernels/sol_attention/pooling.py b/src/maxdiffusion/kernels/sol_attention/pooling.py new file mode 100644 index 000000000..fbd8f9058 --- /dev/null +++ b/src/maxdiffusion/kernels/sol_attention/pooling.py @@ -0,0 +1,116 @@ +""" +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Fused single-pass block pooling of Q, K and V for Sol-Attn on TPU. + +XLA lowers the reshape-and-mean pooling of `routing.pool_blocks` to a relayout +copy plus a reduction per tensor (~0.4 ms each for Wan 2.1 14B shapes on v6e). +This kernel reads each tensor once, straight from the unpadded [H, L, d] +layout, and writes the three block means. +""" + +from __future__ import annotations + +import functools +import math + +import jax +import jax.numpy as jnp +from jax import lax +from jax.experimental import pallas as pl +from jax.experimental.pallas import tpu as pltpu + +NUM_SUBLANES = 8 + + +def slab_rows(block_q: int, block_kv: int) -> int: + """Rows pooled per grid step; each output gets >= 8 pooled rows (sublane tiling).""" + return math.lcm(NUM_SUBLANES * block_q, NUM_SUBLANES * block_kv) + + +def _pool_kernel( + q_ref, k_ref, v_ref, qo_ref, ko_ref, vo_ref, kr_ref, *, seq_len: int, tile: int, block_q: int, block_kv: int +): + """Block means of one `tile`-row slab of q (block_q) and k/v (block_kv), plus key block radii.""" + start = pl.program_id(1) * tile + rows = start + lax.broadcasted_iota(jnp.int32, (tile, 1), 0) + valid = rows < seq_len # The last slab may run past the sequence end. + + def pool(x_ref, out_ref, block): + x = jnp.where(valid, x_ref[...].astype(jnp.float32), 0.0) + sums = x.reshape(tile // block, block, x.shape[-1]).sum(axis=1) + block_start = start + lax.broadcasted_iota(jnp.int32, (tile // block, 1), 0) * block + counts = jnp.clip(seq_len - block_start, 1, block).astype(jnp.float32) + mean = sums / counts + out_ref[...] = mean + return x, mean + + pool(q_ref, qo_ref, block_q) + k, k_mean = pool(k_ref, ko_ref, block_kv) + pool(v_ref, vo_ref, block_kv) + dev = k.reshape(tile // block_kv, block_kv, k.shape[-1]) - k_mean[:, None, :] + dist2 = jnp.where(valid.reshape(tile // block_kv, block_kv, 1), dev * dev, 0.0).sum(axis=2, keepdims=True) + kr_ref[...] = jnp.sqrt(dist2.max(axis=1)) + + +def pool_qkv( + q: jax.Array, + k: jax.Array, + v: jax.Array, + *, + block_q: int, + block_kv: int, + interpret: bool = False, +) -> tuple[jax.Array, jax.Array, jax.Array, jax.Array]: + """Means of q over block_q tokens and of k, v over block_kv tokens. + + Args: + q, k, v: Unpadded [H, L, d] tensors. + block_q, block_kv: Pooling block sizes (multiples of 128). + interpret: Run in Pallas interpret mode. + + Returns: + float32 (q means [H, ceil(L/block_q), d], k and v means [H, ceil(L/block_kv), d], + k radii [H, ceil(L/block_kv)]; see `routing.block_radius`). A ragged last + block averages only its real tokens. + """ + num_heads, seq_len, head_dim = q.shape + tile = slab_rows(block_q, block_kv) + if seq_len < tile: + raise ValueError(f"pool_qkv needs at least {tile} tokens, got {seq_len}.") + nq, nk = -(-seq_len // block_q), -(-seq_len // block_kv) + in_spec = pl.BlockSpec((None, tile, head_dim), lambda h, i: (h, i, 0)) + kernel = functools.partial(_pool_kernel, seq_len=seq_len, tile=tile, block_q=block_q, block_kv=block_kv) + q_pooled, k_pooled, v_pooled, k_radius = pl.pallas_call( + kernel, + grid=(num_heads, -(-seq_len // tile)), + in_specs=[in_spec, in_spec, in_spec], + out_specs=[ + pl.BlockSpec((None, tile // block_q, head_dim), lambda h, i: (h, i, 0)), + pl.BlockSpec((None, tile // block_kv, head_dim), lambda h, i: (h, i, 0)), + pl.BlockSpec((None, tile // block_kv, head_dim), lambda h, i: (h, i, 0)), + pl.BlockSpec((None, tile // block_kv, 1), lambda h, i: (h, i, 0)), + ], + out_shape=[ + jax.ShapeDtypeStruct((num_heads, nq, head_dim), jnp.float32), + jax.ShapeDtypeStruct((num_heads, nk, head_dim), jnp.float32), + jax.ShapeDtypeStruct((num_heads, nk, head_dim), jnp.float32), + jax.ShapeDtypeStruct((num_heads, nk, 1), jnp.float32), + ], + compiler_params=pltpu.CompilerParams(dimension_semantics=("parallel", "parallel")), + interpret=interpret, + name="sol_pool_qkv", + )(q, k, v) + return q_pooled, k_pooled, v_pooled, k_radius[..., 0] diff --git a/src/maxdiffusion/kernels/sol_attention/routing.py b/src/maxdiffusion/kernels/sol_attention/routing.py index 26eb9b888..7b018c07f 100644 --- a/src/maxdiffusion/kernels/sol_attention/routing.py +++ b/src/maxdiffusion/kernels/sol_attention/routing.py @@ -106,6 +106,18 @@ def pool_blocks(x: jax.Array, seq_len: int, block: int) -> jax.Array: return sums / jnp.maximum(counts, 1.0)[:, None] +def block_radius(x: jax.Array, pooled: jax.Array, seq_len: int, block: int) -> jax.Array: + """Max L2 distance of a block's real tokens from the block mean, [..., Nk]. + + Bounds how far any token score can exceed its block's proxy score: + q . k_t <= q . mean(K_j) + ||q|| * radius_j. + """ + *lead, padded_len, d = x.shape + dev = x.astype(jnp.float32).reshape(*lead, padded_len // block, block, d) - pooled[..., None, :] + valid = (jnp.arange(padded_len) < seq_len).reshape(padded_len // block, block) + return jnp.sqrt(jnp.max(jnp.where(valid, jnp.sum(dev * dev, axis=-1), 0.0), axis=-1)) + + def block_proxy_scores(q_pooled: jax.Array, k_pooled: jax.Array, scale: float) -> jax.Array: """Pre-softmax block-proxy logits s_ij (Eq. 3), shape [..., Nq, Nk].""" return ( @@ -186,6 +198,9 @@ class SolRouting: kv_counts: jax.Array # int32 [Nk] k_pooled: jax.Array # f32 [H, Nk, d] mean key per block v_pooled: jax.Array # f32 [H, Nk, d] mean value per block + # f32 [H, Nk] max ||k_t - mean(K_j)|| per block; lets the kernel use a fixed + # softmax offset with an overflow guarantee. None -> 0 (weaker guard). + k_radius: Optional[jax.Array] = None def density(self) -> jax.Array: """Fraction of (query block, real key block) pairs computed exactly.""" @@ -193,6 +208,40 @@ def density(self) -> jax.Array: return jnp.sum(self.selected * valid) / (jnp.sum(valid) * self.selected.shape[0] * self.selected.shape[1]) +def route_pooled( + q_pooled: jax.Array, + k_pooled: jax.Array, + v_pooled: jax.Array, + kv_counts: jax.Array, + *, + scale: float, + beta: float, + always_exact: Optional[jax.Array] = None, + k_radius: Optional[jax.Array] = None, +) -> SolRouting: + """Routes key blocks per query block from already pooled Q/K/V. + + Args: + q_pooled: Mean query per query block [H, Nq, d]. + k_pooled, v_pooled: Mean key/value per key block [H, Nk, d]. + kv_counts: Real tokens per key block [Nk]. + scale: Attention logit scale. + beta: Standardized cutoff (see `threshold_route`). + always_exact: Optional bool [Nk] of key blocks forced into the exact branch. + k_radius: Optional per-block key radius [H, Nk] (see `block_radius`). + """ + scores = block_proxy_scores(q_pooled, k_pooled, scale) + selected, tau = threshold_route(scores, kv_counts, beta, always_exact) + return SolRouting( + selected=selected, + tau=tau, + kv_counts=kv_counts, + k_pooled=k_pooled.astype(jnp.float32), + v_pooled=v_pooled.astype(jnp.float32), + k_radius=k_radius, + ) + + def compute_routing( q: jax.Array, k: jax.Array, @@ -204,7 +253,7 @@ def compute_routing( block_sizes: SolBlockSizes, always_exact: Optional[jax.Array] = None, ) -> SolRouting: - """Pools Q/K/V into blocks and routes key blocks per query block. + """Pools Q/K/V into blocks (pure JAX) and routes key blocks per query block. Args: q, k, v: Zero-padded [H, padded_len, d] tensors. @@ -214,18 +263,16 @@ def compute_routing( block_sizes: Block granularity. always_exact: Optional bool [Nk] of key blocks forced into the exact branch. """ - q_pooled = pool_blocks(q, seq_len, block_sizes.block_q) k_pooled = pool_blocks(k, seq_len, block_sizes.block_kv) - v_pooled = pool_blocks(v, seq_len, block_sizes.block_kv) - kv_counts = block_counts(seq_len, k.shape[-2], block_sizes.block_kv) - scores = block_proxy_scores(q_pooled, k_pooled, scale) - selected, tau = threshold_route(scores, kv_counts, beta, always_exact) - return SolRouting( - selected=selected, - tau=tau, - kv_counts=kv_counts, - k_pooled=k_pooled, - v_pooled=v_pooled, + return route_pooled( + pool_blocks(q, seq_len, block_sizes.block_q), + k_pooled, + pool_blocks(v, seq_len, block_sizes.block_kv), + block_counts(seq_len, k.shape[-2], block_sizes.block_kv), + scale=scale, + beta=beta, + always_exact=always_exact, + k_radius=block_radius(k, k_pooled, seq_len, block_sizes.block_kv), ) diff --git a/src/maxdiffusion/kernels/sol_attention/sol_attention_kernel.py b/src/maxdiffusion/kernels/sol_attention/sol_attention_kernel.py index 655ba1d08..a03da73f6 100644 --- a/src/maxdiffusion/kernels/sol_attention/sol_attention_kernel.py +++ b/src/maxdiffusion/kernels/sol_attention/sol_attention_kernel.py @@ -16,7 +16,7 @@ Pallas TPU kernel for Sol-Attn (arXiv:2607.24027, Algorithm 1). One grid program per (head, query block). Each program keeps a single -online-softmax state and runs two branches into it: +softmax state and runs two branches into it: 1. Approximate branch (outer loop of Algorithm 1): one GEMM of the query block against all pooled keys. Unselected blocks contribute their @@ -27,8 +27,12 @@ query block through a ring of manual HBM->VMEM DMAs, so only routed blocks are ever read. -Routing (pooling + thresholds) runs in a tiny XLA pre-pass, see `routing.py`; -only the compacted per-query-block index rows reach the kernel. +The softmax offset is fixed per query from the pooled scores and key block +radii (see `_sol_kernel`), so the hot loop needs no running max or rescale. + +Pooling runs in a fused single-pass kernel (`pooling.py`) and routing in a tiny +XLA pre-pass (`routing.py`); only the compacted per-query-block index rows +reach this kernel. Inputs need no padding to the block sizes. """ from __future__ import annotations @@ -42,6 +46,7 @@ from jax.experimental import pallas as pl from jax.experimental.pallas import tpu as pltpu +from maxdiffusion.kernels.sol_attention import pooling as sol_pooling from maxdiffusion.kernels.sol_attention import routing as sol_routing NUM_LANES = 128 @@ -51,6 +56,10 @@ TARGET_TILE_KEYS = 1024 NT_DIM_NUMBERS = (((1,), (1,)), ((), ())) LOG2E = 1.4426950408889634 +# Fixed softmax offset (log2 units): max exp2 argument (f32 overflows at 128), +# and the widest bound gap per query tile before falling back to online softmax. +FIXED_M_HEADROOM = 80.0 +FIXED_M_WINDOW = 120.0 _MASK_VALUE = sol_routing.MASK_VALUE _HALF_MASK_VALUE = 0.5 * sol_routing.MASK_VALUE @@ -71,20 +80,21 @@ def _resolve_pipelining(block_kv: int, group_size: Optional[int], num_buffers: O def _sol_kernel( # Inputs. - q_ref, # VMEM [bq, d], pre-scaled by scale * log2(e) + q_ref, # VMEM [bq, d]; the last block may extend past the array end k_pooled_ref, # VMEM [nk_pad, d] v_pooled_ref, # VMEM [nk_pad, d] log_count_ref, # VMEM [nk_pad, 1] f32: log2(tokens in block j), MASK for padding blocks + radius_ref, # VMEM [nk_pad, 1] f32: key block radius (see routing.block_radius) rows_hbm, # HBM [H, Nq, 8, row_cols] int32; each (8, row_cols) row is one tile-aligned DMA - k_hbm, # HBM [H, Lp, d] - v_hbm, # HBM [H, Lp, d] + k_hbm, # HBM [H, L, d], L >= seq_len (no padding needed) + v_hbm, # HBM [H, L, d] # Outputs. - o_ref, # VMEM [d, bq] (transposed) + o_ref, # VMEM [bq, d] # Scratch. row_smem, # SMEM [2, 8, row_cols] int32, double-buffered across grid steps k_buf, # VMEM [num_buffers, bkv, d] v_buf, # VMEM [num_buffers, bkv, d] - bias_scr, # VMEM [nk_pad, 1] f32 + sel_scr, # VMEM [nk_pad, 1] f32: 1 for blocks routed to the exact branch o_scr, # VMEM [d, bq] f32 row_sem, # DMA semaphores [2] kv_sem, # DMA semaphores [2, num_buffers] @@ -93,6 +103,7 @@ def _sol_kernel( block_kv: int, num_buffers: int, group_size: int, + q_scale: float, ): """One (head, query block) program: approximate branch + exact sparse loop.""" # Scores are laid out [keys, queries] so softmax reductions run over @@ -127,10 +138,17 @@ def row_entry(n): count = row_entry(0) + kv_len = k_hbm.shape[1] + + def kv_start(blk): + # A ragged tail block reads the last block_kv rows of an unpadded array; + # the epilogue masks the rows that belong to the previous block. + return jnp.minimum(blk * block_kv, kv_len - block_kv) + def kv_copies(t): """K/V block DMAs for the t-th selected block, into ring slot t % num_buffers.""" slot = lax.rem(t, num_buffers) - start = pl.multiple_of(row_entry(t + 1) * block_kv, block_kv) + start = kv_start(row_entry(t + 1)) k_copy = pltpu.make_async_copy(k_hbm.at[h, pl.ds(start, block_kv), :], k_buf.at[slot], kv_sem.at[0, slot]) v_copy = pltpu.make_async_copy(v_hbm.at[h, pl.ds(start, block_kv), :], v_buf.at[slot], kv_sem.at[1, slot]) return k_copy, v_copy @@ -144,36 +162,58 @@ def start_kv_copies(t): for t in range(num_buffers - group_size): pl.when(t < count)(functools.partial(start_kv_copies, t)) - # Routing mask on chip: selected blocks get no approximate term. - bias_scr[...] = log_count_ref[...] + # Routing flags on chip, from the SMEM index row. + sel_scr[...] = jnp.zeros_like(sel_scr) - def mask_selected(t, carry): - bias_scr[pl.ds(row_entry(t + 1), 1), :] = jnp.full((1, 1), _MASK_VALUE, jnp.float32) + def flag_selected(t, carry): + sel_scr[pl.ds(row_entry(t + 1), 1), :] = jnp.ones((1, 1), jnp.float32) return carry - lax.fori_loop(0, count, mask_selected, 0) + lax.fori_loop(0, count, flag_selected, 0) - q = q_ref[...] + # exp2 everywhere: log2(e) is folded into the query scale and count bias. + qf = q_ref[...].astype(jnp.float32) * q_scale + q = qf.astype(q_ref.dtype) sv_dims = (((0,), (0,)), ((), ())) - # Approximate branch: one GEMM over all pooled keys, zeroth-order term of - # Eqs. (9)-(10) via the log2(count) bias. + # Pooled-key scores [nk_pad, bq]: they drive both the approximate branch + # and the per-query softmax offset m. s = lax.dot_general(k_pooled_ref[...], q, NT_DIM_NUMBERS, preferred_element_type=jnp.float32) - s = s + bias_scr[...] - m = jnp.max(s, axis=0, keepdims=True) - p = jnp.where(s > _HALF_MASK_VALUE, jnp.exp2(s - m), 0.0) + sel = sel_scr[...] > 0.5 + log_count = log_count_ref[...] + approx_logits = s + log_count + # Fixed offset instead of a running max: no per-tile max reduction and no + # rescaling of the accumulators in the hot loop (~20% of kernel time). + # m_est <= the true max (a block's mean score never exceeds its max), and + # ub >= every token score (q.k_t <= q.mean(K_j) + ||q|| * radius_j). With + # m = max(m_est, ub - headroom), no exp2 argument exceeds the headroom (no + # f32 overflow), and if ub - m_est <= window the largest term is + # >= 2^(headroom - window) (no underflow). Tiles with a wider window, e.g. + # from an outlier key that inflates a block radius, fall back to the online + # (running-max) softmax. + ones = jnp.ones((NUM_SUBLANES, qf.shape[1]), jnp.float32) + q_norm = jnp.sqrt(lax.dot_general(ones, qf * qf, NT_DIM_NUMBERS, preferred_element_type=jnp.float32)[:1]) + m_est = jnp.max(jnp.where(sel, s, approx_logits), axis=0, keepdims=True) + ub = jnp.max(jnp.where(sel, s + radius_ref[...] * q_norm, _MASK_VALUE), axis=0, keepdims=True) + fixed = jnp.max(ub - m_est) <= FIXED_M_WINDOW + m = jnp.where(fixed, jnp.maximum(m_est, ub - FIXED_M_HEADROOM), m_est) + + # Approximate branch: zeroth-order term of Eqs. (9)-(10) for unselected, + # non-padding blocks via the log2(count) bias. Its logits are <= m_est. + keep = jnp.logical_and(jnp.logical_not(sel), log_count > _HALF_MASK_VALUE) + p = jnp.where(keep, jnp.exp2(approx_logits - m), 0.0) l = jnp.sum(p, axis=0, keepdims=True) o_scr[...] = lax.dot_general(v_pooled_ref[...], p.astype(v_pooled_ref.dtype), sv_dims, preferred_element_type=jnp.float32) ragged = seq_len % block_kv != 0 last_block = seq_len // block_kv - def exact_blocks(t0, m_prev, l_prev, size: int, prefetch: bool, mask_tail: bool): - """Online-softmax update with selected blocks t0..t0+size-1. + def exact_blocks(t0, carry, size: int, prefetch: bool, mask_tail: bool, online: bool): + """Accumulates selected blocks t0..t0+size-1 into the softmax state. Their ring slots are contiguous, so `size` routed blocks form one [size * bkv, bq] score tile: routing stays fine-grained while each loop - iteration amortizes the softmax rescale over a larger tile. + iteration works on a large tile. carry is (m, l) if online else l. """ slot = lax.rem(t0, num_buffers) copies = [kv_copies(t0 + j) for j in range(size)] @@ -186,19 +226,28 @@ def exact_blocks(t0, m_prev, l_prev, size: int, prefetch: bool, mask_tail: bool) k = k_buf[pl.ds(slot, size)].reshape(size * block_kv, k_buf.shape[-1]) s = lax.dot_general(k, q, NT_DIM_NUMBERS, preferred_element_type=jnp.float32) if mask_tail: - valid = lax.broadcasted_iota(jnp.int32, s.shape, 0) < seq_len - last_block * block_kv + rows = kv_start(last_block) + lax.broadcasted_iota(jnp.int32, s.shape, 0) + valid = jnp.logical_and(rows >= last_block * block_kv, rows < seq_len) s = jnp.where(valid, s, _MASK_VALUE) - m_next = jnp.maximum(m_prev, jnp.max(s, axis=0, keepdims=True)) - alpha = jnp.exp2(m_prev - m_next) + if online: + m_prev, l_prev = carry + m_next = jnp.maximum(m_prev, jnp.max(s, axis=0, keepdims=True)) + alpha = jnp.exp2(m_prev - m_next) + else: + m_next, l_prev, alpha = m, carry, None p = jnp.exp2(s - m_next) - l_next = alpha * l_prev + jnp.sum(p, axis=0, keepdims=True) + l_next = jnp.sum(p, axis=0, keepdims=True) + (l_prev if alpha is None else alpha * l_prev) for _, v_copy in copies: v_copy.wait() v = v_buf[pl.ds(slot, size)].reshape(size * block_kv, v_buf.shape[-1]) - o_scr[...] = alpha * o_scr[...] + lax.dot_general(v, p.astype(v.dtype), sv_dims, preferred_element_type=jnp.float32) - return m_next, l_next - - # Exact branch over routed blocks only, sharing the online-softmax state. + pv = lax.dot_general(v, p.astype(v.dtype), sv_dims, preferred_element_type=jnp.float32) + if online: + o_scr[...] = alpha * o_scr[...] + pv + return m_next, l_next + o_scr[...] += pv + return l_next + + # Exact branch over routed blocks only, sharing the softmax state. # Indices are ascending, so a selected ragged tail block is always last; it # runs as a masked epilogue to keep the hot loop branch-free (a per-block # branch costs ~40% here). Every block's DMA is issued by the prologue or @@ -209,27 +258,35 @@ def exact_blocks(t0, m_prev, l_prev, size: int, prefetch: bool, mask_tail: bool) else: num_main = count num_groups = num_main // group_size - m, l = lax.fori_loop( - 0, - num_groups, - lambda g, c: exact_blocks(g * group_size, *c, size=group_size, prefetch=True, mask_tail=False), - (m, l), - ) - if group_size > 1: - m, l = lax.fori_loop( - num_groups * group_size, - num_main, - lambda t, c: exact_blocks(t, *c, size=1, prefetch=False, mask_tail=False), - (m, l), - ) - if ragged: - m, l = lax.cond( - has_tail, - lambda c: exact_blocks(num_main, *c, size=1, prefetch=False, mask_tail=True), - lambda c: c, - (m, l), + + def exact_branch(online: bool): + step = functools.partial(exact_blocks, online=online) + carry = (m, l) if online else l + carry = lax.fori_loop( + 0, + num_groups, + lambda g, c: step(g * group_size, c, size=group_size, prefetch=True, mask_tail=False), + carry, ) - o_ref[...] = (o_scr[...] * pl.reciprocal(l, approx=False)).astype(o_ref.dtype) + if group_size > 1: + carry = lax.fori_loop( + num_groups * group_size, + num_main, + lambda t, c: step(t, c, size=1, prefetch=False, mask_tail=False), + carry, + ) + if ragged: + carry = lax.cond( + has_tail, + lambda c: step(num_main, c, size=1, prefetch=False, mask_tail=True), + lambda c: c, + carry, + ) + return carry[1] if online else carry + + l = lax.cond(fixed, lambda: exact_branch(online=False), lambda: exact_branch(online=True)) + # Transposing on chip saves an XLA relayout of the whole output. + o_ref[...] = (o_scr[...] * pl.reciprocal(l, approx=False)).T.astype(o_ref.dtype) def sol_attention_from_routing( @@ -246,14 +303,19 @@ def sol_attention_from_routing( group_size: Optional[int] = None, interpret: bool = False, ) -> jax.Array: - """Runs the kernel on padded [H, Lp, d] tensors with precomputed routing. + """Runs the kernel on [H, L, d] tensors (L >= seq_len) with precomputed routing. - Returns the padded [H, Lp, d] output. + The tensors need no padding to the block sizes; zero-padded inputs work too. + + Returns: + The [H, L, d] output (rows past seq_len are unspecified). """ group_size, num_buffers = _resolve_pipelining(block_sizes.block_kv, group_size, num_buffers) - num_heads, padded_len, head_dim = q.shape + num_heads, length, head_dim = q.shape bq, bkv = block_sizes.block_q, block_sizes.block_kv - nq, nk = padded_len // bq, padded_len // bkv + nq, nk = routing.selected.shape[1:] + if nq != -(-length // bq) or nk != -(-length // bkv) or length < max(bq, bkv) or seq_len > length: + raise ValueError(f"Routing {routing.selected.shape} does not match q {q.shape} with blocks {block_sizes}.") nk_pad = _round_up(nk, NUM_LANES) # Row tables are (8, row_cols) tiles so HBM->SMEM DMAs are layout aligned. row_cols = _round_up(-(-(nk + 1) // NUM_SUBLANES), NUM_LANES) @@ -264,16 +326,23 @@ def pad_blocks(x): widths[-2] = (0, nk_pad - nk) return jnp.pad(x, widths) - # exp2 everywhere: fold log2(e) into the query scale and the count bias. - q = (q.astype(jnp.float32) * (scale * LOG2E)).astype(q.dtype) k_pooled = pad_blocks(routing.k_pooled).astype(k.dtype) v_pooled = pad_blocks(routing.v_pooled).astype(v.dtype) counts = jnp.pad(routing.kv_counts, (0, nk_pad - nk)) log_count = jnp.where(counts > 0, jnp.log2(jnp.maximum(counts, 1).astype(jnp.float32)), _MASK_VALUE)[:, None] rows = sol_routing.compact_selection(routing.selected, row_width).reshape(num_heads, nq, NUM_SUBLANES, row_cols) - - kernel = functools.partial(_sol_kernel, seq_len=seq_len, block_kv=bkv, num_buffers=num_buffers, group_size=group_size) - out = pl.pallas_call( + radius = routing.k_radius if routing.k_radius is not None else jnp.zeros((num_heads, nk), jnp.float32) + radius = jnp.pad(radius.astype(jnp.float32), ((0, 0), (0, nk_pad - nk)))[..., None] + + kernel = functools.partial( + _sol_kernel, + seq_len=seq_len, + block_kv=bkv, + num_buffers=num_buffers, + group_size=group_size, + q_scale=scale * LOG2E, + ) + return pl.pallas_call( kernel, grid=(num_heads, nq), in_specs=[ @@ -281,12 +350,13 @@ def pad_blocks(x): pl.BlockSpec((None, nk_pad, head_dim), lambda h, i: (h, 0, 0)), pl.BlockSpec((None, nk_pad, head_dim), lambda h, i: (h, 0, 0)), pl.BlockSpec((nk_pad, 1), lambda h, i: (0, 0)), + pl.BlockSpec((None, nk_pad, 1), lambda h, i: (h, 0, 0)), pl.BlockSpec(memory_space=pltpu.HBM), pl.BlockSpec(memory_space=pltpu.HBM), pl.BlockSpec(memory_space=pltpu.HBM), ], - out_specs=pl.BlockSpec((None, head_dim, bq), lambda h, i: (h, 0, i)), - out_shape=jax.ShapeDtypeStruct((num_heads, head_dim, padded_len), q.dtype), + out_specs=pl.BlockSpec((None, bq, head_dim), lambda h, i: (h, i, 0)), + out_shape=jax.ShapeDtypeStruct(q.shape, q.dtype), scratch_shapes=[ pltpu.SMEM((2, NUM_SUBLANES, row_cols), jnp.int32), pltpu.VMEM((num_buffers, bkv, head_dim), k.dtype), @@ -303,8 +373,34 @@ def pad_blocks(x): ), interpret=interpret, name="sol_attention", - )(q, k_pooled, v_pooled, log_count, rows, k, v) - return jnp.swapaxes(out, 1, 2) + )(q, k_pooled, v_pooled, log_count, radius, rows, k, v) + + +def compute_routing_unpadded( + q: jax.Array, + k: jax.Array, + v: jax.Array, + *, + scale: float, + beta: float, + block_sizes: sol_routing.SolBlockSizes, + always_exact: Optional[jax.Array] = None, + interpret: bool = False, +) -> sol_routing.SolRouting: + """Routing for unpadded [H, L, d] tensors, pooling with the fused kernel.""" + seq_len = q.shape[1] + q_pooled, k_pooled, v_pooled, k_radius = sol_pooling.pool_qkv( + q, k, v, block_q=block_sizes.block_q, block_kv=block_sizes.block_kv, interpret=interpret + ) + kv_counts = sol_routing.block_counts(seq_len, k_pooled.shape[1] * block_sizes.block_kv, block_sizes.block_kv) + return sol_routing.route_pooled( + q_pooled, k_pooled, v_pooled, kv_counts, scale=scale, beta=beta, always_exact=always_exact, k_radius=k_radius + ) + + +def _needs_padding(seq_len: int, block_sizes: sol_routing.SolBlockSizes) -> bool: + """Short or unaligned sequences fall back to zero-padding (cheap at that size).""" + return seq_len % NUM_SUBLANES != 0 or seq_len < sol_pooling.slab_rows(block_sizes.block_q, block_sizes.block_kv) def sol_attention( @@ -335,7 +431,7 @@ def sol_attention( num_buffers: Exact-branch K/V DMA ring depth in blocks (default 2 * group_size). group_size: Selected blocks fused into one score tile per loop step (default: TARGET_TILE_KEYS // block_kv). - interpret: Run the Pallas kernel in interpret mode (CPU testing). + interpret: Run the Pallas kernels in interpret mode (CPU testing). return_density: Also return the exact-branch block density (scalar). Returns: @@ -345,24 +441,24 @@ def sol_attention( raise ValueError(f"Sol-Attn needs matching self-attention q/k/v, got {q.shape}, {k.shape}, {v.shape}.") lead = q.shape[:-2] seq_len, head_dim = q.shape[-2:] - padded_len = sol_routing.padded_length(seq_len, block_sizes) - qp, kp, vp = (sol_routing.pad_sequence(x.reshape(-1, seq_len, head_dim), padded_len) for x in (q, k, v)) - with jax.named_scope("sol_routing"): - routing = sol_routing.compute_routing( - qp, - kp, - vp, - seq_len=seq_len, - scale=scale, - beta=beta, - block_sizes=block_sizes, - always_exact=always_exact, - ) + q, k, v = (x.reshape(-1, seq_len, head_dim) for x in (q, k, v)) + if _needs_padding(seq_len, block_sizes): + padded_len = sol_routing.padded_length(seq_len, block_sizes) + q, k, v = (sol_routing.pad_sequence(x, padded_len) for x in (q, k, v)) + with jax.named_scope("sol_routing"): + routing = sol_routing.compute_routing( + q, k, v, seq_len=seq_len, scale=scale, beta=beta, block_sizes=block_sizes, always_exact=always_exact + ) + else: + with jax.named_scope("sol_routing"): + routing = compute_routing_unpadded( + q, k, v, scale=scale, beta=beta, block_sizes=block_sizes, always_exact=always_exact, interpret=interpret + ) with jax.named_scope("sol_kernel"): out = sol_attention_from_routing( - qp, - kp, - vp, + q, + k, + v, routing, seq_len=seq_len, scale=scale, diff --git a/src/maxdiffusion/models/sol_attention.py b/src/maxdiffusion/models/sol_attention.py index ac763caa8..3459a134a 100644 --- a/src/maxdiffusion/models/sol_attention.py +++ b/src/maxdiffusion/models/sol_attention.py @@ -41,6 +41,7 @@ import flax.linen as nn import jax +import jax.numpy as jnp from jax.sharding import PartitionSpec as P from maxdiffusion.common_types import BATCH, HEAD, LENGTH @@ -207,6 +208,16 @@ def _sequence_axis(spec_entry) -> Optional[str]: raise ValueError(f"Sol-Attn supports sequence sharding over a single mesh axis, got {spec_entry!r}.") +def _head_chunk_ranges(num_heads: int, shards: int, num_chunks: int) -> list[tuple[int, int]]: + """Splits heads into <= num_chunks ranges whose sizes are multiples of `shards`.""" + if num_heads % shards: + raise ValueError(f"Sol-Attn needs heads ({num_heads}) divisible by the sequence shards ({shards}).") + groups = num_heads // shards + num_chunks = max(1, min(num_chunks, groups)) + bounds = [round(c * groups / num_chunks) * shards for c in range(num_chunks + 1)] + return [(a, b) for a, b in zip(bounds[:-1], bounds[1:]) if b > a] + + def sol_sharded_attention(query: jax.Array, key: jax.Array, value: jax.Array, context: Mapping[str, Any]) -> jax.Array: """Sol-Attn entry point for `attention_flax._apply_attention`. @@ -227,6 +238,7 @@ def sol_sharded_attention(query: jax.Array, key: jax.Array, value: jax.Array, co if q.shape != k.shape or q.shape != v.shape: raise ValueError(f"Sol-Attn requires matched self-attention q/k/v, got {q.shape}, {k.shape}, {v.shape}.") + log_density = bool(cfg.get("log_density", False)) kernel = functools.partial( sol_attention_kernel.sol_attention, scale=context["scale"], @@ -235,6 +247,7 @@ def sol_sharded_attention(query: jax.Array, key: jax.Array, value: jax.Array, co vmem_limit_bytes=cfg.get("vmem_limit_bytes"), return_density=True, ) + density = None if mesh is None: out, density = kernel(q, k, v) @@ -247,26 +260,42 @@ def sol_sharded_attention(query: jax.Array, key: jax.Array, value: jax.Array, co raise ValueError("Sol-Attn does not support sharding the head dimension.") seq_axis = _sequence_axis(qspec[2]) mesh_axes = tuple(mesh.axis_names) + shards = mesh.shape[seq_axis] if seq_axis is not None else 1 + chunks = _head_chunk_ranges(q.shape[1], shards, int(context.get("ulysses_attention_chunks") or 1) if seq_axis else 1) @functools.partial( jax.shard_map, mesh=mesh, in_specs=(qspec, kvspec, kvspec), - out_specs=(qspec, P()), + out_specs=(qspec, P()) if log_density else qspec, check_vma=False, ) def local(q, k, v): - if seq_axis is not None: - a2a = functools.partial(jax.lax.all_to_all, axis_name=seq_axis, tiled=True) - q, k, v = (a2a(x, split_axis=1, concat_axis=2) for x in (q, k, v)) - out, density = kernel(q, k, v) - if seq_axis is not None: - out = a2a(out, split_axis=2, concat_axis=1) - return out, jax.lax.pmean(density, mesh_axes) - - out, density = local(q, k, v) - - if cfg.get("log_density", False): + # Head chunks are independent: with async all-to-all enabled, XLA + # overlaps the exchange of chunk c+1 with the kernels of chunk c. + a2a = functools.partial(jax.lax.all_to_all, axis_name=seq_axis, tiled=True) if seq_axis else None + outs, densities = [], [] + for start, stop in chunks: + qc, kc, vc = (x[:, start:stop] for x in (q, k, v)) + if a2a is not None: + qc, kc, vc = (a2a(x, split_axis=1, concat_axis=2) for x in (qc, kc, vc)) + out_c, density_c = kernel(qc, kc, vc) + if a2a is not None: + out_c = a2a(out_c, split_axis=2, concat_axis=1) + outs.append(out_c) + densities.append(density_c) + out = outs[0] if len(outs) == 1 else jnp.concatenate(outs, axis=1) + if not log_density: + return out + # Only pay for the extra collective when the density is requested. + return out, jax.lax.pmean(sum(densities) / len(densities), mesh_axes) + + if log_density: + out, density = local(q, k, v) + else: + out = local(q, k, v) + + if log_density: jax.debug.print( "sol_attention density={d:.4f} layer={l} step={s}", d=density, diff --git a/src/maxdiffusion/tests/sol_attention_test.py b/src/maxdiffusion/tests/sol_attention_test.py index cac3a247e..29ae729eb 100644 --- a/src/maxdiffusion/tests/sol_attention_test.py +++ b/src/maxdiffusion/tests/sol_attention_test.py @@ -17,6 +17,7 @@ Pallas TPU kernel, sharded execution, and model/pipeline config plumbing. """ +import dataclasses from types import SimpleNamespace import flax.linen as nn @@ -29,6 +30,7 @@ from maxdiffusion import aot_cache from maxdiffusion import common_types as ct +from maxdiffusion.kernels.sol_attention import pooling from maxdiffusion.kernels.sol_attention import reference from maxdiffusion.kernels.sol_attention import routing from maxdiffusion.kernels.sol_attention import sol_attention_kernel @@ -195,7 +197,7 @@ def test_kernel_pipelining_variants(group_size, num_buffers): route = routing.compute_routing(qp, kp, vp, seq_len=2500, scale=0.09, beta=0.3, block_sizes=bs) # Force the ragged tail block into every row so the epilogue always runs. sel = route.selected.at[:, :, 2500 // 128].set(True) - route = routing.SolRouting(sel, route.tau, route.kv_counts, route.k_pooled, route.v_pooled) + route = dataclasses.replace(route, selected=sel) out = sol_attention_kernel.sol_attention_from_routing( qp, kp, vp, route, seq_len=2500, scale=0.09, block_sizes=bs, group_size=group_size, num_buffers=num_buffers ) @@ -218,14 +220,59 @@ def test_kernel_handles_empty_and_full_selection(): route = routing.compute_routing(qp, kp, vp, seq_len=700, scale=0.1, beta=1.0, block_sizes=bs) for fill in (False, True): sel = jnp.full_like(route.selected, fill) - forced = routing.SolRouting(sel, route.tau, route.kv_counts, route.k_pooled, route.v_pooled) + forced = dataclasses.replace(route, selected=sel) out = sol_attention_kernel.sol_attention_from_routing(qp, kp, vp, forced, seq_len=700, scale=0.1, block_sizes=bs) want = reference.sol_attention_reference(q, k, v, scale=0.1, beta=1.0, block_sizes=bs, selected=sel) assert _rel_l2(out[:, :700], want) < 1e-2 @requires_tpu -def test_sharded_ulysses_matches_single_device(): +@pytest.mark.parametrize("seq_len,block_q,block_kv", [(9000, 256, 128), (20000, 1024, 256)]) +def test_unpadded_path_matches_reference(seq_len, block_q, block_kv): + """Fused pooling + unpadded kernel (ragged tail read as a shifted window).""" + q, k, v = _video_like_qkv(2, seq_len, 128, seed=11, dtype=jnp.bfloat16) + bs = routing.SolBlockSizes(block_q, block_kv) + route = sol_attention_kernel.compute_routing_unpadded(q, k, v, scale=128**-0.5, beta=1.0, block_sizes=bs) + out = sol_attention_kernel.sol_attention_from_routing(q, k, v, route, seq_len=seq_len, scale=128**-0.5, block_sizes=bs) + padded = routing.padded_length(seq_len, bs) + nq, nk = route.selected.shape[1:] + sel = jnp.pad(route.selected, ((0, 0), (0, padded // block_q - nq), (0, padded // block_kv - nk))) + want = reference.sol_attention_reference(q, k, v, scale=128**-0.5, beta=1.0, block_sizes=bs, selected=sel) + assert _rel_l2(out, want) < 1e-2 + + +@requires_tpu +def test_fused_pooling_matches_reference_pooling(): + q, k, v = _video_like_qkv(2, 9000, 128, seed=2, dtype=jnp.bfloat16) + q_pooled, k_pooled, v_pooled, k_radius = pooling.pool_qkv(q, k, v, block_q=256, block_kv=128) + padded = routing.padded_length(9000, routing.SolBlockSizes(256, 128)) + qp, kp, vp = (routing.pad_sequence(x, padded) for x in (q, k, v)) + want_k = routing.pool_blocks(kp, 9000, 128)[:, : k_pooled.shape[1]] + np.testing.assert_allclose(q_pooled, routing.pool_blocks(qp, 9000, 256)[:, : q_pooled.shape[1]], rtol=1e-5, atol=1e-5) + np.testing.assert_allclose(k_pooled, want_k, rtol=1e-5, atol=1e-5) + np.testing.assert_allclose(v_pooled, routing.pool_blocks(vp, 9000, 128)[:, : v_pooled.shape[1]], rtol=1e-5, atol=1e-5) + want_r = routing.block_radius(kp, routing.pool_blocks(kp, 9000, 128), 9000, 128)[:, : k_radius.shape[1]] + np.testing.assert_allclose(k_radius, want_r, rtol=1e-4, atol=1e-4) + + +@requires_tpu +def test_fixed_offset_survives_outlier_keys(): + """A few keys far above their block mean must neither overflow nor be lost.""" + q, k, v = _video_like_qkv(1, 2048, 128, seed=4, dtype=jnp.float32) + direction = q[0, 100] / jnp.linalg.norm(q[0, 100]) + k = k.at[0, 77].set(60.0 * direction).at[0, 1500].set(-60.0 * direction) + q, k, v = (x.astype(jnp.bfloat16) for x in (q, k, v)) + bs = routing.SolBlockSizes(256, 128) + route = routing.compute_routing(q, k, v, seq_len=2048, scale=0.3, beta=1.0, block_sizes=bs) + out = sol_attention_kernel.sol_attention_from_routing(q, k, v, route, seq_len=2048, scale=0.3, block_sizes=bs) + want = reference.sol_attention_reference(q, k, v, scale=0.3, beta=1.0, block_sizes=bs, selected=route.selected) + assert np.all(np.isfinite(np.asarray(out, np.float32))) + assert _rel_l2(out, want) < 1e-2 + + +@requires_tpu +@pytest.mark.parametrize("head_chunks", [1, 2]) +def test_sharded_ulysses_matches_single_device(head_chunks): devices = jax.devices() if len(devices) < 2: pytest.skip("Requires multiple TPU devices.") @@ -249,6 +296,7 @@ def test_sharded_ulysses_matches_single_device(): "mesh": mesh, "scale": head_dim**-0.5, "attention_mask": None, + "ulysses_attention_chunks": head_chunks, "axis_names_q": (ct.BATCH, ct.SELF_ATTN_HEAD, ct.SELF_ATTN_Q_LENGTH, ct.D_KV), "axis_names_kv": (ct.BATCH, ct.SELF_ATTN_HEAD, ct.SELF_ATTN_KV_LENGTH, ct.D_KV), }