From 9a55a500735c9d1d52ca5ca0803ce3da4f94a9be Mon Sep 17 00:00:00 2001 From: Nina Shvetsova Date: Wed, 11 Nov 2026 15:48:16 +0000 Subject: [PATCH] Fall back to sharding the height in the WAN VAE. The WAN VAE always sharded the width of its activations over the vae_spatial mesh axis. This works for the default 1280x720 landscape size, but users may also want to run inference on portrait videos, e.g. 720x1280 phone recordings. The VAE downsamples their width to 720, 360, 180 and 90, which isn't always divisible by vae_spatial (90 for 4, 180 and 90 for 8). XLA then pads the width, so the shards hold padding and those layers do extra work. This change adds a _spatial_sharding(mesh, shape) helper that shards the height when only the height is divisible by vae_spatial, and the width otherwise, as before. Videos with a divisible width keep the current behavior. --- .../models/wan/autoencoder_kl_wan.py | 42 +++++++++++------ .../models/wan/autoencoder_kl_wan_2p2.py | 7 ++- .../pipelines/wan/wan_pipeline.py | 6 ++- .../pipelines/wan/wan_pipeline_animate.py | 6 ++- .../pipelines/wan/wan_vace_pipeline_2_1.py | 6 ++- src/maxdiffusion/tests/wan/wan_vae_test.py | 47 +++++++++++++++++++ 6 files changed, 90 insertions(+), 24 deletions(-) diff --git a/src/maxdiffusion/models/wan/autoencoder_kl_wan.py b/src/maxdiffusion/models/wan/autoencoder_kl_wan.py index 999f5234b..00e80857a 100644 --- a/src/maxdiffusion/models/wan/autoencoder_kl_wan.py +++ b/src/maxdiffusion/models/wan/autoencoder_kl_wan.py @@ -48,6 +48,27 @@ def _update_cache(cache, idx, value): return cache[:idx] + (value,) + cache[idx + 1 :] +def _spatial_sharding( + mesh: Optional[jax.sharding.Mesh], shape: Sequence[int], batch_axis: Optional[str] = "redundant" +) -> Optional[NamedSharding]: + """Returns the sharding of a (N, D, H, W, C) array over the vae_spatial axis. + + Shards the longer spatial axis: the height of portrait videos, and the width + otherwise. The aspect ratio is the same at every resolution level of the VAE, + so its activations, caches and latents all use the same axis and don't need + to be resharded between levels. For the WAN 720p and 480p sizes, the longer + side (1280 or 832) stays divisible by 4 and 8 shards down to the latents (160 + or 104), while the latents of the shorter side (90 or 60) aren't divisible by + 8. XLA pads the axis if it isn't divisible by vae_spatial. + """ + if mesh is None or "vae_spatial" not in mesh.shape: + return None + _, _, height, width, _ = shape + if height > width: + return NamedSharding(mesh, P(batch_axis, None, "vae_spatial", None, None)) + return NamedSharding(mesh, P(batch_axis, None, None, "vae_spatial", None)) + + # Helper to ensure kernel_size, stride, padding are tuples of 3 integers def _canonicalize_tuple(x: Union[int, Sequence[int]], rank: int, name: str) -> Tuple[int, ...]: """Canonicalizes a value to a tuple of integers.""" @@ -121,7 +142,7 @@ def __init__( def __call__(self, x: jax.Array, cache_x: Optional[jax.Array] = None, idx=-1) -> jax.Array: if self.mesh is not None and "vae_spatial" in self.mesh.shape: - spatial_sharding = NamedSharding(self.mesh, P("redundant", None, None, "vae_spatial", None)) + spatial_sharding = _spatial_sharding(self.mesh, x.shape) if spatial_sharding is not None: x = jax.lax.with_sharding_constraint(x, spatial_sharding) @@ -1169,12 +1190,6 @@ def _encode(self, x: jax.Array, feat_cache: AutoencoderKLWanCache): iter_ = 1 + ((t - 1 + CHUNK_SIZE - 1) // CHUNK_SIZE) if t > 1 else 1 enc_feat_map = feat_cache._enc_feat_map - spatial_sharding = ( - NamedSharding(self.mesh, P("redundant", None, None, "vae_spatial", None)) - if self.mesh is not None and "vae_spatial" in self.mesh.shape - else None - ) - def finalize(out, enc_feat_map): feat_cache._enc_feat_map = enc_feat_map enc = self.quant_conv(out) @@ -1184,6 +1199,7 @@ def finalize(out, enc_feat_map): with jax.named_scope("AutoencoderKLWan_encode_chunk_0"): chunk_0 = x[:, :1, ...] out_0, enc_feat_map, _ = self.encoder(chunk_0, feat_cache=enc_feat_map, feat_idx=0) + spatial_sharding = _spatial_sharding(self.mesh, out_0.shape) if spatial_sharding is not None: out_0 = jax.lax.with_sharding_constraint(out_0, spatial_sharding) @@ -1280,16 +1296,13 @@ def _decode( x = self.post_quant_conv(z) dec_feat_map = feat_cache._feat_map - spatial_sharding = ( - NamedSharding(self.mesh, P("redundant", None, None, "vae_spatial", None)) - if self.mesh is not None and "vae_spatial" in self.mesh.shape - else None - ) + spatial_sharding = _spatial_sharding(self.mesh, x.shape) # First chunk (i=0) with jax.named_scope("AutoencoderKLWan_decode_chunk_0"): + chunk_in_0 = x[:, 0:1, ...] if spatial_sharding is not None: - chunk_in_0 = jax.lax.with_sharding_constraint(x[:, 0:1, ...], spatial_sharding) + chunk_in_0 = jax.lax.with_sharding_constraint(chunk_in_0, spatial_sharding) out_0, dec_feat_map, _ = self.decoder(chunk_in_0, feat_cache=dec_feat_map, feat_idx=0) if spatial_sharding is not None: out_0 = jax.lax.with_sharding_constraint(out_0, spatial_sharding) @@ -1297,8 +1310,9 @@ def _decode( if iter_ > 1: # Run chunk 1 outside scan to properly form the cache shape with jax.named_scope("AutoencoderKLWan_decode_chunk_1"): + chunk_in_1 = x[:, 1:2, ...] if spatial_sharding is not None: - chunk_in_1 = jax.lax.with_sharding_constraint(x[:, 1:2, ...], spatial_sharding) + chunk_in_1 = jax.lax.with_sharding_constraint(chunk_in_1, spatial_sharding) out_chunk_1, dec_feat_map, _ = self.decoder(chunk_in_1, feat_cache=dec_feat_map, feat_idx=0) if spatial_sharding is not None: out_chunk_1 = jax.lax.with_sharding_constraint(out_chunk_1, spatial_sharding) diff --git a/src/maxdiffusion/models/wan/autoencoder_kl_wan_2p2.py b/src/maxdiffusion/models/wan/autoencoder_kl_wan_2p2.py index 841b115c9..c1b300497 100644 --- a/src/maxdiffusion/models/wan/autoencoder_kl_wan_2p2.py +++ b/src/maxdiffusion/models/wan/autoencoder_kl_wan_2p2.py @@ -20,8 +20,7 @@ import jax from jax import tree_util import jax.numpy as jnp -from jax.sharding import NamedSharding, PartitionSpec as P -from maxdiffusion.models.wan.autoencoder_kl_wan import AutoencoderKLWanCache, WanCausalConv3d # pylint: disable=g-importing-member +from maxdiffusion.models.wan.autoencoder_kl_wan import AutoencoderKLWanCache, WanCausalConv3d, _spatial_sharding # pylint: disable=g-importing-member,protected-access from ... import common_types from ...configuration_utils import ConfigMixin @@ -1341,7 +1340,7 @@ def _encode(self, x: jax.Array, feat_cache: AutoencoderKLWanCache): iter_ = 1 + (t - 1) // 4 enc_feat_map = feat_cache._enc_feat_map - spatial_sharding = NamedSharding(self.mesh, P(None, None, None, "vae_spatial", None)) + spatial_sharding = _spatial_sharding(self.mesh, x.shape, batch_axis=None) for i in range(iter_): enc_conv_idx = 0 if i == 0: @@ -1394,7 +1393,7 @@ def _decode( dec_feat_map = feat_cache._feat_map - spatial_sharding = NamedSharding(self.mesh, P(None, None, None, "vae_spatial", None)) + spatial_sharding = _spatial_sharding(self.mesh, x.shape, batch_axis=None) for i in range(iter_): conv_idx = 0 chunk = x[:, i : i + 1, :, :, :] diff --git a/src/maxdiffusion/pipelines/wan/wan_pipeline.py b/src/maxdiffusion/pipelines/wan/wan_pipeline.py index 874158e05..77dc0abdd 100644 --- a/src/maxdiffusion/pipelines/wan/wan_pipeline.py +++ b/src/maxdiffusion/pipelines/wan/wan_pipeline.py @@ -556,9 +556,11 @@ def check_inputs( mesh = getattr(self, "vae_mesh", getattr(self, "mesh", None)) if mesh is not None and hasattr(mesh, "shape"): vae_spatial = mesh.shape.get("vae_spatial", 1) - if vae_spatial > 1 and (width // 8) % vae_spatial != 0: + # The VAE shards the longer spatial axis, see autoencoder_kl_wan._spatial_sharding. + sharded_axis, sharded_size = ("height", height) if height > width else ("width", width) + if vae_spatial > 1 and (sharded_size // 8) % vae_spatial != 0: max_logging.log( - f"Warning: Latent width is not divisible by vae_spatial mesh axis ({vae_spatial})." + f"Warning: Latent {sharded_axis} is not divisible by vae_spatial mesh axis ({vae_spatial})." " VAE spatial sharding will be partially bypassed." ) diff --git a/src/maxdiffusion/pipelines/wan/wan_pipeline_animate.py b/src/maxdiffusion/pipelines/wan/wan_pipeline_animate.py index 19f647e1e..ce4cef6a3 100644 --- a/src/maxdiffusion/pipelines/wan/wan_pipeline_animate.py +++ b/src/maxdiffusion/pipelines/wan/wan_pipeline_animate.py @@ -420,9 +420,11 @@ def check_inputs( mesh = getattr(self, "vae_mesh", getattr(self, "mesh", None)) if mesh is not None and hasattr(mesh, "shape"): vae_spatial = mesh.shape.get("vae_spatial", 1) - if vae_spatial > 1 and (width // 8) % vae_spatial != 0: + # The VAE shards the longer spatial axis, see autoencoder_kl_wan._spatial_sharding. + sharded_axis, sharded_size = ("height", height) if height > width else ("width", width) + if vae_spatial > 1 and (sharded_size // 8) % vae_spatial != 0: max_logging.log( - f"Warning: Latent width is not divisible by vae_spatial mesh axis ({vae_spatial})." + f"Warning: Latent {sharded_axis} is not divisible by vae_spatial mesh axis ({vae_spatial})." " VAE spatial sharding will be partially bypassed." ) diff --git a/src/maxdiffusion/pipelines/wan/wan_vace_pipeline_2_1.py b/src/maxdiffusion/pipelines/wan/wan_vace_pipeline_2_1.py index 229ad793b..879844355 100644 --- a/src/maxdiffusion/pipelines/wan/wan_vace_pipeline_2_1.py +++ b/src/maxdiffusion/pipelines/wan/wan_vace_pipeline_2_1.py @@ -468,9 +468,11 @@ def check_inputs( mesh = getattr(self, "vae_mesh", getattr(self, "mesh", None)) if mesh is not None and hasattr(mesh, "shape"): vae_spatial = mesh.shape.get("vae_spatial", 1) - if vae_spatial > 1 and (width // 8) % vae_spatial != 0: + # The VAE shards the longer spatial axis, see autoencoder_kl_wan._spatial_sharding. + sharded_axis, sharded_size = ("height", height) if height > width else ("width", width) + if vae_spatial > 1 and (sharded_size // 8) % vae_spatial != 0: max_logging.log( - f"Warning: Latent width is not divisible by vae_spatial mesh axis ({vae_spatial})." + f"Warning: Latent {sharded_axis} is not divisible by vae_spatial mesh axis ({vae_spatial})." " VAE spatial sharding will be partially bypassed." ) diff --git a/src/maxdiffusion/tests/wan/wan_vae_test.py b/src/maxdiffusion/tests/wan/wan_vae_test.py index c1bbd26a8..f97b217ae 100644 --- a/src/maxdiffusion/tests/wan/wan_vae_test.py +++ b/src/maxdiffusion/tests/wan/wan_vae_test.py @@ -44,6 +44,7 @@ ZeroPaddedConv2D, WanAttentionBlock, AutoencoderKLWanCache, + _spatial_sharding, ) from maxdiffusion.models.wan.wan_utils import load_wan_vae from maxdiffusion.utils import load_video @@ -506,6 +507,52 @@ def test_wan_encode(self): output = wan_vae.encode(input, feat_cache=vae_cache) assert output.latent_dist.sample(key).shape == (1, 13, 64, 128, 16) + def _create_vae_mesh(self): + devices_array = create_device_mesh(self.config) + vae_spatial = getattr(self.config, "vae_spatial", -1) + if vae_spatial == -1: + vae_spatial = devices_array.size + return Mesh(devices_array.flatten().reshape(-1, vae_spatial), ("redundant", "vae_spatial")) + + def test_spatial_sharding_shards_longer_axis(self): + mesh = self._create_vae_mesh() + width_spec = jax.sharding.PartitionSpec("redundant", None, None, "vae_spatial", None) + height_spec = jax.sharding.PartitionSpec("redundant", None, "vae_spatial", None, None) + + # Landscape and square videos shard the width, portrait videos the height. + self.assertEqual(_spatial_sharding(mesh, (1, 2, 90, 160, 16)).spec, width_spec) + self.assertEqual(_spatial_sharding(mesh, (1, 2, 64, 64, 16)).spec, width_spec) + self.assertEqual(_spatial_sharding(mesh, (1, 2, 160, 90, 16)).spec, height_spec) + # The axis only depends on the aspect ratio, so it is the same at every + # resolution level of the VAE. + for scale in (1, 2, 4, 8): + self.assertEqual(_spatial_sharding(mesh, (1, 2, 1280 // scale, 720 // scale, 96)).spec, height_spec) + self.assertIsNone(_spatial_sharding(None, (1, 2, 160, 90, 16))) + + def test_wan_encode_decode_portrait(self): + key = jax.random.key(0) + rngs = nnx.Rngs(key) + mesh = self._create_vae_mesh() + with mesh, nn_partitioning.axis_rules(self.config.vae_logical_axis_rules): + wan_vae = AutoencoderKLWan( + rngs=rngs, + base_dim=96, + z_dim=16, + dim_mult=[1, 2, 4, 4], + num_res_blocks=2, + attn_scales=[], + temperal_downsample=[False, True, True], + mesh=mesh, + ) + vae_cache = AutoencoderKLWanCache(wan_vae) + # A portrait video whose latent width (3) is not divisible by 4 devices, + # but whose latent height (4) is. + video = jnp.ones((1, 3, 5, 32, 24)) + latents = wan_vae.encode(video, feat_cache=vae_cache).latent_dist.sample(key) + assert latents.shape == (1, 2, 4, 3, 16) + decoded = wan_vae.decode(latents, feat_cache=vae_cache) + assert decoded.sample.shape == (1, 5, 32, 24, 3) + def test_load_checkpoint(self): def vae_encode(video, wan_vae, vae_cache, key): latent = wan_vae.encode(video, feat_cache=vae_cache)