diff --git a/src/maxdiffusion/models/wan/autoencoder_kl_wan.py b/src/maxdiffusion/models/wan/autoencoder_kl_wan.py index 999f5234b..00e80857a 100644 --- a/src/maxdiffusion/models/wan/autoencoder_kl_wan.py +++ b/src/maxdiffusion/models/wan/autoencoder_kl_wan.py @@ -48,6 +48,27 @@ def _update_cache(cache, idx, value): return cache[:idx] + (value,) + cache[idx + 1 :] +def _spatial_sharding( + mesh: Optional[jax.sharding.Mesh], shape: Sequence[int], batch_axis: Optional[str] = "redundant" +) -> Optional[NamedSharding]: + """Returns the sharding of a (N, D, H, W, C) array over the vae_spatial axis. + + Shards the longer spatial axis: the height of portrait videos, and the width + otherwise. The aspect ratio is the same at every resolution level of the VAE, + so its activations, caches and latents all use the same axis and don't need + to be resharded between levels. For the WAN 720p and 480p sizes, the longer + side (1280 or 832) stays divisible by 4 and 8 shards down to the latents (160 + or 104), while the latents of the shorter side (90 or 60) aren't divisible by + 8. XLA pads the axis if it isn't divisible by vae_spatial. + """ + if mesh is None or "vae_spatial" not in mesh.shape: + return None + _, _, height, width, _ = shape + if height > width: + return NamedSharding(mesh, P(batch_axis, None, "vae_spatial", None, None)) + return NamedSharding(mesh, P(batch_axis, None, None, "vae_spatial", None)) + + # Helper to ensure kernel_size, stride, padding are tuples of 3 integers def _canonicalize_tuple(x: Union[int, Sequence[int]], rank: int, name: str) -> Tuple[int, ...]: """Canonicalizes a value to a tuple of integers.""" @@ -121,7 +142,7 @@ def __init__( def __call__(self, x: jax.Array, cache_x: Optional[jax.Array] = None, idx=-1) -> jax.Array: if self.mesh is not None and "vae_spatial" in self.mesh.shape: - spatial_sharding = NamedSharding(self.mesh, P("redundant", None, None, "vae_spatial", None)) + spatial_sharding = _spatial_sharding(self.mesh, x.shape) if spatial_sharding is not None: x = jax.lax.with_sharding_constraint(x, spatial_sharding) @@ -1169,12 +1190,6 @@ def _encode(self, x: jax.Array, feat_cache: AutoencoderKLWanCache): iter_ = 1 + ((t - 1 + CHUNK_SIZE - 1) // CHUNK_SIZE) if t > 1 else 1 enc_feat_map = feat_cache._enc_feat_map - spatial_sharding = ( - NamedSharding(self.mesh, P("redundant", None, None, "vae_spatial", None)) - if self.mesh is not None and "vae_spatial" in self.mesh.shape - else None - ) - def finalize(out, enc_feat_map): feat_cache._enc_feat_map = enc_feat_map enc = self.quant_conv(out) @@ -1184,6 +1199,7 @@ def finalize(out, enc_feat_map): with jax.named_scope("AutoencoderKLWan_encode_chunk_0"): chunk_0 = x[:, :1, ...] out_0, enc_feat_map, _ = self.encoder(chunk_0, feat_cache=enc_feat_map, feat_idx=0) + spatial_sharding = _spatial_sharding(self.mesh, out_0.shape) if spatial_sharding is not None: out_0 = jax.lax.with_sharding_constraint(out_0, spatial_sharding) @@ -1280,16 +1296,13 @@ def _decode( x = self.post_quant_conv(z) dec_feat_map = feat_cache._feat_map - spatial_sharding = ( - NamedSharding(self.mesh, P("redundant", None, None, "vae_spatial", None)) - if self.mesh is not None and "vae_spatial" in self.mesh.shape - else None - ) + spatial_sharding = _spatial_sharding(self.mesh, x.shape) # First chunk (i=0) with jax.named_scope("AutoencoderKLWan_decode_chunk_0"): + chunk_in_0 = x[:, 0:1, ...] if spatial_sharding is not None: - chunk_in_0 = jax.lax.with_sharding_constraint(x[:, 0:1, ...], spatial_sharding) + chunk_in_0 = jax.lax.with_sharding_constraint(chunk_in_0, spatial_sharding) out_0, dec_feat_map, _ = self.decoder(chunk_in_0, feat_cache=dec_feat_map, feat_idx=0) if spatial_sharding is not None: out_0 = jax.lax.with_sharding_constraint(out_0, spatial_sharding) @@ -1297,8 +1310,9 @@ def _decode( if iter_ > 1: # Run chunk 1 outside scan to properly form the cache shape with jax.named_scope("AutoencoderKLWan_decode_chunk_1"): + chunk_in_1 = x[:, 1:2, ...] if spatial_sharding is not None: - chunk_in_1 = jax.lax.with_sharding_constraint(x[:, 1:2, ...], spatial_sharding) + chunk_in_1 = jax.lax.with_sharding_constraint(chunk_in_1, spatial_sharding) out_chunk_1, dec_feat_map, _ = self.decoder(chunk_in_1, feat_cache=dec_feat_map, feat_idx=0) if spatial_sharding is not None: out_chunk_1 = jax.lax.with_sharding_constraint(out_chunk_1, spatial_sharding) diff --git a/src/maxdiffusion/models/wan/autoencoder_kl_wan_2p2.py b/src/maxdiffusion/models/wan/autoencoder_kl_wan_2p2.py index 841b115c9..c1b300497 100644 --- a/src/maxdiffusion/models/wan/autoencoder_kl_wan_2p2.py +++ b/src/maxdiffusion/models/wan/autoencoder_kl_wan_2p2.py @@ -20,8 +20,7 @@ import jax from jax import tree_util import jax.numpy as jnp -from jax.sharding import NamedSharding, PartitionSpec as P -from maxdiffusion.models.wan.autoencoder_kl_wan import AutoencoderKLWanCache, WanCausalConv3d # pylint: disable=g-importing-member +from maxdiffusion.models.wan.autoencoder_kl_wan import AutoencoderKLWanCache, WanCausalConv3d, _spatial_sharding # pylint: disable=g-importing-member,protected-access from ... import common_types from ...configuration_utils import ConfigMixin @@ -1341,7 +1340,7 @@ def _encode(self, x: jax.Array, feat_cache: AutoencoderKLWanCache): iter_ = 1 + (t - 1) // 4 enc_feat_map = feat_cache._enc_feat_map - spatial_sharding = NamedSharding(self.mesh, P(None, None, None, "vae_spatial", None)) + spatial_sharding = _spatial_sharding(self.mesh, x.shape, batch_axis=None) for i in range(iter_): enc_conv_idx = 0 if i == 0: @@ -1394,7 +1393,7 @@ def _decode( dec_feat_map = feat_cache._feat_map - spatial_sharding = NamedSharding(self.mesh, P(None, None, None, "vae_spatial", None)) + spatial_sharding = _spatial_sharding(self.mesh, x.shape, batch_axis=None) for i in range(iter_): conv_idx = 0 chunk = x[:, i : i + 1, :, :, :] diff --git a/src/maxdiffusion/pipelines/wan/wan_pipeline.py b/src/maxdiffusion/pipelines/wan/wan_pipeline.py index 874158e05..77dc0abdd 100644 --- a/src/maxdiffusion/pipelines/wan/wan_pipeline.py +++ b/src/maxdiffusion/pipelines/wan/wan_pipeline.py @@ -556,9 +556,11 @@ def check_inputs( mesh = getattr(self, "vae_mesh", getattr(self, "mesh", None)) if mesh is not None and hasattr(mesh, "shape"): vae_spatial = mesh.shape.get("vae_spatial", 1) - if vae_spatial > 1 and (width // 8) % vae_spatial != 0: + # The VAE shards the longer spatial axis, see autoencoder_kl_wan._spatial_sharding. + sharded_axis, sharded_size = ("height", height) if height > width else ("width", width) + if vae_spatial > 1 and (sharded_size // 8) % vae_spatial != 0: max_logging.log( - f"Warning: Latent width is not divisible by vae_spatial mesh axis ({vae_spatial})." + f"Warning: Latent {sharded_axis} is not divisible by vae_spatial mesh axis ({vae_spatial})." " VAE spatial sharding will be partially bypassed." ) diff --git a/src/maxdiffusion/pipelines/wan/wan_pipeline_animate.py b/src/maxdiffusion/pipelines/wan/wan_pipeline_animate.py index 19f647e1e..ce4cef6a3 100644 --- a/src/maxdiffusion/pipelines/wan/wan_pipeline_animate.py +++ b/src/maxdiffusion/pipelines/wan/wan_pipeline_animate.py @@ -420,9 +420,11 @@ def check_inputs( mesh = getattr(self, "vae_mesh", getattr(self, "mesh", None)) if mesh is not None and hasattr(mesh, "shape"): vae_spatial = mesh.shape.get("vae_spatial", 1) - if vae_spatial > 1 and (width // 8) % vae_spatial != 0: + # The VAE shards the longer spatial axis, see autoencoder_kl_wan._spatial_sharding. + sharded_axis, sharded_size = ("height", height) if height > width else ("width", width) + if vae_spatial > 1 and (sharded_size // 8) % vae_spatial != 0: max_logging.log( - f"Warning: Latent width is not divisible by vae_spatial mesh axis ({vae_spatial})." + f"Warning: Latent {sharded_axis} is not divisible by vae_spatial mesh axis ({vae_spatial})." " VAE spatial sharding will be partially bypassed." ) diff --git a/src/maxdiffusion/pipelines/wan/wan_vace_pipeline_2_1.py b/src/maxdiffusion/pipelines/wan/wan_vace_pipeline_2_1.py index 229ad793b..879844355 100644 --- a/src/maxdiffusion/pipelines/wan/wan_vace_pipeline_2_1.py +++ b/src/maxdiffusion/pipelines/wan/wan_vace_pipeline_2_1.py @@ -468,9 +468,11 @@ def check_inputs( mesh = getattr(self, "vae_mesh", getattr(self, "mesh", None)) if mesh is not None and hasattr(mesh, "shape"): vae_spatial = mesh.shape.get("vae_spatial", 1) - if vae_spatial > 1 and (width // 8) % vae_spatial != 0: + # The VAE shards the longer spatial axis, see autoencoder_kl_wan._spatial_sharding. + sharded_axis, sharded_size = ("height", height) if height > width else ("width", width) + if vae_spatial > 1 and (sharded_size // 8) % vae_spatial != 0: max_logging.log( - f"Warning: Latent width is not divisible by vae_spatial mesh axis ({vae_spatial})." + f"Warning: Latent {sharded_axis} is not divisible by vae_spatial mesh axis ({vae_spatial})." " VAE spatial sharding will be partially bypassed." ) diff --git a/src/maxdiffusion/tests/wan/wan_vae_test.py b/src/maxdiffusion/tests/wan/wan_vae_test.py index c1bbd26a8..f97b217ae 100644 --- a/src/maxdiffusion/tests/wan/wan_vae_test.py +++ b/src/maxdiffusion/tests/wan/wan_vae_test.py @@ -44,6 +44,7 @@ ZeroPaddedConv2D, WanAttentionBlock, AutoencoderKLWanCache, + _spatial_sharding, ) from maxdiffusion.models.wan.wan_utils import load_wan_vae from maxdiffusion.utils import load_video @@ -506,6 +507,52 @@ def test_wan_encode(self): output = wan_vae.encode(input, feat_cache=vae_cache) assert output.latent_dist.sample(key).shape == (1, 13, 64, 128, 16) + def _create_vae_mesh(self): + devices_array = create_device_mesh(self.config) + vae_spatial = getattr(self.config, "vae_spatial", -1) + if vae_spatial == -1: + vae_spatial = devices_array.size + return Mesh(devices_array.flatten().reshape(-1, vae_spatial), ("redundant", "vae_spatial")) + + def test_spatial_sharding_shards_longer_axis(self): + mesh = self._create_vae_mesh() + width_spec = jax.sharding.PartitionSpec("redundant", None, None, "vae_spatial", None) + height_spec = jax.sharding.PartitionSpec("redundant", None, "vae_spatial", None, None) + + # Landscape and square videos shard the width, portrait videos the height. + self.assertEqual(_spatial_sharding(mesh, (1, 2, 90, 160, 16)).spec, width_spec) + self.assertEqual(_spatial_sharding(mesh, (1, 2, 64, 64, 16)).spec, width_spec) + self.assertEqual(_spatial_sharding(mesh, (1, 2, 160, 90, 16)).spec, height_spec) + # The axis only depends on the aspect ratio, so it is the same at every + # resolution level of the VAE. + for scale in (1, 2, 4, 8): + self.assertEqual(_spatial_sharding(mesh, (1, 2, 1280 // scale, 720 // scale, 96)).spec, height_spec) + self.assertIsNone(_spatial_sharding(None, (1, 2, 160, 90, 16))) + + def test_wan_encode_decode_portrait(self): + key = jax.random.key(0) + rngs = nnx.Rngs(key) + mesh = self._create_vae_mesh() + with mesh, nn_partitioning.axis_rules(self.config.vae_logical_axis_rules): + wan_vae = AutoencoderKLWan( + rngs=rngs, + base_dim=96, + z_dim=16, + dim_mult=[1, 2, 4, 4], + num_res_blocks=2, + attn_scales=[], + temperal_downsample=[False, True, True], + mesh=mesh, + ) + vae_cache = AutoencoderKLWanCache(wan_vae) + # A portrait video whose latent width (3) is not divisible by 4 devices, + # but whose latent height (4) is. + video = jnp.ones((1, 3, 5, 32, 24)) + latents = wan_vae.encode(video, feat_cache=vae_cache).latent_dist.sample(key) + assert latents.shape == (1, 2, 4, 3, 16) + decoded = wan_vae.decode(latents, feat_cache=vae_cache) + assert decoded.sample.shape == (1, 5, 32, 24, 3) + def test_load_checkpoint(self): def vae_encode(video, wan_vae, vae_cache, key): latent = wan_vae.encode(video, feat_cache=vae_cache)