Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 28 additions & 14 deletions src/maxdiffusion/models/wan/autoencoder_kl_wan.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,27 @@ def _update_cache(cache, idx, value):
return cache[:idx] + (value,) + cache[idx + 1 :]


def _spatial_sharding(
mesh: Optional[jax.sharding.Mesh], shape: Sequence[int], batch_axis: Optional[str] = "redundant"
) -> Optional[NamedSharding]:
"""Returns the sharding of a (N, D, H, W, C) array over the vae_spatial axis.

Shards the longer spatial axis: the height of portrait videos, and the width
otherwise. The aspect ratio is the same at every resolution level of the VAE,
so its activations, caches and latents all use the same axis and don't need
to be resharded between levels. For the WAN 720p and 480p sizes, the longer
side (1280 or 832) stays divisible by 4 and 8 shards down to the latents (160
or 104), while the latents of the shorter side (90 or 60) aren't divisible by
8. XLA pads the axis if it isn't divisible by vae_spatial.
"""
if mesh is None or "vae_spatial" not in mesh.shape:
return None
_, _, height, width, _ = shape
Comment on lines +64 to +66

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

To prevent potential ValueError during unpacking, we should defensively check that the input shape is indeed 5D (i.e., has a length of 5) before attempting to unpack it. If it is not 5D, we can safely return None to skip applying the sharding constraint.

Suggested change
if mesh is None or "vae_spatial" not in mesh.shape:
return None
_, _, height, width, _ = shape
if mesh is None or "vae_spatial" not in mesh.shape or len(shape) != 5:
return None
_, _, height, width, _ = shape

if height > width:
return NamedSharding(mesh, P(batch_axis, None, "vae_spatial", None, None))
return NamedSharding(mesh, P(batch_axis, None, None, "vae_spatial", None))


# Helper to ensure kernel_size, stride, padding are tuples of 3 integers
def _canonicalize_tuple(x: Union[int, Sequence[int]], rank: int, name: str) -> Tuple[int, ...]:
"""Canonicalizes a value to a tuple of integers."""
Expand Down Expand Up @@ -121,7 +142,7 @@ def __init__(

def __call__(self, x: jax.Array, cache_x: Optional[jax.Array] = None, idx=-1) -> jax.Array:
if self.mesh is not None and "vae_spatial" in self.mesh.shape:
spatial_sharding = NamedSharding(self.mesh, P("redundant", None, None, "vae_spatial", None))
spatial_sharding = _spatial_sharding(self.mesh, x.shape)
if spatial_sharding is not None:
x = jax.lax.with_sharding_constraint(x, spatial_sharding)

Expand Down Expand Up @@ -1169,12 +1190,6 @@ def _encode(self, x: jax.Array, feat_cache: AutoencoderKLWanCache):
iter_ = 1 + ((t - 1 + CHUNK_SIZE - 1) // CHUNK_SIZE) if t > 1 else 1
enc_feat_map = feat_cache._enc_feat_map

spatial_sharding = (
NamedSharding(self.mesh, P("redundant", None, None, "vae_spatial", None))
if self.mesh is not None and "vae_spatial" in self.mesh.shape
else None
)

def finalize(out, enc_feat_map):
feat_cache._enc_feat_map = enc_feat_map
enc = self.quant_conv(out)
Expand All @@ -1184,6 +1199,7 @@ def finalize(out, enc_feat_map):
with jax.named_scope("AutoencoderKLWan_encode_chunk_0"):
chunk_0 = x[:, :1, ...]
out_0, enc_feat_map, _ = self.encoder(chunk_0, feat_cache=enc_feat_map, feat_idx=0)
spatial_sharding = _spatial_sharding(self.mesh, out_0.shape)
if spatial_sharding is not None:
out_0 = jax.lax.with_sharding_constraint(out_0, spatial_sharding)

Expand Down Expand Up @@ -1280,25 +1296,23 @@ def _decode(
x = self.post_quant_conv(z)

dec_feat_map = feat_cache._feat_map
spatial_sharding = (
NamedSharding(self.mesh, P("redundant", None, None, "vae_spatial", None))
if self.mesh is not None and "vae_spatial" in self.mesh.shape
else None
)
spatial_sharding = _spatial_sharding(self.mesh, x.shape)

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

There is a critical bug in _decode where if spatial_sharding is None (e.g., when running on a single device without a mesh), chunk_in_0 (line 1298) and chunk_in_1 (line 1307) are never defined. This will raise a NameError when they are passed to self.decoder on lines 1299 and 1308.

To fix this, you should define chunk_in_0 and chunk_in_1 unconditionally and apply the sharding constraint conditionally. For example:

# First chunk (i=0)
with jax.named_scope("AutoencoderKLWan_decode_chunk_0"):
  chunk_in_0 = x[:, 0:1, ...]
  if spatial_sharding is not None:
    chunk_in_0 = jax.lax.with_sharding_constraint(chunk_in_0, spatial_sharding)
  out_0, dec_feat_map, _ = self.decoder(chunk_in_0, feat_cache=dec_feat_map, feat_idx=0)

And similarly for chunk_in_1.


# First chunk (i=0)
with jax.named_scope("AutoencoderKLWan_decode_chunk_0"):
chunk_in_0 = x[:, 0:1, ...]
if spatial_sharding is not None:
chunk_in_0 = jax.lax.with_sharding_constraint(x[:, 0:1, ...], spatial_sharding)
chunk_in_0 = jax.lax.with_sharding_constraint(chunk_in_0, spatial_sharding)
out_0, dec_feat_map, _ = self.decoder(chunk_in_0, feat_cache=dec_feat_map, feat_idx=0)
if spatial_sharding is not None:
out_0 = jax.lax.with_sharding_constraint(out_0, spatial_sharding)

if iter_ > 1:
# Run chunk 1 outside scan to properly form the cache shape
with jax.named_scope("AutoencoderKLWan_decode_chunk_1"):
chunk_in_1 = x[:, 1:2, ...]
if spatial_sharding is not None:
chunk_in_1 = jax.lax.with_sharding_constraint(x[:, 1:2, ...], spatial_sharding)
chunk_in_1 = jax.lax.with_sharding_constraint(chunk_in_1, spatial_sharding)
out_chunk_1, dec_feat_map, _ = self.decoder(chunk_in_1, feat_cache=dec_feat_map, feat_idx=0)
if spatial_sharding is not None:
out_chunk_1 = jax.lax.with_sharding_constraint(out_chunk_1, spatial_sharding)
Expand Down
7 changes: 3 additions & 4 deletions src/maxdiffusion/models/wan/autoencoder_kl_wan_2p2.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,7 @@
import jax
from jax import tree_util
import jax.numpy as jnp
from jax.sharding import NamedSharding, PartitionSpec as P
from maxdiffusion.models.wan.autoencoder_kl_wan import AutoencoderKLWanCache, WanCausalConv3d # pylint: disable=g-importing-member
from maxdiffusion.models.wan.autoencoder_kl_wan import AutoencoderKLWanCache, WanCausalConv3d, _spatial_sharding # pylint: disable=g-importing-member,protected-access

from ... import common_types
from ...configuration_utils import ConfigMixin
Expand Down Expand Up @@ -1341,7 +1340,7 @@ def _encode(self, x: jax.Array, feat_cache: AutoencoderKLWanCache):
iter_ = 1 + (t - 1) // 4
enc_feat_map = feat_cache._enc_feat_map

spatial_sharding = NamedSharding(self.mesh, P(None, None, None, "vae_spatial", None))
spatial_sharding = _spatial_sharding(self.mesh, x.shape, batch_axis=None)
for i in range(iter_):
enc_conv_idx = 0
if i == 0:
Expand Down Expand Up @@ -1394,7 +1393,7 @@ def _decode(

dec_feat_map = feat_cache._feat_map

spatial_sharding = NamedSharding(self.mesh, P(None, None, None, "vae_spatial", None))
spatial_sharding = _spatial_sharding(self.mesh, x.shape, batch_axis=None)
for i in range(iter_):
conv_idx = 0
chunk = x[:, i : i + 1, :, :, :]
Expand Down
6 changes: 4 additions & 2 deletions src/maxdiffusion/pipelines/wan/wan_pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -556,9 +556,11 @@ def check_inputs(
mesh = getattr(self, "vae_mesh", getattr(self, "mesh", None))
if mesh is not None and hasattr(mesh, "shape"):
vae_spatial = mesh.shape.get("vae_spatial", 1)
if vae_spatial > 1 and (width // 8) % vae_spatial != 0:
# The VAE shards the longer spatial axis, see autoencoder_kl_wan._spatial_sharding.
sharded_axis, sharded_size = ("height", height) if height > width else ("width", width)
if vae_spatial > 1 and (sharded_size // 8) % vae_spatial != 0:
max_logging.log(
f"Warning: Latent width is not divisible by vae_spatial mesh axis ({vae_spatial})."
f"Warning: Latent {sharded_axis} is not divisible by vae_spatial mesh axis ({vae_spatial})."
" VAE spatial sharding will be partially bypassed."
)

Expand Down
6 changes: 4 additions & 2 deletions src/maxdiffusion/pipelines/wan/wan_pipeline_animate.py
Original file line number Diff line number Diff line change
Expand Up @@ -420,9 +420,11 @@ def check_inputs(
mesh = getattr(self, "vae_mesh", getattr(self, "mesh", None))
if mesh is not None and hasattr(mesh, "shape"):
vae_spatial = mesh.shape.get("vae_spatial", 1)
if vae_spatial > 1 and (width // 8) % vae_spatial != 0:
# The VAE shards the longer spatial axis, see autoencoder_kl_wan._spatial_sharding.
sharded_axis, sharded_size = ("height", height) if height > width else ("width", width)
if vae_spatial > 1 and (sharded_size // 8) % vae_spatial != 0:
max_logging.log(
f"Warning: Latent width is not divisible by vae_spatial mesh axis ({vae_spatial})."
f"Warning: Latent {sharded_axis} is not divisible by vae_spatial mesh axis ({vae_spatial})."
" VAE spatial sharding will be partially bypassed."
)

Expand Down
6 changes: 4 additions & 2 deletions src/maxdiffusion/pipelines/wan/wan_vace_pipeline_2_1.py
Original file line number Diff line number Diff line change
Expand Up @@ -468,9 +468,11 @@ def check_inputs(
mesh = getattr(self, "vae_mesh", getattr(self, "mesh", None))
if mesh is not None and hasattr(mesh, "shape"):
vae_spatial = mesh.shape.get("vae_spatial", 1)
if vae_spatial > 1 and (width // 8) % vae_spatial != 0:
# The VAE shards the longer spatial axis, see autoencoder_kl_wan._spatial_sharding.
sharded_axis, sharded_size = ("height", height) if height > width else ("width", width)
if vae_spatial > 1 and (sharded_size // 8) % vae_spatial != 0:
max_logging.log(
f"Warning: Latent width is not divisible by vae_spatial mesh axis ({vae_spatial})."
f"Warning: Latent {sharded_axis} is not divisible by vae_spatial mesh axis ({vae_spatial})."
" VAE spatial sharding will be partially bypassed."
)

Expand Down
47 changes: 47 additions & 0 deletions src/maxdiffusion/tests/wan/wan_vae_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
ZeroPaddedConv2D,
WanAttentionBlock,
AutoencoderKLWanCache,
_spatial_sharding,
)
from maxdiffusion.models.wan.wan_utils import load_wan_vae
from maxdiffusion.utils import load_video
Expand Down Expand Up @@ -506,6 +507,52 @@ def test_wan_encode(self):
output = wan_vae.encode(input, feat_cache=vae_cache)
assert output.latent_dist.sample(key).shape == (1, 13, 64, 128, 16)

def _create_vae_mesh(self):
devices_array = create_device_mesh(self.config)
vae_spatial = getattr(self.config, "vae_spatial", -1)
if vae_spatial == -1:
vae_spatial = devices_array.size
return Mesh(devices_array.flatten().reshape(-1, vae_spatial), ("redundant", "vae_spatial"))

def test_spatial_sharding_shards_longer_axis(self):
mesh = self._create_vae_mesh()
width_spec = jax.sharding.PartitionSpec("redundant", None, None, "vae_spatial", None)
height_spec = jax.sharding.PartitionSpec("redundant", None, "vae_spatial", None, None)

# Landscape and square videos shard the width, portrait videos the height.
self.assertEqual(_spatial_sharding(mesh, (1, 2, 90, 160, 16)).spec, width_spec)
self.assertEqual(_spatial_sharding(mesh, (1, 2, 64, 64, 16)).spec, width_spec)
self.assertEqual(_spatial_sharding(mesh, (1, 2, 160, 90, 16)).spec, height_spec)
# The axis only depends on the aspect ratio, so it is the same at every
# resolution level of the VAE.
for scale in (1, 2, 4, 8):
self.assertEqual(_spatial_sharding(mesh, (1, 2, 1280 // scale, 720 // scale, 96)).spec, height_spec)
self.assertIsNone(_spatial_sharding(None, (1, 2, 160, 90, 16)))

def test_wan_encode_decode_portrait(self):
key = jax.random.key(0)
rngs = nnx.Rngs(key)
mesh = self._create_vae_mesh()
with mesh, nn_partitioning.axis_rules(self.config.vae_logical_axis_rules):
wan_vae = AutoencoderKLWan(
rngs=rngs,
base_dim=96,
z_dim=16,
dim_mult=[1, 2, 4, 4],
num_res_blocks=2,
attn_scales=[],
temperal_downsample=[False, True, True],
mesh=mesh,
)
vae_cache = AutoencoderKLWanCache(wan_vae)
# A portrait video whose latent width (3) is not divisible by 4 devices,
# but whose latent height (4) is.
video = jnp.ones((1, 3, 5, 32, 24))
latents = wan_vae.encode(video, feat_cache=vae_cache).latent_dist.sample(key)
assert latents.shape == (1, 2, 4, 3, 16)
decoded = wan_vae.decode(latents, feat_cache=vae_cache)
assert decoded.sample.shape == (1, 5, 32, 24, 3)

def test_load_checkpoint(self):
def vae_encode(video, wan_vae, vae_cache, key):
latent = wan_vae.encode(video, feat_cache=vae_cache)
Expand Down
Loading