[Feature] Spatially-sharded (SP) decode for the Wan VAE (#4620)

Signed-off-by: Rahul Steiger <rsteiger@nvidia.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
Rahul SteigerandHongsheng Liu authored and GitHub committed 2026-06-27 09:01:25 +08:00
1 parent 47871d5e20
commit 327e9dcaf2
17 files changed
+1416 -11

No files matched your search

+19
View File
@@ -183,6 +183,21 @@ class DiffusionParallelConfig:
vae_patch_parallel_size: int = 1
"""Number of ranks used for VAE patch/tile parallelism (decode/encode)."""
vae_parallel_mode: str = "tile"
"""VAE parallel decode strategy.
- "tile": Patch/tile parallel decode (default). Each rank decodes a subset
of spatial tiles and the results are stitched on rank 0.
- "spatial_shard_height": Spatially-sharded decode that splits decoder
feature maps along height and exchanges halo rows around spatial
convolutions.
- "spatial_shard_width": Same as "spatial_shard_height" but sharded along width.
The "spatial_shard_*" modes are decode-only and currently require
``vae_patch_parallel_size`` to match the DiT group size; otherwise the VAE
falls back to tile-parallel decode at runtime.
"""
use_hsdp: bool = False
"""Enable Hybrid Sharded Data Parallel (HSDP) for model weight sharding."""
@@ -214,6 +229,10 @@ class DiffusionParallelConfig:
f"CFG parallel size must be 1, 2, or 3, but got {self.cfg_parallel_size}"
)
assert self.vae_patch_parallel_size > 0, "VAE patch parallel size must be > 0"
assert self.vae_parallel_mode in {"tile", "spatial_shard_height", "spatial_shard_width"}, (
"vae_parallel_mode must be one of {'tile', 'spatial_shard_height', 'spatial_shard_width'}, "
f"but got {self.vae_parallel_mode!r}."
)
assert self.sequence_parallel_size == self.ulysses_degree * self.ring_degree, (
"Sequence parallel size must be equal to the product of ulysses degree and ring degree,"
f" but got {self.sequence_parallel_size} != {self.ulysses_degree} * {self.ring_degree}"