[Feature] Spatially-sharded (SP) decode for the Wan VAE (#4620)
Signed-off-by: Rahul Steiger <rsteiger@nvidia.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
47871d5e20
commit
327e9dcaf2
17 files changed
+1416
-11
No files matched your search
@@ -5,11 +5,13 @@ from contextlib import nullcontext
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
from diffusers.models.autoencoders import AutoencoderKLWan
|
||||
from diffusers.models.autoencoders.autoencoder_kl_wan import unpatchify
|
||||
from diffusers.models.autoencoders.vae import DecoderOutput
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.distributed.autoencoders import wan_spatial_shard
|
||||
from vllm_omni.diffusion.distributed.autoencoders.distributed_vae_executor import (
|
||||
DistributedOperator,
|
||||
DistributedVaeMixin,
|
||||
@@ -268,10 +270,54 @@ class DistributedAutoencoderKLWan(OmniAutoencoderKLWan, DistributedVaeMixin):
|
||||
dec = torch.clamp(dec, min=-1.0, max=1.0)
|
||||
return dec
|
||||
|
||||
def _spatial_shard_decode_split_dim(self) -> str | None:
|
||||
mode = self.distributed_executor.parallel_mode
|
||||
if mode == "spatial_shard_width":
|
||||
return "width"
|
||||
if mode == "spatial_shard_height":
|
||||
return "height"
|
||||
return None
|
||||
|
||||
def _spatial_shard_decode_enabled(self, z: torch.Tensor) -> bool:
|
||||
split_dim = self._spatial_shard_decode_split_dim()
|
||||
if split_dim is None:
|
||||
return False
|
||||
if z.ndim != 5:
|
||||
logger.warning("Wan VAE spatial sharded decode expects 5D latent input; falling back to tiled decode.")
|
||||
return False
|
||||
if not self.is_distributed_enabled():
|
||||
return False
|
||||
|
||||
group = self.distributed_executor.group
|
||||
world_size = dist.get_world_size(group=group)
|
||||
requested_size = int(self.distributed_executor.parallel_size)
|
||||
if requested_size != world_size:
|
||||
logger.warning(
|
||||
"Wan VAE spatial sharded decode currently requires vae_patch_parallel_size "
|
||||
"to match the DIT group size; falling back to tiled decode. "
|
||||
"requested=%s dit_group=%s split_dim=%s",
|
||||
requested_size,
|
||||
world_size,
|
||||
split_dim,
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
def tiled_decode(self, z: torch.Tensor, return_dict: bool = True):
|
||||
if not self.is_distributed_enabled():
|
||||
return super().tiled_decode(z, return_dict=return_dict)
|
||||
|
||||
if self._spatial_shard_decode_enabled(z):
|
||||
split_dim = self._spatial_shard_decode_split_dim()
|
||||
logger.debug("Decode running with Wan VAE spatial_shard_%s mode", split_dim)
|
||||
return wan_spatial_shard.spatial_shard_decode(
|
||||
self,
|
||||
z,
|
||||
group=self.distributed_executor.group,
|
||||
return_dict=return_dict,
|
||||
split_dim=split_dim,
|
||||
)
|
||||
|
||||
logger.debug("Decode running with distributed executor")
|
||||
result = self.distributed_executor.execute(
|
||||
z,
|
||||
|
||||
Reference in new issue
Block a user