[Perf] Wan2.2 I2V optimization: convert datatype from FP32 to BF16 in vae (#2391)

Signed-off-by: KexiongYu <yukexiong1@huawei.com>
Signed-off-by: Canlin Guo <961750412@qq.com>
Co-authored-by: Canlin Guo <961750412@qq.com>
This commit is contained in:
Y. FisherandCanlin Guo authored and GitHub committed 2026-04-08 03:26:39 +00:00
1 parent cb6a8739c1
commit ec082add35
7 files changed
+123 -11

No files matched your search

@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from contextlib import nullcontext
from typing import Any
import torch
@@ -15,11 +16,38 @@ from vllm_omni.diffusion.distributed.autoencoders.distributed_vae_executor impor
GridSpec,
TileTask,
)
from vllm_omni.platforms import current_omni_platform
logger = init_logger(__name__)
class DistributedAutoencoderKLWan(AutoencoderKLWan, DistributedVaeMixin):
class OmniAutoencoderKLWan(AutoencoderKLWan):
def _execution_context(self):
try:
first_param = next(self.parameters())
except StopIteration:
return nullcontext()
dtype = first_param.dtype
if dtype not in (torch.float16, torch.bfloat16):
return nullcontext()
return current_omni_platform.create_autocast_context(
device_type=first_param.device.type,
dtype=dtype,
enabled=True,
)
def encode(self, x: torch.Tensor, return_dict: bool = True):
with self._execution_context():
return super().encode(x, return_dict=return_dict)
def decode(self, z: torch.Tensor, return_dict: bool = True):
with self._execution_context():
return super().decode(z, return_dict=return_dict)
class DistributedAutoencoderKLWan(OmniAutoencoderKLWan, DistributedVaeMixin):
@classmethod
def from_pretrained(cls, *args: Any, **kwargs: Any):
model = super().from_pretrained(*args, **kwargs)
@@ -84,11 +112,12 @@ class DistributedAutoencoderKLWan(AutoencoderKLWan, DistributedVaeMixin):
"""Decode a single latent tile into RGB space."""
self.clear_cache()
time = []
for k in range(len(task.tensor)):
self._conv_idx = [0]
tile = self.post_quant_conv(task.tensor[k])
decoded = self.decoder(tile, feat_cache=self._feat_map, feat_idx=self._conv_idx, first_chunk=(k == 0))
time.append(decoded)
with self._execution_context():
for k in range(len(task.tensor)):
self._conv_idx = [0]
tile = self.post_quant_conv(task.tensor[k])
decoded = self.decoder(tile, feat_cache=self._feat_map, feat_idx=self._conv_idx, first_chunk=(k == 0))
time.append(decoded)
result = torch.cat(time, dim=2)
return result
@@ -272,7 +272,7 @@ class Wan22Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin, DiffusionPipe
model, subfolder="text_encoder", torch_dtype=dtype, local_files_only=local_files_only
).to(self.device)
self.vae = DistributedAutoencoderKLWan.from_pretrained(
model, subfolder="vae", torch_dtype=torch.float32, local_files_only=local_files_only
model, subfolder="vae", torch_dtype=dtype, local_files_only=local_files_only
).to(self.device)
# Initialize transformers with correct config (weights loaded via load_weights)
@@ -217,7 +217,7 @@ class Wan22I2VPipeline(
# VAE
self.vae = DistributedAutoencoderKLWan.from_pretrained(
model, subfolder="vae", torch_dtype=torch.float32, local_files_only=local_files_only
model, subfolder="vae", torch_dtype=dtype, local_files_only=local_files_only
).to(self.device)
# Transformers (weights loaded via load_weights)
@@ -24,13 +24,13 @@ from typing import Any, cast
import numpy as np
import PIL.Image
import torch
from diffusers import AutoencoderKLWan
from diffusers.utils.torch_utils import randn_tensor
from torch import nn
from transformers import AutoTokenizer, UMT5EncoderModel
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_wan import OmniAutoencoderKLWan
from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
@@ -174,8 +174,8 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin, Progress
).to(self.device)
# VAE
self.vae = AutoencoderKLWan.from_pretrained(
model, subfolder="vae", torch_dtype=torch.float32, local_files_only=local_files_only
self.vae = OmniAutoencoderKLWan.from_pretrained(
model, subfolder="vae", torch_dtype=dtype, local_files_only=local_files_only
).to(self.device)
# Single transformer (TI2V uses dense 5B model, not MoE)