[Model] Add Wan2.2 I2V and TI2V pipeline support (#329)

Signed-off-by: linyueqian <linyueqian@outlook.com>
Signed-off-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
Yueqian LinandHongsheng Liu authored and GitHub committed 2025-12-25 16:45:13 +08:00
1 parent d51f4c086d
commit 3ec35e5409
7 files changed
+1752 -29

No files matched your search

+2 -1
View File
@@ -24,7 +24,8 @@ th {
| `QwenImageEditPlusPipeline` | Qwen-Image-Edit-2509 | `Qwen/Qwen-Image-Edit-2509` |
| `QwenImageLayeredPipeline` | Qwen-Image-Layered | `Qwen/Qwen-Image-Layered` |
|`ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` |
| `WanPipeline` | Wan2.2 | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` |
| `WanPipeline` | Wan2.2-T2V, Wan2.2-TI2V | `Wan-AI/Wan2.2-T2V-A14B-Diffusers`, `Wan-AI/Wan2.2-TI2V-5B-Diffusers` |
| `WanImageToVideoPipeline` | Wan2.2-I2V | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` |
| `OvisImagePipeline` | Ovis-Image | `OvisAI/Ovis-Image` |
|`LongcatImagePipeline` | LongCat-Image | `meituan-longcat/LongCat-Image` |
|`LongCatImageEditPipeline` | LongCat-Image-Edit | `meituan-longcat/LongCat-Image-Edit` |
@@ -0,0 +1,156 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Image-to-Video generation example using Wan2.2 I2V/TI2V models.
Supports:
- Wan2.2-I2V-A14B-Diffusers: MoE model with CLIP image encoder
- Wan2.2-TI2V-5B-Diffusers: Unified T2V+I2V model (dense 5B)
Usage:
# I2V-A14B (MoE)
python image_to_video.py --model Wan-AI/Wan2.2-I2V-A14B-Diffusers \
--image input.jpg --prompt "A cat playing with yarn"
# TI2V-5B (unified)
python image_to_video.py --model Wan-AI/Wan2.2-TI2V-5B-Diffusers \
--image input.jpg --prompt "A cat playing with yarn"
"""
import argparse
from pathlib import Path
import numpy as np
import PIL.Image
import torch
from vllm_omni.entrypoints.omni import Omni
from vllm_omni.utils.platform_utils import detect_device_type, is_npu
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Generate a video from an image with Wan2.2 I2V/TI2V.")
parser.add_argument(
"--model",
default="Wan-AI/Wan2.2-I2V-A14B-Diffusers",
help="Diffusers Wan2.2 I2V model ID or local path.",
)
parser.add_argument("--image", required=True, help="Path to input image.")
parser.add_argument("--prompt", default="", help="Text prompt describing the desired motion.")
parser.add_argument("--negative_prompt", default="", help="Negative prompt.")
parser.add_argument("--seed", type=int, default=42, help="Random seed.")
parser.add_argument("--guidance_scale", type=float, default=5.0, help="CFG scale.")
parser.add_argument(
"--guidance_scale_high", type=float, default=None, help="Optional separate CFG for high-noise (MoE only)."
)
parser.add_argument(
"--height", type=int, default=None, help="Video height (auto-calculated from image if not set)."
)
parser.add_argument("--width", type=int, default=None, help="Video width (auto-calculated from image if not set).")
parser.add_argument("--num_frames", type=int, default=81, help="Number of frames.")
parser.add_argument("--num_inference_steps", type=int, default=50, help="Sampling steps.")
parser.add_argument("--boundary_ratio", type=float, default=0.875, help="Boundary split ratio for MoE models.")
parser.add_argument(
"--flow_shift", type=float, default=5.0, help="Scheduler flow_shift (5.0 for 720p, 12.0 for 480p)."
)
parser.add_argument("--output", type=str, default="i2v_output.mp4", help="Path to save the video (mp4).")
parser.add_argument("--fps", type=int, default=16, help="Frames per second for the output video.")
return parser.parse_args()
def calculate_dimensions(image: PIL.Image.Image, max_area: int = 480 * 832) -> tuple[int, int]:
"""Calculate output dimensions maintaining aspect ratio."""
aspect_ratio = image.height / image.width
mod_value = 16 # Must be divisible by 16
height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value
width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value
return height, width
def main():
args = parse_args()
device = detect_device_type()
generator = torch.Generator(device=device).manual_seed(args.seed)
# Load input image
image = PIL.Image.open(args.image).convert("RGB")
# Calculate dimensions if not provided
height = args.height
width = args.width
if height is None or width is None:
# Default to 480P area for I2V
calc_height, calc_width = calculate_dimensions(image, max_area=480 * 832)
height = height or calc_height
width = width or calc_width
# Resize image to target dimensions
image = image.resize((width, height), PIL.Image.Resampling.LANCZOS)
# Enable VAE memory optimizations on NPU
vae_use_slicing = is_npu()
vae_use_tiling = is_npu()
omni = Omni(
model=args.model,
vae_use_slicing=vae_use_slicing,
vae_use_tiling=vae_use_tiling,
boundary_ratio=args.boundary_ratio,
flow_shift=args.flow_shift,
)
frames = omni.generate(
args.prompt,
negative_prompt=args.negative_prompt,
pil_image=image,
height=height,
width=width,
generator=generator,
guidance_scale=args.guidance_scale,
guidance_scale_2=args.guidance_scale_high,
num_inference_steps=args.num_inference_steps,
num_frames=args.num_frames,
)
output_path = Path(args.output)
output_path.parent.mkdir(parents=True, exist_ok=True)
try:
from diffusers.utils import export_to_video
except ImportError:
raise ImportError("diffusers is required for export_to_video.")
# frames may be np.ndarray (preferred) or torch.Tensor
# export_to_video expects a list of frames with values in [0, 1]
if isinstance(frames, torch.Tensor):
video_tensor = frames.detach().cpu()
if video_tensor.dim() == 5:
# [B, C, F, H, W] or [B, F, H, W, C]
if video_tensor.shape[1] in (3, 4):
video_tensor = video_tensor[0].permute(1, 2, 3, 0)
else:
video_tensor = video_tensor[0]
elif video_tensor.dim() == 4 and video_tensor.shape[0] in (3, 4):
video_tensor = video_tensor.permute(1, 2, 3, 0)
# If float, assume [-1,1] and normalize to [0,1]
if video_tensor.is_floating_point():
video_tensor = video_tensor.clamp(-1, 1) * 0.5 + 0.5
video_array = video_tensor.float().numpy()
else:
video_array = frames
if hasattr(video_array, "shape") and video_array.ndim == 5:
video_array = video_array[0]
# Convert 4D array (frames, H, W, C) to list of frames for export_to_video
if isinstance(video_array, np.ndarray) and video_array.ndim == 4:
video_array = list(video_array)
export_to_video(video_array, str(output_path), fps=args.fps)
print(f"Saved generated video to {output_path}")
if __name__ == "__main__":
main()
+33 -2
View File
@@ -1,4 +1,35 @@
from .pipeline_wan2_2 import Wan22Pipeline, get_wan22_post_process_func
from .pipeline_wan2_2 import (
Wan22Pipeline,
create_transformer_from_config,
get_wan22_post_process_func,
get_wan22_pre_process_func,
load_transformer_config,
retrieve_latents,
)
from .pipeline_wan2_2_i2v import (
Wan22I2VPipeline,
get_wan22_i2v_post_process_func,
get_wan22_i2v_pre_process_func,
)
from .pipeline_wan2_2_ti2v import (
Wan22TI2VPipeline,
get_wan22_ti2v_post_process_func,
get_wan22_ti2v_pre_process_func,
)
from .wan2_2_transformer import WanTransformer3DModel
__all__ = ["Wan22Pipeline", "get_wan22_post_process_func", "WanTransformer3DModel"]
__all__ = [
"Wan22Pipeline",
"get_wan22_post_process_func",
"get_wan22_pre_process_func",
"retrieve_latents",
"load_transformer_config",
"create_transformer_from_config",
"Wan22I2VPipeline",
"get_wan22_i2v_post_process_func",
"get_wan22_i2v_pre_process_func",
"Wan22TI2VPipeline",
"get_wan22_ti2v_post_process_func",
"get_wan22_ti2v_pre_process_func",
"WanTransformer3DModel",
]
@@ -3,9 +3,11 @@
from __future__ import annotations
import json
import os
from collections.abc import Iterable
import PIL.Image
import torch
from diffusers import AutoencoderKLWan, FlowMatchEulerDiscreteScheduler
from diffusers.utils.torch_utils import randn_tensor
@@ -20,6 +22,83 @@ from vllm_omni.diffusion.models.wan2_2.wan2_2_transformer import WanTransformer3
from vllm_omni.diffusion.request import OmniDiffusionRequest
def retrieve_latents(
encoder_output: torch.Tensor,
generator: torch.Generator | None = None,
sample_mode: str = "sample",
):
"""Retrieve latents from VAE encoder output."""
if hasattr(encoder_output, "latent_dist") and sample_mode == "sample":
return encoder_output.latent_dist.sample(generator)
elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax":
return encoder_output.latent_dist.mode()
elif hasattr(encoder_output, "latents"):
return encoder_output.latents
else:
raise AttributeError("Could not access latents of provided encoder_output")
def load_transformer_config(model_path: str, subfolder: str = "transformer", local_files_only: bool = True) -> dict:
"""Load transformer config from model directory or HF Hub."""
if local_files_only:
config_path = os.path.join(model_path, subfolder, "config.json")
if os.path.exists(config_path):
with open(config_path) as f:
return json.load(f)
else:
# Try to download config from HF Hub
try:
from huggingface_hub import hf_hub_download
config_path = hf_hub_download(
repo_id=model_path,
filename=f"{subfolder}/config.json",
)
with open(config_path) as f:
return json.load(f)
except Exception:
pass
return {}
def create_transformer_from_config(config: dict) -> WanTransformer3DModel:
"""Create WanTransformer3DModel from config dict."""
kwargs = {}
if "patch_size" in config:
kwargs["patch_size"] = tuple(config["patch_size"])
if "num_attention_heads" in config:
kwargs["num_attention_heads"] = config["num_attention_heads"]
if "attention_head_dim" in config:
kwargs["attention_head_dim"] = config["attention_head_dim"]
if "in_channels" in config:
kwargs["in_channels"] = config["in_channels"]
if "out_channels" in config:
kwargs["out_channels"] = config["out_channels"]
if "text_dim" in config:
kwargs["text_dim"] = config["text_dim"]
if "freq_dim" in config:
kwargs["freq_dim"] = config["freq_dim"]
if "ffn_dim" in config:
kwargs["ffn_dim"] = config["ffn_dim"]
if "num_layers" in config:
kwargs["num_layers"] = config["num_layers"]
if "cross_attn_norm" in config:
kwargs["cross_attn_norm"] = config["cross_attn_norm"]
if "eps" in config:
kwargs["eps"] = config["eps"]
if "image_dim" in config:
kwargs["image_dim"] = config["image_dim"]
if "added_kv_proj_dim" in config:
kwargs["added_kv_proj_dim"] = config["added_kv_proj_dim"]
if "rope_max_seq_len" in config:
kwargs["rope_max_seq_len"] = config["rope_max_seq_len"]
if "pos_embed_seq_len" in config:
kwargs["pos_embed_seq_len"] = config["pos_embed_seq_len"]
return WanTransformer3DModel(**kwargs)
def get_wan22_post_process_func(
od_config: OmniDiffusionConfig,
):
@@ -38,6 +117,52 @@ def get_wan22_post_process_func(
return post_process_func
def get_wan22_pre_process_func(
od_config: OmniDiffusionConfig,
):
"""Pre-process function for Wan2.2: optionally load and resize input image for I2V mode."""
import numpy as np
from diffusers.video_processor import VideoProcessor
video_processor = VideoProcessor(vae_scale_factor=8)
def pre_process_func(requests: list[OmniDiffusionRequest]) -> list[OmniDiffusionRequest]:
for req in requests:
# Load image if path is provided
if req.image_path is not None and req.pil_image is None:
req.pil_image = PIL.Image.open(req.image_path).convert("RGB")
if req.pil_image is not None:
image = req.pil_image
# Calculate dimensions based on aspect ratio if not provided
if req.height is None or req.width is None:
# Default max area for 720P
max_area = 720 * 1280
aspect_ratio = image.height / image.width
# Calculate dimensions maintaining aspect ratio
mod_value = 16 # Must be divisible by 16
height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value
width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value
if req.height is None:
req.height = height
if req.width is None:
req.width = width
# Resize image to target dimensions
image = image.resize((req.width, req.height), PIL.Image.Resampling.LANCZOS)
req.pil_image = image
# Preprocess for VAE
req.preprocessed_image = video_processor.preprocess(image, height=req.height, width=req.width)
return requests
return pre_process_func
class Wan22Pipeline(nn.Module):
def __init__(
self,
@@ -54,6 +179,19 @@ class Wan22Pipeline(nn.Module):
model = od_config.model
local_files_only = os.path.exists(model)
# Read model_index.json to detect expand_timesteps mode (for TI2V-5B)
self.expand_timesteps = False
if local_files_only:
model_index_path = os.path.join(model, "model_index.json")
if os.path.exists(model_index_path):
with open(model_index_path) as f:
model_index = json.load(f)
self.expand_timesteps = model_index.get("expand_timesteps", False)
# Check if this is a two-stage model (MoE with transformer_2)
transformer_2_path = os.path.join(model, "transformer_2") if local_files_only else None
self.has_transformer_2 = transformer_2_path is not None and os.path.exists(transformer_2_path)
# Set up weights sources for transformer(s)
self.weights_sources = [
DiffusersPipelineLoader.ComponentSource(
@@ -63,14 +201,17 @@ class Wan22Pipeline(nn.Module):
prefix="transformer.",
fall_back_to_pt=True,
),
DiffusersPipelineLoader.ComponentSource(
model_or_path=od_config.model,
subfolder="transformer_2",
revision=None,
prefix="transformer_2.",
fall_back_to_pt=True,
),
]
if self.has_transformer_2:
self.weights_sources.append(
DiffusersPipelineLoader.ComponentSource(
model_or_path=od_config.model,
subfolder="transformer_2",
revision=None,
prefix="transformer_2.",
fall_back_to_pt=True,
)
)
self.tokenizer = AutoTokenizer.from_pretrained(model, subfolder="tokenizer", local_files_only=local_files_only)
self.text_encoder = UMT5EncoderModel.from_pretrained(
@@ -80,9 +221,18 @@ class Wan22Pipeline(nn.Module):
model, subfolder="vae", torch_dtype=torch.float32, local_files_only=local_files_only
).to(self.device)
# Initialize transformers without from_pretrained (weights loaded via load_weights)
self.transformer = WanTransformer3DModel()
self.transformer_2 = WanTransformer3DModel()
# Initialize transformers with correct config (weights loaded via load_weights)
if local_files_only:
transformer_config = load_transformer_config(model, "transformer")
self.transformer = create_transformer_from_config(transformer_config)
if self.has_transformer_2:
transformer_2_config = load_transformer_config(model, "transformer_2")
self.transformer_2 = create_transformer_from_config(transformer_2_config)
else:
self.transformer_2 = None
else:
self.transformer = WanTransformer3DModel()
self.transformer_2 = WanTransformer3DModel() if self.has_transformer_2 else None
self.scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(
model, subfolder="scheduler", local_files_only=local_files_only
@@ -142,6 +292,13 @@ class Wan22Pipeline(nn.Module):
height = req.height or height or 720
width = req.width or width or 1280
num_frames = req.num_frames if req.num_frames else frame_num or 81
# Ensure dimensions are compatible with VAE and patch size
# For expand_timesteps mode, we need latent dims to be even (divisible by patch_size)
patch_size = self.transformer.config.patch_size
mod_value = self.vae_scale_factor_spatial * patch_size[1] # 16*2=32 for TI2V, 8*2=16 for I2V
height = (height // mod_value) * mod_value
width = (width // mod_value) * mod_value
num_steps = req.num_inference_steps or num_inference_steps or 40
guidance_low = guidance_scale if isinstance(guidance_scale, (int, float)) else guidance_scale[0]
@@ -211,19 +368,82 @@ class Wan22Pipeline(nn.Module):
if self.boundary_ratio is not None:
boundary_timestep = self.boundary_ratio * self.scheduler.config.num_train_timesteps
# Latents
num_channels_latents = self.transformer.config.in_channels
latents = self.prepare_latents(
batch_size=prompt_embeds.shape[0],
num_channels_latents=num_channels_latents,
height=height,
width=width,
num_frames=num_frames,
dtype=torch.float32,
device=device,
generator=generator,
latents=req.latents,
)
# Handle I2V mode when expand_timesteps=True and image is provided
image = req.pil_image
latent_condition = None
first_frame_mask = None
if self.expand_timesteps and image is not None:
# I2V mode: encode image and prepare condition
from diffusers.video_processor import VideoProcessor
video_processor = VideoProcessor(vae_scale_factor=self.vae_scale_factor_spatial)
# Preprocess image
if isinstance(image, PIL.Image.Image):
image = image.resize((width, height), PIL.Image.Resampling.LANCZOS)
image_tensor = video_processor.preprocess(image, height=height, width=width)
else:
image_tensor = image
# Use out_channels for noise latents (not in_channels which includes condition)
num_channels_latents = self.transformer.config.out_channels
batch_size = prompt_embeds.shape[0]
# Prepare noise latents
latents = self.prepare_latents(
batch_size=batch_size,
num_channels_latents=num_channels_latents,
height=height,
width=width,
num_frames=num_frames,
dtype=torch.float32,
device=device,
generator=generator,
latents=req.latents,
)
# Encode image condition
num_latent_frames = latents.shape[2]
latent_height = latents.shape[3]
latent_width = latents.shape[4]
image_tensor = image_tensor.unsqueeze(2) # [B, C, 1, H, W]
image_tensor = image_tensor.to(device=device, dtype=self.vae.dtype)
latent_condition = retrieve_latents(self.vae.encode(image_tensor), sample_mode="argmax")
latent_condition = latent_condition.repeat(batch_size, 1, 1, 1, 1)
# Normalize condition latents
latents_mean = (
torch.tensor(self.vae.config.latents_mean)
.view(1, self.vae.config.z_dim, 1, 1, 1)
.to(latent_condition.device, latent_condition.dtype)
)
latents_std = 1.0 / torch.tensor(self.vae.config.latents_std).view(1, self.vae.config.z_dim, 1, 1, 1).to(
latent_condition.device, latent_condition.dtype
)
latent_condition = (latent_condition - latents_mean) * latents_std
latent_condition = latent_condition.to(torch.float32)
# Create mask: 0 for first frame (condition), 1 for rest (to denoise)
first_frame_mask = torch.ones(
1, 1, num_latent_frames, latent_height, latent_width, dtype=torch.float32, device=device
)
first_frame_mask[:, :, 0] = 0
else:
# T2V mode: standard latent preparation
num_channels_latents = self.transformer.config.in_channels
latents = self.prepare_latents(
batch_size=prompt_embeds.shape[0],
num_channels_latents=num_channels_latents,
height=height,
width=width,
num_frames=num_frames,
dtype=torch.float32,
device=device,
generator=generator,
latents=req.latents,
)
if attention_kwargs is None:
attention_kwargs = {}
@@ -233,12 +453,30 @@ class Wan22Pipeline(nn.Module):
self._current_timestep = t
current_model = self.transformer
current_guidance_scale = guidance_low
if boundary_timestep is not None and t < boundary_timestep:
if boundary_timestep is not None and t < boundary_timestep and self.transformer_2 is not None:
current_model = self.transformer_2
current_guidance_scale = guidance_high
latent_model_input = latents.to(dtype)
timestep = t.expand(latents.shape[0])
if self.expand_timesteps and latent_condition is not None:
# I2V mode: blend condition with latents using mask
latent_model_input = (1 - first_frame_mask) * latent_condition + first_frame_mask * latents
latent_model_input = latent_model_input.to(dtype)
# Expand timesteps per patch - use floor division to match patch embedding
patch_size = self.transformer.config.patch_size
num_latent_frames = latents.shape[2]
patch_height = latents.shape[3] // patch_size[1]
patch_width = latents.shape[4] // patch_size[2]
# Create mask at patch resolution (same as hidden states sequence length)
patch_mask = first_frame_mask[:, :, :, :: patch_size[1], :: patch_size[2]]
patch_mask = patch_mask[:, :, :, :patch_height, :patch_width] # Ensure correct dimensions
temp_ts = (patch_mask[0][0] * t).flatten()
timestep = temp_ts.unsqueeze(0).expand(latents.shape[0], -1)
else:
# T2V mode: standard forward
latent_model_input = latents.to(dtype)
timestep = t.expand(latents.shape[0])
noise_pred = current_model(
hidden_states=latent_model_input,
@@ -262,6 +500,10 @@ class Wan22Pipeline(nn.Module):
self._current_timestep = None
# For I2V mode: blend final latents with condition
if self.expand_timesteps and latent_condition is not None:
latents = (1 - first_frame_mask) * latent_condition + first_frame_mask * latents
# Decode
if output_type == "latent":
output = latents
@@ -0,0 +1,702 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import os
from collections.abc import Iterable
import numpy as np
import PIL.Image
import torch
from diffusers import AutoencoderKLWan, FlowMatchEulerDiscreteScheduler
from diffusers.utils.torch_utils import randn_tensor
from torch import nn
from transformers import AutoTokenizer, CLIPImageProcessor, CLIPVisionModel, UMT5EncoderModel
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import (
create_transformer_from_config,
load_transformer_config,
retrieve_latents,
)
from vllm_omni.diffusion.request import OmniDiffusionRequest
def _load_model_index(model: str, local_files_only: bool) -> dict:
"""Load model_index.json from local path or HF Hub."""
if local_files_only:
model_index_path = os.path.join(model, "model_index.json")
if os.path.exists(model_index_path):
import json
with open(model_index_path) as f:
return json.load(f)
else:
try:
import json
from huggingface_hub import hf_hub_download
model_index_path = hf_hub_download(model, "model_index.json")
with open(model_index_path) as f:
return json.load(f)
except Exception:
pass
return {}
def get_wan22_i2v_post_process_func(
od_config: OmniDiffusionConfig,
):
from diffusers.video_processor import VideoProcessor
video_processor = VideoProcessor(vae_scale_factor=8)
def post_process_func(
video: torch.Tensor,
output_type: str = "np",
):
if output_type == "latent":
return video
return video_processor.postprocess_video(video, output_type=output_type)
return post_process_func
def get_wan22_i2v_pre_process_func(
od_config: OmniDiffusionConfig,
):
"""Pre-process function for I2V: load and resize input image."""
from diffusers.video_processor import VideoProcessor
video_processor = VideoProcessor(vae_scale_factor=8)
def pre_process_func(requests: list[OmniDiffusionRequest]) -> list[OmniDiffusionRequest]:
for req in requests:
# Load image if path is provided
if req.image_path is not None and req.pil_image is None:
req.pil_image = PIL.Image.open(req.image_path).convert("RGB")
if req.pil_image is not None:
image = req.pil_image
# Calculate dimensions based on aspect ratio if not provided
if req.height is None or req.width is None:
# Default max area for 480P
max_area = 480 * 832
aspect_ratio = image.height / image.width
# Calculate dimensions maintaining aspect ratio
mod_value = 16 # Must be divisible by 16
height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value
width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value
if req.height is None:
req.height = height
if req.width is None:
req.width = width
# Resize image to target dimensions
image = image.resize((req.width, req.height), PIL.Image.Resampling.LANCZOS)
req.pil_image = image
# Preprocess for VAE
req.preprocessed_image = video_processor.preprocess(image, height=req.height, width=req.width)
return requests
return pre_process_func
class Wan22I2VPipeline(nn.Module):
"""
Wan2.2 Image-to-Video Pipeline.
Supports both Wan2.1-style I2V (with CLIP image embeddings) and
Wan2.2-style I2V (with expand_timesteps for TI2V-5B).
"""
def __init__(
self,
*,
od_config: OmniDiffusionConfig,
prefix: str = "",
):
super().__init__()
self.od_config = od_config
self.device = get_local_device()
dtype = getattr(od_config, "dtype", torch.bfloat16)
model = od_config.model
local_files_only = os.path.exists(model)
# Set up weights sources for transformer(s)
self.weights_sources = [
DiffusersPipelineLoader.ComponentSource(
model_or_path=od_config.model,
subfolder="transformer",
revision=None,
prefix="transformer.",
fall_back_to_pt=True,
),
]
# Load model_index.json to detect available components
model_index = _load_model_index(model, local_files_only)
# Check if this is a two-stage model (MoE with transformer_2)
self.has_transformer_2 = "transformer_2" in model_index
if self.has_transformer_2:
self.weights_sources.append(
DiffusersPipelineLoader.ComponentSource(
model_or_path=od_config.model,
subfolder="transformer_2",
revision=None,
prefix="transformer_2.",
fall_back_to_pt=True,
)
)
# Text encoder
self.tokenizer = AutoTokenizer.from_pretrained(model, subfolder="tokenizer", local_files_only=local_files_only)
self.text_encoder = UMT5EncoderModel.from_pretrained(
model, subfolder="text_encoder", torch_dtype=dtype, local_files_only=local_files_only
).to(self.device)
# Image encoder (CLIP) - optional, for Wan2.1-style I2V
self.has_image_encoder = "image_encoder" in model_index and model_index["image_encoder"][0] is not None
if self.has_image_encoder:
self.image_processor = CLIPImageProcessor.from_pretrained(
model, subfolder="image_processor", local_files_only=local_files_only
)
self.image_encoder = CLIPVisionModel.from_pretrained(
model, subfolder="image_encoder", torch_dtype=dtype, local_files_only=local_files_only
).to(self.device)
else:
self.image_processor = None
self.image_encoder = None
# VAE
self.vae = AutoencoderKLWan.from_pretrained(
model, subfolder="vae", torch_dtype=torch.float32, local_files_only=local_files_only
).to(self.device)
# Transformers (weights loaded via load_weights)
# Load config from model directory or HF Hub to get correct in_channels for I2V models
transformer_config = load_transformer_config(model, "transformer", local_files_only)
self.transformer = create_transformer_from_config(transformer_config)
if self.has_transformer_2:
transformer_2_config = load_transformer_config(model, "transformer_2", local_files_only)
self.transformer_2 = create_transformer_from_config(transformer_2_config)
else:
self.transformer_2 = None
# Scheduler
self.scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(
model, subfolder="scheduler", local_files_only=local_files_only
)
if od_config.flow_shift is not None:
self.scheduler.config.flow_shift = od_config.flow_shift
# VAE scale factors
self.vae_scale_factor_temporal = self.vae.config.scale_factor_temporal if hasattr(self.vae, "config") else 4
self.vae_scale_factor_spatial = self.vae.config.scale_factor_spatial if hasattr(self.vae, "config") else 8
# MoE boundary ratio for two-stage denoising
self.boundary_ratio = od_config.boundary_ratio
# Whether to use expand_timesteps mode (for TI2V-5B style)
self.expand_timesteps = getattr(od_config, "expand_timesteps", False)
self._guidance_scale = None
self._guidance_scale_2 = None
self._num_timesteps = None
self._current_timestep = None
@property
def guidance_scale(self):
return self._guidance_scale
@property
def do_classifier_free_guidance(self):
return self._guidance_scale is not None and self._guidance_scale > 1.0
@property
def num_timesteps(self):
return self._num_timesteps
@property
def current_timestep(self):
return self._current_timestep
def encode_image(
self,
image: PIL.Image.Image | list[PIL.Image.Image],
device: torch.device | None = None,
) -> torch.Tensor:
"""Encode image using CLIP image encoder."""
device = device or self.device
if self.image_encoder is None:
raise ValueError("Image encoder not available for this model.")
pixel_values = self.image_processor(images=image, return_tensors="pt").pixel_values
pixel_values = pixel_values.to(device=device, dtype=self.image_encoder.dtype)
image_embeds = self.image_encoder(pixel_values, output_hidden_states=True)
return image_embeds.hidden_states[-2]
@torch.no_grad()
def forward(
self,
req: OmniDiffusionRequest,
prompt: str | None = None,
negative_prompt: str | None = None,
image: PIL.Image.Image | torch.Tensor | None = None,
height: int | None = None,
width: int | None = None,
num_inference_steps: int | None = None,
guidance_scale: float | tuple[float, float] = 5.0,
frame_num: int | None = None,
output_type: str | None = "np",
generator: torch.Generator | None = None,
prompt_embeds: torch.Tensor | None = None,
negative_prompt_embeds: torch.Tensor | None = None,
image_embeds: torch.Tensor | None = None,
last_image: PIL.Image.Image | torch.Tensor | None = None,
attention_kwargs: dict | None = None,
**kwargs,
) -> DiffusionOutput:
# Get parameters from request or arguments
prompt = req.prompt if req.prompt is not None else prompt
negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt
if prompt is None and prompt_embeds is None:
raise ValueError("Prompt or prompt_embeds is required for Wan2.2 I2V generation.")
# Get image from request
if image is None:
image = req.pil_image
if image is None:
raise ValueError("Image is required for I2V generation.")
height = req.height or height or 480
width = req.width or width or 832
num_frames = req.num_frames if req.num_frames else frame_num or 81
num_steps = req.num_inference_steps or num_inference_steps or 50
# Handle guidance scales
guidance_low = guidance_scale if isinstance(guidance_scale, (int, float)) else guidance_scale[0]
guidance_high = (
req.guidance_scale_2
if req.guidance_scale_2 is not None
else (
guidance_scale[1]
if isinstance(guidance_scale, (list, tuple)) and len(guidance_scale) > 1
else guidance_low
)
)
self._guidance_scale = guidance_low
self._guidance_scale_2 = guidance_high
# Validate inputs
self.check_inputs(
prompt=prompt,
negative_prompt=negative_prompt,
image=image,
height=height,
width=width,
prompt_embeds=prompt_embeds,
negative_prompt_embeds=negative_prompt_embeds,
image_embeds=image_embeds,
guidance_scale_2=guidance_high if self.boundary_ratio is not None else None,
)
# Adjust num_frames to be compatible with VAE temporal scaling
if num_frames % self.vae_scale_factor_temporal != 1:
num_frames = num_frames // self.vae_scale_factor_temporal * self.vae_scale_factor_temporal + 1
num_frames = max(num_frames, 1)
device = self.device
dtype = self.transformer.dtype
# Generator setup
if generator is None:
generator = req.generator
if generator is None and req.seed is not None:
generator = torch.Generator(device=device).manual_seed(req.seed)
# Encode prompts
if prompt_embeds is None:
prompt_embeds, negative_prompt_embeds = self.encode_prompt(
prompt=prompt,
negative_prompt=negative_prompt,
do_classifier_free_guidance=guidance_low > 1.0 or guidance_high > 1.0,
num_videos_per_prompt=req.num_outputs_per_prompt or 1,
max_sequence_length=req.max_sequence_length or 512,
device=device,
dtype=dtype,
)
else:
prompt_embeds = prompt_embeds.to(device=device, dtype=dtype)
if negative_prompt_embeds is not None:
negative_prompt_embeds = negative_prompt_embeds.to(device=device, dtype=dtype)
batch_size = prompt_embeds.shape[0]
# Encode image embeddings (for Wan2.1-style with CLIP)
if self.has_image_encoder and self.transformer.config.image_dim is not None:
if image_embeds is None:
if last_image is None:
image_embeds = self.encode_image(image, device)
else:
image_embeds = self.encode_image([image, last_image], device)
image_embeds = image_embeds.repeat(batch_size, 1, 1)
image_embeds = image_embeds.to(dtype)
else:
image_embeds = None
# Timesteps
self.scheduler.set_timesteps(num_steps, device=device)
timesteps = self.scheduler.timesteps
self._num_timesteps = len(timesteps)
boundary_timestep = None
if self.boundary_ratio is not None:
boundary_timestep = self.boundary_ratio * self.scheduler.config.num_train_timesteps
# Prepare latents (use out_channels=16 for VAE latent, not in_channels=36)
num_channels_latents = self.transformer.config.out_channels
# Preprocess image for VAE
from diffusers.video_processor import VideoProcessor
video_processor = VideoProcessor(vae_scale_factor=self.vae_scale_factor_spatial)
if isinstance(image, PIL.Image.Image):
image_tensor = video_processor.preprocess(image, height=height, width=width)
else:
image_tensor = image
image_tensor = image_tensor.to(device=device, dtype=torch.float32)
# Handle last_image if provided
if last_image is not None:
if isinstance(last_image, PIL.Image.Image):
last_image_tensor = video_processor.preprocess(last_image, height=height, width=width)
else:
last_image_tensor = last_image
last_image_tensor = last_image_tensor.to(device=device, dtype=torch.float32)
else:
last_image_tensor = None
latents, condition, first_frame_mask = self.prepare_latents(
image=image_tensor,
batch_size=batch_size,
num_channels_latents=num_channels_latents,
height=height,
width=width,
num_frames=num_frames,
dtype=torch.float32,
device=device,
generator=generator,
latents=req.latents,
last_image=last_image_tensor,
)
if attention_kwargs is None:
attention_kwargs = {}
# Denoising loop
for t in timesteps:
self._current_timestep = t
# Select model and guidance scale based on timestep
current_model = self.transformer
current_guidance_scale = guidance_low
if boundary_timestep is not None and t < boundary_timestep and self.transformer_2 is not None:
current_model = self.transformer_2
current_guidance_scale = guidance_high
# Prepare latent input
if self.expand_timesteps:
# TI2V-5B style: blend condition with latents using mask
latent_model_input = (1 - first_frame_mask) * condition + first_frame_mask * latents
latent_model_input = latent_model_input.to(dtype)
# Expand timesteps for each patch
temp_ts = (first_frame_mask[0][0][:, ::2, ::2] * t).flatten()
timestep = temp_ts.unsqueeze(0).expand(latents.shape[0], -1)
else:
# Wan2.1 style: concatenate condition with latents
latent_model_input = torch.cat([latents, condition], dim=1).to(dtype)
timestep = t.expand(latents.shape[0])
# Forward pass
noise_pred = current_model(
hidden_states=latent_model_input,
timestep=timestep,
encoder_hidden_states=prompt_embeds,
encoder_hidden_states_image=image_embeds,
attention_kwargs=attention_kwargs,
return_dict=False,
)[0]
# Classifier-free guidance
if current_guidance_scale > 1.0 and negative_prompt_embeds is not None:
noise_uncond = current_model(
hidden_states=latent_model_input,
timestep=timestep,
encoder_hidden_states=negative_prompt_embeds,
encoder_hidden_states_image=image_embeds,
attention_kwargs=attention_kwargs,
return_dict=False,
)[0]
noise_pred = noise_uncond + current_guidance_scale * (noise_pred - noise_uncond)
# Scheduler step
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
self._current_timestep = None
# For expand_timesteps mode, blend final latents with condition
if self.expand_timesteps:
latents = (1 - first_frame_mask) * condition + first_frame_mask * latents
# Decode
if output_type == "latent":
output = latents
else:
latents = latents.to(self.vae.dtype)
latents_mean = (
torch.tensor(self.vae.config.latents_mean)
.view(1, self.vae.config.z_dim, 1, 1, 1)
.to(latents.device, latents.dtype)
)
latents_std = 1.0 / torch.tensor(self.vae.config.latents_std).view(1, self.vae.config.z_dim, 1, 1, 1).to(
latents.device, latents.dtype
)
latents = latents / latents_std + latents_mean
output = self.vae.decode(latents, return_dict=False)[0]
return DiffusionOutput(output=output)
def encode_prompt(
self,
prompt: str | list[str],
negative_prompt: str | list[str] | None = None,
do_classifier_free_guidance: bool = True,
num_videos_per_prompt: int = 1,
max_sequence_length: int = 512,
device: torch.device | None = None,
dtype: torch.dtype | None = None,
):
"""Encode text prompts using T5 text encoder."""
device = device or self.device
dtype = dtype or self.text_encoder.dtype
prompt = [prompt] if isinstance(prompt, str) else prompt
prompt_clean = [self._prompt_clean(p) for p in prompt]
batch_size = len(prompt_clean)
text_inputs = self.tokenizer(
prompt_clean,
padding="max_length",
max_length=max_sequence_length,
truncation=True,
add_special_tokens=True,
return_attention_mask=True,
return_tensors="pt",
)
ids, mask = text_inputs.input_ids, text_inputs.attention_mask
seq_lens = mask.gt(0).sum(dim=1).long()
prompt_embeds = self.text_encoder(ids.to(device), mask.to(device)).last_hidden_state
prompt_embeds = prompt_embeds.to(dtype=dtype, device=device)
prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)]
prompt_embeds = torch.stack(
[torch.cat([u, u.new_zeros(max_sequence_length - u.size(0), u.size(1))]) for u in prompt_embeds], dim=0
)
_, seq_len, _ = prompt_embeds.shape
prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1)
prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1)
negative_prompt_embeds = None
if do_classifier_free_guidance:
negative_prompt = negative_prompt or ""
negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
neg_text_inputs = self.tokenizer(
[self._prompt_clean(p) for p in negative_prompt],
padding="max_length",
max_length=max_sequence_length,
truncation=True,
add_special_tokens=True,
return_attention_mask=True,
return_tensors="pt",
)
ids_neg, mask_neg = neg_text_inputs.input_ids, neg_text_inputs.attention_mask
seq_lens_neg = mask_neg.gt(0).sum(dim=1).long()
negative_prompt_embeds = self.text_encoder(ids_neg.to(device), mask_neg.to(device)).last_hidden_state
negative_prompt_embeds = negative_prompt_embeds.to(dtype=dtype, device=device)
negative_prompt_embeds = [u[:v] for u, v in zip(negative_prompt_embeds, seq_lens_neg)]
negative_prompt_embeds = torch.stack(
[
torch.cat([u, u.new_zeros(max_sequence_length - u.size(0), u.size(1))])
for u in negative_prompt_embeds
],
dim=0,
)
negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_videos_per_prompt, 1)
negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1)
return prompt_embeds, negative_prompt_embeds
@staticmethod
def _prompt_clean(text: str) -> str:
return " ".join(text.strip().split())
def prepare_latents(
self,
image: torch.Tensor,
batch_size: int,
num_channels_latents: int,
height: int,
width: int,
num_frames: int,
dtype: torch.dtype | None,
device: torch.device | None,
generator: torch.Generator | list[torch.Generator] | None,
latents: torch.Tensor | None = None,
last_image: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Prepare latents for I2V generation.
Returns:
latents: Initial noise latents
condition: Encoded image condition (concatenated with mask for non-expand mode)
first_frame_mask: Mask for the first frame (1 for frames to denoise, 0 for condition)
"""
num_latent_frames = (num_frames - 1) // self.vae_scale_factor_temporal + 1
latent_height = height // self.vae_scale_factor_spatial
latent_width = width // self.vae_scale_factor_spatial
shape = (batch_size, num_channels_latents, num_latent_frames, latent_height, latent_width)
if latents is None:
latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
else:
latents = latents.to(device=device, dtype=dtype)
# Prepare image condition
image = image.unsqueeze(2) # [batch, channels, 1, height, width]
if self.expand_timesteps:
# TI2V-5B style: only use first frame as condition
video_condition = image
elif last_image is None:
# Pad with zeros for remaining frames
video_condition = torch.cat(
[image, image.new_zeros(image.shape[0], image.shape[1], num_frames - 1, height, width)], dim=2
)
else:
# First and last frame conditioning
last_image = last_image.unsqueeze(2)
video_condition = torch.cat(
[image, image.new_zeros(image.shape[0], image.shape[1], num_frames - 2, height, width), last_image],
dim=2,
)
video_condition = video_condition.to(device=device, dtype=self.vae.dtype)
# Encode through VAE
latent_condition = retrieve_latents(self.vae.encode(video_condition), sample_mode="argmax")
latent_condition = latent_condition.repeat(batch_size, 1, 1, 1, 1)
# Normalize latents
latents_mean = (
torch.tensor(self.vae.config.latents_mean)
.view(1, self.vae.config.z_dim, 1, 1, 1)
.to(latent_condition.device, latent_condition.dtype)
)
latents_std = 1.0 / torch.tensor(self.vae.config.latents_std).view(1, self.vae.config.z_dim, 1, 1, 1).to(
latent_condition.device, latent_condition.dtype
)
latent_condition = (latent_condition - latents_mean) * latents_std
latent_condition = latent_condition.to(dtype)
if self.expand_timesteps:
# TI2V-5B style: create mask where first frame is 0 (condition), rest is 1 (to denoise)
first_frame_mask = torch.ones(
1, 1, num_latent_frames, latent_height, latent_width, dtype=dtype, device=device
)
first_frame_mask[:, :, 0] = 0
return latents, latent_condition, first_frame_mask
# Wan2.1 style: create mask and concatenate with condition
mask_lat_size = torch.ones(batch_size, 1, num_frames, latent_height, latent_width)
if last_image is None:
mask_lat_size[:, :, list(range(1, num_frames))] = 0
else:
mask_lat_size[:, :, list(range(1, num_frames - 1))] = 0
first_frame_mask = mask_lat_size[:, :, 0:1]
first_frame_mask = torch.repeat_interleave(first_frame_mask, dim=2, repeats=self.vae_scale_factor_temporal)
mask_lat_size = torch.concat([first_frame_mask, mask_lat_size[:, :, 1:, :]], dim=2)
mask_lat_size = mask_lat_size.view(batch_size, -1, self.vae_scale_factor_temporal, latent_height, latent_width)
mask_lat_size = mask_lat_size.transpose(1, 2)
mask_lat_size = mask_lat_size.to(latent_condition.device)
# Concatenate mask with condition for channel dimension
condition = torch.concat([mask_lat_size, latent_condition], dim=1)
# For non-expand mode, first_frame_mask is not used in the same way
first_frame_mask = torch.ones(1, 1, num_latent_frames, latent_height, latent_width, dtype=dtype, device=device)
return latents, condition, first_frame_mask
def check_inputs(
self,
prompt,
negative_prompt,
image,
height,
width,
prompt_embeds=None,
negative_prompt_embeds=None,
image_embeds=None,
guidance_scale_2=None,
):
if image is None and image_embeds is None:
raise ValueError("Provide either `image` or `image_embeds`. Cannot leave both undefined.")
if image is not None and image_embeds is not None:
raise ValueError("Cannot forward both `image` and `image_embeds`. Please provide only one.")
if height % 16 != 0 or width % 16 != 0:
raise ValueError(f"`height` and `width` have to be divisible by 16 but are {height} and {width}.")
if prompt is not None and prompt_embeds is not None:
raise ValueError("Cannot forward both `prompt` and `prompt_embeds`. Please provide only one.")
if negative_prompt is not None and negative_prompt_embeds is not None:
raise ValueError(
"Cannot forward both `negative_prompt` and `negative_prompt_embeds`. Please provide only one."
)
if prompt is None and prompt_embeds is None:
raise ValueError("Provide either `prompt` or `prompt_embeds`.")
if self.boundary_ratio is None and guidance_scale_2 is not None:
raise ValueError("`guidance_scale_2` is only supported when `boundary_ratio` is set.")
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load weights using AutoWeightsLoader for vLLM integration."""
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
@@ -0,0 +1,583 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Wan2.2 TI2V (Text-Image-to-Video) Pipeline.
This pipeline supports the unified TI2V-5B model that can generate videos from:
- Text only (T2V mode)
- Text + Image (I2V mode)
The key difference from the MoE-based I2V pipeline is:
- Single transformer (not MoE with two transformers)
- Uses expand_timesteps mode for image conditioning
- No CLIP image encoder - only VAE encoding for image condition
"""
from __future__ import annotations
import os
from collections.abc import Iterable
import numpy as np
import PIL.Image
import torch
from diffusers import AutoencoderKLWan, FlowMatchEulerDiscreteScheduler
from diffusers.utils.torch_utils import randn_tensor
from torch import nn
from transformers import AutoTokenizer, UMT5EncoderModel
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import retrieve_latents
from vllm_omni.diffusion.models.wan2_2.wan2_2_transformer import WanTransformer3DModel
from vllm_omni.diffusion.request import OmniDiffusionRequest
def get_wan22_ti2v_post_process_func(
od_config: OmniDiffusionConfig,
):
from diffusers.video_processor import VideoProcessor
video_processor = VideoProcessor(vae_scale_factor=8)
def post_process_func(
video: torch.Tensor,
output_type: str = "np",
):
if output_type == "latent":
return video
return video_processor.postprocess_video(video, output_type=output_type)
return post_process_func
def get_wan22_ti2v_pre_process_func(
od_config: OmniDiffusionConfig,
):
"""Pre-process function for TI2V: optionally load and resize input image."""
from diffusers.video_processor import VideoProcessor
video_processor = VideoProcessor(vae_scale_factor=8)
def pre_process_func(requests: list[OmniDiffusionRequest]) -> list[OmniDiffusionRequest]:
for req in requests:
# Load image if path is provided
if req.image_path is not None and req.pil_image is None:
req.pil_image = PIL.Image.open(req.image_path).convert("RGB")
if req.pil_image is not None:
image = req.pil_image
# Calculate dimensions based on aspect ratio if not provided
if req.height is None or req.width is None:
# Default max area for 720P (TI2V-5B default)
max_area = 720 * 1280
aspect_ratio = image.height / image.width
# Calculate dimensions maintaining aspect ratio
mod_value = 16 # Must be divisible by 16
height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value
width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value
if req.height is None:
req.height = height
if req.width is None:
req.width = width
# Resize image to target dimensions
image = image.resize((req.width, req.height), PIL.Image.Resampling.LANCZOS)
req.pil_image = image
# Preprocess for VAE
req.preprocessed_image = video_processor.preprocess(image, height=req.height, width=req.width)
return requests
return pre_process_func
class Wan22TI2VPipeline(nn.Module):
"""
Wan2.2 Text-Image-to-Video (TI2V) Pipeline.
This is a unified pipeline that supports both:
- Text-to-Video (T2V): when no image is provided
- Image-to-Video (I2V): when an image is provided
Uses expand_timesteps mode for I2V conditioning where the first frame
is conditioned on the input image latent.
"""
def __init__(
self,
*,
od_config: OmniDiffusionConfig,
prefix: str = "",
):
super().__init__()
self.od_config = od_config
self.device = get_local_device()
dtype = getattr(od_config, "dtype", torch.bfloat16)
model = od_config.model
local_files_only = os.path.exists(model)
# Set up weights sources for single transformer
self.weights_sources = [
DiffusersPipelineLoader.ComponentSource(
model_or_path=od_config.model,
subfolder="transformer",
revision=None,
prefix="transformer.",
fall_back_to_pt=True,
),
]
# Text encoder
self.tokenizer = AutoTokenizer.from_pretrained(model, subfolder="tokenizer", local_files_only=local_files_only)
self.text_encoder = UMT5EncoderModel.from_pretrained(
model, subfolder="text_encoder", torch_dtype=dtype, local_files_only=local_files_only
).to(self.device)
# VAE
self.vae = AutoencoderKLWan.from_pretrained(
model, subfolder="vae", torch_dtype=torch.float32, local_files_only=local_files_only
).to(self.device)
# Single transformer (TI2V uses dense 5B model, not MoE)
self.transformer = WanTransformer3DModel()
# Scheduler
self.scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(
model, subfolder="scheduler", local_files_only=local_files_only
)
if od_config.flow_shift is not None:
self.scheduler.config.flow_shift = od_config.flow_shift
# VAE scale factors
self.vae_scale_factor_temporal = self.vae.config.scale_factor_temporal if hasattr(self.vae, "config") else 4
self.vae_scale_factor_spatial = self.vae.config.scale_factor_spatial if hasattr(self.vae, "config") else 8
# TI2V always uses expand_timesteps mode
self.expand_timesteps = True
self._guidance_scale = None
self._num_timesteps = None
self._current_timestep = None
@property
def guidance_scale(self):
return self._guidance_scale
@property
def do_classifier_free_guidance(self):
return self._guidance_scale is not None and self._guidance_scale > 1.0
@property
def num_timesteps(self):
return self._num_timesteps
@property
def current_timestep(self):
return self._current_timestep
@torch.no_grad()
def forward(
self,
req: OmniDiffusionRequest,
prompt: str | None = None,
negative_prompt: str | None = None,
image: PIL.Image.Image | torch.Tensor | None = None,
height: int | None = None,
width: int | None = None,
num_inference_steps: int | None = None,
guidance_scale: float = 5.0,
frame_num: int | None = None,
output_type: str | None = "np",
generator: torch.Generator | None = None,
prompt_embeds: torch.Tensor | None = None,
negative_prompt_embeds: torch.Tensor | None = None,
attention_kwargs: dict | None = None,
**kwargs,
) -> DiffusionOutput:
# Get parameters from request or arguments
prompt = req.prompt if req.prompt is not None else prompt
negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt
if prompt is None and prompt_embeds is None:
raise ValueError("Prompt or prompt_embeds is required for Wan2.2 TI2V generation.")
# Get image from request (optional for TI2V)
if image is None:
image = req.pil_image
# Default dimensions for TI2V-5B (720P)
height = req.height or height or 704
width = req.width or width or 1280
num_frames = req.num_frames if req.num_frames else frame_num or 121
num_steps = req.num_inference_steps or num_inference_steps or 50
self._guidance_scale = guidance_scale
# Validate inputs
self.check_inputs(
prompt=prompt,
negative_prompt=negative_prompt,
image=image,
height=height,
width=width,
prompt_embeds=prompt_embeds,
negative_prompt_embeds=negative_prompt_embeds,
)
# Adjust num_frames to be compatible with VAE temporal scaling
if num_frames % self.vae_scale_factor_temporal != 1:
num_frames = num_frames // self.vae_scale_factor_temporal * self.vae_scale_factor_temporal + 1
num_frames = max(num_frames, 1)
device = self.device
dtype = self.transformer.dtype
# Generator setup
if generator is None:
generator = req.generator
if generator is None and req.seed is not None:
generator = torch.Generator(device=device).manual_seed(req.seed)
# Encode prompts
if prompt_embeds is None:
prompt_embeds, negative_prompt_embeds = self.encode_prompt(
prompt=prompt,
negative_prompt=negative_prompt,
do_classifier_free_guidance=guidance_scale > 1.0,
num_videos_per_prompt=req.num_outputs_per_prompt or 1,
max_sequence_length=req.max_sequence_length or 512,
device=device,
dtype=dtype,
)
else:
prompt_embeds = prompt_embeds.to(device=device, dtype=dtype)
if negative_prompt_embeds is not None:
negative_prompt_embeds = negative_prompt_embeds.to(device=device, dtype=dtype)
batch_size = prompt_embeds.shape[0]
# Timesteps
self.scheduler.set_timesteps(num_steps, device=device)
timesteps = self.scheduler.timesteps
self._num_timesteps = len(timesteps)
# Prepare latents
num_channels_latents = self.transformer.config.in_channels
num_latent_frames = (num_frames - 1) // self.vae_scale_factor_temporal + 1
latent_height = height // self.vae_scale_factor_spatial
latent_width = width // self.vae_scale_factor_spatial
# Check if we have an image (I2V mode) or not (T2V mode)
if image is not None:
# I2V mode: prepare latents with image condition
from diffusers.video_processor import VideoProcessor
video_processor = VideoProcessor(vae_scale_factor=self.vae_scale_factor_spatial)
if isinstance(image, PIL.Image.Image):
image_tensor = video_processor.preprocess(image, height=height, width=width)
else:
image_tensor = image
image_tensor = image_tensor.to(device=device, dtype=torch.float32)
latents, latent_condition, first_frame_mask = self.prepare_i2v_latents(
image=image_tensor,
batch_size=batch_size,
num_channels_latents=num_channels_latents,
height=height,
width=width,
num_frames=num_frames,
dtype=torch.float32,
device=device,
generator=generator,
latents=req.latents,
)
else:
# T2V mode: prepare random latents
latents = self.prepare_latents(
batch_size=batch_size,
num_channels_latents=num_channels_latents,
height=height,
width=width,
num_frames=num_frames,
dtype=torch.float32,
device=device,
generator=generator,
latents=req.latents,
)
latent_condition = None
first_frame_mask = torch.ones(
1, 1, num_latent_frames, latent_height, latent_width, dtype=torch.float32, device=device
)
if attention_kwargs is None:
attention_kwargs = {}
# Denoising loop
for t in timesteps:
self._current_timestep = t
# Prepare latent input
if latent_condition is not None:
# I2V mode: blend condition with latents using mask
latent_model_input = (1 - first_frame_mask) * latent_condition + first_frame_mask * latents
latent_model_input = latent_model_input.to(dtype)
# Expand timesteps for each patch (TI2V style)
temp_ts = (first_frame_mask[0][0][:, ::2, ::2] * t).flatten()
timestep = temp_ts.unsqueeze(0).expand(latents.shape[0], -1)
else:
# T2V mode: use latents directly
latent_model_input = latents.to(dtype)
# Expand timesteps for TI2V model architecture
mask = torch.ones(1, 1, num_latent_frames, latent_height, latent_width, device=device)
temp_ts = (mask[0][0][:, ::2, ::2] * t).flatten()
timestep = temp_ts.unsqueeze(0).expand(latents.shape[0], -1)
# Forward pass
noise_pred = self.transformer(
hidden_states=latent_model_input,
timestep=timestep,
encoder_hidden_states=prompt_embeds,
attention_kwargs=attention_kwargs,
return_dict=False,
)[0]
# Classifier-free guidance
if guidance_scale > 1.0 and negative_prompt_embeds is not None:
noise_uncond = self.transformer(
hidden_states=latent_model_input,
timestep=timestep,
encoder_hidden_states=negative_prompt_embeds,
attention_kwargs=attention_kwargs,
return_dict=False,
)[0]
noise_pred = noise_uncond + guidance_scale * (noise_pred - noise_uncond)
# Scheduler step
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
self._current_timestep = None
# For I2V mode, blend final latents with condition
if latent_condition is not None:
latents = (1 - first_frame_mask) * latent_condition + first_frame_mask * latents
# Decode
if output_type == "latent":
output = latents
else:
latents = latents.to(self.vae.dtype)
latents_mean = (
torch.tensor(self.vae.config.latents_mean)
.view(1, self.vae.config.z_dim, 1, 1, 1)
.to(latents.device, latents.dtype)
)
latents_std = 1.0 / torch.tensor(self.vae.config.latents_std).view(1, self.vae.config.z_dim, 1, 1, 1).to(
latents.device, latents.dtype
)
latents = latents / latents_std + latents_mean
output = self.vae.decode(latents, return_dict=False)[0]
return DiffusionOutput(output=output)
def encode_prompt(
self,
prompt: str | list[str],
negative_prompt: str | list[str] | None = None,
do_classifier_free_guidance: bool = True,
num_videos_per_prompt: int = 1,
max_sequence_length: int = 512,
device: torch.device | None = None,
dtype: torch.dtype | None = None,
):
"""Encode text prompts using T5 text encoder."""
device = device or self.device
dtype = dtype or self.text_encoder.dtype
prompt = [prompt] if isinstance(prompt, str) else prompt
prompt_clean = [self._prompt_clean(p) for p in prompt]
batch_size = len(prompt_clean)
text_inputs = self.tokenizer(
prompt_clean,
padding="max_length",
max_length=max_sequence_length,
truncation=True,
add_special_tokens=True,
return_attention_mask=True,
return_tensors="pt",
)
ids, mask = text_inputs.input_ids, text_inputs.attention_mask
seq_lens = mask.gt(0).sum(dim=1).long()
prompt_embeds = self.text_encoder(ids.to(device), mask.to(device)).last_hidden_state
prompt_embeds = prompt_embeds.to(dtype=dtype, device=device)
prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)]
prompt_embeds = torch.stack(
[torch.cat([u, u.new_zeros(max_sequence_length - u.size(0), u.size(1))]) for u in prompt_embeds], dim=0
)
_, seq_len, _ = prompt_embeds.shape
prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1)
prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1)
negative_prompt_embeds = None
if do_classifier_free_guidance:
negative_prompt = negative_prompt or ""
negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
neg_text_inputs = self.tokenizer(
[self._prompt_clean(p) for p in negative_prompt],
padding="max_length",
max_length=max_sequence_length,
truncation=True,
add_special_tokens=True,
return_attention_mask=True,
return_tensors="pt",
)
ids_neg, mask_neg = neg_text_inputs.input_ids, neg_text_inputs.attention_mask
seq_lens_neg = mask_neg.gt(0).sum(dim=1).long()
negative_prompt_embeds = self.text_encoder(ids_neg.to(device), mask_neg.to(device)).last_hidden_state
negative_prompt_embeds = negative_prompt_embeds.to(dtype=dtype, device=device)
negative_prompt_embeds = [u[:v] for u, v in zip(negative_prompt_embeds, seq_lens_neg)]
negative_prompt_embeds = torch.stack(
[
torch.cat([u, u.new_zeros(max_sequence_length - u.size(0), u.size(1))])
for u in negative_prompt_embeds
],
dim=0,
)
negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_videos_per_prompt, 1)
negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1)
return prompt_embeds, negative_prompt_embeds
@staticmethod
def _prompt_clean(text: str) -> str:
return " ".join(text.strip().split())
def prepare_latents(
self,
batch_size: int,
num_channels_latents: int,
height: int,
width: int,
num_frames: int,
dtype: torch.dtype | None,
device: torch.device | None,
generator: torch.Generator | list[torch.Generator] | None,
latents: torch.Tensor | None = None,
) -> torch.Tensor:
"""Prepare random latents for T2V mode."""
if latents is not None:
return latents.to(device=device, dtype=dtype)
num_latent_frames = (num_frames - 1) // self.vae_scale_factor_temporal + 1
shape = (
batch_size,
num_channels_latents,
num_latent_frames,
int(height) // self.vae_scale_factor_spatial,
int(width) // self.vae_scale_factor_spatial,
)
if isinstance(generator, list) and len(generator) != batch_size:
raise ValueError(f"Generator list length {len(generator)} does not match batch size {batch_size}.")
latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
return latents
def prepare_i2v_latents(
self,
image: torch.Tensor,
batch_size: int,
num_channels_latents: int,
height: int,
width: int,
num_frames: int,
dtype: torch.dtype | None,
device: torch.device | None,
generator: torch.Generator | list[torch.Generator] | None,
latents: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Prepare latents for I2V mode with image conditioning.
Returns:
latents: Initial noise latents
latent_condition: Encoded first frame condition
first_frame_mask: Mask (0 for first frame, 1 for rest)
"""
num_latent_frames = (num_frames - 1) // self.vae_scale_factor_temporal + 1
latent_height = height // self.vae_scale_factor_spatial
latent_width = width // self.vae_scale_factor_spatial
shape = (batch_size, num_channels_latents, num_latent_frames, latent_height, latent_width)
if latents is None:
latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
else:
latents = latents.to(device=device, dtype=dtype)
# Prepare first frame condition
image = image.unsqueeze(2) # [batch, channels, 1, height, width]
image = image.to(device=device, dtype=self.vae.dtype)
# Encode through VAE
latent_condition = retrieve_latents(self.vae.encode(image), sample_mode="argmax")
latent_condition = latent_condition.repeat(batch_size, 1, 1, 1, 1)
# Normalize latents
latents_mean = (
torch.tensor(self.vae.config.latents_mean)
.view(1, self.vae.config.z_dim, 1, 1, 1)
.to(latent_condition.device, latent_condition.dtype)
)
latents_std = 1.0 / torch.tensor(self.vae.config.latents_std).view(1, self.vae.config.z_dim, 1, 1, 1).to(
latent_condition.device, latent_condition.dtype
)
latent_condition = (latent_condition - latents_mean) * latents_std
latent_condition = latent_condition.to(dtype)
# Create mask: 0 for first frame (condition), 1 for rest (to denoise)
first_frame_mask = torch.ones(1, 1, num_latent_frames, latent_height, latent_width, dtype=dtype, device=device)
first_frame_mask[:, :, 0] = 0
return latents, latent_condition, first_frame_mask
def check_inputs(
self,
prompt,
negative_prompt,
image,
height,
width,
prompt_embeds=None,
negative_prompt_embeds=None,
):
if height % 16 != 0 or width % 16 != 0:
raise ValueError(f"`height` and `width` have to be divisible by 16 but are {height} and {width}.")
if prompt is not None and prompt_embeds is not None:
raise ValueError("Cannot forward both `prompt` and `prompt_embeds`. Please provide only one.")
if negative_prompt is not None and negative_prompt_embeds is not None:
raise ValueError(
"Cannot forward both `negative_prompt` and `negative_prompt_embeds`. Please provide only one."
)
if prompt is None and prompt_embeds is None:
raise ValueError("Provide either `prompt` or `prompt_embeds`.")
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load weights using AutoWeightsLoader for vLLM integration."""
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
+8
View File
@@ -44,6 +44,11 @@ _DIFFUSION_MODELS = {
"pipeline_wan2_2",
"Wan22Pipeline",
),
"WanImageToVideoPipeline": (
"wan2_2",
"pipeline_wan2_2_i2v",
"Wan22I2VPipeline",
),
"LongCatImagePipeline": (
"longcat_image",
"pipeline_longcat_image",
@@ -94,6 +99,7 @@ _DIFFUSION_POST_PROCESS_FUNCS = {
"ZImagePipeline": "get_post_process_func",
"OvisImagePipeline": "get_ovis_image_post_process_func",
"WanPipeline": "get_wan22_post_process_func",
"WanImageToVideoPipeline": "get_wan22_i2v_post_process_func",
"LongCatImagePipeline": "get_longcat_image_post_process_func",
"LongCatImageEditPipeline": "get_longcat_image_post_process_func",
}
@@ -106,6 +112,8 @@ _DIFFUSION_PRE_PROCESS_FUNCS = {
"QwenImageEditPlusPipeline": "get_qwen_image_edit_plus_pre_process_func",
"LongCatImageEditPipeline": "get_longcat_image_edit_pre_process_func",
"QwenImageLayeredPipeline": "get_qwen_image_layered_pre_process_func",
"WanPipeline": "get_wan22_pre_process_func",
"WanImageToVideoPipeline": "get_wan22_i2v_pre_process_func",
}