[Model] Add Wan2.2 I2V and TI2V pipeline support (#329)
Signed-off-by: linyueqian <linyueqian@outlook.com> Signed-off-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
d51f4c086d
commit
3ec35e5409
7 files changed
+1752
-29
No files matched your search
@@ -24,7 +24,8 @@ th {
|
||||
| `QwenImageEditPlusPipeline` | Qwen-Image-Edit-2509 | `Qwen/Qwen-Image-Edit-2509` |
|
||||
| `QwenImageLayeredPipeline` | Qwen-Image-Layered | `Qwen/Qwen-Image-Layered` |
|
||||
|`ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` |
|
||||
| `WanPipeline` | Wan2.2 | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` |
|
||||
| `WanPipeline` | Wan2.2-T2V, Wan2.2-TI2V | `Wan-AI/Wan2.2-T2V-A14B-Diffusers`, `Wan-AI/Wan2.2-TI2V-5B-Diffusers` |
|
||||
| `WanImageToVideoPipeline` | Wan2.2-I2V | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` |
|
||||
| `OvisImagePipeline` | Ovis-Image | `OvisAI/Ovis-Image` |
|
||||
|`LongcatImagePipeline` | LongCat-Image | `meituan-longcat/LongCat-Image` |
|
||||
|`LongCatImageEditPipeline` | LongCat-Image-Edit | `meituan-longcat/LongCat-Image-Edit` |
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Image-to-Video generation example using Wan2.2 I2V/TI2V models.
|
||||
|
||||
Supports:
|
||||
- Wan2.2-I2V-A14B-Diffusers: MoE model with CLIP image encoder
|
||||
- Wan2.2-TI2V-5B-Diffusers: Unified T2V+I2V model (dense 5B)
|
||||
|
||||
Usage:
|
||||
# I2V-A14B (MoE)
|
||||
python image_to_video.py --model Wan-AI/Wan2.2-I2V-A14B-Diffusers \
|
||||
--image input.jpg --prompt "A cat playing with yarn"
|
||||
|
||||
# TI2V-5B (unified)
|
||||
python image_to_video.py --model Wan-AI/Wan2.2-TI2V-5B-Diffusers \
|
||||
--image input.jpg --prompt "A cat playing with yarn"
|
||||
"""
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import PIL.Image
|
||||
import torch
|
||||
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.utils.platform_utils import detect_device_type, is_npu
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Generate a video from an image with Wan2.2 I2V/TI2V.")
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
default="Wan-AI/Wan2.2-I2V-A14B-Diffusers",
|
||||
help="Diffusers Wan2.2 I2V model ID or local path.",
|
||||
)
|
||||
parser.add_argument("--image", required=True, help="Path to input image.")
|
||||
parser.add_argument("--prompt", default="", help="Text prompt describing the desired motion.")
|
||||
parser.add_argument("--negative_prompt", default="", help="Negative prompt.")
|
||||
parser.add_argument("--seed", type=int, default=42, help="Random seed.")
|
||||
parser.add_argument("--guidance_scale", type=float, default=5.0, help="CFG scale.")
|
||||
parser.add_argument(
|
||||
"--guidance_scale_high", type=float, default=None, help="Optional separate CFG for high-noise (MoE only)."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--height", type=int, default=None, help="Video height (auto-calculated from image if not set)."
|
||||
)
|
||||
parser.add_argument("--width", type=int, default=None, help="Video width (auto-calculated from image if not set).")
|
||||
parser.add_argument("--num_frames", type=int, default=81, help="Number of frames.")
|
||||
parser.add_argument("--num_inference_steps", type=int, default=50, help="Sampling steps.")
|
||||
parser.add_argument("--boundary_ratio", type=float, default=0.875, help="Boundary split ratio for MoE models.")
|
||||
parser.add_argument(
|
||||
"--flow_shift", type=float, default=5.0, help="Scheduler flow_shift (5.0 for 720p, 12.0 for 480p)."
|
||||
)
|
||||
parser.add_argument("--output", type=str, default="i2v_output.mp4", help="Path to save the video (mp4).")
|
||||
parser.add_argument("--fps", type=int, default=16, help="Frames per second for the output video.")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def calculate_dimensions(image: PIL.Image.Image, max_area: int = 480 * 832) -> tuple[int, int]:
|
||||
"""Calculate output dimensions maintaining aspect ratio."""
|
||||
aspect_ratio = image.height / image.width
|
||||
mod_value = 16 # Must be divisible by 16
|
||||
|
||||
height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value
|
||||
width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value
|
||||
|
||||
return height, width
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
device = detect_device_type()
|
||||
generator = torch.Generator(device=device).manual_seed(args.seed)
|
||||
|
||||
# Load input image
|
||||
image = PIL.Image.open(args.image).convert("RGB")
|
||||
|
||||
# Calculate dimensions if not provided
|
||||
height = args.height
|
||||
width = args.width
|
||||
if height is None or width is None:
|
||||
# Default to 480P area for I2V
|
||||
calc_height, calc_width = calculate_dimensions(image, max_area=480 * 832)
|
||||
height = height or calc_height
|
||||
width = width or calc_width
|
||||
|
||||
# Resize image to target dimensions
|
||||
image = image.resize((width, height), PIL.Image.Resampling.LANCZOS)
|
||||
|
||||
# Enable VAE memory optimizations on NPU
|
||||
vae_use_slicing = is_npu()
|
||||
vae_use_tiling = is_npu()
|
||||
|
||||
omni = Omni(
|
||||
model=args.model,
|
||||
vae_use_slicing=vae_use_slicing,
|
||||
vae_use_tiling=vae_use_tiling,
|
||||
boundary_ratio=args.boundary_ratio,
|
||||
flow_shift=args.flow_shift,
|
||||
)
|
||||
|
||||
frames = omni.generate(
|
||||
args.prompt,
|
||||
negative_prompt=args.negative_prompt,
|
||||
pil_image=image,
|
||||
height=height,
|
||||
width=width,
|
||||
generator=generator,
|
||||
guidance_scale=args.guidance_scale,
|
||||
guidance_scale_2=args.guidance_scale_high,
|
||||
num_inference_steps=args.num_inference_steps,
|
||||
num_frames=args.num_frames,
|
||||
)
|
||||
|
||||
output_path = Path(args.output)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
from diffusers.utils import export_to_video
|
||||
except ImportError:
|
||||
raise ImportError("diffusers is required for export_to_video.")
|
||||
|
||||
# frames may be np.ndarray (preferred) or torch.Tensor
|
||||
# export_to_video expects a list of frames with values in [0, 1]
|
||||
if isinstance(frames, torch.Tensor):
|
||||
video_tensor = frames.detach().cpu()
|
||||
if video_tensor.dim() == 5:
|
||||
# [B, C, F, H, W] or [B, F, H, W, C]
|
||||
if video_tensor.shape[1] in (3, 4):
|
||||
video_tensor = video_tensor[0].permute(1, 2, 3, 0)
|
||||
else:
|
||||
video_tensor = video_tensor[0]
|
||||
elif video_tensor.dim() == 4 and video_tensor.shape[0] in (3, 4):
|
||||
video_tensor = video_tensor.permute(1, 2, 3, 0)
|
||||
# If float, assume [-1,1] and normalize to [0,1]
|
||||
if video_tensor.is_floating_point():
|
||||
video_tensor = video_tensor.clamp(-1, 1) * 0.5 + 0.5
|
||||
video_array = video_tensor.float().numpy()
|
||||
else:
|
||||
video_array = frames
|
||||
if hasattr(video_array, "shape") and video_array.ndim == 5:
|
||||
video_array = video_array[0]
|
||||
|
||||
# Convert 4D array (frames, H, W, C) to list of frames for export_to_video
|
||||
if isinstance(video_array, np.ndarray) and video_array.ndim == 4:
|
||||
video_array = list(video_array)
|
||||
|
||||
export_to_video(video_array, str(output_path), fps=args.fps)
|
||||
print(f"Saved generated video to {output_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,4 +1,35 @@
|
||||
from .pipeline_wan2_2 import Wan22Pipeline, get_wan22_post_process_func
|
||||
from .pipeline_wan2_2 import (
|
||||
Wan22Pipeline,
|
||||
create_transformer_from_config,
|
||||
get_wan22_post_process_func,
|
||||
get_wan22_pre_process_func,
|
||||
load_transformer_config,
|
||||
retrieve_latents,
|
||||
)
|
||||
from .pipeline_wan2_2_i2v import (
|
||||
Wan22I2VPipeline,
|
||||
get_wan22_i2v_post_process_func,
|
||||
get_wan22_i2v_pre_process_func,
|
||||
)
|
||||
from .pipeline_wan2_2_ti2v import (
|
||||
Wan22TI2VPipeline,
|
||||
get_wan22_ti2v_post_process_func,
|
||||
get_wan22_ti2v_pre_process_func,
|
||||
)
|
||||
from .wan2_2_transformer import WanTransformer3DModel
|
||||
|
||||
__all__ = ["Wan22Pipeline", "get_wan22_post_process_func", "WanTransformer3DModel"]
|
||||
__all__ = [
|
||||
"Wan22Pipeline",
|
||||
"get_wan22_post_process_func",
|
||||
"get_wan22_pre_process_func",
|
||||
"retrieve_latents",
|
||||
"load_transformer_config",
|
||||
"create_transformer_from_config",
|
||||
"Wan22I2VPipeline",
|
||||
"get_wan22_i2v_post_process_func",
|
||||
"get_wan22_i2v_pre_process_func",
|
||||
"Wan22TI2VPipeline",
|
||||
"get_wan22_ti2v_post_process_func",
|
||||
"get_wan22_ti2v_pre_process_func",
|
||||
"WanTransformer3DModel",
|
||||
]
|
||||
@@ -3,9 +3,11 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from collections.abc import Iterable
|
||||
|
||||
import PIL.Image
|
||||
import torch
|
||||
from diffusers import AutoencoderKLWan, FlowMatchEulerDiscreteScheduler
|
||||
from diffusers.utils.torch_utils import randn_tensor
|
||||
@@ -20,6 +22,83 @@ from vllm_omni.diffusion.models.wan2_2.wan2_2_transformer import WanTransformer3
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
|
||||
|
||||
def retrieve_latents(
|
||||
encoder_output: torch.Tensor,
|
||||
generator: torch.Generator | None = None,
|
||||
sample_mode: str = "sample",
|
||||
):
|
||||
"""Retrieve latents from VAE encoder output."""
|
||||
if hasattr(encoder_output, "latent_dist") and sample_mode == "sample":
|
||||
return encoder_output.latent_dist.sample(generator)
|
||||
elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax":
|
||||
return encoder_output.latent_dist.mode()
|
||||
elif hasattr(encoder_output, "latents"):
|
||||
return encoder_output.latents
|
||||
else:
|
||||
raise AttributeError("Could not access latents of provided encoder_output")
|
||||
|
||||
|
||||
def load_transformer_config(model_path: str, subfolder: str = "transformer", local_files_only: bool = True) -> dict:
|
||||
"""Load transformer config from model directory or HF Hub."""
|
||||
if local_files_only:
|
||||
config_path = os.path.join(model_path, subfolder, "config.json")
|
||||
if os.path.exists(config_path):
|
||||
with open(config_path) as f:
|
||||
return json.load(f)
|
||||
else:
|
||||
# Try to download config from HF Hub
|
||||
try:
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
config_path = hf_hub_download(
|
||||
repo_id=model_path,
|
||||
filename=f"{subfolder}/config.json",
|
||||
)
|
||||
with open(config_path) as f:
|
||||
return json.load(f)
|
||||
except Exception:
|
||||
pass
|
||||
return {}
|
||||
|
||||
|
||||
def create_transformer_from_config(config: dict) -> WanTransformer3DModel:
|
||||
"""Create WanTransformer3DModel from config dict."""
|
||||
kwargs = {}
|
||||
|
||||
if "patch_size" in config:
|
||||
kwargs["patch_size"] = tuple(config["patch_size"])
|
||||
if "num_attention_heads" in config:
|
||||
kwargs["num_attention_heads"] = config["num_attention_heads"]
|
||||
if "attention_head_dim" in config:
|
||||
kwargs["attention_head_dim"] = config["attention_head_dim"]
|
||||
if "in_channels" in config:
|
||||
kwargs["in_channels"] = config["in_channels"]
|
||||
if "out_channels" in config:
|
||||
kwargs["out_channels"] = config["out_channels"]
|
||||
if "text_dim" in config:
|
||||
kwargs["text_dim"] = config["text_dim"]
|
||||
if "freq_dim" in config:
|
||||
kwargs["freq_dim"] = config["freq_dim"]
|
||||
if "ffn_dim" in config:
|
||||
kwargs["ffn_dim"] = config["ffn_dim"]
|
||||
if "num_layers" in config:
|
||||
kwargs["num_layers"] = config["num_layers"]
|
||||
if "cross_attn_norm" in config:
|
||||
kwargs["cross_attn_norm"] = config["cross_attn_norm"]
|
||||
if "eps" in config:
|
||||
kwargs["eps"] = config["eps"]
|
||||
if "image_dim" in config:
|
||||
kwargs["image_dim"] = config["image_dim"]
|
||||
if "added_kv_proj_dim" in config:
|
||||
kwargs["added_kv_proj_dim"] = config["added_kv_proj_dim"]
|
||||
if "rope_max_seq_len" in config:
|
||||
kwargs["rope_max_seq_len"] = config["rope_max_seq_len"]
|
||||
if "pos_embed_seq_len" in config:
|
||||
kwargs["pos_embed_seq_len"] = config["pos_embed_seq_len"]
|
||||
|
||||
return WanTransformer3DModel(**kwargs)
|
||||
|
||||
|
||||
def get_wan22_post_process_func(
|
||||
od_config: OmniDiffusionConfig,
|
||||
):
|
||||
@@ -38,6 +117,52 @@ def get_wan22_post_process_func(
|
||||
return post_process_func
|
||||
|
||||
|
||||
def get_wan22_pre_process_func(
|
||||
od_config: OmniDiffusionConfig,
|
||||
):
|
||||
"""Pre-process function for Wan2.2: optionally load and resize input image for I2V mode."""
|
||||
import numpy as np
|
||||
from diffusers.video_processor import VideoProcessor
|
||||
|
||||
video_processor = VideoProcessor(vae_scale_factor=8)
|
||||
|
||||
def pre_process_func(requests: list[OmniDiffusionRequest]) -> list[OmniDiffusionRequest]:
|
||||
for req in requests:
|
||||
# Load image if path is provided
|
||||
if req.image_path is not None and req.pil_image is None:
|
||||
req.pil_image = PIL.Image.open(req.image_path).convert("RGB")
|
||||
|
||||
if req.pil_image is not None:
|
||||
image = req.pil_image
|
||||
|
||||
# Calculate dimensions based on aspect ratio if not provided
|
||||
if req.height is None or req.width is None:
|
||||
# Default max area for 720P
|
||||
max_area = 720 * 1280
|
||||
aspect_ratio = image.height / image.width
|
||||
|
||||
# Calculate dimensions maintaining aspect ratio
|
||||
mod_value = 16 # Must be divisible by 16
|
||||
height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value
|
||||
width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value
|
||||
|
||||
if req.height is None:
|
||||
req.height = height
|
||||
if req.width is None:
|
||||
req.width = width
|
||||
|
||||
# Resize image to target dimensions
|
||||
image = image.resize((req.width, req.height), PIL.Image.Resampling.LANCZOS)
|
||||
req.pil_image = image
|
||||
|
||||
# Preprocess for VAE
|
||||
req.preprocessed_image = video_processor.preprocess(image, height=req.height, width=req.width)
|
||||
|
||||
return requests
|
||||
|
||||
return pre_process_func
|
||||
|
||||
|
||||
class Wan22Pipeline(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -54,6 +179,19 @@ class Wan22Pipeline(nn.Module):
|
||||
model = od_config.model
|
||||
local_files_only = os.path.exists(model)
|
||||
|
||||
# Read model_index.json to detect expand_timesteps mode (for TI2V-5B)
|
||||
self.expand_timesteps = False
|
||||
if local_files_only:
|
||||
model_index_path = os.path.join(model, "model_index.json")
|
||||
if os.path.exists(model_index_path):
|
||||
with open(model_index_path) as f:
|
||||
model_index = json.load(f)
|
||||
self.expand_timesteps = model_index.get("expand_timesteps", False)
|
||||
|
||||
# Check if this is a two-stage model (MoE with transformer_2)
|
||||
transformer_2_path = os.path.join(model, "transformer_2") if local_files_only else None
|
||||
self.has_transformer_2 = transformer_2_path is not None and os.path.exists(transformer_2_path)
|
||||
|
||||
# Set up weights sources for transformer(s)
|
||||
self.weights_sources = [
|
||||
DiffusersPipelineLoader.ComponentSource(
|
||||
@@ -63,14 +201,17 @@ class Wan22Pipeline(nn.Module):
|
||||
prefix="transformer.",
|
||||
fall_back_to_pt=True,
|
||||
),
|
||||
DiffusersPipelineLoader.ComponentSource(
|
||||
model_or_path=od_config.model,
|
||||
subfolder="transformer_2",
|
||||
revision=None,
|
||||
prefix="transformer_2.",
|
||||
fall_back_to_pt=True,
|
||||
),
|
||||
]
|
||||
if self.has_transformer_2:
|
||||
self.weights_sources.append(
|
||||
DiffusersPipelineLoader.ComponentSource(
|
||||
model_or_path=od_config.model,
|
||||
subfolder="transformer_2",
|
||||
revision=None,
|
||||
prefix="transformer_2.",
|
||||
fall_back_to_pt=True,
|
||||
)
|
||||
)
|
||||
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(model, subfolder="tokenizer", local_files_only=local_files_only)
|
||||
self.text_encoder = UMT5EncoderModel.from_pretrained(
|
||||
@@ -80,9 +221,18 @@ class Wan22Pipeline(nn.Module):
|
||||
model, subfolder="vae", torch_dtype=torch.float32, local_files_only=local_files_only
|
||||
).to(self.device)
|
||||
|
||||
# Initialize transformers without from_pretrained (weights loaded via load_weights)
|
||||
self.transformer = WanTransformer3DModel()
|
||||
self.transformer_2 = WanTransformer3DModel()
|
||||
# Initialize transformers with correct config (weights loaded via load_weights)
|
||||
if local_files_only:
|
||||
transformer_config = load_transformer_config(model, "transformer")
|
||||
self.transformer = create_transformer_from_config(transformer_config)
|
||||
if self.has_transformer_2:
|
||||
transformer_2_config = load_transformer_config(model, "transformer_2")
|
||||
self.transformer_2 = create_transformer_from_config(transformer_2_config)
|
||||
else:
|
||||
self.transformer_2 = None
|
||||
else:
|
||||
self.transformer = WanTransformer3DModel()
|
||||
self.transformer_2 = WanTransformer3DModel() if self.has_transformer_2 else None
|
||||
|
||||
self.scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(
|
||||
model, subfolder="scheduler", local_files_only=local_files_only
|
||||
@@ -142,6 +292,13 @@ class Wan22Pipeline(nn.Module):
|
||||
height = req.height or height or 720
|
||||
width = req.width or width or 1280
|
||||
num_frames = req.num_frames if req.num_frames else frame_num or 81
|
||||
|
||||
# Ensure dimensions are compatible with VAE and patch size
|
||||
# For expand_timesteps mode, we need latent dims to be even (divisible by patch_size)
|
||||
patch_size = self.transformer.config.patch_size
|
||||
mod_value = self.vae_scale_factor_spatial * patch_size[1] # 16*2=32 for TI2V, 8*2=16 for I2V
|
||||
height = (height // mod_value) * mod_value
|
||||
width = (width // mod_value) * mod_value
|
||||
num_steps = req.num_inference_steps or num_inference_steps or 40
|
||||
|
||||
guidance_low = guidance_scale if isinstance(guidance_scale, (int, float)) else guidance_scale[0]
|
||||
@@ -211,19 +368,82 @@ class Wan22Pipeline(nn.Module):
|
||||
if self.boundary_ratio is not None:
|
||||
boundary_timestep = self.boundary_ratio * self.scheduler.config.num_train_timesteps
|
||||
|
||||
# Latents
|
||||
num_channels_latents = self.transformer.config.in_channels
|
||||
latents = self.prepare_latents(
|
||||
batch_size=prompt_embeds.shape[0],
|
||||
num_channels_latents=num_channels_latents,
|
||||
height=height,
|
||||
width=width,
|
||||
num_frames=num_frames,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
generator=generator,
|
||||
latents=req.latents,
|
||||
)
|
||||
# Handle I2V mode when expand_timesteps=True and image is provided
|
||||
image = req.pil_image
|
||||
latent_condition = None
|
||||
first_frame_mask = None
|
||||
|
||||
if self.expand_timesteps and image is not None:
|
||||
# I2V mode: encode image and prepare condition
|
||||
from diffusers.video_processor import VideoProcessor
|
||||
|
||||
video_processor = VideoProcessor(vae_scale_factor=self.vae_scale_factor_spatial)
|
||||
|
||||
# Preprocess image
|
||||
if isinstance(image, PIL.Image.Image):
|
||||
image = image.resize((width, height), PIL.Image.Resampling.LANCZOS)
|
||||
image_tensor = video_processor.preprocess(image, height=height, width=width)
|
||||
else:
|
||||
image_tensor = image
|
||||
|
||||
# Use out_channels for noise latents (not in_channels which includes condition)
|
||||
num_channels_latents = self.transformer.config.out_channels
|
||||
batch_size = prompt_embeds.shape[0]
|
||||
|
||||
# Prepare noise latents
|
||||
latents = self.prepare_latents(
|
||||
batch_size=batch_size,
|
||||
num_channels_latents=num_channels_latents,
|
||||
height=height,
|
||||
width=width,
|
||||
num_frames=num_frames,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
generator=generator,
|
||||
latents=req.latents,
|
||||
)
|
||||
|
||||
# Encode image condition
|
||||
num_latent_frames = latents.shape[2]
|
||||
latent_height = latents.shape[3]
|
||||
latent_width = latents.shape[4]
|
||||
|
||||
image_tensor = image_tensor.unsqueeze(2) # [B, C, 1, H, W]
|
||||
image_tensor = image_tensor.to(device=device, dtype=self.vae.dtype)
|
||||
latent_condition = retrieve_latents(self.vae.encode(image_tensor), sample_mode="argmax")
|
||||
latent_condition = latent_condition.repeat(batch_size, 1, 1, 1, 1)
|
||||
|
||||
# Normalize condition latents
|
||||
latents_mean = (
|
||||
torch.tensor(self.vae.config.latents_mean)
|
||||
.view(1, self.vae.config.z_dim, 1, 1, 1)
|
||||
.to(latent_condition.device, latent_condition.dtype)
|
||||
)
|
||||
latents_std = 1.0 / torch.tensor(self.vae.config.latents_std).view(1, self.vae.config.z_dim, 1, 1, 1).to(
|
||||
latent_condition.device, latent_condition.dtype
|
||||
)
|
||||
latent_condition = (latent_condition - latents_mean) * latents_std
|
||||
latent_condition = latent_condition.to(torch.float32)
|
||||
|
||||
# Create mask: 0 for first frame (condition), 1 for rest (to denoise)
|
||||
first_frame_mask = torch.ones(
|
||||
1, 1, num_latent_frames, latent_height, latent_width, dtype=torch.float32, device=device
|
||||
)
|
||||
first_frame_mask[:, :, 0] = 0
|
||||
else:
|
||||
# T2V mode: standard latent preparation
|
||||
num_channels_latents = self.transformer.config.in_channels
|
||||
latents = self.prepare_latents(
|
||||
batch_size=prompt_embeds.shape[0],
|
||||
num_channels_latents=num_channels_latents,
|
||||
height=height,
|
||||
width=width,
|
||||
num_frames=num_frames,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
generator=generator,
|
||||
latents=req.latents,
|
||||
)
|
||||
|
||||
if attention_kwargs is None:
|
||||
attention_kwargs = {}
|
||||
@@ -233,12 +453,30 @@ class Wan22Pipeline(nn.Module):
|
||||
self._current_timestep = t
|
||||
current_model = self.transformer
|
||||
current_guidance_scale = guidance_low
|
||||
if boundary_timestep is not None and t < boundary_timestep:
|
||||
if boundary_timestep is not None and t < boundary_timestep and self.transformer_2 is not None:
|
||||
current_model = self.transformer_2
|
||||
current_guidance_scale = guidance_high
|
||||
|
||||
latent_model_input = latents.to(dtype)
|
||||
timestep = t.expand(latents.shape[0])
|
||||
if self.expand_timesteps and latent_condition is not None:
|
||||
# I2V mode: blend condition with latents using mask
|
||||
latent_model_input = (1 - first_frame_mask) * latent_condition + first_frame_mask * latents
|
||||
latent_model_input = latent_model_input.to(dtype)
|
||||
|
||||
# Expand timesteps per patch - use floor division to match patch embedding
|
||||
patch_size = self.transformer.config.patch_size
|
||||
num_latent_frames = latents.shape[2]
|
||||
patch_height = latents.shape[3] // patch_size[1]
|
||||
patch_width = latents.shape[4] // patch_size[2]
|
||||
|
||||
# Create mask at patch resolution (same as hidden states sequence length)
|
||||
patch_mask = first_frame_mask[:, :, :, :: patch_size[1], :: patch_size[2]]
|
||||
patch_mask = patch_mask[:, :, :, :patch_height, :patch_width] # Ensure correct dimensions
|
||||
temp_ts = (patch_mask[0][0] * t).flatten()
|
||||
timestep = temp_ts.unsqueeze(0).expand(latents.shape[0], -1)
|
||||
else:
|
||||
# T2V mode: standard forward
|
||||
latent_model_input = latents.to(dtype)
|
||||
timestep = t.expand(latents.shape[0])
|
||||
|
||||
noise_pred = current_model(
|
||||
hidden_states=latent_model_input,
|
||||
@@ -262,6 +500,10 @@ class Wan22Pipeline(nn.Module):
|
||||
|
||||
self._current_timestep = None
|
||||
|
||||
# For I2V mode: blend final latents with condition
|
||||
if self.expand_timesteps and latent_condition is not None:
|
||||
latents = (1 - first_frame_mask) * latent_condition + first_frame_mask * latents
|
||||
|
||||
# Decode
|
||||
if output_type == "latent":
|
||||
output = latents
|
||||
|
||||
@@ -0,0 +1,702 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections.abc import Iterable
|
||||
|
||||
import numpy as np
|
||||
import PIL.Image
|
||||
import torch
|
||||
from diffusers import AutoencoderKLWan, FlowMatchEulerDiscreteScheduler
|
||||
from diffusers.utils.torch_utils import randn_tensor
|
||||
from torch import nn
|
||||
from transformers import AutoTokenizer, CLIPImageProcessor, CLIPVisionModel, UMT5EncoderModel
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import (
|
||||
create_transformer_from_config,
|
||||
load_transformer_config,
|
||||
retrieve_latents,
|
||||
)
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
|
||||
|
||||
def _load_model_index(model: str, local_files_only: bool) -> dict:
|
||||
"""Load model_index.json from local path or HF Hub."""
|
||||
if local_files_only:
|
||||
model_index_path = os.path.join(model, "model_index.json")
|
||||
if os.path.exists(model_index_path):
|
||||
import json
|
||||
|
||||
with open(model_index_path) as f:
|
||||
return json.load(f)
|
||||
else:
|
||||
try:
|
||||
import json
|
||||
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
model_index_path = hf_hub_download(model, "model_index.json")
|
||||
with open(model_index_path) as f:
|
||||
return json.load(f)
|
||||
except Exception:
|
||||
pass
|
||||
return {}
|
||||
|
||||
|
||||
def get_wan22_i2v_post_process_func(
|
||||
od_config: OmniDiffusionConfig,
|
||||
):
|
||||
from diffusers.video_processor import VideoProcessor
|
||||
|
||||
video_processor = VideoProcessor(vae_scale_factor=8)
|
||||
|
||||
def post_process_func(
|
||||
video: torch.Tensor,
|
||||
output_type: str = "np",
|
||||
):
|
||||
if output_type == "latent":
|
||||
return video
|
||||
return video_processor.postprocess_video(video, output_type=output_type)
|
||||
|
||||
return post_process_func
|
||||
|
||||
|
||||
def get_wan22_i2v_pre_process_func(
|
||||
od_config: OmniDiffusionConfig,
|
||||
):
|
||||
"""Pre-process function for I2V: load and resize input image."""
|
||||
from diffusers.video_processor import VideoProcessor
|
||||
|
||||
video_processor = VideoProcessor(vae_scale_factor=8)
|
||||
|
||||
def pre_process_func(requests: list[OmniDiffusionRequest]) -> list[OmniDiffusionRequest]:
|
||||
for req in requests:
|
||||
# Load image if path is provided
|
||||
if req.image_path is not None and req.pil_image is None:
|
||||
req.pil_image = PIL.Image.open(req.image_path).convert("RGB")
|
||||
|
||||
if req.pil_image is not None:
|
||||
image = req.pil_image
|
||||
|
||||
# Calculate dimensions based on aspect ratio if not provided
|
||||
if req.height is None or req.width is None:
|
||||
# Default max area for 480P
|
||||
max_area = 480 * 832
|
||||
aspect_ratio = image.height / image.width
|
||||
|
||||
# Calculate dimensions maintaining aspect ratio
|
||||
mod_value = 16 # Must be divisible by 16
|
||||
height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value
|
||||
width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value
|
||||
|
||||
if req.height is None:
|
||||
req.height = height
|
||||
if req.width is None:
|
||||
req.width = width
|
||||
|
||||
# Resize image to target dimensions
|
||||
image = image.resize((req.width, req.height), PIL.Image.Resampling.LANCZOS)
|
||||
req.pil_image = image
|
||||
|
||||
# Preprocess for VAE
|
||||
req.preprocessed_image = video_processor.preprocess(image, height=req.height, width=req.width)
|
||||
|
||||
return requests
|
||||
|
||||
return pre_process_func
|
||||
|
||||
|
||||
class Wan22I2VPipeline(nn.Module):
|
||||
"""
|
||||
Wan2.2 Image-to-Video Pipeline.
|
||||
|
||||
Supports both Wan2.1-style I2V (with CLIP image embeddings) and
|
||||
Wan2.2-style I2V (with expand_timesteps for TI2V-5B).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
od_config: OmniDiffusionConfig,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.od_config = od_config
|
||||
|
||||
self.device = get_local_device()
|
||||
dtype = getattr(od_config, "dtype", torch.bfloat16)
|
||||
|
||||
model = od_config.model
|
||||
local_files_only = os.path.exists(model)
|
||||
|
||||
# Set up weights sources for transformer(s)
|
||||
self.weights_sources = [
|
||||
DiffusersPipelineLoader.ComponentSource(
|
||||
model_or_path=od_config.model,
|
||||
subfolder="transformer",
|
||||
revision=None,
|
||||
prefix="transformer.",
|
||||
fall_back_to_pt=True,
|
||||
),
|
||||
]
|
||||
|
||||
# Load model_index.json to detect available components
|
||||
model_index = _load_model_index(model, local_files_only)
|
||||
|
||||
# Check if this is a two-stage model (MoE with transformer_2)
|
||||
self.has_transformer_2 = "transformer_2" in model_index
|
||||
|
||||
if self.has_transformer_2:
|
||||
self.weights_sources.append(
|
||||
DiffusersPipelineLoader.ComponentSource(
|
||||
model_or_path=od_config.model,
|
||||
subfolder="transformer_2",
|
||||
revision=None,
|
||||
prefix="transformer_2.",
|
||||
fall_back_to_pt=True,
|
||||
)
|
||||
)
|
||||
|
||||
# Text encoder
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(model, subfolder="tokenizer", local_files_only=local_files_only)
|
||||
self.text_encoder = UMT5EncoderModel.from_pretrained(
|
||||
model, subfolder="text_encoder", torch_dtype=dtype, local_files_only=local_files_only
|
||||
).to(self.device)
|
||||
|
||||
# Image encoder (CLIP) - optional, for Wan2.1-style I2V
|
||||
self.has_image_encoder = "image_encoder" in model_index and model_index["image_encoder"][0] is not None
|
||||
|
||||
if self.has_image_encoder:
|
||||
self.image_processor = CLIPImageProcessor.from_pretrained(
|
||||
model, subfolder="image_processor", local_files_only=local_files_only
|
||||
)
|
||||
self.image_encoder = CLIPVisionModel.from_pretrained(
|
||||
model, subfolder="image_encoder", torch_dtype=dtype, local_files_only=local_files_only
|
||||
).to(self.device)
|
||||
else:
|
||||
self.image_processor = None
|
||||
self.image_encoder = None
|
||||
|
||||
# VAE
|
||||
self.vae = AutoencoderKLWan.from_pretrained(
|
||||
model, subfolder="vae", torch_dtype=torch.float32, local_files_only=local_files_only
|
||||
).to(self.device)
|
||||
|
||||
# Transformers (weights loaded via load_weights)
|
||||
# Load config from model directory or HF Hub to get correct in_channels for I2V models
|
||||
transformer_config = load_transformer_config(model, "transformer", local_files_only)
|
||||
self.transformer = create_transformer_from_config(transformer_config)
|
||||
if self.has_transformer_2:
|
||||
transformer_2_config = load_transformer_config(model, "transformer_2", local_files_only)
|
||||
self.transformer_2 = create_transformer_from_config(transformer_2_config)
|
||||
else:
|
||||
self.transformer_2 = None
|
||||
|
||||
# Scheduler
|
||||
self.scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(
|
||||
model, subfolder="scheduler", local_files_only=local_files_only
|
||||
)
|
||||
if od_config.flow_shift is not None:
|
||||
self.scheduler.config.flow_shift = od_config.flow_shift
|
||||
|
||||
# VAE scale factors
|
||||
self.vae_scale_factor_temporal = self.vae.config.scale_factor_temporal if hasattr(self.vae, "config") else 4
|
||||
self.vae_scale_factor_spatial = self.vae.config.scale_factor_spatial if hasattr(self.vae, "config") else 8
|
||||
|
||||
# MoE boundary ratio for two-stage denoising
|
||||
self.boundary_ratio = od_config.boundary_ratio
|
||||
|
||||
# Whether to use expand_timesteps mode (for TI2V-5B style)
|
||||
self.expand_timesteps = getattr(od_config, "expand_timesteps", False)
|
||||
|
||||
self._guidance_scale = None
|
||||
self._guidance_scale_2 = None
|
||||
self._num_timesteps = None
|
||||
self._current_timestep = None
|
||||
|
||||
@property
|
||||
def guidance_scale(self):
|
||||
return self._guidance_scale
|
||||
|
||||
@property
|
||||
def do_classifier_free_guidance(self):
|
||||
return self._guidance_scale is not None and self._guidance_scale > 1.0
|
||||
|
||||
@property
|
||||
def num_timesteps(self):
|
||||
return self._num_timesteps
|
||||
|
||||
@property
|
||||
def current_timestep(self):
|
||||
return self._current_timestep
|
||||
|
||||
def encode_image(
|
||||
self,
|
||||
image: PIL.Image.Image | list[PIL.Image.Image],
|
||||
device: torch.device | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Encode image using CLIP image encoder."""
|
||||
device = device or self.device
|
||||
if self.image_encoder is None:
|
||||
raise ValueError("Image encoder not available for this model.")
|
||||
|
||||
pixel_values = self.image_processor(images=image, return_tensors="pt").pixel_values
|
||||
pixel_values = pixel_values.to(device=device, dtype=self.image_encoder.dtype)
|
||||
image_embeds = self.image_encoder(pixel_values, output_hidden_states=True)
|
||||
return image_embeds.hidden_states[-2]
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(
|
||||
self,
|
||||
req: OmniDiffusionRequest,
|
||||
prompt: str | None = None,
|
||||
negative_prompt: str | None = None,
|
||||
image: PIL.Image.Image | torch.Tensor | None = None,
|
||||
height: int | None = None,
|
||||
width: int | None = None,
|
||||
num_inference_steps: int | None = None,
|
||||
guidance_scale: float | tuple[float, float] = 5.0,
|
||||
frame_num: int | None = None,
|
||||
output_type: str | None = "np",
|
||||
generator: torch.Generator | None = None,
|
||||
prompt_embeds: torch.Tensor | None = None,
|
||||
negative_prompt_embeds: torch.Tensor | None = None,
|
||||
image_embeds: torch.Tensor | None = None,
|
||||
last_image: PIL.Image.Image | torch.Tensor | None = None,
|
||||
attention_kwargs: dict | None = None,
|
||||
**kwargs,
|
||||
) -> DiffusionOutput:
|
||||
# Get parameters from request or arguments
|
||||
prompt = req.prompt if req.prompt is not None else prompt
|
||||
negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt
|
||||
if prompt is None and prompt_embeds is None:
|
||||
raise ValueError("Prompt or prompt_embeds is required for Wan2.2 I2V generation.")
|
||||
|
||||
# Get image from request
|
||||
if image is None:
|
||||
image = req.pil_image
|
||||
if image is None:
|
||||
raise ValueError("Image is required for I2V generation.")
|
||||
|
||||
height = req.height or height or 480
|
||||
width = req.width or width or 832
|
||||
num_frames = req.num_frames if req.num_frames else frame_num or 81
|
||||
num_steps = req.num_inference_steps or num_inference_steps or 50
|
||||
|
||||
# Handle guidance scales
|
||||
guidance_low = guidance_scale if isinstance(guidance_scale, (int, float)) else guidance_scale[0]
|
||||
guidance_high = (
|
||||
req.guidance_scale_2
|
||||
if req.guidance_scale_2 is not None
|
||||
else (
|
||||
guidance_scale[1]
|
||||
if isinstance(guidance_scale, (list, tuple)) and len(guidance_scale) > 1
|
||||
else guidance_low
|
||||
)
|
||||
)
|
||||
|
||||
self._guidance_scale = guidance_low
|
||||
self._guidance_scale_2 = guidance_high
|
||||
|
||||
# Validate inputs
|
||||
self.check_inputs(
|
||||
prompt=prompt,
|
||||
negative_prompt=negative_prompt,
|
||||
image=image,
|
||||
height=height,
|
||||
width=width,
|
||||
prompt_embeds=prompt_embeds,
|
||||
negative_prompt_embeds=negative_prompt_embeds,
|
||||
image_embeds=image_embeds,
|
||||
guidance_scale_2=guidance_high if self.boundary_ratio is not None else None,
|
||||
)
|
||||
|
||||
# Adjust num_frames to be compatible with VAE temporal scaling
|
||||
if num_frames % self.vae_scale_factor_temporal != 1:
|
||||
num_frames = num_frames // self.vae_scale_factor_temporal * self.vae_scale_factor_temporal + 1
|
||||
num_frames = max(num_frames, 1)
|
||||
|
||||
device = self.device
|
||||
dtype = self.transformer.dtype
|
||||
|
||||
# Generator setup
|
||||
if generator is None:
|
||||
generator = req.generator
|
||||
if generator is None and req.seed is not None:
|
||||
generator = torch.Generator(device=device).manual_seed(req.seed)
|
||||
|
||||
# Encode prompts
|
||||
if prompt_embeds is None:
|
||||
prompt_embeds, negative_prompt_embeds = self.encode_prompt(
|
||||
prompt=prompt,
|
||||
negative_prompt=negative_prompt,
|
||||
do_classifier_free_guidance=guidance_low > 1.0 or guidance_high > 1.0,
|
||||
num_videos_per_prompt=req.num_outputs_per_prompt or 1,
|
||||
max_sequence_length=req.max_sequence_length or 512,
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
)
|
||||
else:
|
||||
prompt_embeds = prompt_embeds.to(device=device, dtype=dtype)
|
||||
if negative_prompt_embeds is not None:
|
||||
negative_prompt_embeds = negative_prompt_embeds.to(device=device, dtype=dtype)
|
||||
|
||||
batch_size = prompt_embeds.shape[0]
|
||||
|
||||
# Encode image embeddings (for Wan2.1-style with CLIP)
|
||||
if self.has_image_encoder and self.transformer.config.image_dim is not None:
|
||||
if image_embeds is None:
|
||||
if last_image is None:
|
||||
image_embeds = self.encode_image(image, device)
|
||||
else:
|
||||
image_embeds = self.encode_image([image, last_image], device)
|
||||
image_embeds = image_embeds.repeat(batch_size, 1, 1)
|
||||
image_embeds = image_embeds.to(dtype)
|
||||
else:
|
||||
image_embeds = None
|
||||
|
||||
# Timesteps
|
||||
self.scheduler.set_timesteps(num_steps, device=device)
|
||||
timesteps = self.scheduler.timesteps
|
||||
self._num_timesteps = len(timesteps)
|
||||
|
||||
boundary_timestep = None
|
||||
if self.boundary_ratio is not None:
|
||||
boundary_timestep = self.boundary_ratio * self.scheduler.config.num_train_timesteps
|
||||
|
||||
# Prepare latents (use out_channels=16 for VAE latent, not in_channels=36)
|
||||
num_channels_latents = self.transformer.config.out_channels
|
||||
|
||||
# Preprocess image for VAE
|
||||
from diffusers.video_processor import VideoProcessor
|
||||
|
||||
video_processor = VideoProcessor(vae_scale_factor=self.vae_scale_factor_spatial)
|
||||
|
||||
if isinstance(image, PIL.Image.Image):
|
||||
image_tensor = video_processor.preprocess(image, height=height, width=width)
|
||||
else:
|
||||
image_tensor = image
|
||||
image_tensor = image_tensor.to(device=device, dtype=torch.float32)
|
||||
|
||||
# Handle last_image if provided
|
||||
if last_image is not None:
|
||||
if isinstance(last_image, PIL.Image.Image):
|
||||
last_image_tensor = video_processor.preprocess(last_image, height=height, width=width)
|
||||
else:
|
||||
last_image_tensor = last_image
|
||||
last_image_tensor = last_image_tensor.to(device=device, dtype=torch.float32)
|
||||
else:
|
||||
last_image_tensor = None
|
||||
|
||||
latents, condition, first_frame_mask = self.prepare_latents(
|
||||
image=image_tensor,
|
||||
batch_size=batch_size,
|
||||
num_channels_latents=num_channels_latents,
|
||||
height=height,
|
||||
width=width,
|
||||
num_frames=num_frames,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
generator=generator,
|
||||
latents=req.latents,
|
||||
last_image=last_image_tensor,
|
||||
)
|
||||
|
||||
if attention_kwargs is None:
|
||||
attention_kwargs = {}
|
||||
|
||||
# Denoising loop
|
||||
for t in timesteps:
|
||||
self._current_timestep = t
|
||||
|
||||
# Select model and guidance scale based on timestep
|
||||
current_model = self.transformer
|
||||
current_guidance_scale = guidance_low
|
||||
if boundary_timestep is not None and t < boundary_timestep and self.transformer_2 is not None:
|
||||
current_model = self.transformer_2
|
||||
current_guidance_scale = guidance_high
|
||||
|
||||
# Prepare latent input
|
||||
if self.expand_timesteps:
|
||||
# TI2V-5B style: blend condition with latents using mask
|
||||
latent_model_input = (1 - first_frame_mask) * condition + first_frame_mask * latents
|
||||
latent_model_input = latent_model_input.to(dtype)
|
||||
|
||||
# Expand timesteps for each patch
|
||||
temp_ts = (first_frame_mask[0][0][:, ::2, ::2] * t).flatten()
|
||||
timestep = temp_ts.unsqueeze(0).expand(latents.shape[0], -1)
|
||||
else:
|
||||
# Wan2.1 style: concatenate condition with latents
|
||||
latent_model_input = torch.cat([latents, condition], dim=1).to(dtype)
|
||||
timestep = t.expand(latents.shape[0])
|
||||
|
||||
# Forward pass
|
||||
noise_pred = current_model(
|
||||
hidden_states=latent_model_input,
|
||||
timestep=timestep,
|
||||
encoder_hidden_states=prompt_embeds,
|
||||
encoder_hidden_states_image=image_embeds,
|
||||
attention_kwargs=attention_kwargs,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
|
||||
# Classifier-free guidance
|
||||
if current_guidance_scale > 1.0 and negative_prompt_embeds is not None:
|
||||
noise_uncond = current_model(
|
||||
hidden_states=latent_model_input,
|
||||
timestep=timestep,
|
||||
encoder_hidden_states=negative_prompt_embeds,
|
||||
encoder_hidden_states_image=image_embeds,
|
||||
attention_kwargs=attention_kwargs,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
noise_pred = noise_uncond + current_guidance_scale * (noise_pred - noise_uncond)
|
||||
|
||||
# Scheduler step
|
||||
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
|
||||
|
||||
self._current_timestep = None
|
||||
|
||||
# For expand_timesteps mode, blend final latents with condition
|
||||
if self.expand_timesteps:
|
||||
latents = (1 - first_frame_mask) * condition + first_frame_mask * latents
|
||||
|
||||
# Decode
|
||||
if output_type == "latent":
|
||||
output = latents
|
||||
else:
|
||||
latents = latents.to(self.vae.dtype)
|
||||
latents_mean = (
|
||||
torch.tensor(self.vae.config.latents_mean)
|
||||
.view(1, self.vae.config.z_dim, 1, 1, 1)
|
||||
.to(latents.device, latents.dtype)
|
||||
)
|
||||
latents_std = 1.0 / torch.tensor(self.vae.config.latents_std).view(1, self.vae.config.z_dim, 1, 1, 1).to(
|
||||
latents.device, latents.dtype
|
||||
)
|
||||
latents = latents / latents_std + latents_mean
|
||||
output = self.vae.decode(latents, return_dict=False)[0]
|
||||
|
||||
return DiffusionOutput(output=output)
|
||||
|
||||
def encode_prompt(
|
||||
self,
|
||||
prompt: str | list[str],
|
||||
negative_prompt: str | list[str] | None = None,
|
||||
do_classifier_free_guidance: bool = True,
|
||||
num_videos_per_prompt: int = 1,
|
||||
max_sequence_length: int = 512,
|
||||
device: torch.device | None = None,
|
||||
dtype: torch.dtype | None = None,
|
||||
):
|
||||
"""Encode text prompts using T5 text encoder."""
|
||||
device = device or self.device
|
||||
dtype = dtype or self.text_encoder.dtype
|
||||
|
||||
prompt = [prompt] if isinstance(prompt, str) else prompt
|
||||
prompt_clean = [self._prompt_clean(p) for p in prompt]
|
||||
batch_size = len(prompt_clean)
|
||||
|
||||
text_inputs = self.tokenizer(
|
||||
prompt_clean,
|
||||
padding="max_length",
|
||||
max_length=max_sequence_length,
|
||||
truncation=True,
|
||||
add_special_tokens=True,
|
||||
return_attention_mask=True,
|
||||
return_tensors="pt",
|
||||
)
|
||||
ids, mask = text_inputs.input_ids, text_inputs.attention_mask
|
||||
seq_lens = mask.gt(0).sum(dim=1).long()
|
||||
|
||||
prompt_embeds = self.text_encoder(ids.to(device), mask.to(device)).last_hidden_state
|
||||
prompt_embeds = prompt_embeds.to(dtype=dtype, device=device)
|
||||
prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)]
|
||||
prompt_embeds = torch.stack(
|
||||
[torch.cat([u, u.new_zeros(max_sequence_length - u.size(0), u.size(1))]) for u in prompt_embeds], dim=0
|
||||
)
|
||||
|
||||
_, seq_len, _ = prompt_embeds.shape
|
||||
prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1)
|
||||
prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1)
|
||||
|
||||
negative_prompt_embeds = None
|
||||
if do_classifier_free_guidance:
|
||||
negative_prompt = negative_prompt or ""
|
||||
negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
|
||||
neg_text_inputs = self.tokenizer(
|
||||
[self._prompt_clean(p) for p in negative_prompt],
|
||||
padding="max_length",
|
||||
max_length=max_sequence_length,
|
||||
truncation=True,
|
||||
add_special_tokens=True,
|
||||
return_attention_mask=True,
|
||||
return_tensors="pt",
|
||||
)
|
||||
ids_neg, mask_neg = neg_text_inputs.input_ids, neg_text_inputs.attention_mask
|
||||
seq_lens_neg = mask_neg.gt(0).sum(dim=1).long()
|
||||
negative_prompt_embeds = self.text_encoder(ids_neg.to(device), mask_neg.to(device)).last_hidden_state
|
||||
negative_prompt_embeds = negative_prompt_embeds.to(dtype=dtype, device=device)
|
||||
negative_prompt_embeds = [u[:v] for u, v in zip(negative_prompt_embeds, seq_lens_neg)]
|
||||
negative_prompt_embeds = torch.stack(
|
||||
[
|
||||
torch.cat([u, u.new_zeros(max_sequence_length - u.size(0), u.size(1))])
|
||||
for u in negative_prompt_embeds
|
||||
],
|
||||
dim=0,
|
||||
)
|
||||
negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_videos_per_prompt, 1)
|
||||
negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1)
|
||||
|
||||
return prompt_embeds, negative_prompt_embeds
|
||||
|
||||
@staticmethod
|
||||
def _prompt_clean(text: str) -> str:
|
||||
return " ".join(text.strip().split())
|
||||
|
||||
def prepare_latents(
|
||||
self,
|
||||
image: torch.Tensor,
|
||||
batch_size: int,
|
||||
num_channels_latents: int,
|
||||
height: int,
|
||||
width: int,
|
||||
num_frames: int,
|
||||
dtype: torch.dtype | None,
|
||||
device: torch.device | None,
|
||||
generator: torch.Generator | list[torch.Generator] | None,
|
||||
latents: torch.Tensor | None = None,
|
||||
last_image: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Prepare latents for I2V generation.
|
||||
|
||||
Returns:
|
||||
latents: Initial noise latents
|
||||
condition: Encoded image condition (concatenated with mask for non-expand mode)
|
||||
first_frame_mask: Mask for the first frame (1 for frames to denoise, 0 for condition)
|
||||
"""
|
||||
num_latent_frames = (num_frames - 1) // self.vae_scale_factor_temporal + 1
|
||||
latent_height = height // self.vae_scale_factor_spatial
|
||||
latent_width = width // self.vae_scale_factor_spatial
|
||||
|
||||
shape = (batch_size, num_channels_latents, num_latent_frames, latent_height, latent_width)
|
||||
|
||||
if latents is None:
|
||||
latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
|
||||
else:
|
||||
latents = latents.to(device=device, dtype=dtype)
|
||||
|
||||
# Prepare image condition
|
||||
image = image.unsqueeze(2) # [batch, channels, 1, height, width]
|
||||
|
||||
if self.expand_timesteps:
|
||||
# TI2V-5B style: only use first frame as condition
|
||||
video_condition = image
|
||||
elif last_image is None:
|
||||
# Pad with zeros for remaining frames
|
||||
video_condition = torch.cat(
|
||||
[image, image.new_zeros(image.shape[0], image.shape[1], num_frames - 1, height, width)], dim=2
|
||||
)
|
||||
else:
|
||||
# First and last frame conditioning
|
||||
last_image = last_image.unsqueeze(2)
|
||||
video_condition = torch.cat(
|
||||
[image, image.new_zeros(image.shape[0], image.shape[1], num_frames - 2, height, width), last_image],
|
||||
dim=2,
|
||||
)
|
||||
|
||||
video_condition = video_condition.to(device=device, dtype=self.vae.dtype)
|
||||
|
||||
# Encode through VAE
|
||||
latent_condition = retrieve_latents(self.vae.encode(video_condition), sample_mode="argmax")
|
||||
latent_condition = latent_condition.repeat(batch_size, 1, 1, 1, 1)
|
||||
|
||||
# Normalize latents
|
||||
latents_mean = (
|
||||
torch.tensor(self.vae.config.latents_mean)
|
||||
.view(1, self.vae.config.z_dim, 1, 1, 1)
|
||||
.to(latent_condition.device, latent_condition.dtype)
|
||||
)
|
||||
latents_std = 1.0 / torch.tensor(self.vae.config.latents_std).view(1, self.vae.config.z_dim, 1, 1, 1).to(
|
||||
latent_condition.device, latent_condition.dtype
|
||||
)
|
||||
latent_condition = (latent_condition - latents_mean) * latents_std
|
||||
latent_condition = latent_condition.to(dtype)
|
||||
|
||||
if self.expand_timesteps:
|
||||
# TI2V-5B style: create mask where first frame is 0 (condition), rest is 1 (to denoise)
|
||||
first_frame_mask = torch.ones(
|
||||
1, 1, num_latent_frames, latent_height, latent_width, dtype=dtype, device=device
|
||||
)
|
||||
first_frame_mask[:, :, 0] = 0
|
||||
return latents, latent_condition, first_frame_mask
|
||||
|
||||
# Wan2.1 style: create mask and concatenate with condition
|
||||
mask_lat_size = torch.ones(batch_size, 1, num_frames, latent_height, latent_width)
|
||||
|
||||
if last_image is None:
|
||||
mask_lat_size[:, :, list(range(1, num_frames))] = 0
|
||||
else:
|
||||
mask_lat_size[:, :, list(range(1, num_frames - 1))] = 0
|
||||
|
||||
first_frame_mask = mask_lat_size[:, :, 0:1]
|
||||
first_frame_mask = torch.repeat_interleave(first_frame_mask, dim=2, repeats=self.vae_scale_factor_temporal)
|
||||
mask_lat_size = torch.concat([first_frame_mask, mask_lat_size[:, :, 1:, :]], dim=2)
|
||||
mask_lat_size = mask_lat_size.view(batch_size, -1, self.vae_scale_factor_temporal, latent_height, latent_width)
|
||||
mask_lat_size = mask_lat_size.transpose(1, 2)
|
||||
mask_lat_size = mask_lat_size.to(latent_condition.device)
|
||||
|
||||
# Concatenate mask with condition for channel dimension
|
||||
condition = torch.concat([mask_lat_size, latent_condition], dim=1)
|
||||
|
||||
# For non-expand mode, first_frame_mask is not used in the same way
|
||||
first_frame_mask = torch.ones(1, 1, num_latent_frames, latent_height, latent_width, dtype=dtype, device=device)
|
||||
|
||||
return latents, condition, first_frame_mask
|
||||
|
||||
def check_inputs(
|
||||
self,
|
||||
prompt,
|
||||
negative_prompt,
|
||||
image,
|
||||
height,
|
||||
width,
|
||||
prompt_embeds=None,
|
||||
negative_prompt_embeds=None,
|
||||
image_embeds=None,
|
||||
guidance_scale_2=None,
|
||||
):
|
||||
if image is None and image_embeds is None:
|
||||
raise ValueError("Provide either `image` or `image_embeds`. Cannot leave both undefined.")
|
||||
|
||||
if image is not None and image_embeds is not None:
|
||||
raise ValueError("Cannot forward both `image` and `image_embeds`. Please provide only one.")
|
||||
|
||||
if height % 16 != 0 or width % 16 != 0:
|
||||
raise ValueError(f"`height` and `width` have to be divisible by 16 but are {height} and {width}.")
|
||||
|
||||
if prompt is not None and prompt_embeds is not None:
|
||||
raise ValueError("Cannot forward both `prompt` and `prompt_embeds`. Please provide only one.")
|
||||
|
||||
if negative_prompt is not None and negative_prompt_embeds is not None:
|
||||
raise ValueError(
|
||||
"Cannot forward both `negative_prompt` and `negative_prompt_embeds`. Please provide only one."
|
||||
)
|
||||
|
||||
if prompt is None and prompt_embeds is None:
|
||||
raise ValueError("Provide either `prompt` or `prompt_embeds`.")
|
||||
|
||||
if self.boundary_ratio is None and guidance_scale_2 is not None:
|
||||
raise ValueError("`guidance_scale_2` is only supported when `boundary_ratio` is set.")
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load weights using AutoWeightsLoader for vLLM integration."""
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
@@ -0,0 +1,583 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Wan2.2 TI2V (Text-Image-to-Video) Pipeline.
|
||||
|
||||
This pipeline supports the unified TI2V-5B model that can generate videos from:
|
||||
- Text only (T2V mode)
|
||||
- Text + Image (I2V mode)
|
||||
|
||||
The key difference from the MoE-based I2V pipeline is:
|
||||
- Single transformer (not MoE with two transformers)
|
||||
- Uses expand_timesteps mode for image conditioning
|
||||
- No CLIP image encoder - only VAE encoding for image condition
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections.abc import Iterable
|
||||
|
||||
import numpy as np
|
||||
import PIL.Image
|
||||
import torch
|
||||
from diffusers import AutoencoderKLWan, FlowMatchEulerDiscreteScheduler
|
||||
from diffusers.utils.torch_utils import randn_tensor
|
||||
from torch import nn
|
||||
from transformers import AutoTokenizer, UMT5EncoderModel
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import retrieve_latents
|
||||
from vllm_omni.diffusion.models.wan2_2.wan2_2_transformer import WanTransformer3DModel
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
|
||||
|
||||
def get_wan22_ti2v_post_process_func(
|
||||
od_config: OmniDiffusionConfig,
|
||||
):
|
||||
from diffusers.video_processor import VideoProcessor
|
||||
|
||||
video_processor = VideoProcessor(vae_scale_factor=8)
|
||||
|
||||
def post_process_func(
|
||||
video: torch.Tensor,
|
||||
output_type: str = "np",
|
||||
):
|
||||
if output_type == "latent":
|
||||
return video
|
||||
return video_processor.postprocess_video(video, output_type=output_type)
|
||||
|
||||
return post_process_func
|
||||
|
||||
|
||||
def get_wan22_ti2v_pre_process_func(
|
||||
od_config: OmniDiffusionConfig,
|
||||
):
|
||||
"""Pre-process function for TI2V: optionally load and resize input image."""
|
||||
from diffusers.video_processor import VideoProcessor
|
||||
|
||||
video_processor = VideoProcessor(vae_scale_factor=8)
|
||||
|
||||
def pre_process_func(requests: list[OmniDiffusionRequest]) -> list[OmniDiffusionRequest]:
|
||||
for req in requests:
|
||||
# Load image if path is provided
|
||||
if req.image_path is not None and req.pil_image is None:
|
||||
req.pil_image = PIL.Image.open(req.image_path).convert("RGB")
|
||||
|
||||
if req.pil_image is not None:
|
||||
image = req.pil_image
|
||||
|
||||
# Calculate dimensions based on aspect ratio if not provided
|
||||
if req.height is None or req.width is None:
|
||||
# Default max area for 720P (TI2V-5B default)
|
||||
max_area = 720 * 1280
|
||||
aspect_ratio = image.height / image.width
|
||||
|
||||
# Calculate dimensions maintaining aspect ratio
|
||||
mod_value = 16 # Must be divisible by 16
|
||||
height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value
|
||||
width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value
|
||||
|
||||
if req.height is None:
|
||||
req.height = height
|
||||
if req.width is None:
|
||||
req.width = width
|
||||
|
||||
# Resize image to target dimensions
|
||||
image = image.resize((req.width, req.height), PIL.Image.Resampling.LANCZOS)
|
||||
req.pil_image = image
|
||||
|
||||
# Preprocess for VAE
|
||||
req.preprocessed_image = video_processor.preprocess(image, height=req.height, width=req.width)
|
||||
|
||||
return requests
|
||||
|
||||
return pre_process_func
|
||||
|
||||
|
||||
class Wan22TI2VPipeline(nn.Module):
|
||||
"""
|
||||
Wan2.2 Text-Image-to-Video (TI2V) Pipeline.
|
||||
|
||||
This is a unified pipeline that supports both:
|
||||
- Text-to-Video (T2V): when no image is provided
|
||||
- Image-to-Video (I2V): when an image is provided
|
||||
|
||||
Uses expand_timesteps mode for I2V conditioning where the first frame
|
||||
is conditioned on the input image latent.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
od_config: OmniDiffusionConfig,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.od_config = od_config
|
||||
|
||||
self.device = get_local_device()
|
||||
dtype = getattr(od_config, "dtype", torch.bfloat16)
|
||||
|
||||
model = od_config.model
|
||||
local_files_only = os.path.exists(model)
|
||||
|
||||
# Set up weights sources for single transformer
|
||||
self.weights_sources = [
|
||||
DiffusersPipelineLoader.ComponentSource(
|
||||
model_or_path=od_config.model,
|
||||
subfolder="transformer",
|
||||
revision=None,
|
||||
prefix="transformer.",
|
||||
fall_back_to_pt=True,
|
||||
),
|
||||
]
|
||||
|
||||
# Text encoder
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(model, subfolder="tokenizer", local_files_only=local_files_only)
|
||||
self.text_encoder = UMT5EncoderModel.from_pretrained(
|
||||
model, subfolder="text_encoder", torch_dtype=dtype, local_files_only=local_files_only
|
||||
).to(self.device)
|
||||
|
||||
# VAE
|
||||
self.vae = AutoencoderKLWan.from_pretrained(
|
||||
model, subfolder="vae", torch_dtype=torch.float32, local_files_only=local_files_only
|
||||
).to(self.device)
|
||||
|
||||
# Single transformer (TI2V uses dense 5B model, not MoE)
|
||||
self.transformer = WanTransformer3DModel()
|
||||
|
||||
# Scheduler
|
||||
self.scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(
|
||||
model, subfolder="scheduler", local_files_only=local_files_only
|
||||
)
|
||||
if od_config.flow_shift is not None:
|
||||
self.scheduler.config.flow_shift = od_config.flow_shift
|
||||
|
||||
# VAE scale factors
|
||||
self.vae_scale_factor_temporal = self.vae.config.scale_factor_temporal if hasattr(self.vae, "config") else 4
|
||||
self.vae_scale_factor_spatial = self.vae.config.scale_factor_spatial if hasattr(self.vae, "config") else 8
|
||||
|
||||
# TI2V always uses expand_timesteps mode
|
||||
self.expand_timesteps = True
|
||||
|
||||
self._guidance_scale = None
|
||||
self._num_timesteps = None
|
||||
self._current_timestep = None
|
||||
|
||||
@property
|
||||
def guidance_scale(self):
|
||||
return self._guidance_scale
|
||||
|
||||
@property
|
||||
def do_classifier_free_guidance(self):
|
||||
return self._guidance_scale is not None and self._guidance_scale > 1.0
|
||||
|
||||
@property
|
||||
def num_timesteps(self):
|
||||
return self._num_timesteps
|
||||
|
||||
@property
|
||||
def current_timestep(self):
|
||||
return self._current_timestep
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(
|
||||
self,
|
||||
req: OmniDiffusionRequest,
|
||||
prompt: str | None = None,
|
||||
negative_prompt: str | None = None,
|
||||
image: PIL.Image.Image | torch.Tensor | None = None,
|
||||
height: int | None = None,
|
||||
width: int | None = None,
|
||||
num_inference_steps: int | None = None,
|
||||
guidance_scale: float = 5.0,
|
||||
frame_num: int | None = None,
|
||||
output_type: str | None = "np",
|
||||
generator: torch.Generator | None = None,
|
||||
prompt_embeds: torch.Tensor | None = None,
|
||||
negative_prompt_embeds: torch.Tensor | None = None,
|
||||
attention_kwargs: dict | None = None,
|
||||
**kwargs,
|
||||
) -> DiffusionOutput:
|
||||
# Get parameters from request or arguments
|
||||
prompt = req.prompt if req.prompt is not None else prompt
|
||||
negative_prompt = req.negative_prompt if req.negative_prompt is not None else negative_prompt
|
||||
if prompt is None and prompt_embeds is None:
|
||||
raise ValueError("Prompt or prompt_embeds is required for Wan2.2 TI2V generation.")
|
||||
|
||||
# Get image from request (optional for TI2V)
|
||||
if image is None:
|
||||
image = req.pil_image
|
||||
|
||||
# Default dimensions for TI2V-5B (720P)
|
||||
height = req.height or height or 704
|
||||
width = req.width or width or 1280
|
||||
num_frames = req.num_frames if req.num_frames else frame_num or 121
|
||||
num_steps = req.num_inference_steps or num_inference_steps or 50
|
||||
|
||||
self._guidance_scale = guidance_scale
|
||||
|
||||
# Validate inputs
|
||||
self.check_inputs(
|
||||
prompt=prompt,
|
||||
negative_prompt=negative_prompt,
|
||||
image=image,
|
||||
height=height,
|
||||
width=width,
|
||||
prompt_embeds=prompt_embeds,
|
||||
negative_prompt_embeds=negative_prompt_embeds,
|
||||
)
|
||||
|
||||
# Adjust num_frames to be compatible with VAE temporal scaling
|
||||
if num_frames % self.vae_scale_factor_temporal != 1:
|
||||
num_frames = num_frames // self.vae_scale_factor_temporal * self.vae_scale_factor_temporal + 1
|
||||
num_frames = max(num_frames, 1)
|
||||
|
||||
device = self.device
|
||||
dtype = self.transformer.dtype
|
||||
|
||||
# Generator setup
|
||||
if generator is None:
|
||||
generator = req.generator
|
||||
if generator is None and req.seed is not None:
|
||||
generator = torch.Generator(device=device).manual_seed(req.seed)
|
||||
|
||||
# Encode prompts
|
||||
if prompt_embeds is None:
|
||||
prompt_embeds, negative_prompt_embeds = self.encode_prompt(
|
||||
prompt=prompt,
|
||||
negative_prompt=negative_prompt,
|
||||
do_classifier_free_guidance=guidance_scale > 1.0,
|
||||
num_videos_per_prompt=req.num_outputs_per_prompt or 1,
|
||||
max_sequence_length=req.max_sequence_length or 512,
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
)
|
||||
else:
|
||||
prompt_embeds = prompt_embeds.to(device=device, dtype=dtype)
|
||||
if negative_prompt_embeds is not None:
|
||||
negative_prompt_embeds = negative_prompt_embeds.to(device=device, dtype=dtype)
|
||||
|
||||
batch_size = prompt_embeds.shape[0]
|
||||
|
||||
# Timesteps
|
||||
self.scheduler.set_timesteps(num_steps, device=device)
|
||||
timesteps = self.scheduler.timesteps
|
||||
self._num_timesteps = len(timesteps)
|
||||
|
||||
# Prepare latents
|
||||
num_channels_latents = self.transformer.config.in_channels
|
||||
num_latent_frames = (num_frames - 1) // self.vae_scale_factor_temporal + 1
|
||||
latent_height = height // self.vae_scale_factor_spatial
|
||||
latent_width = width // self.vae_scale_factor_spatial
|
||||
|
||||
# Check if we have an image (I2V mode) or not (T2V mode)
|
||||
if image is not None:
|
||||
# I2V mode: prepare latents with image condition
|
||||
from diffusers.video_processor import VideoProcessor
|
||||
|
||||
video_processor = VideoProcessor(vae_scale_factor=self.vae_scale_factor_spatial)
|
||||
|
||||
if isinstance(image, PIL.Image.Image):
|
||||
image_tensor = video_processor.preprocess(image, height=height, width=width)
|
||||
else:
|
||||
image_tensor = image
|
||||
image_tensor = image_tensor.to(device=device, dtype=torch.float32)
|
||||
|
||||
latents, latent_condition, first_frame_mask = self.prepare_i2v_latents(
|
||||
image=image_tensor,
|
||||
batch_size=batch_size,
|
||||
num_channels_latents=num_channels_latents,
|
||||
height=height,
|
||||
width=width,
|
||||
num_frames=num_frames,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
generator=generator,
|
||||
latents=req.latents,
|
||||
)
|
||||
else:
|
||||
# T2V mode: prepare random latents
|
||||
latents = self.prepare_latents(
|
||||
batch_size=batch_size,
|
||||
num_channels_latents=num_channels_latents,
|
||||
height=height,
|
||||
width=width,
|
||||
num_frames=num_frames,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
generator=generator,
|
||||
latents=req.latents,
|
||||
)
|
||||
latent_condition = None
|
||||
first_frame_mask = torch.ones(
|
||||
1, 1, num_latent_frames, latent_height, latent_width, dtype=torch.float32, device=device
|
||||
)
|
||||
|
||||
if attention_kwargs is None:
|
||||
attention_kwargs = {}
|
||||
|
||||
# Denoising loop
|
||||
for t in timesteps:
|
||||
self._current_timestep = t
|
||||
|
||||
# Prepare latent input
|
||||
if latent_condition is not None:
|
||||
# I2V mode: blend condition with latents using mask
|
||||
latent_model_input = (1 - first_frame_mask) * latent_condition + first_frame_mask * latents
|
||||
latent_model_input = latent_model_input.to(dtype)
|
||||
|
||||
# Expand timesteps for each patch (TI2V style)
|
||||
temp_ts = (first_frame_mask[0][0][:, ::2, ::2] * t).flatten()
|
||||
timestep = temp_ts.unsqueeze(0).expand(latents.shape[0], -1)
|
||||
else:
|
||||
# T2V mode: use latents directly
|
||||
latent_model_input = latents.to(dtype)
|
||||
|
||||
# Expand timesteps for TI2V model architecture
|
||||
mask = torch.ones(1, 1, num_latent_frames, latent_height, latent_width, device=device)
|
||||
temp_ts = (mask[0][0][:, ::2, ::2] * t).flatten()
|
||||
timestep = temp_ts.unsqueeze(0).expand(latents.shape[0], -1)
|
||||
|
||||
# Forward pass
|
||||
noise_pred = self.transformer(
|
||||
hidden_states=latent_model_input,
|
||||
timestep=timestep,
|
||||
encoder_hidden_states=prompt_embeds,
|
||||
attention_kwargs=attention_kwargs,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
|
||||
# Classifier-free guidance
|
||||
if guidance_scale > 1.0 and negative_prompt_embeds is not None:
|
||||
noise_uncond = self.transformer(
|
||||
hidden_states=latent_model_input,
|
||||
timestep=timestep,
|
||||
encoder_hidden_states=negative_prompt_embeds,
|
||||
attention_kwargs=attention_kwargs,
|
||||
return_dict=False,
|
||||
)[0]
|
||||
noise_pred = noise_uncond + guidance_scale * (noise_pred - noise_uncond)
|
||||
|
||||
# Scheduler step
|
||||
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
|
||||
|
||||
self._current_timestep = None
|
||||
|
||||
# For I2V mode, blend final latents with condition
|
||||
if latent_condition is not None:
|
||||
latents = (1 - first_frame_mask) * latent_condition + first_frame_mask * latents
|
||||
|
||||
# Decode
|
||||
if output_type == "latent":
|
||||
output = latents
|
||||
else:
|
||||
latents = latents.to(self.vae.dtype)
|
||||
latents_mean = (
|
||||
torch.tensor(self.vae.config.latents_mean)
|
||||
.view(1, self.vae.config.z_dim, 1, 1, 1)
|
||||
.to(latents.device, latents.dtype)
|
||||
)
|
||||
latents_std = 1.0 / torch.tensor(self.vae.config.latents_std).view(1, self.vae.config.z_dim, 1, 1, 1).to(
|
||||
latents.device, latents.dtype
|
||||
)
|
||||
latents = latents / latents_std + latents_mean
|
||||
output = self.vae.decode(latents, return_dict=False)[0]
|
||||
|
||||
return DiffusionOutput(output=output)
|
||||
|
||||
def encode_prompt(
|
||||
self,
|
||||
prompt: str | list[str],
|
||||
negative_prompt: str | list[str] | None = None,
|
||||
do_classifier_free_guidance: bool = True,
|
||||
num_videos_per_prompt: int = 1,
|
||||
max_sequence_length: int = 512,
|
||||
device: torch.device | None = None,
|
||||
dtype: torch.dtype | None = None,
|
||||
):
|
||||
"""Encode text prompts using T5 text encoder."""
|
||||
device = device or self.device
|
||||
dtype = dtype or self.text_encoder.dtype
|
||||
|
||||
prompt = [prompt] if isinstance(prompt, str) else prompt
|
||||
prompt_clean = [self._prompt_clean(p) for p in prompt]
|
||||
batch_size = len(prompt_clean)
|
||||
|
||||
text_inputs = self.tokenizer(
|
||||
prompt_clean,
|
||||
padding="max_length",
|
||||
max_length=max_sequence_length,
|
||||
truncation=True,
|
||||
add_special_tokens=True,
|
||||
return_attention_mask=True,
|
||||
return_tensors="pt",
|
||||
)
|
||||
ids, mask = text_inputs.input_ids, text_inputs.attention_mask
|
||||
seq_lens = mask.gt(0).sum(dim=1).long()
|
||||
|
||||
prompt_embeds = self.text_encoder(ids.to(device), mask.to(device)).last_hidden_state
|
||||
prompt_embeds = prompt_embeds.to(dtype=dtype, device=device)
|
||||
prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)]
|
||||
prompt_embeds = torch.stack(
|
||||
[torch.cat([u, u.new_zeros(max_sequence_length - u.size(0), u.size(1))]) for u in prompt_embeds], dim=0
|
||||
)
|
||||
|
||||
_, seq_len, _ = prompt_embeds.shape
|
||||
prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1)
|
||||
prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1)
|
||||
|
||||
negative_prompt_embeds = None
|
||||
if do_classifier_free_guidance:
|
||||
negative_prompt = negative_prompt or ""
|
||||
negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
|
||||
neg_text_inputs = self.tokenizer(
|
||||
[self._prompt_clean(p) for p in negative_prompt],
|
||||
padding="max_length",
|
||||
max_length=max_sequence_length,
|
||||
truncation=True,
|
||||
add_special_tokens=True,
|
||||
return_attention_mask=True,
|
||||
return_tensors="pt",
|
||||
)
|
||||
ids_neg, mask_neg = neg_text_inputs.input_ids, neg_text_inputs.attention_mask
|
||||
seq_lens_neg = mask_neg.gt(0).sum(dim=1).long()
|
||||
negative_prompt_embeds = self.text_encoder(ids_neg.to(device), mask_neg.to(device)).last_hidden_state
|
||||
negative_prompt_embeds = negative_prompt_embeds.to(dtype=dtype, device=device)
|
||||
negative_prompt_embeds = [u[:v] for u, v in zip(negative_prompt_embeds, seq_lens_neg)]
|
||||
negative_prompt_embeds = torch.stack(
|
||||
[
|
||||
torch.cat([u, u.new_zeros(max_sequence_length - u.size(0), u.size(1))])
|
||||
for u in negative_prompt_embeds
|
||||
],
|
||||
dim=0,
|
||||
)
|
||||
negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_videos_per_prompt, 1)
|
||||
negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1)
|
||||
|
||||
return prompt_embeds, negative_prompt_embeds
|
||||
|
||||
@staticmethod
|
||||
def _prompt_clean(text: str) -> str:
|
||||
return " ".join(text.strip().split())
|
||||
|
||||
def prepare_latents(
|
||||
self,
|
||||
batch_size: int,
|
||||
num_channels_latents: int,
|
||||
height: int,
|
||||
width: int,
|
||||
num_frames: int,
|
||||
dtype: torch.dtype | None,
|
||||
device: torch.device | None,
|
||||
generator: torch.Generator | list[torch.Generator] | None,
|
||||
latents: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Prepare random latents for T2V mode."""
|
||||
if latents is not None:
|
||||
return latents.to(device=device, dtype=dtype)
|
||||
|
||||
num_latent_frames = (num_frames - 1) // self.vae_scale_factor_temporal + 1
|
||||
shape = (
|
||||
batch_size,
|
||||
num_channels_latents,
|
||||
num_latent_frames,
|
||||
int(height) // self.vae_scale_factor_spatial,
|
||||
int(width) // self.vae_scale_factor_spatial,
|
||||
)
|
||||
if isinstance(generator, list) and len(generator) != batch_size:
|
||||
raise ValueError(f"Generator list length {len(generator)} does not match batch size {batch_size}.")
|
||||
latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
|
||||
return latents
|
||||
|
||||
def prepare_i2v_latents(
|
||||
self,
|
||||
image: torch.Tensor,
|
||||
batch_size: int,
|
||||
num_channels_latents: int,
|
||||
height: int,
|
||||
width: int,
|
||||
num_frames: int,
|
||||
dtype: torch.dtype | None,
|
||||
device: torch.device | None,
|
||||
generator: torch.Generator | list[torch.Generator] | None,
|
||||
latents: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Prepare latents for I2V mode with image conditioning.
|
||||
|
||||
Returns:
|
||||
latents: Initial noise latents
|
||||
latent_condition: Encoded first frame condition
|
||||
first_frame_mask: Mask (0 for first frame, 1 for rest)
|
||||
"""
|
||||
num_latent_frames = (num_frames - 1) // self.vae_scale_factor_temporal + 1
|
||||
latent_height = height // self.vae_scale_factor_spatial
|
||||
latent_width = width // self.vae_scale_factor_spatial
|
||||
|
||||
shape = (batch_size, num_channels_latents, num_latent_frames, latent_height, latent_width)
|
||||
|
||||
if latents is None:
|
||||
latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
|
||||
else:
|
||||
latents = latents.to(device=device, dtype=dtype)
|
||||
|
||||
# Prepare first frame condition
|
||||
image = image.unsqueeze(2) # [batch, channels, 1, height, width]
|
||||
image = image.to(device=device, dtype=self.vae.dtype)
|
||||
|
||||
# Encode through VAE
|
||||
latent_condition = retrieve_latents(self.vae.encode(image), sample_mode="argmax")
|
||||
latent_condition = latent_condition.repeat(batch_size, 1, 1, 1, 1)
|
||||
|
||||
# Normalize latents
|
||||
latents_mean = (
|
||||
torch.tensor(self.vae.config.latents_mean)
|
||||
.view(1, self.vae.config.z_dim, 1, 1, 1)
|
||||
.to(latent_condition.device, latent_condition.dtype)
|
||||
)
|
||||
latents_std = 1.0 / torch.tensor(self.vae.config.latents_std).view(1, self.vae.config.z_dim, 1, 1, 1).to(
|
||||
latent_condition.device, latent_condition.dtype
|
||||
)
|
||||
latent_condition = (latent_condition - latents_mean) * latents_std
|
||||
latent_condition = latent_condition.to(dtype)
|
||||
|
||||
# Create mask: 0 for first frame (condition), 1 for rest (to denoise)
|
||||
first_frame_mask = torch.ones(1, 1, num_latent_frames, latent_height, latent_width, dtype=dtype, device=device)
|
||||
first_frame_mask[:, :, 0] = 0
|
||||
|
||||
return latents, latent_condition, first_frame_mask
|
||||
|
||||
def check_inputs(
|
||||
self,
|
||||
prompt,
|
||||
negative_prompt,
|
||||
image,
|
||||
height,
|
||||
width,
|
||||
prompt_embeds=None,
|
||||
negative_prompt_embeds=None,
|
||||
):
|
||||
if height % 16 != 0 or width % 16 != 0:
|
||||
raise ValueError(f"`height` and `width` have to be divisible by 16 but are {height} and {width}.")
|
||||
|
||||
if prompt is not None and prompt_embeds is not None:
|
||||
raise ValueError("Cannot forward both `prompt` and `prompt_embeds`. Please provide only one.")
|
||||
|
||||
if negative_prompt is not None and negative_prompt_embeds is not None:
|
||||
raise ValueError(
|
||||
"Cannot forward both `negative_prompt` and `negative_prompt_embeds`. Please provide only one."
|
||||
)
|
||||
|
||||
if prompt is None and prompt_embeds is None:
|
||||
raise ValueError("Provide either `prompt` or `prompt_embeds`.")
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load weights using AutoWeightsLoader for vLLM integration."""
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
@@ -44,6 +44,11 @@ _DIFFUSION_MODELS = {
|
||||
"pipeline_wan2_2",
|
||||
"Wan22Pipeline",
|
||||
),
|
||||
"WanImageToVideoPipeline": (
|
||||
"wan2_2",
|
||||
"pipeline_wan2_2_i2v",
|
||||
"Wan22I2VPipeline",
|
||||
),
|
||||
"LongCatImagePipeline": (
|
||||
"longcat_image",
|
||||
"pipeline_longcat_image",
|
||||
@@ -94,6 +99,7 @@ _DIFFUSION_POST_PROCESS_FUNCS = {
|
||||
"ZImagePipeline": "get_post_process_func",
|
||||
"OvisImagePipeline": "get_ovis_image_post_process_func",
|
||||
"WanPipeline": "get_wan22_post_process_func",
|
||||
"WanImageToVideoPipeline": "get_wan22_i2v_post_process_func",
|
||||
"LongCatImagePipeline": "get_longcat_image_post_process_func",
|
||||
"LongCatImageEditPipeline": "get_longcat_image_post_process_func",
|
||||
}
|
||||
@@ -106,6 +112,8 @@ _DIFFUSION_PRE_PROCESS_FUNCS = {
|
||||
"QwenImageEditPlusPipeline": "get_qwen_image_edit_plus_pre_process_func",
|
||||
"LongCatImageEditPipeline": "get_longcat_image_edit_pre_process_func",
|
||||
"QwenImageLayeredPipeline": "get_qwen_image_layered_pre_process_func",
|
||||
"WanPipeline": "get_wan22_pre_process_func",
|
||||
"WanImageToVideoPipeline": "get_wan22_i2v_pre_process_func",
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user