[Model] Adapt Wan2.2-I2V-A14B via LightX2V offline conversion path (#2134)

Signed-off-by: Celeste-jq <591998922@qq.com>
Co-authored-by: Canlin Guo <canlinguosdu@gmail.com>
This commit is contained in:
Jiaqian LiuandCanlin Guo authored and GitHub committed 2026-04-13 14:20:04 +08:00
1 parent cb4d13a658
commit 8097747a5d
14 files changed
+804 -25

No files matched your search

@@ -24,6 +24,7 @@ from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin, _is_rank_zero
from vllm_omni.diffusion.models.schedulers import FlowUniPCMultistepScheduler
from vllm_omni.diffusion.models.wan2_2.scheduling_wan_euler import WanEulerScheduler
from vllm_omni.diffusion.models.wan2_2.wan2_2_transformer import WanTransformer3DModel
from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin
from vllm_omni.diffusion.request import OmniDiffusionRequest
@@ -32,6 +33,46 @@ from vllm_omni.platforms import current_omni_platform
logger = logging.getLogger(__name__)
DEBUG_PERF = False
WAN_SAMPLE_SOLVER_CHOICES = {"unipc", "euler"}
def build_wan_scheduler(sample_solver: str, flow_shift: float) -> Any:
if sample_solver == "unipc":
return FlowUniPCMultistepScheduler(
num_train_timesteps=1000,
shift=flow_shift,
prediction_type="flow_prediction",
)
if sample_solver == "euler":
return WanEulerScheduler(
num_train_timesteps=1000,
shift=flow_shift,
)
raise ValueError(
f"Unsupported Wan sample_solver: {sample_solver}. Expected one of: {sorted(WAN_SAMPLE_SOLVER_CHOICES)}"
)
def resolve_wan_sample_solver(req: OmniDiffusionRequest, default: str = "unipc") -> str:
extra_args = getattr(req.sampling_params, "extra_args", {}) or {}
raw = extra_args.get("sample_solver", default)
sample_solver = str(raw).strip().lower()
if sample_solver not in WAN_SAMPLE_SOLVER_CHOICES:
raise ValueError(f"Invalid sample_solver={raw!r}. Expected one of: {sorted(WAN_SAMPLE_SOLVER_CHOICES)}")
return sample_solver
def resolve_wan_flow_shift(req: OmniDiffusionRequest, od_config: OmniDiffusionConfig) -> float:
extra_args = getattr(req.sampling_params, "extra_args", {}) or {}
raw_flow_shift = extra_args.get("flow_shift")
if raw_flow_shift is None:
raw_flow_shift = od_config.flow_shift if od_config.flow_shift is not None else 5.0
try:
return float(raw_flow_shift)
except (TypeError, ValueError) as exc:
raise ValueError(f"Invalid flow_shift={raw_flow_shift!r}. flow_shift must be a float.") from exc
def retrieve_latents(
@@ -296,13 +337,9 @@ class Wan22Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin, DiffusionPipe
else:
raise RuntimeError("No transformer loaded")
# Initialize UniPC scheduler
flow_shift = od_config.flow_shift if od_config.flow_shift is not None else 5.0 # default for 720p
self.scheduler = FlowUniPCMultistepScheduler(
num_train_timesteps=1000,
shift=flow_shift,
prediction_type="flow_prediction",
)
self._sample_solver = "unipc"
self._flow_shift = od_config.flow_shift if od_config.flow_shift is not None else 5.0
self.scheduler = build_wan_scheduler(self._sample_solver, self._flow_shift)
self.vae_scale_factor_temporal = self.vae.config.scale_factor_temporal if getattr(self, "vae", None) else 4
self.vae_scale_factor_spatial = self.vae.config.scale_factor_spatial if getattr(self, "vae", None) else 8
@@ -462,6 +499,13 @@ class Wan22Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin, DiffusionPipe
current_omni_platform.synchronize()
_t_text_enc_ms = (time.perf_counter() - _t_text_enc_start) * 1000
sample_solver = resolve_wan_sample_solver(req, default=self._sample_solver)
flow_shift = resolve_wan_flow_shift(req, self.od_config)
if sample_solver != self._sample_solver or abs(flow_shift - self._flow_shift) > 1e-6:
self.scheduler = build_wan_scheduler(sample_solver, flow_shift)
self._sample_solver = sample_solver
self._flow_shift = flow_shift
# Timesteps
self.scheduler.set_timesteps(num_steps, device=device)
timesteps = self.scheduler.timesteps
@@ -24,10 +24,12 @@ from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.interface import SupportImageInput
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin, _is_rank_zero
from vllm_omni.diffusion.models.schedulers import FlowUniPCMultistepScheduler
from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import (
build_wan_scheduler,
create_transformer_from_config,
load_transformer_config,
resolve_wan_flow_shift,
resolve_wan_sample_solver,
retrieve_latents,
)
from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin
@@ -230,13 +232,9 @@ class Wan22I2VPipeline(
else:
self.transformer_2 = None
# Initialize UniPC scheduler
flow_shift = od_config.flow_shift if od_config.flow_shift is not None else 5.0 # default for 720p
self.scheduler = FlowUniPCMultistepScheduler(
num_train_timesteps=1000,
shift=flow_shift,
prediction_type="flow_prediction",
)
self._sample_solver = "unipc"
self._flow_shift = od_config.flow_shift if od_config.flow_shift is not None else 5.0
self.scheduler = build_wan_scheduler(self._sample_solver, self._flow_shift)
# VAE scale factors
self.vae_scale_factor_temporal = self.vae.config.scale_factor_temporal if hasattr(self.vae, "config") else 4
@@ -440,6 +438,13 @@ class Wan22I2VPipeline(
current_omni_platform.synchronize()
_t_img_enc_ms = (time.perf_counter() - _t_img_enc_start) * 1000
sample_solver = resolve_wan_sample_solver(req, default=self._sample_solver)
flow_shift = resolve_wan_flow_shift(req, self.od_config)
if sample_solver != self._sample_solver or abs(flow_shift - self._flow_shift) > 1e-6:
self.scheduler = build_wan_scheduler(sample_solver, flow_shift)
self._sample_solver = sample_solver
self._flow_shift = flow_shift
# Timesteps
self.scheduler.set_timesteps(num_steps, device=device)
timesteps = self.scheduler.timesteps
@@ -36,10 +36,12 @@ from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.models.interface import SupportImageInput
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin
from vllm_omni.diffusion.models.schedulers import FlowUniPCMultistepScheduler
from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import (
build_wan_scheduler,
create_transformer_from_config,
load_transformer_config,
resolve_wan_flow_shift,
resolve_wan_sample_solver,
retrieve_latents,
)
from vllm_omni.diffusion.request import OmniDiffusionRequest
@@ -183,13 +185,9 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin, Progress
transformer_config = load_transformer_config(model, "transformer", local_files_only)
self.transformer = create_transformer_from_config(transformer_config)
# Initialize UniPC scheduler
flow_shift = od_config.flow_shift if od_config.flow_shift is not None else 5.0 # default for 720p
self.scheduler = FlowUniPCMultistepScheduler(
num_train_timesteps=1000,
shift=flow_shift,
prediction_type="flow_prediction",
)
self._sample_solver = "unipc"
self._flow_shift = od_config.flow_shift if od_config.flow_shift is not None else 5.0
self.scheduler = build_wan_scheduler(self._sample_solver, self._flow_shift)
# VAE scale factors
self.vae_scale_factor_temporal = self.vae.config.scale_factor_temporal if hasattr(self.vae, "config") else 4
@@ -323,6 +321,13 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin, Progress
batch_size = prompt_embeds.shape[0]
sample_solver = resolve_wan_sample_solver(req, default=self._sample_solver)
flow_shift = resolve_wan_flow_shift(req, self.od_config)
if sample_solver != self._sample_solver or abs(flow_shift - self._flow_shift) > 1e-6:
self.scheduler = build_wan_scheduler(sample_solver, flow_shift)
self._sample_solver = sample_solver
self._flow_shift = flow_shift
# Timesteps
self.scheduler.set_timesteps(num_steps, device=device)
timesteps = self.scheduler.timesteps
@@ -0,0 +1,147 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
from dataclasses import dataclass
from types import SimpleNamespace
import numpy as np
import torch
@dataclass
class WanEulerSchedulerOutput:
prev_sample: torch.FloatTensor
def _unsqueeze_to_ndim(in_tensor: torch.Tensor, target_ndim: int) -> torch.Tensor:
if in_tensor.ndim >= target_ndim:
return in_tensor
return in_tensor[(...,) + (None,) * (target_ndim - in_tensor.ndim)]
def _get_timesteps(num_steps: int, max_steps: int = 1000) -> np.ndarray:
# Keep num_steps + 1 points so Euler update can always access sigma_next.
return np.linspace(max_steps, 0, num_steps + 1, dtype=np.float32)
def _timestep_shift(timesteps: torch.Tensor, shift: float = 1.0) -> torch.Tensor:
return shift * timesteps / (1 + (shift - 1) * timesteps)
class WanEulerScheduler:
order = 1
def __init__(
self,
num_train_timesteps: int = 1000,
shift: float = 1.0,
device: torch.device | str = "cpu",
) -> None:
self.num_train_timesteps = int(num_train_timesteps)
self._shift = float(shift)
self.device = device
self.config = SimpleNamespace(num_train_timesteps=self.num_train_timesteps)
self.init_noise_sigma = 1.0
self._step_index: int | None = None
self._begin_index: int | None = None
self.timesteps = torch.empty(0, dtype=torch.float32)
self.sigmas = torch.empty(0, dtype=torch.float32)
self.timesteps_ori = torch.empty(0, dtype=torch.float32)
self.set_timesteps(num_inference_steps=self.num_train_timesteps, device=self.device)
@property
def step_index(self) -> int | None:
return self._step_index
@property
def begin_index(self) -> int | None:
return self._begin_index
def set_begin_index(self, begin_index: int = 0) -> None:
self._begin_index = int(begin_index)
def index_for_timestep(self, timestep: torch.Tensor) -> int:
indices = (self.timesteps == timestep).nonzero()
if len(indices) > 0:
pos = 1 if len(indices) > 1 else 0
return int(indices[pos].item())
# Fallback for tiny float drift
return int(torch.argmin(torch.abs(self.timesteps - timestep)).item())
def _init_step_index(self, timestep: float | torch.Tensor) -> None:
if self.begin_index is None:
if isinstance(timestep, torch.Tensor):
timestep_t = timestep.to(self.timesteps.device, dtype=self.timesteps.dtype)
else:
timestep_t = torch.tensor(timestep, device=self.timesteps.device, dtype=self.timesteps.dtype)
self._step_index = self.index_for_timestep(timestep_t)
else:
self._step_index = self._begin_index
def set_shift(self, shift: float = 1.0) -> None:
# Compute shifted sigma schedule on [0, 1].
sigmas_full = self.timesteps_ori / float(self.num_train_timesteps)
sigmas_full = _timestep_shift(sigmas_full, shift=float(shift))
self.sigmas = sigmas_full
# Public timesteps are the first N points; next point is consumed as sigma_next.
self.timesteps = self.sigmas[:-1] * self.num_train_timesteps
self._shift = float(shift)
def set_timesteps(
self,
num_inference_steps: int,
device: torch.device | str | int | None = None,
**kwargs, # noqa: ARG002 - kept for scheduler API compatibility
) -> None:
timesteps = _get_timesteps(
num_steps=int(num_inference_steps),
max_steps=self.num_train_timesteps,
)
self.timesteps_ori = torch.from_numpy(timesteps).to(
dtype=torch.float32,
device=device or self.device,
)
self.set_shift(self._shift)
self._step_index = None
self._begin_index = None
def scale_model_input(self, sample: torch.Tensor, timestep: int | None = None) -> torch.Tensor: # noqa: ARG002
return sample
def step(
self,
model_output: torch.FloatTensor,
timestep: float | torch.FloatTensor,
sample: torch.FloatTensor,
return_dict: bool = True,
**kwargs, # noqa: ARG002 - kept for scheduler API compatibility
) -> WanEulerSchedulerOutput | tuple[torch.FloatTensor]:
if isinstance(timestep, (int, torch.IntTensor, torch.LongTensor)):
raise ValueError(
"Passing integer indices as timesteps is not supported. Use one value from scheduler.timesteps instead."
)
if self.step_index is None:
self._init_step_index(timestep)
assert self._step_index is not None
sample_fp32 = sample.to(torch.float32)
sigma = _unsqueeze_to_ndim(self.sigmas[self._step_index], sample_fp32.ndim).to(sample_fp32.device)
sigma_next = _unsqueeze_to_ndim(self.sigmas[self._step_index + 1], sample_fp32.ndim).to(sample_fp32.device)
prev_sample = sample_fp32 + (sigma_next - sigma) * model_output
prev_sample = prev_sample.to(model_output.dtype)
self._step_index += 1
if not return_dict:
return (prev_sample,)
return WanEulerSchedulerOutput(prev_sample=prev_sample)
def __len__(self) -> int:
return self.num_train_timesteps
@@ -1015,6 +1015,14 @@ class WanTransformer3DModel(nn.Module):
if ".to_out.0." in lookup_name:
lookup_name = lookup_name.replace(".to_out.0.", ".to_out.")
# Compatibility: some Wan conversion pipelines still keep
# block modulation keys as `blocks.N.modulation` instead of
# `blocks.N.scale_shift_table`.
if lookup_name.endswith(".modulation"):
modulation_alias = lookup_name[: -len(".modulation")] + ".scale_shift_table"
if modulation_alias in params_dict:
lookup_name = modulation_alias
if lookup_name not in params_dict:
logger.warning(f"Skipping weight {original_name} -> {lookup_name}")
continue