[Model] Adapt Wan2.2-I2V-A14B via LightX2V offline conversion path (#2134)
Signed-off-by: Celeste-jq <591998922@qq.com> Co-authored-by: Canlin Guo <canlinguosdu@gmail.com>
This commit is contained in:
1 parent
cb4d13a658
commit
8097747a5d
14 files changed
+804
-25
No files matched your search
@@ -24,6 +24,7 @@ from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin, _is_rank_zero
|
||||
from vllm_omni.diffusion.models.schedulers import FlowUniPCMultistepScheduler
|
||||
from vllm_omni.diffusion.models.wan2_2.scheduling_wan_euler import WanEulerScheduler
|
||||
from vllm_omni.diffusion.models.wan2_2.wan2_2_transformer import WanTransformer3DModel
|
||||
from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
@@ -32,6 +33,46 @@ from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
DEBUG_PERF = False
|
||||
WAN_SAMPLE_SOLVER_CHOICES = {"unipc", "euler"}
|
||||
|
||||
|
||||
def build_wan_scheduler(sample_solver: str, flow_shift: float) -> Any:
|
||||
if sample_solver == "unipc":
|
||||
return FlowUniPCMultistepScheduler(
|
||||
num_train_timesteps=1000,
|
||||
shift=flow_shift,
|
||||
prediction_type="flow_prediction",
|
||||
)
|
||||
if sample_solver == "euler":
|
||||
return WanEulerScheduler(
|
||||
num_train_timesteps=1000,
|
||||
shift=flow_shift,
|
||||
)
|
||||
|
||||
raise ValueError(
|
||||
f"Unsupported Wan sample_solver: {sample_solver}. Expected one of: {sorted(WAN_SAMPLE_SOLVER_CHOICES)}"
|
||||
)
|
||||
|
||||
|
||||
def resolve_wan_sample_solver(req: OmniDiffusionRequest, default: str = "unipc") -> str:
|
||||
extra_args = getattr(req.sampling_params, "extra_args", {}) or {}
|
||||
raw = extra_args.get("sample_solver", default)
|
||||
sample_solver = str(raw).strip().lower()
|
||||
if sample_solver not in WAN_SAMPLE_SOLVER_CHOICES:
|
||||
raise ValueError(f"Invalid sample_solver={raw!r}. Expected one of: {sorted(WAN_SAMPLE_SOLVER_CHOICES)}")
|
||||
return sample_solver
|
||||
|
||||
|
||||
def resolve_wan_flow_shift(req: OmniDiffusionRequest, od_config: OmniDiffusionConfig) -> float:
|
||||
extra_args = getattr(req.sampling_params, "extra_args", {}) or {}
|
||||
raw_flow_shift = extra_args.get("flow_shift")
|
||||
if raw_flow_shift is None:
|
||||
raw_flow_shift = od_config.flow_shift if od_config.flow_shift is not None else 5.0
|
||||
|
||||
try:
|
||||
return float(raw_flow_shift)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError(f"Invalid flow_shift={raw_flow_shift!r}. flow_shift must be a float.") from exc
|
||||
|
||||
|
||||
def retrieve_latents(
|
||||
@@ -296,13 +337,9 @@ class Wan22Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin, DiffusionPipe
|
||||
else:
|
||||
raise RuntimeError("No transformer loaded")
|
||||
|
||||
# Initialize UniPC scheduler
|
||||
flow_shift = od_config.flow_shift if od_config.flow_shift is not None else 5.0 # default for 720p
|
||||
self.scheduler = FlowUniPCMultistepScheduler(
|
||||
num_train_timesteps=1000,
|
||||
shift=flow_shift,
|
||||
prediction_type="flow_prediction",
|
||||
)
|
||||
self._sample_solver = "unipc"
|
||||
self._flow_shift = od_config.flow_shift if od_config.flow_shift is not None else 5.0
|
||||
self.scheduler = build_wan_scheduler(self._sample_solver, self._flow_shift)
|
||||
|
||||
self.vae_scale_factor_temporal = self.vae.config.scale_factor_temporal if getattr(self, "vae", None) else 4
|
||||
self.vae_scale_factor_spatial = self.vae.config.scale_factor_spatial if getattr(self, "vae", None) else 8
|
||||
@@ -462,6 +499,13 @@ class Wan22Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin, DiffusionPipe
|
||||
current_omni_platform.synchronize()
|
||||
_t_text_enc_ms = (time.perf_counter() - _t_text_enc_start) * 1000
|
||||
|
||||
sample_solver = resolve_wan_sample_solver(req, default=self._sample_solver)
|
||||
flow_shift = resolve_wan_flow_shift(req, self.od_config)
|
||||
if sample_solver != self._sample_solver or abs(flow_shift - self._flow_shift) > 1e-6:
|
||||
self.scheduler = build_wan_scheduler(sample_solver, flow_shift)
|
||||
self._sample_solver = sample_solver
|
||||
self._flow_shift = flow_shift
|
||||
|
||||
# Timesteps
|
||||
self.scheduler.set_timesteps(num_steps, device=device)
|
||||
timesteps = self.scheduler.timesteps
|
||||
|
||||
@@ -24,10 +24,12 @@ from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.models.interface import SupportImageInput
|
||||
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin, _is_rank_zero
|
||||
from vllm_omni.diffusion.models.schedulers import FlowUniPCMultistepScheduler
|
||||
from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import (
|
||||
build_wan_scheduler,
|
||||
create_transformer_from_config,
|
||||
load_transformer_config,
|
||||
resolve_wan_flow_shift,
|
||||
resolve_wan_sample_solver,
|
||||
retrieve_latents,
|
||||
)
|
||||
from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin
|
||||
@@ -230,13 +232,9 @@ class Wan22I2VPipeline(
|
||||
else:
|
||||
self.transformer_2 = None
|
||||
|
||||
# Initialize UniPC scheduler
|
||||
flow_shift = od_config.flow_shift if od_config.flow_shift is not None else 5.0 # default for 720p
|
||||
self.scheduler = FlowUniPCMultistepScheduler(
|
||||
num_train_timesteps=1000,
|
||||
shift=flow_shift,
|
||||
prediction_type="flow_prediction",
|
||||
)
|
||||
self._sample_solver = "unipc"
|
||||
self._flow_shift = od_config.flow_shift if od_config.flow_shift is not None else 5.0
|
||||
self.scheduler = build_wan_scheduler(self._sample_solver, self._flow_shift)
|
||||
|
||||
# VAE scale factors
|
||||
self.vae_scale_factor_temporal = self.vae.config.scale_factor_temporal if hasattr(self.vae, "config") else 4
|
||||
@@ -440,6 +438,13 @@ class Wan22I2VPipeline(
|
||||
current_omni_platform.synchronize()
|
||||
_t_img_enc_ms = (time.perf_counter() - _t_img_enc_start) * 1000
|
||||
|
||||
sample_solver = resolve_wan_sample_solver(req, default=self._sample_solver)
|
||||
flow_shift = resolve_wan_flow_shift(req, self.od_config)
|
||||
if sample_solver != self._sample_solver or abs(flow_shift - self._flow_shift) > 1e-6:
|
||||
self.scheduler = build_wan_scheduler(sample_solver, flow_shift)
|
||||
self._sample_solver = sample_solver
|
||||
self._flow_shift = flow_shift
|
||||
|
||||
# Timesteps
|
||||
self.scheduler.set_timesteps(num_steps, device=device)
|
||||
timesteps = self.scheduler.timesteps
|
||||
|
||||
@@ -36,10 +36,12 @@ from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.models.interface import SupportImageInput
|
||||
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin
|
||||
from vllm_omni.diffusion.models.schedulers import FlowUniPCMultistepScheduler
|
||||
from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import (
|
||||
build_wan_scheduler,
|
||||
create_transformer_from_config,
|
||||
load_transformer_config,
|
||||
resolve_wan_flow_shift,
|
||||
resolve_wan_sample_solver,
|
||||
retrieve_latents,
|
||||
)
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
@@ -183,13 +185,9 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin, Progress
|
||||
transformer_config = load_transformer_config(model, "transformer", local_files_only)
|
||||
self.transformer = create_transformer_from_config(transformer_config)
|
||||
|
||||
# Initialize UniPC scheduler
|
||||
flow_shift = od_config.flow_shift if od_config.flow_shift is not None else 5.0 # default for 720p
|
||||
self.scheduler = FlowUniPCMultistepScheduler(
|
||||
num_train_timesteps=1000,
|
||||
shift=flow_shift,
|
||||
prediction_type="flow_prediction",
|
||||
)
|
||||
self._sample_solver = "unipc"
|
||||
self._flow_shift = od_config.flow_shift if od_config.flow_shift is not None else 5.0
|
||||
self.scheduler = build_wan_scheduler(self._sample_solver, self._flow_shift)
|
||||
|
||||
# VAE scale factors
|
||||
self.vae_scale_factor_temporal = self.vae.config.scale_factor_temporal if hasattr(self.vae, "config") else 4
|
||||
@@ -323,6 +321,13 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin, Progress
|
||||
|
||||
batch_size = prompt_embeds.shape[0]
|
||||
|
||||
sample_solver = resolve_wan_sample_solver(req, default=self._sample_solver)
|
||||
flow_shift = resolve_wan_flow_shift(req, self.od_config)
|
||||
if sample_solver != self._sample_solver or abs(flow_shift - self._flow_shift) > 1e-6:
|
||||
self.scheduler = build_wan_scheduler(sample_solver, flow_shift)
|
||||
self._sample_solver = sample_solver
|
||||
self._flow_shift = flow_shift
|
||||
|
||||
# Timesteps
|
||||
self.scheduler.set_timesteps(num_steps, device=device)
|
||||
timesteps = self.scheduler.timesteps
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
|
||||
@dataclass
|
||||
class WanEulerSchedulerOutput:
|
||||
prev_sample: torch.FloatTensor
|
||||
|
||||
|
||||
def _unsqueeze_to_ndim(in_tensor: torch.Tensor, target_ndim: int) -> torch.Tensor:
|
||||
if in_tensor.ndim >= target_ndim:
|
||||
return in_tensor
|
||||
return in_tensor[(...,) + (None,) * (target_ndim - in_tensor.ndim)]
|
||||
|
||||
|
||||
def _get_timesteps(num_steps: int, max_steps: int = 1000) -> np.ndarray:
|
||||
# Keep num_steps + 1 points so Euler update can always access sigma_next.
|
||||
return np.linspace(max_steps, 0, num_steps + 1, dtype=np.float32)
|
||||
|
||||
|
||||
def _timestep_shift(timesteps: torch.Tensor, shift: float = 1.0) -> torch.Tensor:
|
||||
return shift * timesteps / (1 + (shift - 1) * timesteps)
|
||||
|
||||
|
||||
class WanEulerScheduler:
|
||||
order = 1
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_train_timesteps: int = 1000,
|
||||
shift: float = 1.0,
|
||||
device: torch.device | str = "cpu",
|
||||
) -> None:
|
||||
self.num_train_timesteps = int(num_train_timesteps)
|
||||
self._shift = float(shift)
|
||||
self.device = device
|
||||
self.config = SimpleNamespace(num_train_timesteps=self.num_train_timesteps)
|
||||
self.init_noise_sigma = 1.0
|
||||
|
||||
self._step_index: int | None = None
|
||||
self._begin_index: int | None = None
|
||||
|
||||
self.timesteps = torch.empty(0, dtype=torch.float32)
|
||||
self.sigmas = torch.empty(0, dtype=torch.float32)
|
||||
self.timesteps_ori = torch.empty(0, dtype=torch.float32)
|
||||
|
||||
self.set_timesteps(num_inference_steps=self.num_train_timesteps, device=self.device)
|
||||
|
||||
@property
|
||||
def step_index(self) -> int | None:
|
||||
return self._step_index
|
||||
|
||||
@property
|
||||
def begin_index(self) -> int | None:
|
||||
return self._begin_index
|
||||
|
||||
def set_begin_index(self, begin_index: int = 0) -> None:
|
||||
self._begin_index = int(begin_index)
|
||||
|
||||
def index_for_timestep(self, timestep: torch.Tensor) -> int:
|
||||
indices = (self.timesteps == timestep).nonzero()
|
||||
if len(indices) > 0:
|
||||
pos = 1 if len(indices) > 1 else 0
|
||||
return int(indices[pos].item())
|
||||
# Fallback for tiny float drift
|
||||
return int(torch.argmin(torch.abs(self.timesteps - timestep)).item())
|
||||
|
||||
def _init_step_index(self, timestep: float | torch.Tensor) -> None:
|
||||
if self.begin_index is None:
|
||||
if isinstance(timestep, torch.Tensor):
|
||||
timestep_t = timestep.to(self.timesteps.device, dtype=self.timesteps.dtype)
|
||||
else:
|
||||
timestep_t = torch.tensor(timestep, device=self.timesteps.device, dtype=self.timesteps.dtype)
|
||||
self._step_index = self.index_for_timestep(timestep_t)
|
||||
else:
|
||||
self._step_index = self._begin_index
|
||||
|
||||
def set_shift(self, shift: float = 1.0) -> None:
|
||||
# Compute shifted sigma schedule on [0, 1].
|
||||
sigmas_full = self.timesteps_ori / float(self.num_train_timesteps)
|
||||
sigmas_full = _timestep_shift(sigmas_full, shift=float(shift))
|
||||
self.sigmas = sigmas_full
|
||||
# Public timesteps are the first N points; next point is consumed as sigma_next.
|
||||
self.timesteps = self.sigmas[:-1] * self.num_train_timesteps
|
||||
self._shift = float(shift)
|
||||
|
||||
def set_timesteps(
|
||||
self,
|
||||
num_inference_steps: int,
|
||||
device: torch.device | str | int | None = None,
|
||||
**kwargs, # noqa: ARG002 - kept for scheduler API compatibility
|
||||
) -> None:
|
||||
timesteps = _get_timesteps(
|
||||
num_steps=int(num_inference_steps),
|
||||
max_steps=self.num_train_timesteps,
|
||||
)
|
||||
self.timesteps_ori = torch.from_numpy(timesteps).to(
|
||||
dtype=torch.float32,
|
||||
device=device or self.device,
|
||||
)
|
||||
self.set_shift(self._shift)
|
||||
self._step_index = None
|
||||
self._begin_index = None
|
||||
|
||||
def scale_model_input(self, sample: torch.Tensor, timestep: int | None = None) -> torch.Tensor: # noqa: ARG002
|
||||
return sample
|
||||
|
||||
def step(
|
||||
self,
|
||||
model_output: torch.FloatTensor,
|
||||
timestep: float | torch.FloatTensor,
|
||||
sample: torch.FloatTensor,
|
||||
return_dict: bool = True,
|
||||
**kwargs, # noqa: ARG002 - kept for scheduler API compatibility
|
||||
) -> WanEulerSchedulerOutput | tuple[torch.FloatTensor]:
|
||||
if isinstance(timestep, (int, torch.IntTensor, torch.LongTensor)):
|
||||
raise ValueError(
|
||||
"Passing integer indices as timesteps is not supported. Use one value from scheduler.timesteps instead."
|
||||
)
|
||||
|
||||
if self.step_index is None:
|
||||
self._init_step_index(timestep)
|
||||
assert self._step_index is not None
|
||||
|
||||
sample_fp32 = sample.to(torch.float32)
|
||||
sigma = _unsqueeze_to_ndim(self.sigmas[self._step_index], sample_fp32.ndim).to(sample_fp32.device)
|
||||
sigma_next = _unsqueeze_to_ndim(self.sigmas[self._step_index + 1], sample_fp32.ndim).to(sample_fp32.device)
|
||||
|
||||
prev_sample = sample_fp32 + (sigma_next - sigma) * model_output
|
||||
prev_sample = prev_sample.to(model_output.dtype)
|
||||
|
||||
self._step_index += 1
|
||||
|
||||
if not return_dict:
|
||||
return (prev_sample,)
|
||||
return WanEulerSchedulerOutput(prev_sample=prev_sample)
|
||||
|
||||
def __len__(self) -> int:
|
||||
return self.num_train_timesteps
|
||||
@@ -1015,6 +1015,14 @@ class WanTransformer3DModel(nn.Module):
|
||||
if ".to_out.0." in lookup_name:
|
||||
lookup_name = lookup_name.replace(".to_out.0.", ".to_out.")
|
||||
|
||||
# Compatibility: some Wan conversion pipelines still keep
|
||||
# block modulation keys as `blocks.N.modulation` instead of
|
||||
# `blocks.N.scale_shift_table`.
|
||||
if lookup_name.endswith(".modulation"):
|
||||
modulation_alias = lookup_name[: -len(".modulation")] + ".scale_shift_table"
|
||||
if modulation_alias in params_dict:
|
||||
lookup_name = modulation_alias
|
||||
|
||||
if lookup_name not in params_dict:
|
||||
logger.warning(f"Skipping weight {original_name} -> {lookup_name}")
|
||||
continue
|
||||
|
||||
Reference in new issue
Block a user