[Refactor] refactor wan2.2 diffuse && add ut (#2672)

Signed-off-by: bjf-frz <frz123db@gmail.com>
This commit is contained in:
bjf-frz authored and GitHub committed 2026-04-17 20:13:49 +08:00
1 parent 6b7be88cd5
commit 18ac679763
10 files changed
+1028 -246

No files matched your search

+80
View File
@@ -0,0 +1,80 @@
from __future__ import annotations
from contextlib import contextmanager
from types import SimpleNamespace
import torch
from torch import nn
class StubTransformer(nn.Module):
def __init__(self, *, name: str = "transformer", in_channels: int = 4, out_channels: int = 4) -> None:
super().__init__()
self.name = name
self.config = SimpleNamespace(
patch_size=(1, 2, 2),
in_channels=in_channels,
out_channels=out_channels,
image_dim=None,
)
@property
def dtype(self) -> torch.dtype:
return torch.float32
def forward(self, **kwargs):
hidden_states = kwargs["hidden_states"]
return (torch.zeros_like(hidden_states[:, : self.config.out_channels]),)
class StubScheduler:
def __init__(self, timesteps: list[int]) -> None:
self.timesteps = torch.tensor(timesteps, dtype=torch.int64)
self.config = SimpleNamespace(num_train_timesteps=1000)
self.set_timesteps_calls: list[tuple[int, torch.device]] = []
def set_timesteps(self, num_steps: int, device: torch.device) -> None:
self.set_timesteps_calls.append((num_steps, device))
class StubVAE:
dtype = torch.float32
def __init__(self, z_dim: int = 4) -> None:
self.config = SimpleNamespace(
z_dim=z_dim,
scale_factor_temporal=4,
scale_factor_spatial=8,
latents_mean=[0.0] * z_dim,
latents_std=[1.0] * z_dim,
)
def encode(self, video: torch.Tensor):
latent_frames = (video.shape[2] + self.config.scale_factor_temporal - 1) // self.config.scale_factor_temporal
latent_height = video.shape[-2] // self.config.scale_factor_spatial
latent_width = video.shape[-1] // self.config.scale_factor_spatial
latents = torch.ones(
video.shape[0],
self.config.z_dim,
latent_frames,
latent_height,
latent_width,
dtype=video.dtype,
device=video.device,
)
return SimpleNamespace(latents=latents)
def decode(self, latents: torch.Tensor, return_dict: bool = False):
del return_dict
return (latents,)
@contextmanager
def noop_progress_bar(*args, **kwargs):
del args, kwargs
class Bar:
def update(self) -> None:
return None
yield Bar()
@@ -0,0 +1,126 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from PIL import Image
from torch import nn
from tests.diffusion.models.wan2_2.conftest import StubTransformer, StubVAE, noop_progress_bar
from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2_i2v import (
Wan22I2VPipeline,
get_wan22_i2v_pre_process_func,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
def _make_i2v_pipeline(*, expand_timesteps: bool) -> Wan22I2VPipeline:
pipeline = object.__new__(Wan22I2VPipeline)
nn.Module.__init__(pipeline)
pipeline.device = torch.device("cpu")
pipeline.transformer = StubTransformer(name="high", in_channels=8, out_channels=4)
pipeline.transformer_2 = StubTransformer(name="low", in_channels=8, out_channels=4)
pipeline.vae = StubVAE(z_dim=4)
pipeline.vae_scale_factor_temporal = 4
pipeline.vae_scale_factor_spatial = 8
pipeline.expand_timesteps = expand_timesteps
pipeline.progress_bar = noop_progress_bar
return pipeline
def test_i2v_preprocess_requires_image_and_resizes_to_480p_aspect() -> None:
preprocess = get_wan22_i2v_pre_process_func(SimpleNamespace())
request = SimpleNamespace(
prompts=[{"prompt": "p", "multi_modal_data": {"image": Image.new("RGB", (320, 160), "red")}}],
sampling_params=SimpleNamespace(height=None, width=None),
)
result = preprocess(request)
prompt = result.prompts[0]
assert result.sampling_params.height == 432
assert result.sampling_params.width == 880
assert prompt["multi_modal_data"]["image"].size == (880, 432)
assert prompt["additional_information"]["preprocessed_image"].shape[-2:] == (432, 880)
missing_image = SimpleNamespace(
prompts=[{"prompt": "p", "multi_modal_data": {}}],
sampling_params=SimpleNamespace(height=None, width=None),
)
with pytest.raises(ValueError, match="No image is provided"):
preprocess(missing_image)
def test_i2v_diffuse_selects_stage_guidance_and_expands_timesteps() -> None:
pipeline = _make_i2v_pipeline(expand_timesteps=True)
latents = torch.zeros(1, 4, 2, 4, 4)
condition = torch.ones_like(latents)
first_frame_mask = torch.ones(1, 1, 2, 4, 4)
first_frame_mask[:, :, 0] = 0
timesteps = torch.tensor([900, 100])
calls = []
def fake_predict_noise_maybe_with_cfg(**kwargs):
positive = kwargs["positive_kwargs"]
calls.append(
{
"model": positive["current_model"].name,
"scale": kwargs["true_cfg_scale"],
"timestep_shape": tuple(positive["timestep"].shape),
"timestep_values": positive["timestep"].clone(),
"hidden_states": positive["hidden_states"].clone(),
}
)
return torch.ones_like(latents)
pipeline.predict_noise_maybe_with_cfg = fake_predict_noise_maybe_with_cfg # type: ignore[method-assign]
pipeline.scheduler_step_maybe_with_cfg = lambda noise, t, current, cfg: current + noise # type: ignore[method-assign]
result = pipeline.diffuse(
latents=latents,
timesteps=timesteps,
prompt_embeds=torch.zeros(1, 2, 3),
negative_prompt_embeds=None,
image_embeds=None,
guidance_low=1.0,
guidance_high=2.0,
boundary_timestep=500.0,
dtype=torch.float32,
attention_kwargs={},
condition=condition,
first_frame_mask=first_frame_mask,
)
assert [call["model"] for call in calls] == ["high", "low"]
assert [call["scale"] for call in calls] == [1.0, 2.0]
assert calls[0]["timestep_shape"] == (1, 8)
timestep_dtype = calls[0]["timestep_values"].dtype
torch.testing.assert_close(calls[0]["timestep_values"][0, :4], torch.zeros(4, dtype=timestep_dtype))
torch.testing.assert_close(calls[0]["timestep_values"][0, 4:], torch.full((4,), 900, dtype=timestep_dtype))
torch.testing.assert_close(calls[0]["hidden_states"][:, :, 0], torch.ones(1, 4, 4, 4))
torch.testing.assert_close(result, torch.full_like(latents, 2.0))
def test_i2v_prepare_latents_builds_expand_condition_and_first_frame_mask() -> None:
pipeline = _make_i2v_pipeline(expand_timesteps=True)
latents, condition, first_frame_mask = pipeline.prepare_latents(
image=torch.zeros(1, 3, 16, 16),
batch_size=1,
num_channels_latents=4,
height=16,
width=16,
num_frames=5,
dtype=torch.float32,
device=torch.device("cpu"),
generator=torch.Generator(device="cpu").manual_seed(0),
)
assert latents.shape == (1, 4, 2, 2, 2)
assert condition.shape == (1, 4, 1, 2, 2)
assert first_frame_mask.shape == (1, 1, 2, 2, 2)
assert first_frame_mask[:, :, 0].sum() == 0
assert first_frame_mask[:, :, 1].sum() == 4
@@ -0,0 +1,155 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from contextlib import contextmanager
from types import SimpleNamespace
import pytest
import torch
from torch import nn
from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import Wan22Pipeline
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
class _StubTransformer(nn.Module):
@property
def dtype(self) -> torch.dtype:
return torch.float32
class _StubScheduler:
def __init__(self, timesteps: list[int]) -> None:
self.timesteps = torch.tensor(timesteps, dtype=torch.int64)
self.config = SimpleNamespace(num_train_timesteps=1000)
self.set_timesteps_calls: list[tuple[int, torch.device]] = []
def set_timesteps(self, num_steps: int, device: torch.device) -> None:
self.set_timesteps_calls.append((num_steps, device))
@contextmanager
def _noop_progress_bar(*args, **kwargs):
del args, kwargs
class _Bar:
def update(self) -> None:
return None
yield _Bar()
def _make_pipeline() -> Wan22Pipeline:
pipeline = object.__new__(Wan22Pipeline)
nn.Module.__init__(pipeline)
pipeline.device = torch.device("cpu")
pipeline.transformer = _StubTransformer()
pipeline.transformer_2 = None
pipeline.transformer_config = SimpleNamespace(patch_size=(1, 2, 2), in_channels=4, out_channels=4)
pipeline.scheduler = _StubScheduler([9, 5])
pipeline.od_config = SimpleNamespace(flow_shift=5.0)
pipeline._sample_solver = "unipc"
pipeline._flow_shift = 5.0
pipeline.vae_scale_factor_temporal = 4
pipeline.vae_scale_factor_spatial = 8
pipeline.boundary_ratio = 0.875
pipeline.expand_timesteps = False
pipeline._guidance_scale = None
pipeline._guidance_scale_2 = None
pipeline._num_timesteps = None
pipeline._current_timestep = None
pipeline.check_inputs = lambda **kwargs: None
pipeline.prepare_latents = lambda **kwargs: torch.zeros((1, 4, 1, 8, 8), dtype=torch.float32)
pipeline.progress_bar = _noop_progress_bar
return pipeline
def test_forward_delegates_denoising_to_diffuse(monkeypatch) -> None:
pipeline = _make_pipeline()
prompt_embeds = torch.randn(1, 8)
captured: dict[str, object] = {}
def _fake_diffuse(**kwargs):
captured.update(kwargs)
return kwargs["latents"] + 1
pipeline.diffuse = _fake_diffuse # type: ignore[method-assign]
req = SimpleNamespace(
prompts=["prompt"],
sampling_params=SimpleNamespace(
height=None,
width=None,
num_frames=1,
num_inference_steps=2,
guidance_scale_provided=False,
guidance_scale=None,
guidance_scale_2=None,
boundary_ratio=None,
generator=None,
seed=None,
num_outputs_per_prompt=1,
max_sequence_length=32,
latents=None,
extra_args={},
),
)
output = pipeline.forward(req, prompt_embeds=prompt_embeds, output_type="latent", guidance_scale=1.0)
assert torch.equal(output.output, torch.ones((1, 4, 1, 8, 8)))
assert torch.equal(captured["timesteps"], pipeline.scheduler.timesteps)
assert captured["guidance_low"] == 1.0
assert captured["guidance_high"] == 1.0
assert captured["boundary_timestep"] == pytest.approx(875.0)
assert captured["latent_condition"] is None
assert captured["first_frame_mask"] is None
assert pipeline.scheduler.set_timesteps_calls == [(2, torch.device("cpu"))]
def test_diffuse_runs_prediction_and_scheduler_for_each_timestep() -> None:
pipeline = _make_pipeline()
latents = torch.zeros((1, 1, 1, 2, 2), dtype=torch.float32)
timesteps = torch.tensor([7, 3], dtype=torch.int64)
prompt_embeds = torch.randn(1, 8)
predict_calls: list[dict[str, object]] = []
scheduler_calls: list[tuple[float, int, float, bool]] = []
def _fake_predict_noise_maybe_with_cfg(**kwargs):
predict_calls.append(kwargs)
timestep = kwargs["positive_kwargs"]["timestep"]
assert isinstance(timestep, torch.Tensor)
return torch.full_like(latents, float(timestep[0].item()))
def _fake_scheduler_step_maybe_with_cfg(noise_pred, t, current_latents, do_true_cfg):
scheduler_calls.append(
(float(noise_pred[0, 0, 0, 0, 0]), int(t.item()), float(current_latents.sum()), do_true_cfg)
)
return current_latents + noise_pred
pipeline.predict_noise_maybe_with_cfg = _fake_predict_noise_maybe_with_cfg # type: ignore[method-assign]
pipeline.scheduler_step_maybe_with_cfg = _fake_scheduler_step_maybe_with_cfg # type: ignore[method-assign]
result = pipeline.diffuse(
latents=latents,
timesteps=timesteps,
prompt_embeds=prompt_embeds,
negative_prompt_embeds=None,
guidance_low=1.0,
guidance_high=2.0,
boundary_timestep=5.0,
dtype=torch.float32,
attention_kwargs={},
)
assert len(predict_calls) == 2
assert predict_calls[0]["true_cfg_scale"] == 1.0
assert predict_calls[1]["true_cfg_scale"] == 2.0
assert scheduler_calls == [
(7.0, 7, 0.0, False),
(3.0, 3, 28.0, False),
]
assert torch.equal(result, torch.full_like(latents, 10.0))
@@ -0,0 +1,81 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from types import SimpleNamespace
import pytest
import torch
import vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 as wan22_module
from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import (
create_transformer_from_config,
load_transformer_config,
retrieve_latents,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
class _LatentDist:
def sample(self, generator):
assert isinstance(generator, torch.Generator)
return torch.tensor([1.0])
def mode(self):
return torch.tensor([2.0])
def test_retrieve_latents_supports_sample_mode_argmax_and_direct_latents() -> None:
generator = torch.Generator(device="cpu")
assert retrieve_latents(SimpleNamespace(latent_dist=_LatentDist()), generator).item() == 1.0
assert retrieve_latents(SimpleNamespace(latent_dist=_LatentDist()), sample_mode="argmax").item() == 2.0
torch.testing.assert_close(retrieve_latents(SimpleNamespace(latents=torch.tensor([3.0]))), torch.tensor([3.0]))
def test_retrieve_latents_rejects_unknown_encoder_output() -> None:
with pytest.raises(AttributeError, match="Could not access latents"):
retrieve_latents(SimpleNamespace())
def test_load_transformer_config_reads_local_subfolder_config(tmp_path) -> None:
config_dir = tmp_path / "transformer_2"
config_dir.mkdir(parents=True)
(config_dir / "config.json").write_text(json.dumps({"patch_size": [1, 2, 2], "num_layers": 2}))
assert load_transformer_config(str(tmp_path), "transformer_2") == {"patch_size": [1, 2, 2], "num_layers": 2}
assert load_transformer_config(str(tmp_path), "missing") == {}
def test_create_transformer_from_config_maps_supported_keys(monkeypatch) -> None:
captured = {}
class FakeTransformer:
def __init__(self, **kwargs) -> None:
captured.update(kwargs)
monkeypatch.setattr(wan22_module, "WanTransformer3DModel", FakeTransformer)
transformer = create_transformer_from_config(
{
"patch_size": [1, 2, 2],
"num_attention_heads": 8,
"attention_head_dim": 128,
"in_channels": 16,
"out_channels": 16,
"text_dim": 4096,
"vace_layers": [0],
"ignored": "value",
}
)
assert isinstance(transformer, FakeTransformer)
assert captured == {
"patch_size": (1, 2, 2),
"num_attention_heads": 8,
"attention_head_dim": 128,
"in_channels": 16,
"out_channels": 16,
"text_dim": 4096,
}
@@ -0,0 +1,98 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from PIL import Image
from torch import nn
from tests.diffusion.models.wan2_2.conftest import StubTransformer, StubVAE, noop_progress_bar
from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2_ti2v import (
Wan22TI2VPipeline,
get_wan22_ti2v_pre_process_func,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
def _make_ti2v_pipeline() -> Wan22TI2VPipeline:
pipeline = object.__new__(Wan22TI2VPipeline)
nn.Module.__init__(pipeline)
pipeline.device = torch.device("cpu")
pipeline.transformer = StubTransformer(in_channels=4, out_channels=4)
pipeline.vae = StubVAE(z_dim=4)
pipeline.vae_scale_factor_temporal = 4
pipeline.vae_scale_factor_spatial = 8
pipeline.progress_bar = noop_progress_bar
return pipeline
def test_ti2v_preprocess_uses_720p_area_for_image_condition() -> None:
preprocess = get_wan22_ti2v_pre_process_func(SimpleNamespace())
request = SimpleNamespace(
prompts=[{"prompt": "p", "multi_modal_data": {"image": Image.new("RGB", (320, 160), "blue")}}],
sampling_params=SimpleNamespace(height=None, width=None),
)
result = preprocess(request)
assert result.sampling_params.height == 672
assert result.sampling_params.width == 1344
assert result.prompts[0]["multi_modal_data"]["image"].size == (1344, 672)
assert result.prompts[0]["additional_information"]["preprocessed_image"].shape[-2:] == (672, 1344)
def test_ti2v_diffuse_without_image_condition_expands_patch_timesteps() -> None:
pipeline = _make_ti2v_pipeline()
latents = torch.zeros(1, 4, 2, 4, 4)
calls = []
def fake_predict_noise_maybe_with_cfg(**kwargs):
calls.append(kwargs)
return torch.ones_like(latents)
pipeline.predict_noise_maybe_with_cfg = fake_predict_noise_maybe_with_cfg # type: ignore[method-assign]
pipeline.scheduler_step_maybe_with_cfg = lambda noise, t, current, cfg: current + noise # type: ignore[method-assign]
result = pipeline.diffuse(
latents=latents,
timesteps=torch.tensor([7]),
prompt_embeds=torch.zeros(1, 2, 3),
negative_prompt_embeds=torch.zeros(1, 2, 3),
guidance_scale=3.0,
dtype=torch.float32,
attention_kwargs={"a": "b"},
num_latent_frames=2,
latent_height=4,
latent_width=4,
)
positive = calls[0]["positive_kwargs"]
assert calls[0]["do_true_cfg"] is True
assert positive["timestep"].shape == (1, 8)
torch.testing.assert_close(positive["timestep"], torch.full((1, 8), 7, dtype=positive["timestep"].dtype))
torch.testing.assert_close(positive["hidden_states"], latents)
torch.testing.assert_close(result, torch.ones_like(latents))
def test_ti2v_prepare_i2v_latents_encodes_condition_and_masks_first_frame() -> None:
pipeline = _make_ti2v_pipeline()
latents, latent_condition, first_frame_mask = pipeline.prepare_i2v_latents(
image=torch.zeros(1, 3, 16, 16),
batch_size=1,
num_channels_latents=4,
height=16,
width=16,
num_frames=5,
dtype=torch.float32,
device=torch.device("cpu"),
generator=None,
latents=torch.zeros(1, 4, 2, 2, 2),
)
torch.testing.assert_close(latents, torch.zeros(1, 4, 2, 2, 2))
assert latent_condition.shape == (1, 4, 1, 2, 2)
assert first_frame_mask[:, :, 0].sum() == 0
assert first_frame_mask[:, :, 1].sum() == 4
@@ -0,0 +1,137 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from PIL import Image
from torch import nn
from tests.diffusion.models.wan2_2.conftest import StubTransformer, StubVAE, noop_progress_bar
from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2_vace import (
Wan22VACEPipeline,
create_vace_transformer_from_config,
get_wan22_vace_pre_process_func,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
def _make_vace_pipeline() -> Wan22VACEPipeline:
pipeline = object.__new__(Wan22VACEPipeline)
nn.Module.__init__(pipeline)
pipeline.device = torch.device("cpu")
pipeline.transformer = StubTransformer(in_channels=4, out_channels=4)
pipeline.transformer_config = pipeline.transformer.config
pipeline.vae = StubVAE(z_dim=4)
pipeline.vae_scale_factor_temporal = 4
pipeline.vae_scale_factor_spatial = 8
pipeline.progress_bar = noop_progress_bar
return pipeline
def test_vace_preprocess_collects_reference_video_and_mask_inputs() -> None:
preprocess = get_wan22_vace_pre_process_func(SimpleNamespace())
ref = Image.new("RGB", (320, 160), "green")
frame = Image.new("RGB", (64, 64), "black")
mask = Image.new("L", (64, 64), 255)
request = SimpleNamespace(
prompts=[
{
"prompt": "p",
"multi_modal_data": {
"image": ref,
"video": [frame],
"mask": mask,
},
}
],
sampling_params=SimpleNamespace(height=None, width=None),
)
result = preprocess(request)
additional_info = result.prompts[0]["additional_information"]
assert result.sampling_params.height == 432
assert result.sampling_params.width == 880
assert additional_info["reference_images"] == [ref]
assert additional_info["source_video"] == [frame]
assert additional_info["mask"] == [mask]
def test_create_vace_transformer_from_config_maps_vace_specific_keys(monkeypatch) -> None:
captured = {}
class FakeVACETransformer:
def __init__(self, **kwargs) -> None:
captured.update(kwargs)
monkeypatch.setattr(
"vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2_vace.WanVACETransformer3DModel",
FakeVACETransformer,
)
transformer = create_vace_transformer_from_config(
{
"patch_size": [1, 2, 2],
"in_channels": 96,
"out_channels": 16,
"vace_layers": [0, 1, 2],
"vace_in_channels": 132,
"unknown": "ignored",
}
)
assert isinstance(transformer, FakeVACETransformer)
assert captured == {
"patch_size": (1, 2, 2),
"in_channels": 96,
"out_channels": 16,
"vace_layers": [0, 1, 2],
"vace_in_channels": 132,
}
def test_vace_prepare_masks_encodes_spatial_stride_and_reference_padding() -> None:
pipeline = _make_vace_pipeline()
mask = torch.ones(1, 3, 5, 16, 16)
reference_images = [[torch.zeros(3, 16, 16), torch.zeros(3, 16, 16)]]
encoded = pipeline.prepare_masks(mask, reference_images)
assert encoded.shape == (1, 64, 4, 2, 2)
torch.testing.assert_close(encoded[:, :, :2], torch.zeros(1, 64, 2, 2, 2))
torch.testing.assert_close(encoded[:, :, 2:], torch.ones(1, 64, 2, 2, 2))
def test_vace_diffuse_passes_context_and_scale_to_cfg_branches() -> None:
pipeline = _make_vace_pipeline()
latents = torch.zeros(1, 4, 1, 2, 2)
vace_context = torch.ones(1, 12, 1, 2, 2)
calls = []
def fake_predict_noise_maybe_with_cfg(**kwargs):
calls.append(kwargs)
return torch.ones_like(latents)
pipeline.predict_noise_maybe_with_cfg = fake_predict_noise_maybe_with_cfg # type: ignore[method-assign]
pipeline.scheduler_step_maybe_with_cfg = lambda noise, t, current, cfg: current + noise # type: ignore[method-assign]
result = pipeline.diffuse(
latents=latents,
timesteps=torch.tensor([5]),
prompt_embeds=torch.zeros(1, 2, 3),
negative_prompt_embeds=torch.zeros(1, 2, 3),
guidance_scale=4.0,
dtype=torch.float32,
attention_kwargs={},
vace_context=vace_context,
vace_context_scale=0.75,
)
assert calls[0]["do_true_cfg"] is True
assert calls[0]["true_cfg_scale"] == 4.0
assert calls[0]["positive_kwargs"]["vace_context"] is vace_context
assert calls[0]["negative_kwargs"]["vace_context_scale"] == 0.75
torch.testing.assert_close(result, torch.ones_like(latents))