[Refactor] refactor wan2.2 diffuse && add ut (#2672)
Signed-off-by: bjf-frz <frz123db@gmail.com>
This commit is contained in:
1 parent
6b7be88cd5
commit
18ac679763
10 files changed
+1028
-246
No files matched your search
@@ -0,0 +1,80 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import contextmanager
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
|
||||
class StubTransformer(nn.Module):
|
||||
def __init__(self, *, name: str = "transformer", in_channels: int = 4, out_channels: int = 4) -> None:
|
||||
super().__init__()
|
||||
self.name = name
|
||||
self.config = SimpleNamespace(
|
||||
patch_size=(1, 2, 2),
|
||||
in_channels=in_channels,
|
||||
out_channels=out_channels,
|
||||
image_dim=None,
|
||||
)
|
||||
|
||||
@property
|
||||
def dtype(self) -> torch.dtype:
|
||||
return torch.float32
|
||||
|
||||
def forward(self, **kwargs):
|
||||
hidden_states = kwargs["hidden_states"]
|
||||
return (torch.zeros_like(hidden_states[:, : self.config.out_channels]),)
|
||||
|
||||
|
||||
class StubScheduler:
|
||||
def __init__(self, timesteps: list[int]) -> None:
|
||||
self.timesteps = torch.tensor(timesteps, dtype=torch.int64)
|
||||
self.config = SimpleNamespace(num_train_timesteps=1000)
|
||||
self.set_timesteps_calls: list[tuple[int, torch.device]] = []
|
||||
|
||||
def set_timesteps(self, num_steps: int, device: torch.device) -> None:
|
||||
self.set_timesteps_calls.append((num_steps, device))
|
||||
|
||||
|
||||
class StubVAE:
|
||||
dtype = torch.float32
|
||||
|
||||
def __init__(self, z_dim: int = 4) -> None:
|
||||
self.config = SimpleNamespace(
|
||||
z_dim=z_dim,
|
||||
scale_factor_temporal=4,
|
||||
scale_factor_spatial=8,
|
||||
latents_mean=[0.0] * z_dim,
|
||||
latents_std=[1.0] * z_dim,
|
||||
)
|
||||
|
||||
def encode(self, video: torch.Tensor):
|
||||
latent_frames = (video.shape[2] + self.config.scale_factor_temporal - 1) // self.config.scale_factor_temporal
|
||||
latent_height = video.shape[-2] // self.config.scale_factor_spatial
|
||||
latent_width = video.shape[-1] // self.config.scale_factor_spatial
|
||||
latents = torch.ones(
|
||||
video.shape[0],
|
||||
self.config.z_dim,
|
||||
latent_frames,
|
||||
latent_height,
|
||||
latent_width,
|
||||
dtype=video.dtype,
|
||||
device=video.device,
|
||||
)
|
||||
return SimpleNamespace(latents=latents)
|
||||
|
||||
def decode(self, latents: torch.Tensor, return_dict: bool = False):
|
||||
del return_dict
|
||||
return (latents,)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def noop_progress_bar(*args, **kwargs):
|
||||
del args, kwargs
|
||||
|
||||
class Bar:
|
||||
def update(self) -> None:
|
||||
return None
|
||||
|
||||
yield Bar()
|
||||
@@ -0,0 +1,126 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
from torch import nn
|
||||
|
||||
from tests.diffusion.models.wan2_2.conftest import StubTransformer, StubVAE, noop_progress_bar
|
||||
from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2_i2v import (
|
||||
Wan22I2VPipeline,
|
||||
get_wan22_i2v_pre_process_func,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
|
||||
def _make_i2v_pipeline(*, expand_timesteps: bool) -> Wan22I2VPipeline:
|
||||
pipeline = object.__new__(Wan22I2VPipeline)
|
||||
nn.Module.__init__(pipeline)
|
||||
pipeline.device = torch.device("cpu")
|
||||
pipeline.transformer = StubTransformer(name="high", in_channels=8, out_channels=4)
|
||||
pipeline.transformer_2 = StubTransformer(name="low", in_channels=8, out_channels=4)
|
||||
pipeline.vae = StubVAE(z_dim=4)
|
||||
pipeline.vae_scale_factor_temporal = 4
|
||||
pipeline.vae_scale_factor_spatial = 8
|
||||
pipeline.expand_timesteps = expand_timesteps
|
||||
pipeline.progress_bar = noop_progress_bar
|
||||
return pipeline
|
||||
|
||||
|
||||
def test_i2v_preprocess_requires_image_and_resizes_to_480p_aspect() -> None:
|
||||
preprocess = get_wan22_i2v_pre_process_func(SimpleNamespace())
|
||||
request = SimpleNamespace(
|
||||
prompts=[{"prompt": "p", "multi_modal_data": {"image": Image.new("RGB", (320, 160), "red")}}],
|
||||
sampling_params=SimpleNamespace(height=None, width=None),
|
||||
)
|
||||
|
||||
result = preprocess(request)
|
||||
prompt = result.prompts[0]
|
||||
|
||||
assert result.sampling_params.height == 432
|
||||
assert result.sampling_params.width == 880
|
||||
assert prompt["multi_modal_data"]["image"].size == (880, 432)
|
||||
assert prompt["additional_information"]["preprocessed_image"].shape[-2:] == (432, 880)
|
||||
|
||||
missing_image = SimpleNamespace(
|
||||
prompts=[{"prompt": "p", "multi_modal_data": {}}],
|
||||
sampling_params=SimpleNamespace(height=None, width=None),
|
||||
)
|
||||
with pytest.raises(ValueError, match="No image is provided"):
|
||||
preprocess(missing_image)
|
||||
|
||||
|
||||
def test_i2v_diffuse_selects_stage_guidance_and_expands_timesteps() -> None:
|
||||
pipeline = _make_i2v_pipeline(expand_timesteps=True)
|
||||
latents = torch.zeros(1, 4, 2, 4, 4)
|
||||
condition = torch.ones_like(latents)
|
||||
first_frame_mask = torch.ones(1, 1, 2, 4, 4)
|
||||
first_frame_mask[:, :, 0] = 0
|
||||
timesteps = torch.tensor([900, 100])
|
||||
|
||||
calls = []
|
||||
|
||||
def fake_predict_noise_maybe_with_cfg(**kwargs):
|
||||
positive = kwargs["positive_kwargs"]
|
||||
calls.append(
|
||||
{
|
||||
"model": positive["current_model"].name,
|
||||
"scale": kwargs["true_cfg_scale"],
|
||||
"timestep_shape": tuple(positive["timestep"].shape),
|
||||
"timestep_values": positive["timestep"].clone(),
|
||||
"hidden_states": positive["hidden_states"].clone(),
|
||||
}
|
||||
)
|
||||
return torch.ones_like(latents)
|
||||
|
||||
pipeline.predict_noise_maybe_with_cfg = fake_predict_noise_maybe_with_cfg # type: ignore[method-assign]
|
||||
pipeline.scheduler_step_maybe_with_cfg = lambda noise, t, current, cfg: current + noise # type: ignore[method-assign]
|
||||
|
||||
result = pipeline.diffuse(
|
||||
latents=latents,
|
||||
timesteps=timesteps,
|
||||
prompt_embeds=torch.zeros(1, 2, 3),
|
||||
negative_prompt_embeds=None,
|
||||
image_embeds=None,
|
||||
guidance_low=1.0,
|
||||
guidance_high=2.0,
|
||||
boundary_timestep=500.0,
|
||||
dtype=torch.float32,
|
||||
attention_kwargs={},
|
||||
condition=condition,
|
||||
first_frame_mask=first_frame_mask,
|
||||
)
|
||||
|
||||
assert [call["model"] for call in calls] == ["high", "low"]
|
||||
assert [call["scale"] for call in calls] == [1.0, 2.0]
|
||||
assert calls[0]["timestep_shape"] == (1, 8)
|
||||
timestep_dtype = calls[0]["timestep_values"].dtype
|
||||
torch.testing.assert_close(calls[0]["timestep_values"][0, :4], torch.zeros(4, dtype=timestep_dtype))
|
||||
torch.testing.assert_close(calls[0]["timestep_values"][0, 4:], torch.full((4,), 900, dtype=timestep_dtype))
|
||||
torch.testing.assert_close(calls[0]["hidden_states"][:, :, 0], torch.ones(1, 4, 4, 4))
|
||||
torch.testing.assert_close(result, torch.full_like(latents, 2.0))
|
||||
|
||||
|
||||
def test_i2v_prepare_latents_builds_expand_condition_and_first_frame_mask() -> None:
|
||||
pipeline = _make_i2v_pipeline(expand_timesteps=True)
|
||||
latents, condition, first_frame_mask = pipeline.prepare_latents(
|
||||
image=torch.zeros(1, 3, 16, 16),
|
||||
batch_size=1,
|
||||
num_channels_latents=4,
|
||||
height=16,
|
||||
width=16,
|
||||
num_frames=5,
|
||||
dtype=torch.float32,
|
||||
device=torch.device("cpu"),
|
||||
generator=torch.Generator(device="cpu").manual_seed(0),
|
||||
)
|
||||
|
||||
assert latents.shape == (1, 4, 2, 2, 2)
|
||||
assert condition.shape == (1, 4, 1, 2, 2)
|
||||
assert first_frame_mask.shape == (1, 1, 2, 2, 2)
|
||||
assert first_frame_mask[:, :, 0].sum() == 0
|
||||
assert first_frame_mask[:, :, 1].sum() == 4
|
||||
@@ -0,0 +1,155 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from contextlib import contextmanager
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import Wan22Pipeline
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
|
||||
class _StubTransformer(nn.Module):
|
||||
@property
|
||||
def dtype(self) -> torch.dtype:
|
||||
return torch.float32
|
||||
|
||||
|
||||
class _StubScheduler:
|
||||
def __init__(self, timesteps: list[int]) -> None:
|
||||
self.timesteps = torch.tensor(timesteps, dtype=torch.int64)
|
||||
self.config = SimpleNamespace(num_train_timesteps=1000)
|
||||
self.set_timesteps_calls: list[tuple[int, torch.device]] = []
|
||||
|
||||
def set_timesteps(self, num_steps: int, device: torch.device) -> None:
|
||||
self.set_timesteps_calls.append((num_steps, device))
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _noop_progress_bar(*args, **kwargs):
|
||||
del args, kwargs
|
||||
|
||||
class _Bar:
|
||||
def update(self) -> None:
|
||||
return None
|
||||
|
||||
yield _Bar()
|
||||
|
||||
|
||||
def _make_pipeline() -> Wan22Pipeline:
|
||||
pipeline = object.__new__(Wan22Pipeline)
|
||||
nn.Module.__init__(pipeline)
|
||||
pipeline.device = torch.device("cpu")
|
||||
pipeline.transformer = _StubTransformer()
|
||||
pipeline.transformer_2 = None
|
||||
pipeline.transformer_config = SimpleNamespace(patch_size=(1, 2, 2), in_channels=4, out_channels=4)
|
||||
pipeline.scheduler = _StubScheduler([9, 5])
|
||||
pipeline.od_config = SimpleNamespace(flow_shift=5.0)
|
||||
pipeline._sample_solver = "unipc"
|
||||
pipeline._flow_shift = 5.0
|
||||
pipeline.vae_scale_factor_temporal = 4
|
||||
pipeline.vae_scale_factor_spatial = 8
|
||||
pipeline.boundary_ratio = 0.875
|
||||
pipeline.expand_timesteps = False
|
||||
pipeline._guidance_scale = None
|
||||
pipeline._guidance_scale_2 = None
|
||||
pipeline._num_timesteps = None
|
||||
pipeline._current_timestep = None
|
||||
pipeline.check_inputs = lambda **kwargs: None
|
||||
pipeline.prepare_latents = lambda **kwargs: torch.zeros((1, 4, 1, 8, 8), dtype=torch.float32)
|
||||
pipeline.progress_bar = _noop_progress_bar
|
||||
return pipeline
|
||||
|
||||
|
||||
def test_forward_delegates_denoising_to_diffuse(monkeypatch) -> None:
|
||||
pipeline = _make_pipeline()
|
||||
|
||||
prompt_embeds = torch.randn(1, 8)
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def _fake_diffuse(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return kwargs["latents"] + 1
|
||||
|
||||
pipeline.diffuse = _fake_diffuse # type: ignore[method-assign]
|
||||
|
||||
req = SimpleNamespace(
|
||||
prompts=["prompt"],
|
||||
sampling_params=SimpleNamespace(
|
||||
height=None,
|
||||
width=None,
|
||||
num_frames=1,
|
||||
num_inference_steps=2,
|
||||
guidance_scale_provided=False,
|
||||
guidance_scale=None,
|
||||
guidance_scale_2=None,
|
||||
boundary_ratio=None,
|
||||
generator=None,
|
||||
seed=None,
|
||||
num_outputs_per_prompt=1,
|
||||
max_sequence_length=32,
|
||||
latents=None,
|
||||
extra_args={},
|
||||
),
|
||||
)
|
||||
|
||||
output = pipeline.forward(req, prompt_embeds=prompt_embeds, output_type="latent", guidance_scale=1.0)
|
||||
|
||||
assert torch.equal(output.output, torch.ones((1, 4, 1, 8, 8)))
|
||||
assert torch.equal(captured["timesteps"], pipeline.scheduler.timesteps)
|
||||
assert captured["guidance_low"] == 1.0
|
||||
assert captured["guidance_high"] == 1.0
|
||||
assert captured["boundary_timestep"] == pytest.approx(875.0)
|
||||
assert captured["latent_condition"] is None
|
||||
assert captured["first_frame_mask"] is None
|
||||
assert pipeline.scheduler.set_timesteps_calls == [(2, torch.device("cpu"))]
|
||||
|
||||
|
||||
def test_diffuse_runs_prediction_and_scheduler_for_each_timestep() -> None:
|
||||
pipeline = _make_pipeline()
|
||||
latents = torch.zeros((1, 1, 1, 2, 2), dtype=torch.float32)
|
||||
timesteps = torch.tensor([7, 3], dtype=torch.int64)
|
||||
prompt_embeds = torch.randn(1, 8)
|
||||
|
||||
predict_calls: list[dict[str, object]] = []
|
||||
scheduler_calls: list[tuple[float, int, float, bool]] = []
|
||||
|
||||
def _fake_predict_noise_maybe_with_cfg(**kwargs):
|
||||
predict_calls.append(kwargs)
|
||||
timestep = kwargs["positive_kwargs"]["timestep"]
|
||||
assert isinstance(timestep, torch.Tensor)
|
||||
return torch.full_like(latents, float(timestep[0].item()))
|
||||
|
||||
def _fake_scheduler_step_maybe_with_cfg(noise_pred, t, current_latents, do_true_cfg):
|
||||
scheduler_calls.append(
|
||||
(float(noise_pred[0, 0, 0, 0, 0]), int(t.item()), float(current_latents.sum()), do_true_cfg)
|
||||
)
|
||||
return current_latents + noise_pred
|
||||
|
||||
pipeline.predict_noise_maybe_with_cfg = _fake_predict_noise_maybe_with_cfg # type: ignore[method-assign]
|
||||
pipeline.scheduler_step_maybe_with_cfg = _fake_scheduler_step_maybe_with_cfg # type: ignore[method-assign]
|
||||
|
||||
result = pipeline.diffuse(
|
||||
latents=latents,
|
||||
timesteps=timesteps,
|
||||
prompt_embeds=prompt_embeds,
|
||||
negative_prompt_embeds=None,
|
||||
guidance_low=1.0,
|
||||
guidance_high=2.0,
|
||||
boundary_timestep=5.0,
|
||||
dtype=torch.float32,
|
||||
attention_kwargs={},
|
||||
)
|
||||
|
||||
assert len(predict_calls) == 2
|
||||
assert predict_calls[0]["true_cfg_scale"] == 1.0
|
||||
assert predict_calls[1]["true_cfg_scale"] == 2.0
|
||||
assert scheduler_calls == [
|
||||
(7.0, 7, 0.0, False),
|
||||
(3.0, 3, 28.0, False),
|
||||
]
|
||||
assert torch.equal(result, torch.full_like(latents, 10.0))
|
||||
@@ -0,0 +1,81 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 as wan22_module
|
||||
from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import (
|
||||
create_transformer_from_config,
|
||||
load_transformer_config,
|
||||
retrieve_latents,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
|
||||
class _LatentDist:
|
||||
def sample(self, generator):
|
||||
assert isinstance(generator, torch.Generator)
|
||||
return torch.tensor([1.0])
|
||||
|
||||
def mode(self):
|
||||
return torch.tensor([2.0])
|
||||
|
||||
|
||||
def test_retrieve_latents_supports_sample_mode_argmax_and_direct_latents() -> None:
|
||||
generator = torch.Generator(device="cpu")
|
||||
|
||||
assert retrieve_latents(SimpleNamespace(latent_dist=_LatentDist()), generator).item() == 1.0
|
||||
assert retrieve_latents(SimpleNamespace(latent_dist=_LatentDist()), sample_mode="argmax").item() == 2.0
|
||||
torch.testing.assert_close(retrieve_latents(SimpleNamespace(latents=torch.tensor([3.0]))), torch.tensor([3.0]))
|
||||
|
||||
|
||||
def test_retrieve_latents_rejects_unknown_encoder_output() -> None:
|
||||
with pytest.raises(AttributeError, match="Could not access latents"):
|
||||
retrieve_latents(SimpleNamespace())
|
||||
|
||||
|
||||
def test_load_transformer_config_reads_local_subfolder_config(tmp_path) -> None:
|
||||
config_dir = tmp_path / "transformer_2"
|
||||
config_dir.mkdir(parents=True)
|
||||
(config_dir / "config.json").write_text(json.dumps({"patch_size": [1, 2, 2], "num_layers": 2}))
|
||||
|
||||
assert load_transformer_config(str(tmp_path), "transformer_2") == {"patch_size": [1, 2, 2], "num_layers": 2}
|
||||
assert load_transformer_config(str(tmp_path), "missing") == {}
|
||||
|
||||
|
||||
def test_create_transformer_from_config_maps_supported_keys(monkeypatch) -> None:
|
||||
captured = {}
|
||||
|
||||
class FakeTransformer:
|
||||
def __init__(self, **kwargs) -> None:
|
||||
captured.update(kwargs)
|
||||
|
||||
monkeypatch.setattr(wan22_module, "WanTransformer3DModel", FakeTransformer)
|
||||
|
||||
transformer = create_transformer_from_config(
|
||||
{
|
||||
"patch_size": [1, 2, 2],
|
||||
"num_attention_heads": 8,
|
||||
"attention_head_dim": 128,
|
||||
"in_channels": 16,
|
||||
"out_channels": 16,
|
||||
"text_dim": 4096,
|
||||
"vace_layers": [0],
|
||||
"ignored": "value",
|
||||
}
|
||||
)
|
||||
|
||||
assert isinstance(transformer, FakeTransformer)
|
||||
assert captured == {
|
||||
"patch_size": (1, 2, 2),
|
||||
"num_attention_heads": 8,
|
||||
"attention_head_dim": 128,
|
||||
"in_channels": 16,
|
||||
"out_channels": 16,
|
||||
"text_dim": 4096,
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
from torch import nn
|
||||
|
||||
from tests.diffusion.models.wan2_2.conftest import StubTransformer, StubVAE, noop_progress_bar
|
||||
from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2_ti2v import (
|
||||
Wan22TI2VPipeline,
|
||||
get_wan22_ti2v_pre_process_func,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
|
||||
def _make_ti2v_pipeline() -> Wan22TI2VPipeline:
|
||||
pipeline = object.__new__(Wan22TI2VPipeline)
|
||||
nn.Module.__init__(pipeline)
|
||||
pipeline.device = torch.device("cpu")
|
||||
pipeline.transformer = StubTransformer(in_channels=4, out_channels=4)
|
||||
pipeline.vae = StubVAE(z_dim=4)
|
||||
pipeline.vae_scale_factor_temporal = 4
|
||||
pipeline.vae_scale_factor_spatial = 8
|
||||
pipeline.progress_bar = noop_progress_bar
|
||||
return pipeline
|
||||
|
||||
|
||||
def test_ti2v_preprocess_uses_720p_area_for_image_condition() -> None:
|
||||
preprocess = get_wan22_ti2v_pre_process_func(SimpleNamespace())
|
||||
request = SimpleNamespace(
|
||||
prompts=[{"prompt": "p", "multi_modal_data": {"image": Image.new("RGB", (320, 160), "blue")}}],
|
||||
sampling_params=SimpleNamespace(height=None, width=None),
|
||||
)
|
||||
|
||||
result = preprocess(request)
|
||||
|
||||
assert result.sampling_params.height == 672
|
||||
assert result.sampling_params.width == 1344
|
||||
assert result.prompts[0]["multi_modal_data"]["image"].size == (1344, 672)
|
||||
assert result.prompts[0]["additional_information"]["preprocessed_image"].shape[-2:] == (672, 1344)
|
||||
|
||||
|
||||
def test_ti2v_diffuse_without_image_condition_expands_patch_timesteps() -> None:
|
||||
pipeline = _make_ti2v_pipeline()
|
||||
latents = torch.zeros(1, 4, 2, 4, 4)
|
||||
calls = []
|
||||
|
||||
def fake_predict_noise_maybe_with_cfg(**kwargs):
|
||||
calls.append(kwargs)
|
||||
return torch.ones_like(latents)
|
||||
|
||||
pipeline.predict_noise_maybe_with_cfg = fake_predict_noise_maybe_with_cfg # type: ignore[method-assign]
|
||||
pipeline.scheduler_step_maybe_with_cfg = lambda noise, t, current, cfg: current + noise # type: ignore[method-assign]
|
||||
|
||||
result = pipeline.diffuse(
|
||||
latents=latents,
|
||||
timesteps=torch.tensor([7]),
|
||||
prompt_embeds=torch.zeros(1, 2, 3),
|
||||
negative_prompt_embeds=torch.zeros(1, 2, 3),
|
||||
guidance_scale=3.0,
|
||||
dtype=torch.float32,
|
||||
attention_kwargs={"a": "b"},
|
||||
num_latent_frames=2,
|
||||
latent_height=4,
|
||||
latent_width=4,
|
||||
)
|
||||
|
||||
positive = calls[0]["positive_kwargs"]
|
||||
assert calls[0]["do_true_cfg"] is True
|
||||
assert positive["timestep"].shape == (1, 8)
|
||||
torch.testing.assert_close(positive["timestep"], torch.full((1, 8), 7, dtype=positive["timestep"].dtype))
|
||||
torch.testing.assert_close(positive["hidden_states"], latents)
|
||||
torch.testing.assert_close(result, torch.ones_like(latents))
|
||||
|
||||
|
||||
def test_ti2v_prepare_i2v_latents_encodes_condition_and_masks_first_frame() -> None:
|
||||
pipeline = _make_ti2v_pipeline()
|
||||
latents, latent_condition, first_frame_mask = pipeline.prepare_i2v_latents(
|
||||
image=torch.zeros(1, 3, 16, 16),
|
||||
batch_size=1,
|
||||
num_channels_latents=4,
|
||||
height=16,
|
||||
width=16,
|
||||
num_frames=5,
|
||||
dtype=torch.float32,
|
||||
device=torch.device("cpu"),
|
||||
generator=None,
|
||||
latents=torch.zeros(1, 4, 2, 2, 2),
|
||||
)
|
||||
|
||||
torch.testing.assert_close(latents, torch.zeros(1, 4, 2, 2, 2))
|
||||
assert latent_condition.shape == (1, 4, 1, 2, 2)
|
||||
assert first_frame_mask[:, :, 0].sum() == 0
|
||||
assert first_frame_mask[:, :, 1].sum() == 4
|
||||
@@ -0,0 +1,137 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
from torch import nn
|
||||
|
||||
from tests.diffusion.models.wan2_2.conftest import StubTransformer, StubVAE, noop_progress_bar
|
||||
from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2_vace import (
|
||||
Wan22VACEPipeline,
|
||||
create_vace_transformer_from_config,
|
||||
get_wan22_vace_pre_process_func,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
|
||||
def _make_vace_pipeline() -> Wan22VACEPipeline:
|
||||
pipeline = object.__new__(Wan22VACEPipeline)
|
||||
nn.Module.__init__(pipeline)
|
||||
pipeline.device = torch.device("cpu")
|
||||
pipeline.transformer = StubTransformer(in_channels=4, out_channels=4)
|
||||
pipeline.transformer_config = pipeline.transformer.config
|
||||
pipeline.vae = StubVAE(z_dim=4)
|
||||
pipeline.vae_scale_factor_temporal = 4
|
||||
pipeline.vae_scale_factor_spatial = 8
|
||||
pipeline.progress_bar = noop_progress_bar
|
||||
return pipeline
|
||||
|
||||
|
||||
def test_vace_preprocess_collects_reference_video_and_mask_inputs() -> None:
|
||||
preprocess = get_wan22_vace_pre_process_func(SimpleNamespace())
|
||||
ref = Image.new("RGB", (320, 160), "green")
|
||||
frame = Image.new("RGB", (64, 64), "black")
|
||||
mask = Image.new("L", (64, 64), 255)
|
||||
request = SimpleNamespace(
|
||||
prompts=[
|
||||
{
|
||||
"prompt": "p",
|
||||
"multi_modal_data": {
|
||||
"image": ref,
|
||||
"video": [frame],
|
||||
"mask": mask,
|
||||
},
|
||||
}
|
||||
],
|
||||
sampling_params=SimpleNamespace(height=None, width=None),
|
||||
)
|
||||
|
||||
result = preprocess(request)
|
||||
additional_info = result.prompts[0]["additional_information"]
|
||||
|
||||
assert result.sampling_params.height == 432
|
||||
assert result.sampling_params.width == 880
|
||||
assert additional_info["reference_images"] == [ref]
|
||||
assert additional_info["source_video"] == [frame]
|
||||
assert additional_info["mask"] == [mask]
|
||||
|
||||
|
||||
def test_create_vace_transformer_from_config_maps_vace_specific_keys(monkeypatch) -> None:
|
||||
captured = {}
|
||||
|
||||
class FakeVACETransformer:
|
||||
def __init__(self, **kwargs) -> None:
|
||||
captured.update(kwargs)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2_vace.WanVACETransformer3DModel",
|
||||
FakeVACETransformer,
|
||||
)
|
||||
|
||||
transformer = create_vace_transformer_from_config(
|
||||
{
|
||||
"patch_size": [1, 2, 2],
|
||||
"in_channels": 96,
|
||||
"out_channels": 16,
|
||||
"vace_layers": [0, 1, 2],
|
||||
"vace_in_channels": 132,
|
||||
"unknown": "ignored",
|
||||
}
|
||||
)
|
||||
|
||||
assert isinstance(transformer, FakeVACETransformer)
|
||||
assert captured == {
|
||||
"patch_size": (1, 2, 2),
|
||||
"in_channels": 96,
|
||||
"out_channels": 16,
|
||||
"vace_layers": [0, 1, 2],
|
||||
"vace_in_channels": 132,
|
||||
}
|
||||
|
||||
|
||||
def test_vace_prepare_masks_encodes_spatial_stride_and_reference_padding() -> None:
|
||||
pipeline = _make_vace_pipeline()
|
||||
mask = torch.ones(1, 3, 5, 16, 16)
|
||||
reference_images = [[torch.zeros(3, 16, 16), torch.zeros(3, 16, 16)]]
|
||||
|
||||
encoded = pipeline.prepare_masks(mask, reference_images)
|
||||
|
||||
assert encoded.shape == (1, 64, 4, 2, 2)
|
||||
torch.testing.assert_close(encoded[:, :, :2], torch.zeros(1, 64, 2, 2, 2))
|
||||
torch.testing.assert_close(encoded[:, :, 2:], torch.ones(1, 64, 2, 2, 2))
|
||||
|
||||
|
||||
def test_vace_diffuse_passes_context_and_scale_to_cfg_branches() -> None:
|
||||
pipeline = _make_vace_pipeline()
|
||||
latents = torch.zeros(1, 4, 1, 2, 2)
|
||||
vace_context = torch.ones(1, 12, 1, 2, 2)
|
||||
calls = []
|
||||
|
||||
def fake_predict_noise_maybe_with_cfg(**kwargs):
|
||||
calls.append(kwargs)
|
||||
return torch.ones_like(latents)
|
||||
|
||||
pipeline.predict_noise_maybe_with_cfg = fake_predict_noise_maybe_with_cfg # type: ignore[method-assign]
|
||||
pipeline.scheduler_step_maybe_with_cfg = lambda noise, t, current, cfg: current + noise # type: ignore[method-assign]
|
||||
|
||||
result = pipeline.diffuse(
|
||||
latents=latents,
|
||||
timesteps=torch.tensor([5]),
|
||||
prompt_embeds=torch.zeros(1, 2, 3),
|
||||
negative_prompt_embeds=torch.zeros(1, 2, 3),
|
||||
guidance_scale=4.0,
|
||||
dtype=torch.float32,
|
||||
attention_kwargs={},
|
||||
vace_context=vace_context,
|
||||
vace_context_scale=0.75,
|
||||
)
|
||||
|
||||
assert calls[0]["do_true_cfg"] is True
|
||||
assert calls[0]["true_cfg_scale"] == 4.0
|
||||
assert calls[0]["positive_kwargs"]["vace_context"] is vace_context
|
||||
assert calls[0]["negative_kwargs"]["vace_context_scale"] == 0.75
|
||||
torch.testing.assert_close(result, torch.ones_like(latents))
|
||||
Reference in new issue
Block a user