diff --git a/tests/diffusion/models/wan2_2/conftest.py b/tests/diffusion/models/wan2_2/conftest.py new file mode 100644 index 000000000..f836fa545 --- /dev/null +++ b/tests/diffusion/models/wan2_2/conftest.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from contextlib import contextmanager +from types import SimpleNamespace + +import torch +from torch import nn + + +class StubTransformer(nn.Module): + def __init__(self, *, name: str = "transformer", in_channels: int = 4, out_channels: int = 4) -> None: + super().__init__() + self.name = name + self.config = SimpleNamespace( + patch_size=(1, 2, 2), + in_channels=in_channels, + out_channels=out_channels, + image_dim=None, + ) + + @property + def dtype(self) -> torch.dtype: + return torch.float32 + + def forward(self, **kwargs): + hidden_states = kwargs["hidden_states"] + return (torch.zeros_like(hidden_states[:, : self.config.out_channels]),) + + +class StubScheduler: + def __init__(self, timesteps: list[int]) -> None: + self.timesteps = torch.tensor(timesteps, dtype=torch.int64) + self.config = SimpleNamespace(num_train_timesteps=1000) + self.set_timesteps_calls: list[tuple[int, torch.device]] = [] + + def set_timesteps(self, num_steps: int, device: torch.device) -> None: + self.set_timesteps_calls.append((num_steps, device)) + + +class StubVAE: + dtype = torch.float32 + + def __init__(self, z_dim: int = 4) -> None: + self.config = SimpleNamespace( + z_dim=z_dim, + scale_factor_temporal=4, + scale_factor_spatial=8, + latents_mean=[0.0] * z_dim, + latents_std=[1.0] * z_dim, + ) + + def encode(self, video: torch.Tensor): + latent_frames = (video.shape[2] + self.config.scale_factor_temporal - 1) // self.config.scale_factor_temporal + latent_height = video.shape[-2] // self.config.scale_factor_spatial + latent_width = video.shape[-1] // self.config.scale_factor_spatial + latents = torch.ones( + video.shape[0], + self.config.z_dim, + latent_frames, + latent_height, + latent_width, + dtype=video.dtype, + device=video.device, + ) + return SimpleNamespace(latents=latents) + + def decode(self, latents: torch.Tensor, return_dict: bool = False): + del return_dict + return (latents,) + + +@contextmanager +def noop_progress_bar(*args, **kwargs): + del args, kwargs + + class Bar: + def update(self) -> None: + return None + + yield Bar() diff --git a/tests/diffusion/models/wan2_2/test_wan22_i2v_pipeline.py b/tests/diffusion/models/wan2_2/test_wan22_i2v_pipeline.py new file mode 100644 index 000000000..04e834ac4 --- /dev/null +++ b/tests/diffusion/models/wan2_2/test_wan22_i2v_pipeline.py @@ -0,0 +1,126 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace + +import pytest +import torch +from PIL import Image +from torch import nn + +from tests.diffusion.models.wan2_2.conftest import StubTransformer, StubVAE, noop_progress_bar +from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2_i2v import ( + Wan22I2VPipeline, + get_wan22_i2v_pre_process_func, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion] + + +def _make_i2v_pipeline(*, expand_timesteps: bool) -> Wan22I2VPipeline: + pipeline = object.__new__(Wan22I2VPipeline) + nn.Module.__init__(pipeline) + pipeline.device = torch.device("cpu") + pipeline.transformer = StubTransformer(name="high", in_channels=8, out_channels=4) + pipeline.transformer_2 = StubTransformer(name="low", in_channels=8, out_channels=4) + pipeline.vae = StubVAE(z_dim=4) + pipeline.vae_scale_factor_temporal = 4 + pipeline.vae_scale_factor_spatial = 8 + pipeline.expand_timesteps = expand_timesteps + pipeline.progress_bar = noop_progress_bar + return pipeline + + +def test_i2v_preprocess_requires_image_and_resizes_to_480p_aspect() -> None: + preprocess = get_wan22_i2v_pre_process_func(SimpleNamespace()) + request = SimpleNamespace( + prompts=[{"prompt": "p", "multi_modal_data": {"image": Image.new("RGB", (320, 160), "red")}}], + sampling_params=SimpleNamespace(height=None, width=None), + ) + + result = preprocess(request) + prompt = result.prompts[0] + + assert result.sampling_params.height == 432 + assert result.sampling_params.width == 880 + assert prompt["multi_modal_data"]["image"].size == (880, 432) + assert prompt["additional_information"]["preprocessed_image"].shape[-2:] == (432, 880) + + missing_image = SimpleNamespace( + prompts=[{"prompt": "p", "multi_modal_data": {}}], + sampling_params=SimpleNamespace(height=None, width=None), + ) + with pytest.raises(ValueError, match="No image is provided"): + preprocess(missing_image) + + +def test_i2v_diffuse_selects_stage_guidance_and_expands_timesteps() -> None: + pipeline = _make_i2v_pipeline(expand_timesteps=True) + latents = torch.zeros(1, 4, 2, 4, 4) + condition = torch.ones_like(latents) + first_frame_mask = torch.ones(1, 1, 2, 4, 4) + first_frame_mask[:, :, 0] = 0 + timesteps = torch.tensor([900, 100]) + + calls = [] + + def fake_predict_noise_maybe_with_cfg(**kwargs): + positive = kwargs["positive_kwargs"] + calls.append( + { + "model": positive["current_model"].name, + "scale": kwargs["true_cfg_scale"], + "timestep_shape": tuple(positive["timestep"].shape), + "timestep_values": positive["timestep"].clone(), + "hidden_states": positive["hidden_states"].clone(), + } + ) + return torch.ones_like(latents) + + pipeline.predict_noise_maybe_with_cfg = fake_predict_noise_maybe_with_cfg # type: ignore[method-assign] + pipeline.scheduler_step_maybe_with_cfg = lambda noise, t, current, cfg: current + noise # type: ignore[method-assign] + + result = pipeline.diffuse( + latents=latents, + timesteps=timesteps, + prompt_embeds=torch.zeros(1, 2, 3), + negative_prompt_embeds=None, + image_embeds=None, + guidance_low=1.0, + guidance_high=2.0, + boundary_timestep=500.0, + dtype=torch.float32, + attention_kwargs={}, + condition=condition, + first_frame_mask=first_frame_mask, + ) + + assert [call["model"] for call in calls] == ["high", "low"] + assert [call["scale"] for call in calls] == [1.0, 2.0] + assert calls[0]["timestep_shape"] == (1, 8) + timestep_dtype = calls[0]["timestep_values"].dtype + torch.testing.assert_close(calls[0]["timestep_values"][0, :4], torch.zeros(4, dtype=timestep_dtype)) + torch.testing.assert_close(calls[0]["timestep_values"][0, 4:], torch.full((4,), 900, dtype=timestep_dtype)) + torch.testing.assert_close(calls[0]["hidden_states"][:, :, 0], torch.ones(1, 4, 4, 4)) + torch.testing.assert_close(result, torch.full_like(latents, 2.0)) + + +def test_i2v_prepare_latents_builds_expand_condition_and_first_frame_mask() -> None: + pipeline = _make_i2v_pipeline(expand_timesteps=True) + latents, condition, first_frame_mask = pipeline.prepare_latents( + image=torch.zeros(1, 3, 16, 16), + batch_size=1, + num_channels_latents=4, + height=16, + width=16, + num_frames=5, + dtype=torch.float32, + device=torch.device("cpu"), + generator=torch.Generator(device="cpu").manual_seed(0), + ) + + assert latents.shape == (1, 4, 2, 2, 2) + assert condition.shape == (1, 4, 1, 2, 2) + assert first_frame_mask.shape == (1, 1, 2, 2, 2) + assert first_frame_mask[:, :, 0].sum() == 0 + assert first_frame_mask[:, :, 1].sum() == 4 diff --git a/tests/diffusion/models/wan2_2/test_wan22_pipeline_diffuse.py b/tests/diffusion/models/wan2_2/test_wan22_pipeline_diffuse.py new file mode 100644 index 000000000..54bb672ef --- /dev/null +++ b/tests/diffusion/models/wan2_2/test_wan22_pipeline_diffuse.py @@ -0,0 +1,155 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from contextlib import contextmanager +from types import SimpleNamespace + +import pytest +import torch +from torch import nn + +from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import Wan22Pipeline + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion] + + +class _StubTransformer(nn.Module): + @property + def dtype(self) -> torch.dtype: + return torch.float32 + + +class _StubScheduler: + def __init__(self, timesteps: list[int]) -> None: + self.timesteps = torch.tensor(timesteps, dtype=torch.int64) + self.config = SimpleNamespace(num_train_timesteps=1000) + self.set_timesteps_calls: list[tuple[int, torch.device]] = [] + + def set_timesteps(self, num_steps: int, device: torch.device) -> None: + self.set_timesteps_calls.append((num_steps, device)) + + +@contextmanager +def _noop_progress_bar(*args, **kwargs): + del args, kwargs + + class _Bar: + def update(self) -> None: + return None + + yield _Bar() + + +def _make_pipeline() -> Wan22Pipeline: + pipeline = object.__new__(Wan22Pipeline) + nn.Module.__init__(pipeline) + pipeline.device = torch.device("cpu") + pipeline.transformer = _StubTransformer() + pipeline.transformer_2 = None + pipeline.transformer_config = SimpleNamespace(patch_size=(1, 2, 2), in_channels=4, out_channels=4) + pipeline.scheduler = _StubScheduler([9, 5]) + pipeline.od_config = SimpleNamespace(flow_shift=5.0) + pipeline._sample_solver = "unipc" + pipeline._flow_shift = 5.0 + pipeline.vae_scale_factor_temporal = 4 + pipeline.vae_scale_factor_spatial = 8 + pipeline.boundary_ratio = 0.875 + pipeline.expand_timesteps = False + pipeline._guidance_scale = None + pipeline._guidance_scale_2 = None + pipeline._num_timesteps = None + pipeline._current_timestep = None + pipeline.check_inputs = lambda **kwargs: None + pipeline.prepare_latents = lambda **kwargs: torch.zeros((1, 4, 1, 8, 8), dtype=torch.float32) + pipeline.progress_bar = _noop_progress_bar + return pipeline + + +def test_forward_delegates_denoising_to_diffuse(monkeypatch) -> None: + pipeline = _make_pipeline() + + prompt_embeds = torch.randn(1, 8) + captured: dict[str, object] = {} + + def _fake_diffuse(**kwargs): + captured.update(kwargs) + return kwargs["latents"] + 1 + + pipeline.diffuse = _fake_diffuse # type: ignore[method-assign] + + req = SimpleNamespace( + prompts=["prompt"], + sampling_params=SimpleNamespace( + height=None, + width=None, + num_frames=1, + num_inference_steps=2, + guidance_scale_provided=False, + guidance_scale=None, + guidance_scale_2=None, + boundary_ratio=None, + generator=None, + seed=None, + num_outputs_per_prompt=1, + max_sequence_length=32, + latents=None, + extra_args={}, + ), + ) + + output = pipeline.forward(req, prompt_embeds=prompt_embeds, output_type="latent", guidance_scale=1.0) + + assert torch.equal(output.output, torch.ones((1, 4, 1, 8, 8))) + assert torch.equal(captured["timesteps"], pipeline.scheduler.timesteps) + assert captured["guidance_low"] == 1.0 + assert captured["guidance_high"] == 1.0 + assert captured["boundary_timestep"] == pytest.approx(875.0) + assert captured["latent_condition"] is None + assert captured["first_frame_mask"] is None + assert pipeline.scheduler.set_timesteps_calls == [(2, torch.device("cpu"))] + + +def test_diffuse_runs_prediction_and_scheduler_for_each_timestep() -> None: + pipeline = _make_pipeline() + latents = torch.zeros((1, 1, 1, 2, 2), dtype=torch.float32) + timesteps = torch.tensor([7, 3], dtype=torch.int64) + prompt_embeds = torch.randn(1, 8) + + predict_calls: list[dict[str, object]] = [] + scheduler_calls: list[tuple[float, int, float, bool]] = [] + + def _fake_predict_noise_maybe_with_cfg(**kwargs): + predict_calls.append(kwargs) + timestep = kwargs["positive_kwargs"]["timestep"] + assert isinstance(timestep, torch.Tensor) + return torch.full_like(latents, float(timestep[0].item())) + + def _fake_scheduler_step_maybe_with_cfg(noise_pred, t, current_latents, do_true_cfg): + scheduler_calls.append( + (float(noise_pred[0, 0, 0, 0, 0]), int(t.item()), float(current_latents.sum()), do_true_cfg) + ) + return current_latents + noise_pred + + pipeline.predict_noise_maybe_with_cfg = _fake_predict_noise_maybe_with_cfg # type: ignore[method-assign] + pipeline.scheduler_step_maybe_with_cfg = _fake_scheduler_step_maybe_with_cfg # type: ignore[method-assign] + + result = pipeline.diffuse( + latents=latents, + timesteps=timesteps, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=None, + guidance_low=1.0, + guidance_high=2.0, + boundary_timestep=5.0, + dtype=torch.float32, + attention_kwargs={}, + ) + + assert len(predict_calls) == 2 + assert predict_calls[0]["true_cfg_scale"] == 1.0 + assert predict_calls[1]["true_cfg_scale"] == 2.0 + assert scheduler_calls == [ + (7.0, 7, 0.0, False), + (3.0, 3, 28.0, False), + ] + assert torch.equal(result, torch.full_like(latents, 10.0)) diff --git a/tests/diffusion/models/wan2_2/test_wan22_pipeline_helpers.py b/tests/diffusion/models/wan2_2/test_wan22_pipeline_helpers.py new file mode 100644 index 000000000..314717869 --- /dev/null +++ b/tests/diffusion/models/wan2_2/test_wan22_pipeline_helpers.py @@ -0,0 +1,81 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import json +from types import SimpleNamespace + +import pytest +import torch + +import vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 as wan22_module +from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import ( + create_transformer_from_config, + load_transformer_config, + retrieve_latents, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion] + + +class _LatentDist: + def sample(self, generator): + assert isinstance(generator, torch.Generator) + return torch.tensor([1.0]) + + def mode(self): + return torch.tensor([2.0]) + + +def test_retrieve_latents_supports_sample_mode_argmax_and_direct_latents() -> None: + generator = torch.Generator(device="cpu") + + assert retrieve_latents(SimpleNamespace(latent_dist=_LatentDist()), generator).item() == 1.0 + assert retrieve_latents(SimpleNamespace(latent_dist=_LatentDist()), sample_mode="argmax").item() == 2.0 + torch.testing.assert_close(retrieve_latents(SimpleNamespace(latents=torch.tensor([3.0]))), torch.tensor([3.0])) + + +def test_retrieve_latents_rejects_unknown_encoder_output() -> None: + with pytest.raises(AttributeError, match="Could not access latents"): + retrieve_latents(SimpleNamespace()) + + +def test_load_transformer_config_reads_local_subfolder_config(tmp_path) -> None: + config_dir = tmp_path / "transformer_2" + config_dir.mkdir(parents=True) + (config_dir / "config.json").write_text(json.dumps({"patch_size": [1, 2, 2], "num_layers": 2})) + + assert load_transformer_config(str(tmp_path), "transformer_2") == {"patch_size": [1, 2, 2], "num_layers": 2} + assert load_transformer_config(str(tmp_path), "missing") == {} + + +def test_create_transformer_from_config_maps_supported_keys(monkeypatch) -> None: + captured = {} + + class FakeTransformer: + def __init__(self, **kwargs) -> None: + captured.update(kwargs) + + monkeypatch.setattr(wan22_module, "WanTransformer3DModel", FakeTransformer) + + transformer = create_transformer_from_config( + { + "patch_size": [1, 2, 2], + "num_attention_heads": 8, + "attention_head_dim": 128, + "in_channels": 16, + "out_channels": 16, + "text_dim": 4096, + "vace_layers": [0], + "ignored": "value", + } + ) + + assert isinstance(transformer, FakeTransformer) + assert captured == { + "patch_size": (1, 2, 2), + "num_attention_heads": 8, + "attention_head_dim": 128, + "in_channels": 16, + "out_channels": 16, + "text_dim": 4096, + } diff --git a/tests/diffusion/models/wan2_2/test_wan22_ti2v_pipeline.py b/tests/diffusion/models/wan2_2/test_wan22_ti2v_pipeline.py new file mode 100644 index 000000000..983350c4c --- /dev/null +++ b/tests/diffusion/models/wan2_2/test_wan22_ti2v_pipeline.py @@ -0,0 +1,98 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace + +import pytest +import torch +from PIL import Image +from torch import nn + +from tests.diffusion.models.wan2_2.conftest import StubTransformer, StubVAE, noop_progress_bar +from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2_ti2v import ( + Wan22TI2VPipeline, + get_wan22_ti2v_pre_process_func, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion] + + +def _make_ti2v_pipeline() -> Wan22TI2VPipeline: + pipeline = object.__new__(Wan22TI2VPipeline) + nn.Module.__init__(pipeline) + pipeline.device = torch.device("cpu") + pipeline.transformer = StubTransformer(in_channels=4, out_channels=4) + pipeline.vae = StubVAE(z_dim=4) + pipeline.vae_scale_factor_temporal = 4 + pipeline.vae_scale_factor_spatial = 8 + pipeline.progress_bar = noop_progress_bar + return pipeline + + +def test_ti2v_preprocess_uses_720p_area_for_image_condition() -> None: + preprocess = get_wan22_ti2v_pre_process_func(SimpleNamespace()) + request = SimpleNamespace( + prompts=[{"prompt": "p", "multi_modal_data": {"image": Image.new("RGB", (320, 160), "blue")}}], + sampling_params=SimpleNamespace(height=None, width=None), + ) + + result = preprocess(request) + + assert result.sampling_params.height == 672 + assert result.sampling_params.width == 1344 + assert result.prompts[0]["multi_modal_data"]["image"].size == (1344, 672) + assert result.prompts[0]["additional_information"]["preprocessed_image"].shape[-2:] == (672, 1344) + + +def test_ti2v_diffuse_without_image_condition_expands_patch_timesteps() -> None: + pipeline = _make_ti2v_pipeline() + latents = torch.zeros(1, 4, 2, 4, 4) + calls = [] + + def fake_predict_noise_maybe_with_cfg(**kwargs): + calls.append(kwargs) + return torch.ones_like(latents) + + pipeline.predict_noise_maybe_with_cfg = fake_predict_noise_maybe_with_cfg # type: ignore[method-assign] + pipeline.scheduler_step_maybe_with_cfg = lambda noise, t, current, cfg: current + noise # type: ignore[method-assign] + + result = pipeline.diffuse( + latents=latents, + timesteps=torch.tensor([7]), + prompt_embeds=torch.zeros(1, 2, 3), + negative_prompt_embeds=torch.zeros(1, 2, 3), + guidance_scale=3.0, + dtype=torch.float32, + attention_kwargs={"a": "b"}, + num_latent_frames=2, + latent_height=4, + latent_width=4, + ) + + positive = calls[0]["positive_kwargs"] + assert calls[0]["do_true_cfg"] is True + assert positive["timestep"].shape == (1, 8) + torch.testing.assert_close(positive["timestep"], torch.full((1, 8), 7, dtype=positive["timestep"].dtype)) + torch.testing.assert_close(positive["hidden_states"], latents) + torch.testing.assert_close(result, torch.ones_like(latents)) + + +def test_ti2v_prepare_i2v_latents_encodes_condition_and_masks_first_frame() -> None: + pipeline = _make_ti2v_pipeline() + latents, latent_condition, first_frame_mask = pipeline.prepare_i2v_latents( + image=torch.zeros(1, 3, 16, 16), + batch_size=1, + num_channels_latents=4, + height=16, + width=16, + num_frames=5, + dtype=torch.float32, + device=torch.device("cpu"), + generator=None, + latents=torch.zeros(1, 4, 2, 2, 2), + ) + + torch.testing.assert_close(latents, torch.zeros(1, 4, 2, 2, 2)) + assert latent_condition.shape == (1, 4, 1, 2, 2) + assert first_frame_mask[:, :, 0].sum() == 0 + assert first_frame_mask[:, :, 1].sum() == 4 diff --git a/tests/diffusion/models/wan2_2/test_wan22_vace_pipeline.py b/tests/diffusion/models/wan2_2/test_wan22_vace_pipeline.py new file mode 100644 index 000000000..9fa9b67c4 --- /dev/null +++ b/tests/diffusion/models/wan2_2/test_wan22_vace_pipeline.py @@ -0,0 +1,137 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace + +import pytest +import torch +from PIL import Image +from torch import nn + +from tests.diffusion.models.wan2_2.conftest import StubTransformer, StubVAE, noop_progress_bar +from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2_vace import ( + Wan22VACEPipeline, + create_vace_transformer_from_config, + get_wan22_vace_pre_process_func, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion] + + +def _make_vace_pipeline() -> Wan22VACEPipeline: + pipeline = object.__new__(Wan22VACEPipeline) + nn.Module.__init__(pipeline) + pipeline.device = torch.device("cpu") + pipeline.transformer = StubTransformer(in_channels=4, out_channels=4) + pipeline.transformer_config = pipeline.transformer.config + pipeline.vae = StubVAE(z_dim=4) + pipeline.vae_scale_factor_temporal = 4 + pipeline.vae_scale_factor_spatial = 8 + pipeline.progress_bar = noop_progress_bar + return pipeline + + +def test_vace_preprocess_collects_reference_video_and_mask_inputs() -> None: + preprocess = get_wan22_vace_pre_process_func(SimpleNamespace()) + ref = Image.new("RGB", (320, 160), "green") + frame = Image.new("RGB", (64, 64), "black") + mask = Image.new("L", (64, 64), 255) + request = SimpleNamespace( + prompts=[ + { + "prompt": "p", + "multi_modal_data": { + "image": ref, + "video": [frame], + "mask": mask, + }, + } + ], + sampling_params=SimpleNamespace(height=None, width=None), + ) + + result = preprocess(request) + additional_info = result.prompts[0]["additional_information"] + + assert result.sampling_params.height == 432 + assert result.sampling_params.width == 880 + assert additional_info["reference_images"] == [ref] + assert additional_info["source_video"] == [frame] + assert additional_info["mask"] == [mask] + + +def test_create_vace_transformer_from_config_maps_vace_specific_keys(monkeypatch) -> None: + captured = {} + + class FakeVACETransformer: + def __init__(self, **kwargs) -> None: + captured.update(kwargs) + + monkeypatch.setattr( + "vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2_vace.WanVACETransformer3DModel", + FakeVACETransformer, + ) + + transformer = create_vace_transformer_from_config( + { + "patch_size": [1, 2, 2], + "in_channels": 96, + "out_channels": 16, + "vace_layers": [0, 1, 2], + "vace_in_channels": 132, + "unknown": "ignored", + } + ) + + assert isinstance(transformer, FakeVACETransformer) + assert captured == { + "patch_size": (1, 2, 2), + "in_channels": 96, + "out_channels": 16, + "vace_layers": [0, 1, 2], + "vace_in_channels": 132, + } + + +def test_vace_prepare_masks_encodes_spatial_stride_and_reference_padding() -> None: + pipeline = _make_vace_pipeline() + mask = torch.ones(1, 3, 5, 16, 16) + reference_images = [[torch.zeros(3, 16, 16), torch.zeros(3, 16, 16)]] + + encoded = pipeline.prepare_masks(mask, reference_images) + + assert encoded.shape == (1, 64, 4, 2, 2) + torch.testing.assert_close(encoded[:, :, :2], torch.zeros(1, 64, 2, 2, 2)) + torch.testing.assert_close(encoded[:, :, 2:], torch.ones(1, 64, 2, 2, 2)) + + +def test_vace_diffuse_passes_context_and_scale_to_cfg_branches() -> None: + pipeline = _make_vace_pipeline() + latents = torch.zeros(1, 4, 1, 2, 2) + vace_context = torch.ones(1, 12, 1, 2, 2) + calls = [] + + def fake_predict_noise_maybe_with_cfg(**kwargs): + calls.append(kwargs) + return torch.ones_like(latents) + + pipeline.predict_noise_maybe_with_cfg = fake_predict_noise_maybe_with_cfg # type: ignore[method-assign] + pipeline.scheduler_step_maybe_with_cfg = lambda noise, t, current, cfg: current + noise # type: ignore[method-assign] + + result = pipeline.diffuse( + latents=latents, + timesteps=torch.tensor([5]), + prompt_embeds=torch.zeros(1, 2, 3), + negative_prompt_embeds=torch.zeros(1, 2, 3), + guidance_scale=4.0, + dtype=torch.float32, + attention_kwargs={}, + vace_context=vace_context, + vace_context_scale=0.75, + ) + + assert calls[0]["do_true_cfg"] is True + assert calls[0]["true_cfg_scale"] == 4.0 + assert calls[0]["positive_kwargs"]["vace_context"] is vace_context + assert calls[0]["negative_kwargs"]["vace_context_scale"] == 0.75 + torch.testing.assert_close(result, torch.ones_like(latents)) diff --git a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py index e1249e889..652425d50 100644 --- a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py +++ b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py @@ -392,6 +392,102 @@ class Wan22Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin, DiffusionPipe def current_timestep(self): return self._current_timestep + def diffuse( + self, + latents: torch.Tensor, + timesteps: torch.Tensor, + prompt_embeds: torch.Tensor, + negative_prompt_embeds: torch.Tensor | None, + guidance_low: float, + guidance_high: float, + boundary_timestep: float | None, + dtype: torch.dtype, + attention_kwargs: dict[str, Any], + latent_condition: torch.Tensor | None = None, + first_frame_mask: torch.Tensor | None = None, + ) -> torch.Tensor: + with self.progress_bar(total=len(timesteps)) as pbar: + for t in timesteps: + self._current_timestep = t + + # Select model based on timestep and boundary_ratio + # High noise stage (t >= boundary_timestep): use transformer + # Low noise stage (t < boundary_timestep): use transformer_2 + if boundary_timestep is not None and t < boundary_timestep: + # Low noise stage - always use guidance_high for this stage + current_guidance_scale = guidance_high + if self.transformer_2 is not None: + current_model = self.transformer_2 + elif self.transformer is not None: + # Fallback to transformer if transformer_2 not loaded + current_model = self.transformer + else: + raise RuntimeError("No transformer available for low-noise stage") + else: + # High noise stage - always use guidance_low for this stage + current_guidance_scale = guidance_low + if self.transformer is not None: + current_model = self.transformer + elif self.transformer_2 is not None: + # Fallback to transformer_2 if transformer not loaded + current_model = self.transformer_2 + else: + raise RuntimeError("No transformer available for high-noise stage") + + if self.expand_timesteps and latent_condition is not None: + # I2V mode: blend condition with latents using mask + latent_model_input = (1 - first_frame_mask) * latent_condition + first_frame_mask * latents + latent_model_input = latent_model_input.to(dtype) + + # Expand timesteps per patch - use floor division to match patch embedding + patch_size = self.transformer_config.patch_size + patch_height = latents.shape[3] // patch_size[1] + patch_width = latents.shape[4] // patch_size[2] + + # Create mask at patch resolution (same as hidden states sequence length) + patch_mask = first_frame_mask[:, :, :, :: patch_size[1], :: patch_size[2]] + patch_mask = patch_mask[:, :, :, :patch_height, :patch_width] # Ensure correct dimensions + temp_ts = (patch_mask[0][0] * t).flatten() + timestep = temp_ts.unsqueeze(0).expand(latents.shape[0], -1) + else: + # T2V mode: standard forward + latent_model_input = latents.to(dtype) + timestep = t.expand(latents.shape[0]) + + do_true_cfg = current_guidance_scale > 1.0 and negative_prompt_embeds is not None + positive_kwargs = { + "hidden_states": latent_model_input, + "timestep": timestep, + "encoder_hidden_states": prompt_embeds, + "attention_kwargs": attention_kwargs, + "return_dict": False, + "current_model": current_model, + } + if do_true_cfg: + negative_kwargs = { + "hidden_states": latent_model_input, + "timestep": timestep, + "encoder_hidden_states": negative_prompt_embeds, + "attention_kwargs": attention_kwargs, + "return_dict": False, + "current_model": current_model, + } + else: + negative_kwargs = None + + noise_pred = self.predict_noise_maybe_with_cfg( + do_true_cfg=do_true_cfg, + true_cfg_scale=current_guidance_scale, + positive_kwargs=positive_kwargs, + negative_kwargs=negative_kwargs, + cfg_normalize=False, + ) + + latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg) + pbar.update() + + return latents + def forward( self, req: OmniDiffusionRequest, @@ -635,90 +731,19 @@ class Wan22Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin, DiffusionPipe if DEBUG_PERF: _t_denoise_start = time.perf_counter() - with self.progress_bar(total=len(timesteps)) as pbar: - for t in timesteps: - self._current_timestep = t - - # Select model based on timestep and boundary_ratio - # High noise stage (t >= boundary_timestep): use transformer - # Low noise stage (t < boundary_timestep): use transformer_2 - if boundary_timestep is not None and t < boundary_timestep: - # Low noise stage - always use guidance_high for this stage - current_guidance_scale = guidance_high - if self.transformer_2 is not None: - current_model = self.transformer_2 - elif self.transformer is not None: - # Fallback to transformer if transformer_2 not loaded - current_model = self.transformer - else: - raise RuntimeError("No transformer available for low-noise stage") - else: - # High noise stage - always use guidance_low for this stage - current_guidance_scale = guidance_low - if self.transformer is not None: - current_model = self.transformer - elif self.transformer_2 is not None: - # Fallback to transformer_2 if transformer not loaded - current_model = self.transformer_2 - else: - raise RuntimeError("No transformer available for high-noise stage") - - if self.expand_timesteps and latent_condition is not None: - # I2V mode: blend condition with latents using mask - latent_model_input = (1 - first_frame_mask) * latent_condition + first_frame_mask * latents - latent_model_input = latent_model_input.to(dtype) - - # Expand timesteps per patch - use floor division to match patch embedding - patch_size = self.transformer_config.patch_size - num_latent_frames = latents.shape[2] - patch_height = latents.shape[3] // patch_size[1] - patch_width = latents.shape[4] // patch_size[2] - - # Create mask at patch resolution (same as hidden states sequence length) - patch_mask = first_frame_mask[:, :, :, :: patch_size[1], :: patch_size[2]] - patch_mask = patch_mask[:, :, :, :patch_height, :patch_width] # Ensure correct dimensions - temp_ts = (patch_mask[0][0] * t).flatten() - timestep = temp_ts.unsqueeze(0).expand(latents.shape[0], -1) - else: - # T2V mode: standard forward - latent_model_input = latents.to(dtype) - timestep = t.expand(latents.shape[0]) - - do_true_cfg = current_guidance_scale > 1.0 and negative_prompt_embeds is not None - # Prepare kwargs for positive and negative predictions - positive_kwargs = { - "hidden_states": latent_model_input, - "timestep": timestep, - "encoder_hidden_states": prompt_embeds, - "attention_kwargs": attention_kwargs, - "return_dict": False, - "current_model": current_model, - } - if do_true_cfg: - negative_kwargs = { - "hidden_states": latent_model_input, - "timestep": timestep, - "encoder_hidden_states": negative_prompt_embeds, - "attention_kwargs": attention_kwargs, - "return_dict": False, - "current_model": current_model, - } - else: - negative_kwargs = None - - # Predict noise with automatic CFG parallel handling - noise_pred = self.predict_noise_maybe_with_cfg( - do_true_cfg=do_true_cfg, - true_cfg_scale=current_guidance_scale, - positive_kwargs=positive_kwargs, - negative_kwargs=negative_kwargs, - cfg_normalize=False, - ) - - # Compute the previous noisy sample x_t -> x_t-1 with automatic CFG sync - latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg) - - pbar.update() + latents = self.diffuse( + latents=latents, + timesteps=timesteps, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + guidance_low=guidance_low, + guidance_high=guidance_high, + boundary_timestep=boundary_timestep, + dtype=dtype, + attention_kwargs=attention_kwargs, + latent_condition=latent_condition, + first_frame_mask=first_frame_mask, + ) # Wan2.2 is prone to out of memory errors when predicting large videos # so we empty the cache here to avoid OOM before vae decoding. diff --git a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py index ca042ca22..24e8965a3 100644 --- a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py +++ b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py @@ -289,6 +289,82 @@ class Wan22I2VPipeline( def current_timestep(self): return self._current_timestep + def diffuse( + self, + latents: torch.Tensor, + timesteps: torch.Tensor, + prompt_embeds: torch.Tensor, + negative_prompt_embeds: torch.Tensor | None, + image_embeds: torch.Tensor | None, + guidance_low: float, + guidance_high: float, + boundary_timestep: float | None, + dtype: torch.dtype, + attention_kwargs: dict[str, Any], + condition: torch.Tensor, + first_frame_mask: torch.Tensor, + ) -> torch.Tensor: + with self.progress_bar(total=len(timesteps)) as pbar: + for t in timesteps: + self._current_timestep = t + + # Select model and guidance scale based on timestep + current_model = self.transformer + current_guidance_scale = guidance_low + if boundary_timestep is not None and t < boundary_timestep and self.transformer_2 is not None: + current_model = self.transformer_2 + current_guidance_scale = guidance_high + + # Prepare latent input + if self.expand_timesteps: + # TI2V-5B style: blend condition with latents using mask + latent_model_input = (1 - first_frame_mask) * condition + first_frame_mask * latents + latent_model_input = latent_model_input.to(dtype) + + # Expand timesteps for each patch + temp_ts = (first_frame_mask[0][0][:, ::2, ::2] * t).flatten() + timestep = temp_ts.unsqueeze(0).expand(latents.shape[0], -1) + else: + # Wan2.1 style: concatenate condition with latents + latent_model_input = torch.cat([latents, condition], dim=1).to(dtype) + timestep = t.expand(latents.shape[0]) + + do_true_cfg = current_guidance_scale > 1.0 and negative_prompt_embeds is not None + positive_kwargs = { + "hidden_states": latent_model_input, + "timestep": timestep, + "encoder_hidden_states": prompt_embeds, + "encoder_hidden_states_image": image_embeds, + "attention_kwargs": attention_kwargs, + "return_dict": False, + "current_model": current_model, + } + if do_true_cfg: + negative_kwargs = { + "hidden_states": latent_model_input, + "timestep": timestep, + "encoder_hidden_states": negative_prompt_embeds, + "encoder_hidden_states_image": image_embeds, + "attention_kwargs": attention_kwargs, + "return_dict": False, + "current_model": current_model, + } + else: + negative_kwargs = None + + noise_pred = self.predict_noise_maybe_with_cfg( + do_true_cfg=do_true_cfg, + true_cfg_scale=current_guidance_scale, + positive_kwargs=positive_kwargs, + negative_kwargs=negative_kwargs, + cfg_normalize=False, + ) + + latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg) + pbar.update() + + return latents + def encode_image( self, image: PIL.Image.Image | list[PIL.Image.Image], @@ -522,68 +598,20 @@ class Wan22I2VPipeline( if DEBUG_PERF: _t_denoise_start = time.perf_counter() - with self.progress_bar(total=len(timesteps)) as pbar: - for t in timesteps: - self._current_timestep = t - - # Select model and guidance scale based on timestep - current_model = self.transformer - current_guidance_scale = guidance_low - if boundary_timestep is not None and t < boundary_timestep and self.transformer_2 is not None: - current_model = self.transformer_2 - current_guidance_scale = guidance_high - - # Prepare latent input - if self.expand_timesteps: - # TI2V-5B style: blend condition with latents using mask - latent_model_input = (1 - first_frame_mask) * condition + first_frame_mask * latents - latent_model_input = latent_model_input.to(dtype) - - # Expand timesteps for each patch - temp_ts = (first_frame_mask[0][0][:, ::2, ::2] * t).flatten() - timestep = temp_ts.unsqueeze(0).expand(latents.shape[0], -1) - else: - # Wan2.1 style: concatenate condition with latents - latent_model_input = torch.cat([latents, condition], dim=1).to(dtype) - timestep = t.expand(latents.shape[0]) - - do_true_cfg = current_guidance_scale > 1.0 and negative_prompt_embeds is not None - # Prepare kwargs for positive and negative predictions - positive_kwargs = { - "hidden_states": latent_model_input, - "timestep": timestep, - "encoder_hidden_states": prompt_embeds, - "encoder_hidden_states_image": image_embeds, - "attention_kwargs": attention_kwargs, - "return_dict": False, - "current_model": current_model, - } - if do_true_cfg: - negative_kwargs = { - "hidden_states": latent_model_input, - "timestep": timestep, - "encoder_hidden_states": negative_prompt_embeds, - "encoder_hidden_states_image": image_embeds, - "attention_kwargs": attention_kwargs, - "return_dict": False, - "current_model": current_model, - } - else: - negative_kwargs = None - - # Predict noise with automatic CFG parallel handling - noise_pred = self.predict_noise_maybe_with_cfg( - do_true_cfg=do_true_cfg, - true_cfg_scale=current_guidance_scale, - positive_kwargs=positive_kwargs, - negative_kwargs=negative_kwargs, - cfg_normalize=False, - ) - - # Compute the previous noisy sample x_t -> x_t-1 with automatic CFG sync - latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg) - - pbar.update() + latents = self.diffuse( + latents=latents, + timesteps=timesteps, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + image_embeds=image_embeds, + guidance_low=guidance_low, + guidance_high=guidance_high, + boundary_timestep=boundary_timestep, + dtype=dtype, + attention_kwargs=attention_kwargs, + condition=condition, + first_frame_mask=first_frame_mask, + ) # Wan2.2 is prone to out of memory errors when predicting large videos # so we empty the cache here to avoid OOM before vae decoding. diff --git a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_ti2v.py b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_ti2v.py index 6cbd6d2d6..dba76ba8a 100644 --- a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_ti2v.py +++ b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_ti2v.py @@ -235,6 +235,77 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin, Progress def current_timestep(self): return self._current_timestep + def diffuse( + self, + latents: torch.Tensor, + timesteps: torch.Tensor, + prompt_embeds: torch.Tensor, + negative_prompt_embeds: torch.Tensor | None, + guidance_scale: float, + dtype: torch.dtype, + attention_kwargs: dict[str, Any], + num_latent_frames: int, + latent_height: int, + latent_width: int, + latent_condition: torch.Tensor | None = None, + first_frame_mask: torch.Tensor | None = None, + ) -> torch.Tensor: + with self.progress_bar(total=len(timesteps)) as pbar: + for t in timesteps: + self._current_timestep = t + + # Prepare latent input + if latent_condition is not None: + # I2V mode: blend condition with latents using mask + latent_model_input = (1 - first_frame_mask) * latent_condition + first_frame_mask * latents + latent_model_input = latent_model_input.to(dtype) + + # Expand timesteps for each patch (TI2V style) + temp_ts = (first_frame_mask[0][0][:, ::2, ::2] * t).flatten() + timestep = temp_ts.unsqueeze(0).expand(latents.shape[0], -1) + else: + # T2V mode: use latents directly + latent_model_input = latents.to(dtype) + + # Expand timesteps for TI2V model architecture + mask = torch.ones(1, 1, num_latent_frames, latent_height, latent_width, device=latents.device) + temp_ts = (mask[0][0][:, ::2, ::2] * t).flatten() + timestep = temp_ts.unsqueeze(0).expand(latents.shape[0], -1) + + do_true_cfg = guidance_scale > 1.0 and negative_prompt_embeds is not None + positive_kwargs = { + "hidden_states": latent_model_input, + "timestep": timestep, + "encoder_hidden_states": prompt_embeds, + "attention_kwargs": attention_kwargs, + "return_dict": False, + "current_model": self.transformer, + } + if do_true_cfg: + negative_kwargs = { + "hidden_states": latent_model_input, + "timestep": timestep, + "encoder_hidden_states": negative_prompt_embeds, + "attention_kwargs": attention_kwargs, + "return_dict": False, + "current_model": self.transformer, + } + else: + negative_kwargs = None + + noise_pred = self.predict_noise_maybe_with_cfg( + do_true_cfg=do_true_cfg, + true_cfg_scale=guidance_scale, + positive_kwargs=positive_kwargs, + negative_kwargs=negative_kwargs, + cfg_normalize=False, + ) + + latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg) + pbar.update() + + return latents + def forward( self, req: OmniDiffusionRequest, @@ -405,64 +476,20 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin, Progress if attention_kwargs is None: attention_kwargs = {} - # Denoising loop - with self.progress_bar(total=len(timesteps)) as pbar: - for t in timesteps: - self._current_timestep = t - - # Prepare latent input - if latent_condition is not None: - # I2V mode: blend condition with latents using mask - latent_model_input = (1 - first_frame_mask) * latent_condition + first_frame_mask * latents - latent_model_input = latent_model_input.to(dtype) - - # Expand timesteps for each patch (TI2V style) - temp_ts = (first_frame_mask[0][0][:, ::2, ::2] * t).flatten() - timestep = temp_ts.unsqueeze(0).expand(latents.shape[0], -1) - else: - # T2V mode: use latents directly - latent_model_input = latents.to(dtype) - - # Expand timesteps for TI2V model architecture - mask = torch.ones(1, 1, num_latent_frames, latent_height, latent_width, device=device) - temp_ts = (mask[0][0][:, ::2, ::2] * t).flatten() - timestep = temp_ts.unsqueeze(0).expand(latents.shape[0], -1) - - do_true_cfg = guidance_scale > 1.0 and negative_prompt_embeds is not None - # Prepare kwargs for positive and negative predictions - positive_kwargs = { - "hidden_states": latent_model_input, - "timestep": timestep, - "encoder_hidden_states": prompt_embeds, - "attention_kwargs": attention_kwargs, - "return_dict": False, - "current_model": self.transformer, - } - if do_true_cfg: - negative_kwargs = { - "hidden_states": latent_model_input, - "timestep": timestep, - "encoder_hidden_states": negative_prompt_embeds, - "attention_kwargs": attention_kwargs, - "return_dict": False, - "current_model": self.transformer, - } - else: - negative_kwargs = None - - # Predict noise with automatic CFG parallel handling - noise_pred = self.predict_noise_maybe_with_cfg( - do_true_cfg=do_true_cfg, - true_cfg_scale=guidance_scale, - positive_kwargs=positive_kwargs, - negative_kwargs=negative_kwargs, - cfg_normalize=False, - ) - - # Compute the previous noisy sample x_t -> x_t-1 with automatic CFG sync - latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg) - - pbar.update() + latents = self.diffuse( + latents=latents, + timesteps=timesteps, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + guidance_scale=guidance_scale, + dtype=dtype, + attention_kwargs=attention_kwargs, + num_latent_frames=num_latent_frames, + latent_height=latent_height, + latent_width=latent_width, + latent_condition=latent_condition, + first_frame_mask=first_frame_mask, + ) # Wan2.2 is prone to out of memory errors when predicting large videos # so we empty the cache here to avoid OOM before vae decoding. diff --git a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_vace.py b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_vace.py index b661108cc..11408e2d2 100644 --- a/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_vace.py +++ b/vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_vace.py @@ -176,6 +176,62 @@ class Wan22VACEPipeline(Wan22Pipeline, SupportImageInput): """Build VACE transformer directly from config dict.""" return create_vace_transformer_from_config(config) + def diffuse( + self, + latents: torch.Tensor, + timesteps: torch.Tensor, + prompt_embeds: torch.Tensor, + negative_prompt_embeds: torch.Tensor | None, + guidance_scale: float, + dtype: torch.dtype, + attention_kwargs: dict[str, object], + vace_context: torch.Tensor | None, + vace_context_scale: float, + ) -> torch.Tensor: + with self.progress_bar(total=len(timesteps)) as pbar: + for t in timesteps: + self._current_timestep = t + latent_model_input = latents.to(dtype) + timestep = t.expand(latents.shape[0]) + + do_true_cfg = guidance_scale > 1.0 and negative_prompt_embeds is not None + + positive_kwargs = { + "hidden_states": latent_model_input, + "timestep": timestep, + "encoder_hidden_states": prompt_embeds, + "attention_kwargs": attention_kwargs, + "vace_context": vace_context, + "vace_context_scale": vace_context_scale, + "return_dict": False, + } + negative_kwargs = ( + { + "hidden_states": latent_model_input, + "timestep": timestep, + "encoder_hidden_states": negative_prompt_embeds, + "attention_kwargs": attention_kwargs, + "vace_context": vace_context, + "vace_context_scale": vace_context_scale, + "return_dict": False, + } + if do_true_cfg + else None + ) + + noise_pred = self.predict_noise_maybe_with_cfg( + do_true_cfg=do_true_cfg, + true_cfg_scale=guidance_scale, + positive_kwargs=positive_kwargs, + negative_kwargs=negative_kwargs, + cfg_normalize=False, + ) + + latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg) + pbar.update() + + return latents + def check_inputs( self, prompt, @@ -572,48 +628,17 @@ class Wan22VACEPipeline(Wan22Pipeline, SupportImageInput): timesteps = self.scheduler.timesteps self._num_timesteps = len(timesteps) - # Denoising loop - with self.progress_bar(total=len(timesteps)) as pbar: - for t in timesteps: - self._current_timestep = t - latent_model_input = latents.to(dtype) - timestep = t.expand(latents.shape[0]) - - do_true_cfg = guidance_scale > 1.0 and negative_prompt_embeds is not None - - positive_kwargs = { - "hidden_states": latent_model_input, - "timestep": timestep, - "encoder_hidden_states": prompt_embeds, - "attention_kwargs": attention_kwargs, - "vace_context": vace_context, - "vace_context_scale": vace_context_scale, - "return_dict": False, - } - negative_kwargs = ( - { - "hidden_states": latent_model_input, - "timestep": timestep, - "encoder_hidden_states": negative_prompt_embeds, - "attention_kwargs": attention_kwargs, - "vace_context": vace_context, - "vace_context_scale": vace_context_scale, - "return_dict": False, - } - if do_true_cfg - else None - ) - - noise_pred = self.predict_noise_maybe_with_cfg( - do_true_cfg=do_true_cfg, - true_cfg_scale=guidance_scale, - positive_kwargs=positive_kwargs, - negative_kwargs=negative_kwargs, - cfg_normalize=False, - ) - - latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg) - pbar.update() + latents = self.diffuse( + latents=latents, + timesteps=timesteps, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + guidance_scale=guidance_scale, + dtype=dtype, + attention_kwargs=attention_kwargs, + vace_context=vace_context, + vace_context_scale=vace_context_scale, + ) self._current_timestep = None