[Refactor] refactor wan2.2 diffuse && add ut (#2672)
Signed-off-by: bjf-frz <frz123db@gmail.com>
This commit is contained in:
1 parent
6b7be88cd5
commit
18ac679763
10 files changed
+1028
-246
No files matched your search
@@ -0,0 +1,80 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import contextmanager
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
|
||||
class StubTransformer(nn.Module):
|
||||
def __init__(self, *, name: str = "transformer", in_channels: int = 4, out_channels: int = 4) -> None:
|
||||
super().__init__()
|
||||
self.name = name
|
||||
self.config = SimpleNamespace(
|
||||
patch_size=(1, 2, 2),
|
||||
in_channels=in_channels,
|
||||
out_channels=out_channels,
|
||||
image_dim=None,
|
||||
)
|
||||
|
||||
@property
|
||||
def dtype(self) -> torch.dtype:
|
||||
return torch.float32
|
||||
|
||||
def forward(self, **kwargs):
|
||||
hidden_states = kwargs["hidden_states"]
|
||||
return (torch.zeros_like(hidden_states[:, : self.config.out_channels]),)
|
||||
|
||||
|
||||
class StubScheduler:
|
||||
def __init__(self, timesteps: list[int]) -> None:
|
||||
self.timesteps = torch.tensor(timesteps, dtype=torch.int64)
|
||||
self.config = SimpleNamespace(num_train_timesteps=1000)
|
||||
self.set_timesteps_calls: list[tuple[int, torch.device]] = []
|
||||
|
||||
def set_timesteps(self, num_steps: int, device: torch.device) -> None:
|
||||
self.set_timesteps_calls.append((num_steps, device))
|
||||
|
||||
|
||||
class StubVAE:
|
||||
dtype = torch.float32
|
||||
|
||||
def __init__(self, z_dim: int = 4) -> None:
|
||||
self.config = SimpleNamespace(
|
||||
z_dim=z_dim,
|
||||
scale_factor_temporal=4,
|
||||
scale_factor_spatial=8,
|
||||
latents_mean=[0.0] * z_dim,
|
||||
latents_std=[1.0] * z_dim,
|
||||
)
|
||||
|
||||
def encode(self, video: torch.Tensor):
|
||||
latent_frames = (video.shape[2] + self.config.scale_factor_temporal - 1) // self.config.scale_factor_temporal
|
||||
latent_height = video.shape[-2] // self.config.scale_factor_spatial
|
||||
latent_width = video.shape[-1] // self.config.scale_factor_spatial
|
||||
latents = torch.ones(
|
||||
video.shape[0],
|
||||
self.config.z_dim,
|
||||
latent_frames,
|
||||
latent_height,
|
||||
latent_width,
|
||||
dtype=video.dtype,
|
||||
device=video.device,
|
||||
)
|
||||
return SimpleNamespace(latents=latents)
|
||||
|
||||
def decode(self, latents: torch.Tensor, return_dict: bool = False):
|
||||
del return_dict
|
||||
return (latents,)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def noop_progress_bar(*args, **kwargs):
|
||||
del args, kwargs
|
||||
|
||||
class Bar:
|
||||
def update(self) -> None:
|
||||
return None
|
||||
|
||||
yield Bar()
|
||||
@@ -0,0 +1,126 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
from torch import nn
|
||||
|
||||
from tests.diffusion.models.wan2_2.conftest import StubTransformer, StubVAE, noop_progress_bar
|
||||
from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2_i2v import (
|
||||
Wan22I2VPipeline,
|
||||
get_wan22_i2v_pre_process_func,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
|
||||
def _make_i2v_pipeline(*, expand_timesteps: bool) -> Wan22I2VPipeline:
|
||||
pipeline = object.__new__(Wan22I2VPipeline)
|
||||
nn.Module.__init__(pipeline)
|
||||
pipeline.device = torch.device("cpu")
|
||||
pipeline.transformer = StubTransformer(name="high", in_channels=8, out_channels=4)
|
||||
pipeline.transformer_2 = StubTransformer(name="low", in_channels=8, out_channels=4)
|
||||
pipeline.vae = StubVAE(z_dim=4)
|
||||
pipeline.vae_scale_factor_temporal = 4
|
||||
pipeline.vae_scale_factor_spatial = 8
|
||||
pipeline.expand_timesteps = expand_timesteps
|
||||
pipeline.progress_bar = noop_progress_bar
|
||||
return pipeline
|
||||
|
||||
|
||||
def test_i2v_preprocess_requires_image_and_resizes_to_480p_aspect() -> None:
|
||||
preprocess = get_wan22_i2v_pre_process_func(SimpleNamespace())
|
||||
request = SimpleNamespace(
|
||||
prompts=[{"prompt": "p", "multi_modal_data": {"image": Image.new("RGB", (320, 160), "red")}}],
|
||||
sampling_params=SimpleNamespace(height=None, width=None),
|
||||
)
|
||||
|
||||
result = preprocess(request)
|
||||
prompt = result.prompts[0]
|
||||
|
||||
assert result.sampling_params.height == 432
|
||||
assert result.sampling_params.width == 880
|
||||
assert prompt["multi_modal_data"]["image"].size == (880, 432)
|
||||
assert prompt["additional_information"]["preprocessed_image"].shape[-2:] == (432, 880)
|
||||
|
||||
missing_image = SimpleNamespace(
|
||||
prompts=[{"prompt": "p", "multi_modal_data": {}}],
|
||||
sampling_params=SimpleNamespace(height=None, width=None),
|
||||
)
|
||||
with pytest.raises(ValueError, match="No image is provided"):
|
||||
preprocess(missing_image)
|
||||
|
||||
|
||||
def test_i2v_diffuse_selects_stage_guidance_and_expands_timesteps() -> None:
|
||||
pipeline = _make_i2v_pipeline(expand_timesteps=True)
|
||||
latents = torch.zeros(1, 4, 2, 4, 4)
|
||||
condition = torch.ones_like(latents)
|
||||
first_frame_mask = torch.ones(1, 1, 2, 4, 4)
|
||||
first_frame_mask[:, :, 0] = 0
|
||||
timesteps = torch.tensor([900, 100])
|
||||
|
||||
calls = []
|
||||
|
||||
def fake_predict_noise_maybe_with_cfg(**kwargs):
|
||||
positive = kwargs["positive_kwargs"]
|
||||
calls.append(
|
||||
{
|
||||
"model": positive["current_model"].name,
|
||||
"scale": kwargs["true_cfg_scale"],
|
||||
"timestep_shape": tuple(positive["timestep"].shape),
|
||||
"timestep_values": positive["timestep"].clone(),
|
||||
"hidden_states": positive["hidden_states"].clone(),
|
||||
}
|
||||
)
|
||||
return torch.ones_like(latents)
|
||||
|
||||
pipeline.predict_noise_maybe_with_cfg = fake_predict_noise_maybe_with_cfg # type: ignore[method-assign]
|
||||
pipeline.scheduler_step_maybe_with_cfg = lambda noise, t, current, cfg: current + noise # type: ignore[method-assign]
|
||||
|
||||
result = pipeline.diffuse(
|
||||
latents=latents,
|
||||
timesteps=timesteps,
|
||||
prompt_embeds=torch.zeros(1, 2, 3),
|
||||
negative_prompt_embeds=None,
|
||||
image_embeds=None,
|
||||
guidance_low=1.0,
|
||||
guidance_high=2.0,
|
||||
boundary_timestep=500.0,
|
||||
dtype=torch.float32,
|
||||
attention_kwargs={},
|
||||
condition=condition,
|
||||
first_frame_mask=first_frame_mask,
|
||||
)
|
||||
|
||||
assert [call["model"] for call in calls] == ["high", "low"]
|
||||
assert [call["scale"] for call in calls] == [1.0, 2.0]
|
||||
assert calls[0]["timestep_shape"] == (1, 8)
|
||||
timestep_dtype = calls[0]["timestep_values"].dtype
|
||||
torch.testing.assert_close(calls[0]["timestep_values"][0, :4], torch.zeros(4, dtype=timestep_dtype))
|
||||
torch.testing.assert_close(calls[0]["timestep_values"][0, 4:], torch.full((4,), 900, dtype=timestep_dtype))
|
||||
torch.testing.assert_close(calls[0]["hidden_states"][:, :, 0], torch.ones(1, 4, 4, 4))
|
||||
torch.testing.assert_close(result, torch.full_like(latents, 2.0))
|
||||
|
||||
|
||||
def test_i2v_prepare_latents_builds_expand_condition_and_first_frame_mask() -> None:
|
||||
pipeline = _make_i2v_pipeline(expand_timesteps=True)
|
||||
latents, condition, first_frame_mask = pipeline.prepare_latents(
|
||||
image=torch.zeros(1, 3, 16, 16),
|
||||
batch_size=1,
|
||||
num_channels_latents=4,
|
||||
height=16,
|
||||
width=16,
|
||||
num_frames=5,
|
||||
dtype=torch.float32,
|
||||
device=torch.device("cpu"),
|
||||
generator=torch.Generator(device="cpu").manual_seed(0),
|
||||
)
|
||||
|
||||
assert latents.shape == (1, 4, 2, 2, 2)
|
||||
assert condition.shape == (1, 4, 1, 2, 2)
|
||||
assert first_frame_mask.shape == (1, 1, 2, 2, 2)
|
||||
assert first_frame_mask[:, :, 0].sum() == 0
|
||||
assert first_frame_mask[:, :, 1].sum() == 4
|
||||
@@ -0,0 +1,155 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from contextlib import contextmanager
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import Wan22Pipeline
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
|
||||
class _StubTransformer(nn.Module):
|
||||
@property
|
||||
def dtype(self) -> torch.dtype:
|
||||
return torch.float32
|
||||
|
||||
|
||||
class _StubScheduler:
|
||||
def __init__(self, timesteps: list[int]) -> None:
|
||||
self.timesteps = torch.tensor(timesteps, dtype=torch.int64)
|
||||
self.config = SimpleNamespace(num_train_timesteps=1000)
|
||||
self.set_timesteps_calls: list[tuple[int, torch.device]] = []
|
||||
|
||||
def set_timesteps(self, num_steps: int, device: torch.device) -> None:
|
||||
self.set_timesteps_calls.append((num_steps, device))
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _noop_progress_bar(*args, **kwargs):
|
||||
del args, kwargs
|
||||
|
||||
class _Bar:
|
||||
def update(self) -> None:
|
||||
return None
|
||||
|
||||
yield _Bar()
|
||||
|
||||
|
||||
def _make_pipeline() -> Wan22Pipeline:
|
||||
pipeline = object.__new__(Wan22Pipeline)
|
||||
nn.Module.__init__(pipeline)
|
||||
pipeline.device = torch.device("cpu")
|
||||
pipeline.transformer = _StubTransformer()
|
||||
pipeline.transformer_2 = None
|
||||
pipeline.transformer_config = SimpleNamespace(patch_size=(1, 2, 2), in_channels=4, out_channels=4)
|
||||
pipeline.scheduler = _StubScheduler([9, 5])
|
||||
pipeline.od_config = SimpleNamespace(flow_shift=5.0)
|
||||
pipeline._sample_solver = "unipc"
|
||||
pipeline._flow_shift = 5.0
|
||||
pipeline.vae_scale_factor_temporal = 4
|
||||
pipeline.vae_scale_factor_spatial = 8
|
||||
pipeline.boundary_ratio = 0.875
|
||||
pipeline.expand_timesteps = False
|
||||
pipeline._guidance_scale = None
|
||||
pipeline._guidance_scale_2 = None
|
||||
pipeline._num_timesteps = None
|
||||
pipeline._current_timestep = None
|
||||
pipeline.check_inputs = lambda **kwargs: None
|
||||
pipeline.prepare_latents = lambda **kwargs: torch.zeros((1, 4, 1, 8, 8), dtype=torch.float32)
|
||||
pipeline.progress_bar = _noop_progress_bar
|
||||
return pipeline
|
||||
|
||||
|
||||
def test_forward_delegates_denoising_to_diffuse(monkeypatch) -> None:
|
||||
pipeline = _make_pipeline()
|
||||
|
||||
prompt_embeds = torch.randn(1, 8)
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def _fake_diffuse(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return kwargs["latents"] + 1
|
||||
|
||||
pipeline.diffuse = _fake_diffuse # type: ignore[method-assign]
|
||||
|
||||
req = SimpleNamespace(
|
||||
prompts=["prompt"],
|
||||
sampling_params=SimpleNamespace(
|
||||
height=None,
|
||||
width=None,
|
||||
num_frames=1,
|
||||
num_inference_steps=2,
|
||||
guidance_scale_provided=False,
|
||||
guidance_scale=None,
|
||||
guidance_scale_2=None,
|
||||
boundary_ratio=None,
|
||||
generator=None,
|
||||
seed=None,
|
||||
num_outputs_per_prompt=1,
|
||||
max_sequence_length=32,
|
||||
latents=None,
|
||||
extra_args={},
|
||||
),
|
||||
)
|
||||
|
||||
output = pipeline.forward(req, prompt_embeds=prompt_embeds, output_type="latent", guidance_scale=1.0)
|
||||
|
||||
assert torch.equal(output.output, torch.ones((1, 4, 1, 8, 8)))
|
||||
assert torch.equal(captured["timesteps"], pipeline.scheduler.timesteps)
|
||||
assert captured["guidance_low"] == 1.0
|
||||
assert captured["guidance_high"] == 1.0
|
||||
assert captured["boundary_timestep"] == pytest.approx(875.0)
|
||||
assert captured["latent_condition"] is None
|
||||
assert captured["first_frame_mask"] is None
|
||||
assert pipeline.scheduler.set_timesteps_calls == [(2, torch.device("cpu"))]
|
||||
|
||||
|
||||
def test_diffuse_runs_prediction_and_scheduler_for_each_timestep() -> None:
|
||||
pipeline = _make_pipeline()
|
||||
latents = torch.zeros((1, 1, 1, 2, 2), dtype=torch.float32)
|
||||
timesteps = torch.tensor([7, 3], dtype=torch.int64)
|
||||
prompt_embeds = torch.randn(1, 8)
|
||||
|
||||
predict_calls: list[dict[str, object]] = []
|
||||
scheduler_calls: list[tuple[float, int, float, bool]] = []
|
||||
|
||||
def _fake_predict_noise_maybe_with_cfg(**kwargs):
|
||||
predict_calls.append(kwargs)
|
||||
timestep = kwargs["positive_kwargs"]["timestep"]
|
||||
assert isinstance(timestep, torch.Tensor)
|
||||
return torch.full_like(latents, float(timestep[0].item()))
|
||||
|
||||
def _fake_scheduler_step_maybe_with_cfg(noise_pred, t, current_latents, do_true_cfg):
|
||||
scheduler_calls.append(
|
||||
(float(noise_pred[0, 0, 0, 0, 0]), int(t.item()), float(current_latents.sum()), do_true_cfg)
|
||||
)
|
||||
return current_latents + noise_pred
|
||||
|
||||
pipeline.predict_noise_maybe_with_cfg = _fake_predict_noise_maybe_with_cfg # type: ignore[method-assign]
|
||||
pipeline.scheduler_step_maybe_with_cfg = _fake_scheduler_step_maybe_with_cfg # type: ignore[method-assign]
|
||||
|
||||
result = pipeline.diffuse(
|
||||
latents=latents,
|
||||
timesteps=timesteps,
|
||||
prompt_embeds=prompt_embeds,
|
||||
negative_prompt_embeds=None,
|
||||
guidance_low=1.0,
|
||||
guidance_high=2.0,
|
||||
boundary_timestep=5.0,
|
||||
dtype=torch.float32,
|
||||
attention_kwargs={},
|
||||
)
|
||||
|
||||
assert len(predict_calls) == 2
|
||||
assert predict_calls[0]["true_cfg_scale"] == 1.0
|
||||
assert predict_calls[1]["true_cfg_scale"] == 2.0
|
||||
assert scheduler_calls == [
|
||||
(7.0, 7, 0.0, False),
|
||||
(3.0, 3, 28.0, False),
|
||||
]
|
||||
assert torch.equal(result, torch.full_like(latents, 10.0))
|
||||
@@ -0,0 +1,81 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 as wan22_module
|
||||
from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import (
|
||||
create_transformer_from_config,
|
||||
load_transformer_config,
|
||||
retrieve_latents,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
|
||||
class _LatentDist:
|
||||
def sample(self, generator):
|
||||
assert isinstance(generator, torch.Generator)
|
||||
return torch.tensor([1.0])
|
||||
|
||||
def mode(self):
|
||||
return torch.tensor([2.0])
|
||||
|
||||
|
||||
def test_retrieve_latents_supports_sample_mode_argmax_and_direct_latents() -> None:
|
||||
generator = torch.Generator(device="cpu")
|
||||
|
||||
assert retrieve_latents(SimpleNamespace(latent_dist=_LatentDist()), generator).item() == 1.0
|
||||
assert retrieve_latents(SimpleNamespace(latent_dist=_LatentDist()), sample_mode="argmax").item() == 2.0
|
||||
torch.testing.assert_close(retrieve_latents(SimpleNamespace(latents=torch.tensor([3.0]))), torch.tensor([3.0]))
|
||||
|
||||
|
||||
def test_retrieve_latents_rejects_unknown_encoder_output() -> None:
|
||||
with pytest.raises(AttributeError, match="Could not access latents"):
|
||||
retrieve_latents(SimpleNamespace())
|
||||
|
||||
|
||||
def test_load_transformer_config_reads_local_subfolder_config(tmp_path) -> None:
|
||||
config_dir = tmp_path / "transformer_2"
|
||||
config_dir.mkdir(parents=True)
|
||||
(config_dir / "config.json").write_text(json.dumps({"patch_size": [1, 2, 2], "num_layers": 2}))
|
||||
|
||||
assert load_transformer_config(str(tmp_path), "transformer_2") == {"patch_size": [1, 2, 2], "num_layers": 2}
|
||||
assert load_transformer_config(str(tmp_path), "missing") == {}
|
||||
|
||||
|
||||
def test_create_transformer_from_config_maps_supported_keys(monkeypatch) -> None:
|
||||
captured = {}
|
||||
|
||||
class FakeTransformer:
|
||||
def __init__(self, **kwargs) -> None:
|
||||
captured.update(kwargs)
|
||||
|
||||
monkeypatch.setattr(wan22_module, "WanTransformer3DModel", FakeTransformer)
|
||||
|
||||
transformer = create_transformer_from_config(
|
||||
{
|
||||
"patch_size": [1, 2, 2],
|
||||
"num_attention_heads": 8,
|
||||
"attention_head_dim": 128,
|
||||
"in_channels": 16,
|
||||
"out_channels": 16,
|
||||
"text_dim": 4096,
|
||||
"vace_layers": [0],
|
||||
"ignored": "value",
|
||||
}
|
||||
)
|
||||
|
||||
assert isinstance(transformer, FakeTransformer)
|
||||
assert captured == {
|
||||
"patch_size": (1, 2, 2),
|
||||
"num_attention_heads": 8,
|
||||
"attention_head_dim": 128,
|
||||
"in_channels": 16,
|
||||
"out_channels": 16,
|
||||
"text_dim": 4096,
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
from torch import nn
|
||||
|
||||
from tests.diffusion.models.wan2_2.conftest import StubTransformer, StubVAE, noop_progress_bar
|
||||
from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2_ti2v import (
|
||||
Wan22TI2VPipeline,
|
||||
get_wan22_ti2v_pre_process_func,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
|
||||
def _make_ti2v_pipeline() -> Wan22TI2VPipeline:
|
||||
pipeline = object.__new__(Wan22TI2VPipeline)
|
||||
nn.Module.__init__(pipeline)
|
||||
pipeline.device = torch.device("cpu")
|
||||
pipeline.transformer = StubTransformer(in_channels=4, out_channels=4)
|
||||
pipeline.vae = StubVAE(z_dim=4)
|
||||
pipeline.vae_scale_factor_temporal = 4
|
||||
pipeline.vae_scale_factor_spatial = 8
|
||||
pipeline.progress_bar = noop_progress_bar
|
||||
return pipeline
|
||||
|
||||
|
||||
def test_ti2v_preprocess_uses_720p_area_for_image_condition() -> None:
|
||||
preprocess = get_wan22_ti2v_pre_process_func(SimpleNamespace())
|
||||
request = SimpleNamespace(
|
||||
prompts=[{"prompt": "p", "multi_modal_data": {"image": Image.new("RGB", (320, 160), "blue")}}],
|
||||
sampling_params=SimpleNamespace(height=None, width=None),
|
||||
)
|
||||
|
||||
result = preprocess(request)
|
||||
|
||||
assert result.sampling_params.height == 672
|
||||
assert result.sampling_params.width == 1344
|
||||
assert result.prompts[0]["multi_modal_data"]["image"].size == (1344, 672)
|
||||
assert result.prompts[0]["additional_information"]["preprocessed_image"].shape[-2:] == (672, 1344)
|
||||
|
||||
|
||||
def test_ti2v_diffuse_without_image_condition_expands_patch_timesteps() -> None:
|
||||
pipeline = _make_ti2v_pipeline()
|
||||
latents = torch.zeros(1, 4, 2, 4, 4)
|
||||
calls = []
|
||||
|
||||
def fake_predict_noise_maybe_with_cfg(**kwargs):
|
||||
calls.append(kwargs)
|
||||
return torch.ones_like(latents)
|
||||
|
||||
pipeline.predict_noise_maybe_with_cfg = fake_predict_noise_maybe_with_cfg # type: ignore[method-assign]
|
||||
pipeline.scheduler_step_maybe_with_cfg = lambda noise, t, current, cfg: current + noise # type: ignore[method-assign]
|
||||
|
||||
result = pipeline.diffuse(
|
||||
latents=latents,
|
||||
timesteps=torch.tensor([7]),
|
||||
prompt_embeds=torch.zeros(1, 2, 3),
|
||||
negative_prompt_embeds=torch.zeros(1, 2, 3),
|
||||
guidance_scale=3.0,
|
||||
dtype=torch.float32,
|
||||
attention_kwargs={"a": "b"},
|
||||
num_latent_frames=2,
|
||||
latent_height=4,
|
||||
latent_width=4,
|
||||
)
|
||||
|
||||
positive = calls[0]["positive_kwargs"]
|
||||
assert calls[0]["do_true_cfg"] is True
|
||||
assert positive["timestep"].shape == (1, 8)
|
||||
torch.testing.assert_close(positive["timestep"], torch.full((1, 8), 7, dtype=positive["timestep"].dtype))
|
||||
torch.testing.assert_close(positive["hidden_states"], latents)
|
||||
torch.testing.assert_close(result, torch.ones_like(latents))
|
||||
|
||||
|
||||
def test_ti2v_prepare_i2v_latents_encodes_condition_and_masks_first_frame() -> None:
|
||||
pipeline = _make_ti2v_pipeline()
|
||||
latents, latent_condition, first_frame_mask = pipeline.prepare_i2v_latents(
|
||||
image=torch.zeros(1, 3, 16, 16),
|
||||
batch_size=1,
|
||||
num_channels_latents=4,
|
||||
height=16,
|
||||
width=16,
|
||||
num_frames=5,
|
||||
dtype=torch.float32,
|
||||
device=torch.device("cpu"),
|
||||
generator=None,
|
||||
latents=torch.zeros(1, 4, 2, 2, 2),
|
||||
)
|
||||
|
||||
torch.testing.assert_close(latents, torch.zeros(1, 4, 2, 2, 2))
|
||||
assert latent_condition.shape == (1, 4, 1, 2, 2)
|
||||
assert first_frame_mask[:, :, 0].sum() == 0
|
||||
assert first_frame_mask[:, :, 1].sum() == 4
|
||||
@@ -0,0 +1,137 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
from torch import nn
|
||||
|
||||
from tests.diffusion.models.wan2_2.conftest import StubTransformer, StubVAE, noop_progress_bar
|
||||
from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2_vace import (
|
||||
Wan22VACEPipeline,
|
||||
create_vace_transformer_from_config,
|
||||
get_wan22_vace_pre_process_func,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
|
||||
def _make_vace_pipeline() -> Wan22VACEPipeline:
|
||||
pipeline = object.__new__(Wan22VACEPipeline)
|
||||
nn.Module.__init__(pipeline)
|
||||
pipeline.device = torch.device("cpu")
|
||||
pipeline.transformer = StubTransformer(in_channels=4, out_channels=4)
|
||||
pipeline.transformer_config = pipeline.transformer.config
|
||||
pipeline.vae = StubVAE(z_dim=4)
|
||||
pipeline.vae_scale_factor_temporal = 4
|
||||
pipeline.vae_scale_factor_spatial = 8
|
||||
pipeline.progress_bar = noop_progress_bar
|
||||
return pipeline
|
||||
|
||||
|
||||
def test_vace_preprocess_collects_reference_video_and_mask_inputs() -> None:
|
||||
preprocess = get_wan22_vace_pre_process_func(SimpleNamespace())
|
||||
ref = Image.new("RGB", (320, 160), "green")
|
||||
frame = Image.new("RGB", (64, 64), "black")
|
||||
mask = Image.new("L", (64, 64), 255)
|
||||
request = SimpleNamespace(
|
||||
prompts=[
|
||||
{
|
||||
"prompt": "p",
|
||||
"multi_modal_data": {
|
||||
"image": ref,
|
||||
"video": [frame],
|
||||
"mask": mask,
|
||||
},
|
||||
}
|
||||
],
|
||||
sampling_params=SimpleNamespace(height=None, width=None),
|
||||
)
|
||||
|
||||
result = preprocess(request)
|
||||
additional_info = result.prompts[0]["additional_information"]
|
||||
|
||||
assert result.sampling_params.height == 432
|
||||
assert result.sampling_params.width == 880
|
||||
assert additional_info["reference_images"] == [ref]
|
||||
assert additional_info["source_video"] == [frame]
|
||||
assert additional_info["mask"] == [mask]
|
||||
|
||||
|
||||
def test_create_vace_transformer_from_config_maps_vace_specific_keys(monkeypatch) -> None:
|
||||
captured = {}
|
||||
|
||||
class FakeVACETransformer:
|
||||
def __init__(self, **kwargs) -> None:
|
||||
captured.update(kwargs)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2_vace.WanVACETransformer3DModel",
|
||||
FakeVACETransformer,
|
||||
)
|
||||
|
||||
transformer = create_vace_transformer_from_config(
|
||||
{
|
||||
"patch_size": [1, 2, 2],
|
||||
"in_channels": 96,
|
||||
"out_channels": 16,
|
||||
"vace_layers": [0, 1, 2],
|
||||
"vace_in_channels": 132,
|
||||
"unknown": "ignored",
|
||||
}
|
||||
)
|
||||
|
||||
assert isinstance(transformer, FakeVACETransformer)
|
||||
assert captured == {
|
||||
"patch_size": (1, 2, 2),
|
||||
"in_channels": 96,
|
||||
"out_channels": 16,
|
||||
"vace_layers": [0, 1, 2],
|
||||
"vace_in_channels": 132,
|
||||
}
|
||||
|
||||
|
||||
def test_vace_prepare_masks_encodes_spatial_stride_and_reference_padding() -> None:
|
||||
pipeline = _make_vace_pipeline()
|
||||
mask = torch.ones(1, 3, 5, 16, 16)
|
||||
reference_images = [[torch.zeros(3, 16, 16), torch.zeros(3, 16, 16)]]
|
||||
|
||||
encoded = pipeline.prepare_masks(mask, reference_images)
|
||||
|
||||
assert encoded.shape == (1, 64, 4, 2, 2)
|
||||
torch.testing.assert_close(encoded[:, :, :2], torch.zeros(1, 64, 2, 2, 2))
|
||||
torch.testing.assert_close(encoded[:, :, 2:], torch.ones(1, 64, 2, 2, 2))
|
||||
|
||||
|
||||
def test_vace_diffuse_passes_context_and_scale_to_cfg_branches() -> None:
|
||||
pipeline = _make_vace_pipeline()
|
||||
latents = torch.zeros(1, 4, 1, 2, 2)
|
||||
vace_context = torch.ones(1, 12, 1, 2, 2)
|
||||
calls = []
|
||||
|
||||
def fake_predict_noise_maybe_with_cfg(**kwargs):
|
||||
calls.append(kwargs)
|
||||
return torch.ones_like(latents)
|
||||
|
||||
pipeline.predict_noise_maybe_with_cfg = fake_predict_noise_maybe_with_cfg # type: ignore[method-assign]
|
||||
pipeline.scheduler_step_maybe_with_cfg = lambda noise, t, current, cfg: current + noise # type: ignore[method-assign]
|
||||
|
||||
result = pipeline.diffuse(
|
||||
latents=latents,
|
||||
timesteps=torch.tensor([5]),
|
||||
prompt_embeds=torch.zeros(1, 2, 3),
|
||||
negative_prompt_embeds=torch.zeros(1, 2, 3),
|
||||
guidance_scale=4.0,
|
||||
dtype=torch.float32,
|
||||
attention_kwargs={},
|
||||
vace_context=vace_context,
|
||||
vace_context_scale=0.75,
|
||||
)
|
||||
|
||||
assert calls[0]["do_true_cfg"] is True
|
||||
assert calls[0]["true_cfg_scale"] == 4.0
|
||||
assert calls[0]["positive_kwargs"]["vace_context"] is vace_context
|
||||
assert calls[0]["negative_kwargs"]["vace_context_scale"] == 0.75
|
||||
torch.testing.assert_close(result, torch.ones_like(latents))
|
||||
@@ -392,6 +392,102 @@ class Wan22Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin, DiffusionPipe
|
||||
def current_timestep(self):
|
||||
return self._current_timestep
|
||||
|
||||
def diffuse(
|
||||
self,
|
||||
latents: torch.Tensor,
|
||||
timesteps: torch.Tensor,
|
||||
prompt_embeds: torch.Tensor,
|
||||
negative_prompt_embeds: torch.Tensor | None,
|
||||
guidance_low: float,
|
||||
guidance_high: float,
|
||||
boundary_timestep: float | None,
|
||||
dtype: torch.dtype,
|
||||
attention_kwargs: dict[str, Any],
|
||||
latent_condition: torch.Tensor | None = None,
|
||||
first_frame_mask: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
with self.progress_bar(total=len(timesteps)) as pbar:
|
||||
for t in timesteps:
|
||||
self._current_timestep = t
|
||||
|
||||
# Select model based on timestep and boundary_ratio
|
||||
# High noise stage (t >= boundary_timestep): use transformer
|
||||
# Low noise stage (t < boundary_timestep): use transformer_2
|
||||
if boundary_timestep is not None and t < boundary_timestep:
|
||||
# Low noise stage - always use guidance_high for this stage
|
||||
current_guidance_scale = guidance_high
|
||||
if self.transformer_2 is not None:
|
||||
current_model = self.transformer_2
|
||||
elif self.transformer is not None:
|
||||
# Fallback to transformer if transformer_2 not loaded
|
||||
current_model = self.transformer
|
||||
else:
|
||||
raise RuntimeError("No transformer available for low-noise stage")
|
||||
else:
|
||||
# High noise stage - always use guidance_low for this stage
|
||||
current_guidance_scale = guidance_low
|
||||
if self.transformer is not None:
|
||||
current_model = self.transformer
|
||||
elif self.transformer_2 is not None:
|
||||
# Fallback to transformer_2 if transformer not loaded
|
||||
current_model = self.transformer_2
|
||||
else:
|
||||
raise RuntimeError("No transformer available for high-noise stage")
|
||||
|
||||
if self.expand_timesteps and latent_condition is not None:
|
||||
# I2V mode: blend condition with latents using mask
|
||||
latent_model_input = (1 - first_frame_mask) * latent_condition + first_frame_mask * latents
|
||||
latent_model_input = latent_model_input.to(dtype)
|
||||
|
||||
# Expand timesteps per patch - use floor division to match patch embedding
|
||||
patch_size = self.transformer_config.patch_size
|
||||
patch_height = latents.shape[3] // patch_size[1]
|
||||
patch_width = latents.shape[4] // patch_size[2]
|
||||
|
||||
# Create mask at patch resolution (same as hidden states sequence length)
|
||||
patch_mask = first_frame_mask[:, :, :, :: patch_size[1], :: patch_size[2]]
|
||||
patch_mask = patch_mask[:, :, :, :patch_height, :patch_width] # Ensure correct dimensions
|
||||
temp_ts = (patch_mask[0][0] * t).flatten()
|
||||
timestep = temp_ts.unsqueeze(0).expand(latents.shape[0], -1)
|
||||
else:
|
||||
# T2V mode: standard forward
|
||||
latent_model_input = latents.to(dtype)
|
||||
timestep = t.expand(latents.shape[0])
|
||||
|
||||
do_true_cfg = current_guidance_scale > 1.0 and negative_prompt_embeds is not None
|
||||
positive_kwargs = {
|
||||
"hidden_states": latent_model_input,
|
||||
"timestep": timestep,
|
||||
"encoder_hidden_states": prompt_embeds,
|
||||
"attention_kwargs": attention_kwargs,
|
||||
"return_dict": False,
|
||||
"current_model": current_model,
|
||||
}
|
||||
if do_true_cfg:
|
||||
negative_kwargs = {
|
||||
"hidden_states": latent_model_input,
|
||||
"timestep": timestep,
|
||||
"encoder_hidden_states": negative_prompt_embeds,
|
||||
"attention_kwargs": attention_kwargs,
|
||||
"return_dict": False,
|
||||
"current_model": current_model,
|
||||
}
|
||||
else:
|
||||
negative_kwargs = None
|
||||
|
||||
noise_pred = self.predict_noise_maybe_with_cfg(
|
||||
do_true_cfg=do_true_cfg,
|
||||
true_cfg_scale=current_guidance_scale,
|
||||
positive_kwargs=positive_kwargs,
|
||||
negative_kwargs=negative_kwargs,
|
||||
cfg_normalize=False,
|
||||
)
|
||||
|
||||
latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg)
|
||||
pbar.update()
|
||||
|
||||
return latents
|
||||
|
||||
def forward(
|
||||
self,
|
||||
req: OmniDiffusionRequest,
|
||||
@@ -635,90 +731,19 @@ class Wan22Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin, DiffusionPipe
|
||||
|
||||
if DEBUG_PERF:
|
||||
_t_denoise_start = time.perf_counter()
|
||||
with self.progress_bar(total=len(timesteps)) as pbar:
|
||||
for t in timesteps:
|
||||
self._current_timestep = t
|
||||
|
||||
# Select model based on timestep and boundary_ratio
|
||||
# High noise stage (t >= boundary_timestep): use transformer
|
||||
# Low noise stage (t < boundary_timestep): use transformer_2
|
||||
if boundary_timestep is not None and t < boundary_timestep:
|
||||
# Low noise stage - always use guidance_high for this stage
|
||||
current_guidance_scale = guidance_high
|
||||
if self.transformer_2 is not None:
|
||||
current_model = self.transformer_2
|
||||
elif self.transformer is not None:
|
||||
# Fallback to transformer if transformer_2 not loaded
|
||||
current_model = self.transformer
|
||||
else:
|
||||
raise RuntimeError("No transformer available for low-noise stage")
|
||||
else:
|
||||
# High noise stage - always use guidance_low for this stage
|
||||
current_guidance_scale = guidance_low
|
||||
if self.transformer is not None:
|
||||
current_model = self.transformer
|
||||
elif self.transformer_2 is not None:
|
||||
# Fallback to transformer_2 if transformer not loaded
|
||||
current_model = self.transformer_2
|
||||
else:
|
||||
raise RuntimeError("No transformer available for high-noise stage")
|
||||
|
||||
if self.expand_timesteps and latent_condition is not None:
|
||||
# I2V mode: blend condition with latents using mask
|
||||
latent_model_input = (1 - first_frame_mask) * latent_condition + first_frame_mask * latents
|
||||
latent_model_input = latent_model_input.to(dtype)
|
||||
|
||||
# Expand timesteps per patch - use floor division to match patch embedding
|
||||
patch_size = self.transformer_config.patch_size
|
||||
num_latent_frames = latents.shape[2]
|
||||
patch_height = latents.shape[3] // patch_size[1]
|
||||
patch_width = latents.shape[4] // patch_size[2]
|
||||
|
||||
# Create mask at patch resolution (same as hidden states sequence length)
|
||||
patch_mask = first_frame_mask[:, :, :, :: patch_size[1], :: patch_size[2]]
|
||||
patch_mask = patch_mask[:, :, :, :patch_height, :patch_width] # Ensure correct dimensions
|
||||
temp_ts = (patch_mask[0][0] * t).flatten()
|
||||
timestep = temp_ts.unsqueeze(0).expand(latents.shape[0], -1)
|
||||
else:
|
||||
# T2V mode: standard forward
|
||||
latent_model_input = latents.to(dtype)
|
||||
timestep = t.expand(latents.shape[0])
|
||||
|
||||
do_true_cfg = current_guidance_scale > 1.0 and negative_prompt_embeds is not None
|
||||
# Prepare kwargs for positive and negative predictions
|
||||
positive_kwargs = {
|
||||
"hidden_states": latent_model_input,
|
||||
"timestep": timestep,
|
||||
"encoder_hidden_states": prompt_embeds,
|
||||
"attention_kwargs": attention_kwargs,
|
||||
"return_dict": False,
|
||||
"current_model": current_model,
|
||||
}
|
||||
if do_true_cfg:
|
||||
negative_kwargs = {
|
||||
"hidden_states": latent_model_input,
|
||||
"timestep": timestep,
|
||||
"encoder_hidden_states": negative_prompt_embeds,
|
||||
"attention_kwargs": attention_kwargs,
|
||||
"return_dict": False,
|
||||
"current_model": current_model,
|
||||
}
|
||||
else:
|
||||
negative_kwargs = None
|
||||
|
||||
# Predict noise with automatic CFG parallel handling
|
||||
noise_pred = self.predict_noise_maybe_with_cfg(
|
||||
do_true_cfg=do_true_cfg,
|
||||
true_cfg_scale=current_guidance_scale,
|
||||
positive_kwargs=positive_kwargs,
|
||||
negative_kwargs=negative_kwargs,
|
||||
cfg_normalize=False,
|
||||
)
|
||||
|
||||
# Compute the previous noisy sample x_t -> x_t-1 with automatic CFG sync
|
||||
latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg)
|
||||
|
||||
pbar.update()
|
||||
latents = self.diffuse(
|
||||
latents=latents,
|
||||
timesteps=timesteps,
|
||||
prompt_embeds=prompt_embeds,
|
||||
negative_prompt_embeds=negative_prompt_embeds,
|
||||
guidance_low=guidance_low,
|
||||
guidance_high=guidance_high,
|
||||
boundary_timestep=boundary_timestep,
|
||||
dtype=dtype,
|
||||
attention_kwargs=attention_kwargs,
|
||||
latent_condition=latent_condition,
|
||||
first_frame_mask=first_frame_mask,
|
||||
)
|
||||
|
||||
# Wan2.2 is prone to out of memory errors when predicting large videos
|
||||
# so we empty the cache here to avoid OOM before vae decoding.
|
||||
|
||||
@@ -289,6 +289,82 @@ class Wan22I2VPipeline(
|
||||
def current_timestep(self):
|
||||
return self._current_timestep
|
||||
|
||||
def diffuse(
|
||||
self,
|
||||
latents: torch.Tensor,
|
||||
timesteps: torch.Tensor,
|
||||
prompt_embeds: torch.Tensor,
|
||||
negative_prompt_embeds: torch.Tensor | None,
|
||||
image_embeds: torch.Tensor | None,
|
||||
guidance_low: float,
|
||||
guidance_high: float,
|
||||
boundary_timestep: float | None,
|
||||
dtype: torch.dtype,
|
||||
attention_kwargs: dict[str, Any],
|
||||
condition: torch.Tensor,
|
||||
first_frame_mask: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
with self.progress_bar(total=len(timesteps)) as pbar:
|
||||
for t in timesteps:
|
||||
self._current_timestep = t
|
||||
|
||||
# Select model and guidance scale based on timestep
|
||||
current_model = self.transformer
|
||||
current_guidance_scale = guidance_low
|
||||
if boundary_timestep is not None and t < boundary_timestep and self.transformer_2 is not None:
|
||||
current_model = self.transformer_2
|
||||
current_guidance_scale = guidance_high
|
||||
|
||||
# Prepare latent input
|
||||
if self.expand_timesteps:
|
||||
# TI2V-5B style: blend condition with latents using mask
|
||||
latent_model_input = (1 - first_frame_mask) * condition + first_frame_mask * latents
|
||||
latent_model_input = latent_model_input.to(dtype)
|
||||
|
||||
# Expand timesteps for each patch
|
||||
temp_ts = (first_frame_mask[0][0][:, ::2, ::2] * t).flatten()
|
||||
timestep = temp_ts.unsqueeze(0).expand(latents.shape[0], -1)
|
||||
else:
|
||||
# Wan2.1 style: concatenate condition with latents
|
||||
latent_model_input = torch.cat([latents, condition], dim=1).to(dtype)
|
||||
timestep = t.expand(latents.shape[0])
|
||||
|
||||
do_true_cfg = current_guidance_scale > 1.0 and negative_prompt_embeds is not None
|
||||
positive_kwargs = {
|
||||
"hidden_states": latent_model_input,
|
||||
"timestep": timestep,
|
||||
"encoder_hidden_states": prompt_embeds,
|
||||
"encoder_hidden_states_image": image_embeds,
|
||||
"attention_kwargs": attention_kwargs,
|
||||
"return_dict": False,
|
||||
"current_model": current_model,
|
||||
}
|
||||
if do_true_cfg:
|
||||
negative_kwargs = {
|
||||
"hidden_states": latent_model_input,
|
||||
"timestep": timestep,
|
||||
"encoder_hidden_states": negative_prompt_embeds,
|
||||
"encoder_hidden_states_image": image_embeds,
|
||||
"attention_kwargs": attention_kwargs,
|
||||
"return_dict": False,
|
||||
"current_model": current_model,
|
||||
}
|
||||
else:
|
||||
negative_kwargs = None
|
||||
|
||||
noise_pred = self.predict_noise_maybe_with_cfg(
|
||||
do_true_cfg=do_true_cfg,
|
||||
true_cfg_scale=current_guidance_scale,
|
||||
positive_kwargs=positive_kwargs,
|
||||
negative_kwargs=negative_kwargs,
|
||||
cfg_normalize=False,
|
||||
)
|
||||
|
||||
latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg)
|
||||
pbar.update()
|
||||
|
||||
return latents
|
||||
|
||||
def encode_image(
|
||||
self,
|
||||
image: PIL.Image.Image | list[PIL.Image.Image],
|
||||
@@ -522,68 +598,20 @@ class Wan22I2VPipeline(
|
||||
|
||||
if DEBUG_PERF:
|
||||
_t_denoise_start = time.perf_counter()
|
||||
with self.progress_bar(total=len(timesteps)) as pbar:
|
||||
for t in timesteps:
|
||||
self._current_timestep = t
|
||||
|
||||
# Select model and guidance scale based on timestep
|
||||
current_model = self.transformer
|
||||
current_guidance_scale = guidance_low
|
||||
if boundary_timestep is not None and t < boundary_timestep and self.transformer_2 is not None:
|
||||
current_model = self.transformer_2
|
||||
current_guidance_scale = guidance_high
|
||||
|
||||
# Prepare latent input
|
||||
if self.expand_timesteps:
|
||||
# TI2V-5B style: blend condition with latents using mask
|
||||
latent_model_input = (1 - first_frame_mask) * condition + first_frame_mask * latents
|
||||
latent_model_input = latent_model_input.to(dtype)
|
||||
|
||||
# Expand timesteps for each patch
|
||||
temp_ts = (first_frame_mask[0][0][:, ::2, ::2] * t).flatten()
|
||||
timestep = temp_ts.unsqueeze(0).expand(latents.shape[0], -1)
|
||||
else:
|
||||
# Wan2.1 style: concatenate condition with latents
|
||||
latent_model_input = torch.cat([latents, condition], dim=1).to(dtype)
|
||||
timestep = t.expand(latents.shape[0])
|
||||
|
||||
do_true_cfg = current_guidance_scale > 1.0 and negative_prompt_embeds is not None
|
||||
# Prepare kwargs for positive and negative predictions
|
||||
positive_kwargs = {
|
||||
"hidden_states": latent_model_input,
|
||||
"timestep": timestep,
|
||||
"encoder_hidden_states": prompt_embeds,
|
||||
"encoder_hidden_states_image": image_embeds,
|
||||
"attention_kwargs": attention_kwargs,
|
||||
"return_dict": False,
|
||||
"current_model": current_model,
|
||||
}
|
||||
if do_true_cfg:
|
||||
negative_kwargs = {
|
||||
"hidden_states": latent_model_input,
|
||||
"timestep": timestep,
|
||||
"encoder_hidden_states": negative_prompt_embeds,
|
||||
"encoder_hidden_states_image": image_embeds,
|
||||
"attention_kwargs": attention_kwargs,
|
||||
"return_dict": False,
|
||||
"current_model": current_model,
|
||||
}
|
||||
else:
|
||||
negative_kwargs = None
|
||||
|
||||
# Predict noise with automatic CFG parallel handling
|
||||
noise_pred = self.predict_noise_maybe_with_cfg(
|
||||
do_true_cfg=do_true_cfg,
|
||||
true_cfg_scale=current_guidance_scale,
|
||||
positive_kwargs=positive_kwargs,
|
||||
negative_kwargs=negative_kwargs,
|
||||
cfg_normalize=False,
|
||||
)
|
||||
|
||||
# Compute the previous noisy sample x_t -> x_t-1 with automatic CFG sync
|
||||
latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg)
|
||||
|
||||
pbar.update()
|
||||
latents = self.diffuse(
|
||||
latents=latents,
|
||||
timesteps=timesteps,
|
||||
prompt_embeds=prompt_embeds,
|
||||
negative_prompt_embeds=negative_prompt_embeds,
|
||||
image_embeds=image_embeds,
|
||||
guidance_low=guidance_low,
|
||||
guidance_high=guidance_high,
|
||||
boundary_timestep=boundary_timestep,
|
||||
dtype=dtype,
|
||||
attention_kwargs=attention_kwargs,
|
||||
condition=condition,
|
||||
first_frame_mask=first_frame_mask,
|
||||
)
|
||||
|
||||
# Wan2.2 is prone to out of memory errors when predicting large videos
|
||||
# so we empty the cache here to avoid OOM before vae decoding.
|
||||
|
||||
@@ -235,6 +235,77 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin, Progress
|
||||
def current_timestep(self):
|
||||
return self._current_timestep
|
||||
|
||||
def diffuse(
|
||||
self,
|
||||
latents: torch.Tensor,
|
||||
timesteps: torch.Tensor,
|
||||
prompt_embeds: torch.Tensor,
|
||||
negative_prompt_embeds: torch.Tensor | None,
|
||||
guidance_scale: float,
|
||||
dtype: torch.dtype,
|
||||
attention_kwargs: dict[str, Any],
|
||||
num_latent_frames: int,
|
||||
latent_height: int,
|
||||
latent_width: int,
|
||||
latent_condition: torch.Tensor | None = None,
|
||||
first_frame_mask: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
with self.progress_bar(total=len(timesteps)) as pbar:
|
||||
for t in timesteps:
|
||||
self._current_timestep = t
|
||||
|
||||
# Prepare latent input
|
||||
if latent_condition is not None:
|
||||
# I2V mode: blend condition with latents using mask
|
||||
latent_model_input = (1 - first_frame_mask) * latent_condition + first_frame_mask * latents
|
||||
latent_model_input = latent_model_input.to(dtype)
|
||||
|
||||
# Expand timesteps for each patch (TI2V style)
|
||||
temp_ts = (first_frame_mask[0][0][:, ::2, ::2] * t).flatten()
|
||||
timestep = temp_ts.unsqueeze(0).expand(latents.shape[0], -1)
|
||||
else:
|
||||
# T2V mode: use latents directly
|
||||
latent_model_input = latents.to(dtype)
|
||||
|
||||
# Expand timesteps for TI2V model architecture
|
||||
mask = torch.ones(1, 1, num_latent_frames, latent_height, latent_width, device=latents.device)
|
||||
temp_ts = (mask[0][0][:, ::2, ::2] * t).flatten()
|
||||
timestep = temp_ts.unsqueeze(0).expand(latents.shape[0], -1)
|
||||
|
||||
do_true_cfg = guidance_scale > 1.0 and negative_prompt_embeds is not None
|
||||
positive_kwargs = {
|
||||
"hidden_states": latent_model_input,
|
||||
"timestep": timestep,
|
||||
"encoder_hidden_states": prompt_embeds,
|
||||
"attention_kwargs": attention_kwargs,
|
||||
"return_dict": False,
|
||||
"current_model": self.transformer,
|
||||
}
|
||||
if do_true_cfg:
|
||||
negative_kwargs = {
|
||||
"hidden_states": latent_model_input,
|
||||
"timestep": timestep,
|
||||
"encoder_hidden_states": negative_prompt_embeds,
|
||||
"attention_kwargs": attention_kwargs,
|
||||
"return_dict": False,
|
||||
"current_model": self.transformer,
|
||||
}
|
||||
else:
|
||||
negative_kwargs = None
|
||||
|
||||
noise_pred = self.predict_noise_maybe_with_cfg(
|
||||
do_true_cfg=do_true_cfg,
|
||||
true_cfg_scale=guidance_scale,
|
||||
positive_kwargs=positive_kwargs,
|
||||
negative_kwargs=negative_kwargs,
|
||||
cfg_normalize=False,
|
||||
)
|
||||
|
||||
latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg)
|
||||
pbar.update()
|
||||
|
||||
return latents
|
||||
|
||||
def forward(
|
||||
self,
|
||||
req: OmniDiffusionRequest,
|
||||
@@ -405,64 +476,20 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin, Progress
|
||||
if attention_kwargs is None:
|
||||
attention_kwargs = {}
|
||||
|
||||
# Denoising loop
|
||||
with self.progress_bar(total=len(timesteps)) as pbar:
|
||||
for t in timesteps:
|
||||
self._current_timestep = t
|
||||
|
||||
# Prepare latent input
|
||||
if latent_condition is not None:
|
||||
# I2V mode: blend condition with latents using mask
|
||||
latent_model_input = (1 - first_frame_mask) * latent_condition + first_frame_mask * latents
|
||||
latent_model_input = latent_model_input.to(dtype)
|
||||
|
||||
# Expand timesteps for each patch (TI2V style)
|
||||
temp_ts = (first_frame_mask[0][0][:, ::2, ::2] * t).flatten()
|
||||
timestep = temp_ts.unsqueeze(0).expand(latents.shape[0], -1)
|
||||
else:
|
||||
# T2V mode: use latents directly
|
||||
latent_model_input = latents.to(dtype)
|
||||
|
||||
# Expand timesteps for TI2V model architecture
|
||||
mask = torch.ones(1, 1, num_latent_frames, latent_height, latent_width, device=device)
|
||||
temp_ts = (mask[0][0][:, ::2, ::2] * t).flatten()
|
||||
timestep = temp_ts.unsqueeze(0).expand(latents.shape[0], -1)
|
||||
|
||||
do_true_cfg = guidance_scale > 1.0 and negative_prompt_embeds is not None
|
||||
# Prepare kwargs for positive and negative predictions
|
||||
positive_kwargs = {
|
||||
"hidden_states": latent_model_input,
|
||||
"timestep": timestep,
|
||||
"encoder_hidden_states": prompt_embeds,
|
||||
"attention_kwargs": attention_kwargs,
|
||||
"return_dict": False,
|
||||
"current_model": self.transformer,
|
||||
}
|
||||
if do_true_cfg:
|
||||
negative_kwargs = {
|
||||
"hidden_states": latent_model_input,
|
||||
"timestep": timestep,
|
||||
"encoder_hidden_states": negative_prompt_embeds,
|
||||
"attention_kwargs": attention_kwargs,
|
||||
"return_dict": False,
|
||||
"current_model": self.transformer,
|
||||
}
|
||||
else:
|
||||
negative_kwargs = None
|
||||
|
||||
# Predict noise with automatic CFG parallel handling
|
||||
noise_pred = self.predict_noise_maybe_with_cfg(
|
||||
do_true_cfg=do_true_cfg,
|
||||
true_cfg_scale=guidance_scale,
|
||||
positive_kwargs=positive_kwargs,
|
||||
negative_kwargs=negative_kwargs,
|
||||
cfg_normalize=False,
|
||||
)
|
||||
|
||||
# Compute the previous noisy sample x_t -> x_t-1 with automatic CFG sync
|
||||
latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg)
|
||||
|
||||
pbar.update()
|
||||
latents = self.diffuse(
|
||||
latents=latents,
|
||||
timesteps=timesteps,
|
||||
prompt_embeds=prompt_embeds,
|
||||
negative_prompt_embeds=negative_prompt_embeds,
|
||||
guidance_scale=guidance_scale,
|
||||
dtype=dtype,
|
||||
attention_kwargs=attention_kwargs,
|
||||
num_latent_frames=num_latent_frames,
|
||||
latent_height=latent_height,
|
||||
latent_width=latent_width,
|
||||
latent_condition=latent_condition,
|
||||
first_frame_mask=first_frame_mask,
|
||||
)
|
||||
|
||||
# Wan2.2 is prone to out of memory errors when predicting large videos
|
||||
# so we empty the cache here to avoid OOM before vae decoding.
|
||||
|
||||
@@ -176,6 +176,62 @@ class Wan22VACEPipeline(Wan22Pipeline, SupportImageInput):
|
||||
"""Build VACE transformer directly from config dict."""
|
||||
return create_vace_transformer_from_config(config)
|
||||
|
||||
def diffuse(
|
||||
self,
|
||||
latents: torch.Tensor,
|
||||
timesteps: torch.Tensor,
|
||||
prompt_embeds: torch.Tensor,
|
||||
negative_prompt_embeds: torch.Tensor | None,
|
||||
guidance_scale: float,
|
||||
dtype: torch.dtype,
|
||||
attention_kwargs: dict[str, object],
|
||||
vace_context: torch.Tensor | None,
|
||||
vace_context_scale: float,
|
||||
) -> torch.Tensor:
|
||||
with self.progress_bar(total=len(timesteps)) as pbar:
|
||||
for t in timesteps:
|
||||
self._current_timestep = t
|
||||
latent_model_input = latents.to(dtype)
|
||||
timestep = t.expand(latents.shape[0])
|
||||
|
||||
do_true_cfg = guidance_scale > 1.0 and negative_prompt_embeds is not None
|
||||
|
||||
positive_kwargs = {
|
||||
"hidden_states": latent_model_input,
|
||||
"timestep": timestep,
|
||||
"encoder_hidden_states": prompt_embeds,
|
||||
"attention_kwargs": attention_kwargs,
|
||||
"vace_context": vace_context,
|
||||
"vace_context_scale": vace_context_scale,
|
||||
"return_dict": False,
|
||||
}
|
||||
negative_kwargs = (
|
||||
{
|
||||
"hidden_states": latent_model_input,
|
||||
"timestep": timestep,
|
||||
"encoder_hidden_states": negative_prompt_embeds,
|
||||
"attention_kwargs": attention_kwargs,
|
||||
"vace_context": vace_context,
|
||||
"vace_context_scale": vace_context_scale,
|
||||
"return_dict": False,
|
||||
}
|
||||
if do_true_cfg
|
||||
else None
|
||||
)
|
||||
|
||||
noise_pred = self.predict_noise_maybe_with_cfg(
|
||||
do_true_cfg=do_true_cfg,
|
||||
true_cfg_scale=guidance_scale,
|
||||
positive_kwargs=positive_kwargs,
|
||||
negative_kwargs=negative_kwargs,
|
||||
cfg_normalize=False,
|
||||
)
|
||||
|
||||
latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg)
|
||||
pbar.update()
|
||||
|
||||
return latents
|
||||
|
||||
def check_inputs(
|
||||
self,
|
||||
prompt,
|
||||
@@ -572,48 +628,17 @@ class Wan22VACEPipeline(Wan22Pipeline, SupportImageInput):
|
||||
timesteps = self.scheduler.timesteps
|
||||
self._num_timesteps = len(timesteps)
|
||||
|
||||
# Denoising loop
|
||||
with self.progress_bar(total=len(timesteps)) as pbar:
|
||||
for t in timesteps:
|
||||
self._current_timestep = t
|
||||
latent_model_input = latents.to(dtype)
|
||||
timestep = t.expand(latents.shape[0])
|
||||
|
||||
do_true_cfg = guidance_scale > 1.0 and negative_prompt_embeds is not None
|
||||
|
||||
positive_kwargs = {
|
||||
"hidden_states": latent_model_input,
|
||||
"timestep": timestep,
|
||||
"encoder_hidden_states": prompt_embeds,
|
||||
"attention_kwargs": attention_kwargs,
|
||||
"vace_context": vace_context,
|
||||
"vace_context_scale": vace_context_scale,
|
||||
"return_dict": False,
|
||||
}
|
||||
negative_kwargs = (
|
||||
{
|
||||
"hidden_states": latent_model_input,
|
||||
"timestep": timestep,
|
||||
"encoder_hidden_states": negative_prompt_embeds,
|
||||
"attention_kwargs": attention_kwargs,
|
||||
"vace_context": vace_context,
|
||||
"vace_context_scale": vace_context_scale,
|
||||
"return_dict": False,
|
||||
}
|
||||
if do_true_cfg
|
||||
else None
|
||||
)
|
||||
|
||||
noise_pred = self.predict_noise_maybe_with_cfg(
|
||||
do_true_cfg=do_true_cfg,
|
||||
true_cfg_scale=guidance_scale,
|
||||
positive_kwargs=positive_kwargs,
|
||||
negative_kwargs=negative_kwargs,
|
||||
cfg_normalize=False,
|
||||
)
|
||||
|
||||
latents = self.scheduler_step_maybe_with_cfg(noise_pred, t, latents, do_true_cfg)
|
||||
pbar.update()
|
||||
latents = self.diffuse(
|
||||
latents=latents,
|
||||
timesteps=timesteps,
|
||||
prompt_embeds=prompt_embeds,
|
||||
negative_prompt_embeds=negative_prompt_embeds,
|
||||
guidance_scale=guidance_scale,
|
||||
dtype=dtype,
|
||||
attention_kwargs=attention_kwargs,
|
||||
vace_context=vace_context,
|
||||
vace_context_scale=vace_context_scale,
|
||||
)
|
||||
|
||||
self._current_timestep = None
|
||||
|
||||
|
||||
Reference in new issue
Block a user