[codex][release/v0.18.0.post1] revert Wan2.2 pipeline changes from #2878 (#2937)

Signed-off-by: david6666666 <530634352@qq.com>
This commit is contained in:
WeiQing Chen authored and GitHub committed 2026-04-20 19:22:11 +08:00
1 parent 8a1ff4e994
commit 89f733d941
6 files changed
+32 -372

No files matched your search

@@ -1,190 +0,0 @@
from types import SimpleNamespace
import PIL.Image
import pytest
import torch
from torch import nn
from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import (
WAN22_MAX_SEQUENCE_LENGTH,
Wan22Pipeline,
)
from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2_i2v import (
Wan22I2VPipeline,
)
from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2_ti2v import (
Wan22TI2VPipeline,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
class _RejectingTextEncoder:
dtype = torch.float32
def __call__(self, *args, **kwargs):
raise AssertionError("text encoder should not run for prompts that exceed max_sequence_length")
class _FakeTokenBatch:
def __init__(self, total_sequence_length: int, padded_length: int | None = None):
padded_length = padded_length or total_sequence_length
attention_mask = torch.zeros((1, padded_length), dtype=torch.long)
attention_mask[:, :total_sequence_length] = 1
self.input_ids = attention_mask.clone()
self.attention_mask = attention_mask
class _FakeTokenizer:
def __init__(self, total_sequence_length: int | list[int]):
if isinstance(total_sequence_length, list):
self.total_sequence_lengths = list(total_sequence_length)
else:
self.total_sequence_lengths = [total_sequence_length]
def __call__(self, *args, **kwargs):
if len(self.total_sequence_lengths) > 1:
total_sequence_length = self.total_sequence_lengths.pop(0)
else:
total_sequence_length = self.total_sequence_lengths[0]
return _FakeTokenBatch(total_sequence_length, kwargs.get("max_length"))
class _RecordingTextEncoder:
dtype = torch.float32
def __init__(self):
self.input_lengths: list[int] = []
def __call__(self, input_ids, attention_mask):
self.input_lengths.append(input_ids.shape[1])
return SimpleNamespace(last_hidden_state=torch.zeros((1, input_ids.shape[1], 4), dtype=torch.float32))
PIPELINE_CASES = [
pytest.param(Wan22Pipeline, id="wan22-t2v"),
pytest.param(Wan22I2VPipeline, id="wan22-i2v"),
pytest.param(Wan22TI2VPipeline, id="wan22-ti2v"),
]
def _make_pipeline(pipeline_class: type, *, total_sequence_length: int):
pipeline = object.__new__(pipeline_class)
nn.Module.__init__(pipeline)
pipeline.device = torch.device("cpu")
pipeline.text_encoder = _RejectingTextEncoder()
pipeline.tokenizer = _FakeTokenizer(total_sequence_length)
pipeline.tokenizer_max_length = WAN22_MAX_SEQUENCE_LENGTH
return pipeline
@pytest.mark.parametrize("pipeline_class", PIPELINE_CASES)
def test_encode_prompt_rejects_prompt_longer_than_default_max_sequence_length(pipeline_class: type):
pipeline = _make_pipeline(pipeline_class, total_sequence_length=WAN22_MAX_SEQUENCE_LENGTH + 1)
with pytest.raises(
ValueError,
match=rf"got {WAN22_MAX_SEQUENCE_LENGTH + 1} tokens, but `max_sequence_length` is {WAN22_MAX_SEQUENCE_LENGTH}",
):
pipeline.encode_prompt(prompt="prompt")
@pytest.mark.parametrize("pipeline_class", PIPELINE_CASES)
def test_encode_prompt_rejects_prompt_longer_than_explicit_max_sequence_length(pipeline_class: type):
pipeline = _make_pipeline(pipeline_class, total_sequence_length=17)
with pytest.raises(ValueError, match=r"got 17 tokens, but `max_sequence_length` is 16"):
pipeline.encode_prompt(prompt="prompt", max_sequence_length=16)
@pytest.mark.parametrize("pipeline_class", PIPELINE_CASES)
def test_encode_prompt_uses_actual_prompt_length_for_text_encoder_padding(pipeline_class: type):
pipeline = object.__new__(pipeline_class)
nn.Module.__init__(pipeline)
pipeline.device = torch.device("cpu")
pipeline.text_encoder = _RecordingTextEncoder()
# prompt validation, prompt encode, negative validation, negative encode
pipeline.tokenizer = _FakeTokenizer([17, 17, 9, 9])
pipeline.tokenizer_max_length = WAN22_MAX_SEQUENCE_LENGTH
prompt_embeds, negative_prompt_embeds = pipeline.encode_prompt(
prompt="prompt",
negative_prompt="neg",
do_classifier_free_guidance=True,
)
assert pipeline.text_encoder.input_lengths == [17, 17]
assert prompt_embeds.shape[1] == 17
assert negative_prompt_embeds is not None
assert negative_prompt_embeds.shape[1] == 17
def _sampling_params(**overrides):
defaults = dict(
height=None,
width=None,
num_frames=None,
num_inference_steps=None,
generator=None,
guidance_scale_provided=False,
guidance_scale_2=None,
boundary_ratio=None,
num_outputs_per_prompt=0,
max_sequence_length=None,
seed=None,
extra_args={},
prompt_embeds=None,
negative_prompt_embeds=None,
)
defaults.update(overrides)
return SimpleNamespace(**defaults)
@pytest.mark.parametrize(
("pipeline_class", "prompt_value", "forward_kwargs"),
[
pytest.param(Wan22Pipeline, "prompt", {}, id="wan22-t2v"),
pytest.param(
Wan22I2VPipeline,
{"prompt": "prompt", "multi_modal_data": {"image": PIL.Image.new("RGB", (64, 64))}},
{"image": PIL.Image.new("RGB", (64, 64))},
id="wan22-i2v",
),
pytest.param(
Wan22TI2VPipeline,
{"prompt": "prompt", "multi_modal_data": {"image": PIL.Image.new("RGB", (64, 64))}},
{"image": PIL.Image.new("RGB", (64, 64))},
id="wan22-ti2v",
),
],
)
def test_forward_defaults_to_wan22_tokenizer_max_length(
pipeline_class: type,
prompt_value,
forward_kwargs,
):
pipeline = object.__new__(pipeline_class)
nn.Module.__init__(pipeline)
pipeline.tokenizer_max_length = WAN22_MAX_SEQUENCE_LENGTH
pipeline.boundary_ratio = None
pipeline.vae_scale_factor_temporal = 4
pipeline.vae_scale_factor_spatial = 8
pipeline.transformer_config = SimpleNamespace(patch_size=(1, 2, 2))
captured = {}
def _fake_check_inputs(*args, **kwargs):
captured["max_sequence_length"] = kwargs["max_sequence_length"]
raise RuntimeError("stop after capture")
pipeline.check_inputs = _fake_check_inputs
req = SimpleNamespace(
prompts=[prompt_value],
sampling_params=_sampling_params(),
)
with pytest.raises(RuntimeError, match="stop after capture"):
pipeline.forward(req, **forward_kwargs)
assert captured["max_sequence_length"] == WAN22_MAX_SEQUENCE_LENGTH
@@ -1,7 +1,6 @@
from importlib.util import find_spec
import torch
import torch.nn as nn
from vllm.logger import init_logger
from vllm_omni.diffusion.layers.custom_op import CustomOp
@@ -28,15 +28,11 @@ from vllm_omni.diffusion.models.wan2_2.wan2_2_transformer import WanTransformer3
from vllm_omni.diffusion.postprocess import interpolate_video_tensor
from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.utils.prompt_utils import (
validate_prompt_sequence_lengths,
)
from vllm_omni.inputs.data import OmniTextPrompt
from vllm_omni.platforms import current_omni_platform
logger = logging.getLogger(__name__)
DEBUG_PERF = False
WAN22_MAX_SEQUENCE_LENGTH = 2048
def retrieve_latents(
@@ -252,7 +248,6 @@ class Wan22Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin, DiffusionPipe
pass
self.boundary_ratio = od_config.boundary_ratio
self.tokenizer_max_length = WAN22_MAX_SEQUENCE_LENGTH
# Determine which transformers to load based on boundary_ratio
# boundary_ratio=1.0: only load transformer_2 (low-noise stage only)
@@ -428,7 +423,6 @@ class Wan22Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin, DiffusionPipe
negative_prompt_embeds=negative_prompt_embeds,
guidance_scale_2=guidance_high if boundary_ratio is not None else None,
boundary_ratio=boundary_ratio,
max_sequence_length=req.sampling_params.max_sequence_length or self.tokenizer_max_length,
)
if num_frames % self.vae_scale_factor_temporal != 1:
@@ -462,7 +456,7 @@ class Wan22Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin, DiffusionPipe
negative_prompt=negative_prompt,
do_classifier_free_guidance=guidance_low > 1.0 or guidance_high > 1.0,
num_videos_per_prompt=req.sampling_params.num_outputs_per_prompt or 1,
max_sequence_length=req.sampling_params.max_sequence_length or self.tokenizer_max_length,
max_sequence_length=req.sampling_params.max_sequence_length or 512,
device=device,
dtype=dtype,
)
@@ -749,7 +743,7 @@ class Wan22Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin, DiffusionPipe
negative_prompt: str | list[str] | None = None,
do_classifier_free_guidance: bool = True,
num_videos_per_prompt: int = 1,
max_sequence_length: int = WAN22_MAX_SEQUENCE_LENGTH,
max_sequence_length: int = 512,
device: torch.device | None = None,
dtype: torch.dtype | None = None,
):
@@ -759,51 +753,11 @@ class Wan22Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin, DiffusionPipe
prompt = [prompt] if isinstance(prompt, str) else prompt
prompt_clean = [self._prompt_clean(p) for p in prompt]
batch_size = len(prompt_clean)
text_inputs_untruncated = self.tokenizer(
prompt_clean,
padding=True,
truncation=False,
add_special_tokens=True,
return_attention_mask=True,
return_tensors="pt",
)
validate_prompt_sequence_lengths(
text_inputs_untruncated.attention_mask,
max_sequence_length=max_sequence_length,
supported_max_sequence_length=self.tokenizer_max_length,
error_context="for Wan2.2 text encoding",
)
prompt_encode_length = max(int(text_inputs_untruncated.attention_mask.sum(dim=1).max().item()), 1)
negative_prompt_embeds = None
if do_classifier_free_guidance:
negative_prompt = negative_prompt or ""
negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
negative_prompt_clean = [self._prompt_clean(p) for p in negative_prompt]
neg_text_inputs_untruncated = self.tokenizer(
negative_prompt_clean,
padding=True,
truncation=False,
add_special_tokens=True,
return_attention_mask=True,
return_tensors="pt",
)
validate_prompt_sequence_lengths(
neg_text_inputs_untruncated.attention_mask,
max_sequence_length=max_sequence_length,
supported_max_sequence_length=self.tokenizer_max_length,
prompt_name="negative_prompt",
error_context="for Wan2.2 text encoding",
)
prompt_encode_length = max(
prompt_encode_length,
int(neg_text_inputs_untruncated.attention_mask.sum(dim=1).max().item()),
)
text_inputs = self.tokenizer(
prompt_clean,
padding="max_length",
max_length=prompt_encode_length,
max_length=max_sequence_length,
truncation=True,
add_special_tokens=True,
return_attention_mask=True,
@@ -816,18 +770,21 @@ class Wan22Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin, DiffusionPipe
prompt_embeds = prompt_embeds.to(dtype=dtype, device=device)
prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)]
prompt_embeds = torch.stack(
[torch.cat([u, u.new_zeros(prompt_encode_length - u.size(0), u.size(1))]) for u in prompt_embeds], dim=0
[torch.cat([u, u.new_zeros(max_sequence_length - u.size(0), u.size(1))]) for u in prompt_embeds], dim=0
)
_, seq_len, _ = prompt_embeds.shape
prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1)
prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1)
negative_prompt_embeds = None
if do_classifier_free_guidance:
negative_prompt = negative_prompt or ""
negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
neg_text_inputs = self.tokenizer(
negative_prompt_clean,
[self._prompt_clean(p) for p in negative_prompt],
padding="max_length",
max_length=prompt_encode_length,
max_length=max_sequence_length,
truncation=True,
add_special_tokens=True,
return_attention_mask=True,
@@ -840,7 +797,7 @@ class Wan22Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin, DiffusionPipe
negative_prompt_embeds = [u[:v] for u, v in zip(negative_prompt_embeds, seq_lens_neg)]
negative_prompt_embeds = torch.stack(
[
torch.cat([u, u.new_zeros(prompt_encode_length - u.size(0), u.size(1))])
torch.cat([u, u.new_zeros(max_sequence_length - u.size(0), u.size(1))])
for u in negative_prompt_embeds
],
dim=0,
@@ -897,7 +854,6 @@ class Wan22Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin, DiffusionPipe
negative_prompt_embeds=None,
guidance_scale_2=None,
boundary_ratio=None,
max_sequence_length=None,
):
if height % 16 != 0 or width % 16 != 0:
raise ValueError(f"`height` and `width` have to be divisible by 16 but are {height} and {width}.")
@@ -924,10 +880,5 @@ class Wan22Pipeline(nn.Module, CFGParallelMixin, ProgressBarMixin, DiffusionPipe
):
raise ValueError(f"`negative_prompt` has to be of type `str` or `list` but is {type(negative_prompt)}")
if max_sequence_length is not None and max_sequence_length > self.tokenizer_max_length:
raise ValueError(
f"`max_sequence_length` cannot be greater than {self.tokenizer_max_length} but is {max_sequence_length}"
)
if boundary_ratio is None and guidance_scale_2 is not None:
raise ValueError("`guidance_scale_2` is only supported when `boundary_ratio` is set.")
@@ -27,7 +27,6 @@ from vllm_omni.diffusion.models.interface import SupportImageInput
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin, _is_rank_zero
from vllm_omni.diffusion.models.schedulers import FlowUniPCMultistepScheduler
from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import (
WAN22_MAX_SEQUENCE_LENGTH,
create_transformer_from_config,
load_transformer_config,
retrieve_latents,
@@ -35,9 +34,6 @@ from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import (
from vllm_omni.diffusion.postprocess import interpolate_video_tensor
from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.utils.prompt_utils import (
validate_prompt_sequence_lengths,
)
from vllm_omni.inputs.data import OmniTextPrompt
from vllm_omni.platforms import current_omni_platform
@@ -216,7 +212,6 @@ class Wan22I2VPipeline(
# Text encoder
self.tokenizer = AutoTokenizer.from_pretrained(model, subfolder="tokenizer", local_files_only=local_files_only)
self.tokenizer_max_length = WAN22_MAX_SEQUENCE_LENGTH
self.text_encoder = UMT5EncoderModel.from_pretrained(
model, subfolder="text_encoder", torch_dtype=dtype, local_files_only=local_files_only
).to(self.device)
@@ -400,7 +395,6 @@ class Wan22I2VPipeline(
image_embeds=image_embeds,
guidance_scale_2=guidance_high if boundary_ratio is not None else None,
boundary_ratio=boundary_ratio,
max_sequence_length=req.sampling_params.max_sequence_length or self.tokenizer_max_length,
)
# Adjust num_frames to be compatible with VAE temporal scaling
@@ -429,7 +423,7 @@ class Wan22I2VPipeline(
negative_prompt=negative_prompt,
do_classifier_free_guidance=guidance_low > 1.0 or guidance_high > 1.0,
num_videos_per_prompt=req.sampling_params.num_outputs_per_prompt or 1,
max_sequence_length=req.sampling_params.max_sequence_length or self.tokenizer_max_length,
max_sequence_length=req.sampling_params.max_sequence_length or 512,
device=device,
dtype=dtype,
)
@@ -489,7 +483,7 @@ class Wan22I2VPipeline(
# Handle last_image if provided
if last_image is not None:
if isinstance(last_image, PIL.Image.Image):
image = TF.to_tensor(last_image).to(device)
last_image = TF.to_tensor(last_image).to(device)
last_image_tensor = video_processor.preprocess(last_image, height=height, width=width)
else:
last_image_tensor = last_image
@@ -664,7 +658,7 @@ class Wan22I2VPipeline(
negative_prompt: str | list[str] | None = None,
do_classifier_free_guidance: bool = True,
num_videos_per_prompt: int = 1,
max_sequence_length: int = WAN22_MAX_SEQUENCE_LENGTH,
max_sequence_length: int = 512,
device: torch.device | None = None,
dtype: torch.dtype | None = None,
):
@@ -675,51 +669,11 @@ class Wan22I2VPipeline(
prompt = [prompt] if isinstance(prompt, str) else prompt
prompt_clean = [self._prompt_clean(p) for p in prompt]
batch_size = len(prompt_clean)
text_inputs_untruncated = self.tokenizer(
prompt_clean,
padding=True,
truncation=False,
add_special_tokens=True,
return_attention_mask=True,
return_tensors="pt",
)
validate_prompt_sequence_lengths(
text_inputs_untruncated.attention_mask,
max_sequence_length=max_sequence_length,
supported_max_sequence_length=self.tokenizer_max_length,
error_context="for Wan2.2 text encoding",
)
prompt_encode_length = max(int(text_inputs_untruncated.attention_mask.sum(dim=1).max().item()), 1)
negative_prompt_embeds = None
if do_classifier_free_guidance:
negative_prompt = negative_prompt or ""
negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
negative_prompt_clean = [self._prompt_clean(p) for p in negative_prompt]
neg_text_inputs_untruncated = self.tokenizer(
negative_prompt_clean,
padding=True,
truncation=False,
add_special_tokens=True,
return_attention_mask=True,
return_tensors="pt",
)
validate_prompt_sequence_lengths(
neg_text_inputs_untruncated.attention_mask,
max_sequence_length=max_sequence_length,
supported_max_sequence_length=self.tokenizer_max_length,
prompt_name="negative_prompt",
error_context="for Wan2.2 text encoding",
)
prompt_encode_length = max(
prompt_encode_length,
int(neg_text_inputs_untruncated.attention_mask.sum(dim=1).max().item()),
)
text_inputs = self.tokenizer(
prompt_clean,
padding="max_length",
max_length=prompt_encode_length,
max_length=max_sequence_length,
truncation=True,
add_special_tokens=True,
return_attention_mask=True,
@@ -732,18 +686,21 @@ class Wan22I2VPipeline(
prompt_embeds = prompt_embeds.to(dtype=dtype, device=device)
prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)]
prompt_embeds = torch.stack(
[torch.cat([u, u.new_zeros(prompt_encode_length - u.size(0), u.size(1))]) for u in prompt_embeds], dim=0
[torch.cat([u, u.new_zeros(max_sequence_length - u.size(0), u.size(1))]) for u in prompt_embeds], dim=0
)
_, seq_len, _ = prompt_embeds.shape
prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1)
prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1)
negative_prompt_embeds = None
if do_classifier_free_guidance:
negative_prompt = negative_prompt or ""
negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
neg_text_inputs = self.tokenizer(
negative_prompt_clean,
[self._prompt_clean(p) for p in negative_prompt],
padding="max_length",
max_length=prompt_encode_length,
max_length=max_sequence_length,
truncation=True,
add_special_tokens=True,
return_attention_mask=True,
@@ -756,7 +713,7 @@ class Wan22I2VPipeline(
negative_prompt_embeds = [u[:v] for u, v in zip(negative_prompt_embeds, seq_lens_neg)]
negative_prompt_embeds = torch.stack(
[
torch.cat([u, u.new_zeros(prompt_encode_length - u.size(0), u.size(1))])
torch.cat([u, u.new_zeros(max_sequence_length - u.size(0), u.size(1))])
for u in negative_prompt_embeds
],
dim=0,
@@ -885,7 +842,6 @@ class Wan22I2VPipeline(
image_embeds=None,
guidance_scale_2=None,
boundary_ratio=None,
max_sequence_length=None,
):
if image is None and image_embeds is None:
raise ValueError("Provide either `image` or `image_embeds`. Cannot leave both undefined.")
@@ -907,11 +863,6 @@ class Wan22I2VPipeline(
if prompt is None and prompt_embeds is None:
raise ValueError("Provide either `prompt` or `prompt_embeds`.")
if max_sequence_length is not None and max_sequence_length > self.tokenizer_max_length:
raise ValueError(
f"`max_sequence_length` cannot be greater than {self.tokenizer_max_length} but is {max_sequence_length}"
)
if boundary_ratio is None and guidance_scale_2 is not None:
raise ValueError("`guidance_scale_2` is only supported when `boundary_ratio` is set.")
@@ -38,16 +38,12 @@ from vllm_omni.diffusion.models.interface import SupportImageInput
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin
from vllm_omni.diffusion.models.schedulers import FlowUniPCMultistepScheduler
from vllm_omni.diffusion.models.wan2_2.pipeline_wan2_2 import (
WAN22_MAX_SEQUENCE_LENGTH,
create_transformer_from_config,
load_transformer_config,
retrieve_latents,
)
from vllm_omni.diffusion.postprocess import interpolate_video_tensor
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.utils.prompt_utils import (
validate_prompt_sequence_lengths,
)
from vllm_omni.inputs.data import OmniTextPrompt
from vllm_omni.platforms import current_omni_platform
@@ -187,7 +183,6 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin, Progress
# Text encoder
self.tokenizer = AutoTokenizer.from_pretrained(model, subfolder="tokenizer", local_files_only=local_files_only)
self.tokenizer_max_length = WAN22_MAX_SEQUENCE_LENGTH
self.text_encoder = UMT5EncoderModel.from_pretrained(
model, subfolder="text_encoder", torch_dtype=dtype, local_files_only=local_files_only
).to(self.device)
@@ -308,7 +303,6 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin, Progress
width=width,
prompt_embeds=prompt_embeds,
negative_prompt_embeds=negative_prompt_embeds,
max_sequence_length=req.sampling_params.max_sequence_length or self.tokenizer_max_length,
)
# Adjust num_frames to be compatible with VAE temporal scaling
@@ -332,7 +326,7 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin, Progress
negative_prompt=negative_prompt,
do_classifier_free_guidance=guidance_scale > 1.0,
num_videos_per_prompt=req.sampling_params.num_outputs_per_prompt or 1,
max_sequence_length=req.sampling_params.max_sequence_length or self.tokenizer_max_length,
max_sequence_length=req.sampling_params.max_sequence_length or 512,
device=device,
dtype=dtype,
)
@@ -508,7 +502,7 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin, Progress
negative_prompt: str | list[str] | None = None,
do_classifier_free_guidance: bool = True,
num_videos_per_prompt: int = 1,
max_sequence_length: int = WAN22_MAX_SEQUENCE_LENGTH,
max_sequence_length: int = 512,
device: torch.device | None = None,
dtype: torch.dtype | None = None,
):
@@ -519,51 +513,11 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin, Progress
prompt = [prompt] if isinstance(prompt, str) else prompt
prompt_clean = [self._prompt_clean(p) for p in prompt]
batch_size = len(prompt_clean)
text_inputs_untruncated = self.tokenizer(
prompt_clean,
padding=True,
truncation=False,
add_special_tokens=True,
return_attention_mask=True,
return_tensors="pt",
)
validate_prompt_sequence_lengths(
text_inputs_untruncated.attention_mask,
max_sequence_length=max_sequence_length,
supported_max_sequence_length=self.tokenizer_max_length,
error_context="for Wan2.2 text encoding",
)
prompt_encode_length = max(int(text_inputs_untruncated.attention_mask.sum(dim=1).max().item()), 1)
negative_prompt_embeds = None
if do_classifier_free_guidance:
negative_prompt = negative_prompt or ""
negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
negative_prompt_clean = [self._prompt_clean(p) for p in negative_prompt]
neg_text_inputs_untruncated = self.tokenizer(
negative_prompt_clean,
padding=True,
truncation=False,
add_special_tokens=True,
return_attention_mask=True,
return_tensors="pt",
)
validate_prompt_sequence_lengths(
neg_text_inputs_untruncated.attention_mask,
max_sequence_length=max_sequence_length,
supported_max_sequence_length=self.tokenizer_max_length,
prompt_name="negative_prompt",
error_context="for Wan2.2 text encoding",
)
prompt_encode_length = max(
prompt_encode_length,
int(neg_text_inputs_untruncated.attention_mask.sum(dim=1).max().item()),
)
text_inputs = self.tokenizer(
prompt_clean,
padding="max_length",
max_length=prompt_encode_length,
max_length=max_sequence_length,
truncation=True,
add_special_tokens=True,
return_attention_mask=True,
@@ -576,18 +530,21 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin, Progress
prompt_embeds = prompt_embeds.to(dtype=dtype, device=device)
prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)]
prompt_embeds = torch.stack(
[torch.cat([u, u.new_zeros(prompt_encode_length - u.size(0), u.size(1))]) for u in prompt_embeds], dim=0
[torch.cat([u, u.new_zeros(max_sequence_length - u.size(0), u.size(1))]) for u in prompt_embeds], dim=0
)
_, seq_len, _ = prompt_embeds.shape
prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1)
prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1)
negative_prompt_embeds = None
if do_classifier_free_guidance:
negative_prompt = negative_prompt or ""
negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
neg_text_inputs = self.tokenizer(
negative_prompt_clean,
[self._prompt_clean(p) for p in negative_prompt],
padding="max_length",
max_length=prompt_encode_length,
max_length=max_sequence_length,
truncation=True,
add_special_tokens=True,
return_attention_mask=True,
@@ -600,7 +557,7 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin, Progress
negative_prompt_embeds = [u[:v] for u, v in zip(negative_prompt_embeds, seq_lens_neg)]
negative_prompt_embeds = torch.stack(
[
torch.cat([u, u.new_zeros(prompt_encode_length - u.size(0), u.size(1))])
torch.cat([u, u.new_zeros(max_sequence_length - u.size(0), u.size(1))])
for u in negative_prompt_embeds
],
dim=0,
@@ -710,7 +667,6 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin, Progress
width,
prompt_embeds=None,
negative_prompt_embeds=None,
max_sequence_length=None,
):
if height % 16 != 0 or width % 16 != 0:
raise ValueError(f"`height` and `width` have to be divisible by 16 but are {height} and {width}.")
@@ -726,11 +682,6 @@ class Wan22TI2VPipeline(nn.Module, SupportImageInput, CFGParallelMixin, Progress
if prompt is None and prompt_embeds is None:
raise ValueError("Provide either `prompt` or `prompt_embeds`.")
if max_sequence_length is not None and max_sequence_length > self.tokenizer_max_length:
raise ValueError(
f"`max_sequence_length` cannot be greater than {self.tokenizer_max_length} but is {max_sequence_length}"
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load weights using AutoWeightsLoader for vLLM integration."""
loader = AutoWeightsLoader(self)
@@ -11,7 +11,6 @@ import torch.nn.functional as F
from diffusers.models.attention import FeedForward
from diffusers.models.embeddings import PixArtAlphaTextProjection, TimestepEmbedding, Timesteps
from diffusers.models.modeling_outputs import Transformer2DModelOutput
from diffusers.models.normalization import FP32LayerNorm
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
@@ -24,14 +23,13 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.layers.adalayernorm import AdaLayerNorm
from vllm_omni.diffusion.layers.norm import LayerNorm, RMSNorm
from vllm_omni.diffusion.distributed.sp_plan import (
SequenceParallelInput,
SequenceParallelOutput,
)
from vllm_omni.diffusion.forward_context import get_forward_context
from vllm_omni.diffusion.layers.adalayernorm import AdaLayerNorm
from vllm_omni.diffusion.layers.norm import LayerNorm, RMSNorm
logger = init_logger(__name__)