[Diffusion] Refactor LTX2 to use unified CFG parallel framework (#2160)
Signed-off-by: Yangshen Deng <yangshen.d@outlook.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
ec082add35
commit
1cd5210440
5 files changed
+764
-424
No files matched your search
@@ -0,0 +1,217 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_ltx2_baseline_eager",
|
||||
"description": "Single-device baseline with enforce-eager (no torch.compile)",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Lightricks/LTX-2",
|
||||
"serve_args": {
|
||||
"enforce-eager": true,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "256x256_145f_steps6",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 256,
|
||||
"height": 256,
|
||||
"num-frames": 145,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 6,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
},
|
||||
{
|
||||
"name": "480x768_41f_steps20",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 768,
|
||||
"height": 480,
|
||||
"num-frames": 41,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
{
|
||||
"test_name": "test_ltx2_torch_compile",
|
||||
"description": "Single-device with torch.compile (default, no enforce-eager)",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Lightricks/LTX-2",
|
||||
"serve_args": {
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "256x256_145f_steps6",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 256,
|
||||
"height": 256,
|
||||
"num-frames": 145,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 6,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
},
|
||||
{
|
||||
"name": "480x768_41f_steps20",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 768,
|
||||
"height": 480,
|
||||
"num-frames": 41,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
{
|
||||
"test_name": "test_ltx2_cfg2_eager",
|
||||
"description": "CFG-parallel=2 with enforce-eager",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Lightricks/LTX-2",
|
||||
"serve_args": {
|
||||
"cfg-parallel-size": 2,
|
||||
"enforce-eager": true,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "256x256_145f_steps6",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 256,
|
||||
"height": 256,
|
||||
"num-frames": 145,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 6,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
},
|
||||
{
|
||||
"name": "480x768_41f_steps20",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 768,
|
||||
"height": 480,
|
||||
"num-frames": 41,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
{
|
||||
"test_name": "test_ltx2_cfg2_compile",
|
||||
"description": "CFG-parallel=2 with torch.compile",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Lightricks/LTX-2",
|
||||
"serve_args": {
|
||||
"cfg-parallel-size": 2,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "256x256_145f_steps6",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 256,
|
||||
"height": 256,
|
||||
"num-frames": 145,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 6,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
},
|
||||
{
|
||||
"name": "480x768_41f_steps20",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 768,
|
||||
"height": 480,
|
||||
"num-frames": 41,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
{
|
||||
"test_name": "test_ltx2_cache_dit_eager",
|
||||
"description": "CacheDiT with enforce-eager",
|
||||
"server_type": "vllm-omni",
|
||||
"server_params": {
|
||||
"model": "Lightricks/LTX-2",
|
||||
"serve_args": {
|
||||
"cache-backend": "cache_dit",
|
||||
"enforce-eager": true,
|
||||
"enable-diffusion-pipeline-profiler": true
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"name": "256x256_145f_steps6",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 256,
|
||||
"height": 256,
|
||||
"num-frames": 145,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 6,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
},
|
||||
{
|
||||
"name": "480x768_41f_steps20",
|
||||
"dataset": "random",
|
||||
"task": "t2v",
|
||||
"backend": "v1/videos",
|
||||
"width": 768,
|
||||
"height": 480,
|
||||
"num-frames": 41,
|
||||
"fps": 24,
|
||||
"num-inference-steps": 20,
|
||||
"num-prompts": 3,
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,58 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _make_pipeline(sequence_parallel_size: int = 1) -> LTX2Pipeline:
|
||||
pipeline = object.__new__(LTX2Pipeline)
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
pipeline.audio_vae_temporal_compression_ratio = 4
|
||||
pipeline.audio_vae_mel_compression_ratio = 4
|
||||
pipeline.od_config = SimpleNamespace(parallel_config=SimpleNamespace(sequence_parallel_size=sequence_parallel_size))
|
||||
# Mock audio_vae with identity normalization (mean=0, std=1) so
|
||||
# _normalize_audio_latents is a no-op and test values are preserved.
|
||||
pipeline.audio_vae = SimpleNamespace(
|
||||
latents_mean=torch.tensor(0.0),
|
||||
latents_std=torch.tensor(1.0),
|
||||
)
|
||||
return pipeline
|
||||
|
||||
|
||||
def test_prepare_audio_latents_pads_packed_sequence_dim_for_provided_latents():
|
||||
pipeline = _make_pipeline(sequence_parallel_size=4)
|
||||
latents = torch.arange(40, dtype=torch.float32).view(1, 10, 4)
|
||||
|
||||
padded, original_num_frames, padded_num_frames = pipeline.prepare_audio_latents(
|
||||
batch_size=1,
|
||||
num_channels_latents=2,
|
||||
num_mel_bins=8,
|
||||
audio_latent_length=10,
|
||||
dtype=torch.float32,
|
||||
device=torch.device("cpu"),
|
||||
latents=latents,
|
||||
)
|
||||
|
||||
assert original_num_frames == 10
|
||||
assert padded_num_frames == 12
|
||||
assert padded.shape == (1, 12, 4)
|
||||
torch.testing.assert_close(padded[:, :10], latents)
|
||||
torch.testing.assert_close(padded[:, 10:], torch.zeros(1, 2, 4))
|
||||
|
||||
|
||||
def test_unpad_audio_latents_restores_original_frames_before_unpack():
|
||||
pipeline = _make_pipeline()
|
||||
original = torch.arange(40, dtype=torch.float32).view(1, 10, 4)
|
||||
padded = torch.cat([original, torch.full((1, 2, 4), 999.0)], dim=1)
|
||||
|
||||
unpadded = pipeline._unpad_audio_latents(padded, 10)
|
||||
unpacked = pipeline._unpack_audio_latents(unpadded, latent_length=10, num_mel_bins=2)
|
||||
expected = pipeline._unpack_audio_latents(original, latent_length=10, num_mel_bins=2)
|
||||
|
||||
assert unpacked.shape == (1, 2, 10, 2)
|
||||
assert not (unpacked == 999.0).any()
|
||||
torch.testing.assert_close(unpacked, expected)
|
||||
@@ -0,0 +1,243 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from tests.utils import hardware_test
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
T2V_EXAMPLE = REPO_ROOT / "examples" / "offline_inference" / "text_to_video" / "text_to_video.py"
|
||||
I2V_EXAMPLE = REPO_ROOT / "examples" / "offline_inference" / "image_to_video" / "image_to_video.py"
|
||||
|
||||
T2V_PROMPT = (
|
||||
"At sunrise, a glowing paper lantern boat drifts through a narrow canal between mossy stone walls, "
|
||||
"soft fog above the water, the camera slowly gliding forward as golden reflections shimmer across "
|
||||
"the ripples, cinematic, realistic, highly detailed."
|
||||
)
|
||||
T2V_NEGATIVE_PROMPT = "worst quality, blurry, jittery motion, distorted, oversaturated, artifacts"
|
||||
I2V_PROMPT = "A cinematic dolly shot of a boat drifting on calm water at sunset"
|
||||
I2V_NEGATIVE_PROMPT = "worst quality, blurry, jittery motion"
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
|
||||
def _get_ltx2_model() -> str:
|
||||
return os.environ.get("VLLM_TEST_LTX2_MODEL", "Lightricks/LTX-2")
|
||||
|
||||
|
||||
def _md5(path: Path) -> str:
|
||||
digest = hashlib.md5(usedforsecurity=False)
|
||||
with path.open("rb") as f:
|
||||
for chunk in iter(lambda: f.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _make_deterministic_test_image(path: Path) -> None:
|
||||
"""Create a deterministic 256x256 test image for I2V tests."""
|
||||
rng = np.random.RandomState(42)
|
||||
img = Image.fromarray(rng.randint(0, 255, (256, 256, 3), dtype=np.uint8))
|
||||
img.save(path)
|
||||
|
||||
|
||||
def _run_and_check(cmd: list[str], env: dict, output_path: Path, expected_md5: str) -> None:
|
||||
result = subprocess.run(cmd, cwd=REPO_ROOT, env=env, capture_output=True, text=True, check=False)
|
||||
assert result.returncode == 0, (
|
||||
f"Command failed (exit {result.returncode}).\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}"
|
||||
)
|
||||
generated_md5 = _md5(output_path)
|
||||
assert generated_md5 == expected_md5, (
|
||||
f"Unexpected output md5: {generated_md5} != {expected_md5}.\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}"
|
||||
)
|
||||
|
||||
|
||||
# ── T2V tests ──
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parallel
|
||||
@pytest.mark.slow
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=2)
|
||||
def test_ltx2_t2v_cfg_parallel(tmp_path: Path):
|
||||
"""T2V with CFG=4.0, cfg-parallel-size=2."""
|
||||
output = tmp_path / "t2v_cfg4.mp4"
|
||||
env = os.environ.copy()
|
||||
env.setdefault("CUDA_VISIBLE_DEVICES", "0,1")
|
||||
cmd = [
|
||||
sys.executable,
|
||||
str(T2V_EXAMPLE),
|
||||
"--model",
|
||||
_get_ltx2_model(),
|
||||
"--prompt",
|
||||
T2V_PROMPT,
|
||||
"--negative-prompt",
|
||||
T2V_NEGATIVE_PROMPT,
|
||||
"--height",
|
||||
"256",
|
||||
"--width",
|
||||
"256",
|
||||
"--num-frames",
|
||||
"145",
|
||||
"--num-inference-steps",
|
||||
"6",
|
||||
"--guidance-scale",
|
||||
"4.0",
|
||||
"--frame-rate",
|
||||
"24",
|
||||
"--fps",
|
||||
"24",
|
||||
"--seed",
|
||||
"42",
|
||||
"--cfg-parallel-size",
|
||||
"2",
|
||||
"--enforce-eager",
|
||||
"--output",
|
||||
str(output),
|
||||
]
|
||||
_run_and_check(cmd, env, output, expected_md5="08e606b9c522fee4b6f30cee8b77db40")
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.slow
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_ltx2_t2v_no_cfg(tmp_path: Path):
|
||||
"""T2V with CFG=1.0 (no classifier-free guidance)."""
|
||||
output = tmp_path / "t2v_nocfg.mp4"
|
||||
env = os.environ.copy()
|
||||
env.setdefault("CUDA_VISIBLE_DEVICES", "0")
|
||||
cmd = [
|
||||
sys.executable,
|
||||
str(T2V_EXAMPLE),
|
||||
"--model",
|
||||
_get_ltx2_model(),
|
||||
"--prompt",
|
||||
T2V_PROMPT,
|
||||
"--height",
|
||||
"256",
|
||||
"--width",
|
||||
"256",
|
||||
"--num-frames",
|
||||
"145",
|
||||
"--num-inference-steps",
|
||||
"6",
|
||||
"--guidance-scale",
|
||||
"1.0",
|
||||
"--frame-rate",
|
||||
"24",
|
||||
"--fps",
|
||||
"24",
|
||||
"--seed",
|
||||
"42",
|
||||
"--enforce-eager",
|
||||
"--output",
|
||||
str(output),
|
||||
]
|
||||
_run_and_check(cmd, env, output, expected_md5="a83994b94b6e67c54a524e0383c45ce8")
|
||||
|
||||
|
||||
# ── I2V tests ──
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parallel
|
||||
@pytest.mark.slow
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=2)
|
||||
def test_ltx2_i2v_cfg_parallel(tmp_path: Path):
|
||||
"""I2V with CFG=4.0, cfg-parallel-size=2."""
|
||||
test_image = tmp_path / "test_input.png"
|
||||
_make_deterministic_test_image(test_image)
|
||||
output = tmp_path / "i2v_cfg4.mp4"
|
||||
env = os.environ.copy()
|
||||
env.setdefault("CUDA_VISIBLE_DEVICES", "0,1")
|
||||
cmd = [
|
||||
sys.executable,
|
||||
str(I2V_EXAMPLE),
|
||||
"--model",
|
||||
_get_ltx2_model(),
|
||||
"--model-class-name",
|
||||
"LTX2ImageToVideoPipeline",
|
||||
"--image",
|
||||
str(test_image),
|
||||
"--prompt",
|
||||
I2V_PROMPT,
|
||||
"--negative-prompt",
|
||||
I2V_NEGATIVE_PROMPT,
|
||||
"--height",
|
||||
"256",
|
||||
"--width",
|
||||
"256",
|
||||
"--num-frames",
|
||||
"73",
|
||||
"--num-inference-steps",
|
||||
"6",
|
||||
"--guidance-scale",
|
||||
"4.0",
|
||||
"--frame-rate",
|
||||
"24",
|
||||
"--fps",
|
||||
"24",
|
||||
"--seed",
|
||||
"42",
|
||||
"--cfg-parallel-size",
|
||||
"2",
|
||||
"--enforce-eager",
|
||||
"--output",
|
||||
str(output),
|
||||
]
|
||||
_run_and_check(cmd, env, output, expected_md5="aed7e56084b36373244d8f839b16d115")
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.slow
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_ltx2_i2v_no_cfg(tmp_path: Path):
|
||||
"""I2V with CFG=1.0 (no classifier-free guidance)."""
|
||||
test_image = tmp_path / "test_input.png"
|
||||
_make_deterministic_test_image(test_image)
|
||||
output = tmp_path / "i2v_nocfg.mp4"
|
||||
env = os.environ.copy()
|
||||
env.setdefault("CUDA_VISIBLE_DEVICES", "0")
|
||||
cmd = [
|
||||
sys.executable,
|
||||
str(I2V_EXAMPLE),
|
||||
"--model",
|
||||
_get_ltx2_model(),
|
||||
"--model-class-name",
|
||||
"LTX2ImageToVideoPipeline",
|
||||
"--image",
|
||||
str(test_image),
|
||||
"--prompt",
|
||||
I2V_PROMPT,
|
||||
"--height",
|
||||
"256",
|
||||
"--width",
|
||||
"256",
|
||||
"--num-frames",
|
||||
"73",
|
||||
"--num-inference-steps",
|
||||
"6",
|
||||
"--guidance-scale",
|
||||
"1.0",
|
||||
"--frame-rate",
|
||||
"24",
|
||||
"--fps",
|
||||
"24",
|
||||
"--seed",
|
||||
"42",
|
||||
"--enforce-eager",
|
||||
"--output",
|
||||
str(output),
|
||||
]
|
||||
_run_and_check(cmd, env, output, expected_md5="81b21ede12753e9e14a357a6c548b666")
|
||||
Reference in new issue
Block a user