[Diffusion] Refactor LTX2 to use unified CFG parallel framework (#2160)

Signed-off-by: Yangshen Deng <yangshen.d@outlook.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Yangshen DengandClaude Opus 4.6 authored and GitHub committed 2026-04-08 11:44:04 +08:00
1 parent ec082add35
commit 1cd5210440
5 files changed
+764 -424

No files matched your search

@@ -0,0 +1,217 @@
[
{
"test_name": "test_ltx2_baseline_eager",
"description": "Single-device baseline with enforce-eager (no torch.compile)",
"server_type": "vllm-omni",
"server_params": {
"model": "Lightricks/LTX-2",
"serve_args": {
"enforce-eager": true,
"enable-diffusion-pipeline-profiler": true
}
},
"benchmark_params": [
{
"name": "256x256_145f_steps6",
"dataset": "random",
"task": "t2v",
"backend": "v1/videos",
"width": 256,
"height": 256,
"num-frames": 145,
"fps": 24,
"num-inference-steps": 6,
"num-prompts": 3,
"max-concurrency": 1,
"enable-negative-prompt": true
},
{
"name": "480x768_41f_steps20",
"dataset": "random",
"task": "t2v",
"backend": "v1/videos",
"width": 768,
"height": 480,
"num-frames": 41,
"fps": 24,
"num-inference-steps": 20,
"num-prompts": 3,
"max-concurrency": 1,
"enable-negative-prompt": true
}
]
},
{
"test_name": "test_ltx2_torch_compile",
"description": "Single-device with torch.compile (default, no enforce-eager)",
"server_type": "vllm-omni",
"server_params": {
"model": "Lightricks/LTX-2",
"serve_args": {
"enable-diffusion-pipeline-profiler": true
}
},
"benchmark_params": [
{
"name": "256x256_145f_steps6",
"dataset": "random",
"task": "t2v",
"backend": "v1/videos",
"width": 256,
"height": 256,
"num-frames": 145,
"fps": 24,
"num-inference-steps": 6,
"num-prompts": 3,
"max-concurrency": 1,
"enable-negative-prompt": true
},
{
"name": "480x768_41f_steps20",
"dataset": "random",
"task": "t2v",
"backend": "v1/videos",
"width": 768,
"height": 480,
"num-frames": 41,
"fps": 24,
"num-inference-steps": 20,
"num-prompts": 3,
"max-concurrency": 1,
"enable-negative-prompt": true
}
]
},
{
"test_name": "test_ltx2_cfg2_eager",
"description": "CFG-parallel=2 with enforce-eager",
"server_type": "vllm-omni",
"server_params": {
"model": "Lightricks/LTX-2",
"serve_args": {
"cfg-parallel-size": 2,
"enforce-eager": true,
"enable-diffusion-pipeline-profiler": true
}
},
"benchmark_params": [
{
"name": "256x256_145f_steps6",
"dataset": "random",
"task": "t2v",
"backend": "v1/videos",
"width": 256,
"height": 256,
"num-frames": 145,
"fps": 24,
"num-inference-steps": 6,
"num-prompts": 3,
"max-concurrency": 1,
"enable-negative-prompt": true
},
{
"name": "480x768_41f_steps20",
"dataset": "random",
"task": "t2v",
"backend": "v1/videos",
"width": 768,
"height": 480,
"num-frames": 41,
"fps": 24,
"num-inference-steps": 20,
"num-prompts": 3,
"max-concurrency": 1,
"enable-negative-prompt": true
}
]
},
{
"test_name": "test_ltx2_cfg2_compile",
"description": "CFG-parallel=2 with torch.compile",
"server_type": "vllm-omni",
"server_params": {
"model": "Lightricks/LTX-2",
"serve_args": {
"cfg-parallel-size": 2,
"enable-diffusion-pipeline-profiler": true
}
},
"benchmark_params": [
{
"name": "256x256_145f_steps6",
"dataset": "random",
"task": "t2v",
"backend": "v1/videos",
"width": 256,
"height": 256,
"num-frames": 145,
"fps": 24,
"num-inference-steps": 6,
"num-prompts": 3,
"max-concurrency": 1,
"enable-negative-prompt": true
},
{
"name": "480x768_41f_steps20",
"dataset": "random",
"task": "t2v",
"backend": "v1/videos",
"width": 768,
"height": 480,
"num-frames": 41,
"fps": 24,
"num-inference-steps": 20,
"num-prompts": 3,
"max-concurrency": 1,
"enable-negative-prompt": true
}
]
},
{
"test_name": "test_ltx2_cache_dit_eager",
"description": "CacheDiT with enforce-eager",
"server_type": "vllm-omni",
"server_params": {
"model": "Lightricks/LTX-2",
"serve_args": {
"cache-backend": "cache_dit",
"enforce-eager": true,
"enable-diffusion-pipeline-profiler": true
}
},
"benchmark_params": [
{
"name": "256x256_145f_steps6",
"dataset": "random",
"task": "t2v",
"backend": "v1/videos",
"width": 256,
"height": 256,
"num-frames": 145,
"fps": 24,
"num-inference-steps": 6,
"num-prompts": 3,
"max-concurrency": 1,
"enable-negative-prompt": true
},
{
"name": "480x768_41f_steps20",
"dataset": "random",
"task": "t2v",
"backend": "v1/videos",
"width": 768,
"height": 480,
"num-frames": 41,
"fps": 24,
"num-inference-steps": 20,
"num-prompts": 3,
"max-concurrency": 1,
"enable-negative-prompt": true
}
]
}
]
@@ -0,0 +1,58 @@
from types import SimpleNamespace
import pytest
import torch
from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _make_pipeline(sequence_parallel_size: int = 1) -> LTX2Pipeline:
pipeline = object.__new__(LTX2Pipeline)
torch.nn.Module.__init__(pipeline)
pipeline.audio_vae_temporal_compression_ratio = 4
pipeline.audio_vae_mel_compression_ratio = 4
pipeline.od_config = SimpleNamespace(parallel_config=SimpleNamespace(sequence_parallel_size=sequence_parallel_size))
# Mock audio_vae with identity normalization (mean=0, std=1) so
# _normalize_audio_latents is a no-op and test values are preserved.
pipeline.audio_vae = SimpleNamespace(
latents_mean=torch.tensor(0.0),
latents_std=torch.tensor(1.0),
)
return pipeline
def test_prepare_audio_latents_pads_packed_sequence_dim_for_provided_latents():
pipeline = _make_pipeline(sequence_parallel_size=4)
latents = torch.arange(40, dtype=torch.float32).view(1, 10, 4)
padded, original_num_frames, padded_num_frames = pipeline.prepare_audio_latents(
batch_size=1,
num_channels_latents=2,
num_mel_bins=8,
audio_latent_length=10,
dtype=torch.float32,
device=torch.device("cpu"),
latents=latents,
)
assert original_num_frames == 10
assert padded_num_frames == 12
assert padded.shape == (1, 12, 4)
torch.testing.assert_close(padded[:, :10], latents)
torch.testing.assert_close(padded[:, 10:], torch.zeros(1, 2, 4))
def test_unpad_audio_latents_restores_original_frames_before_unpack():
pipeline = _make_pipeline()
original = torch.arange(40, dtype=torch.float32).view(1, 10, 4)
padded = torch.cat([original, torch.full((1, 2, 4), 999.0)], dim=1)
unpadded = pipeline._unpad_audio_latents(padded, 10)
unpacked = pipeline._unpack_audio_latents(unpadded, latent_length=10, num_mel_bins=2)
expected = pipeline._unpack_audio_latents(original, latent_length=10, num_mel_bins=2)
assert unpacked.shape == (1, 2, 10, 2)
assert not (unpacked == 999.0).any()
torch.testing.assert_close(unpacked, expected)
@@ -0,0 +1,243 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import hashlib
import os
import subprocess
import sys
from pathlib import Path
import numpy as np
import pytest
from PIL import Image
from tests.utils import hardware_test
REPO_ROOT = Path(__file__).resolve().parents[3]
T2V_EXAMPLE = REPO_ROOT / "examples" / "offline_inference" / "text_to_video" / "text_to_video.py"
I2V_EXAMPLE = REPO_ROOT / "examples" / "offline_inference" / "image_to_video" / "image_to_video.py"
T2V_PROMPT = (
"At sunrise, a glowing paper lantern boat drifts through a narrow canal between mossy stone walls, "
"soft fog above the water, the camera slowly gliding forward as golden reflections shimmer across "
"the ripples, cinematic, realistic, highly detailed."
)
T2V_NEGATIVE_PROMPT = "worst quality, blurry, jittery motion, distorted, oversaturated, artifacts"
I2V_PROMPT = "A cinematic dolly shot of a boat drifting on calm water at sunset"
I2V_NEGATIVE_PROMPT = "worst quality, blurry, jittery motion"
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
def _get_ltx2_model() -> str:
return os.environ.get("VLLM_TEST_LTX2_MODEL", "Lightricks/LTX-2")
def _md5(path: Path) -> str:
digest = hashlib.md5(usedforsecurity=False)
with path.open("rb") as f:
for chunk in iter(lambda: f.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _make_deterministic_test_image(path: Path) -> None:
"""Create a deterministic 256x256 test image for I2V tests."""
rng = np.random.RandomState(42)
img = Image.fromarray(rng.randint(0, 255, (256, 256, 3), dtype=np.uint8))
img.save(path)
def _run_and_check(cmd: list[str], env: dict, output_path: Path, expected_md5: str) -> None:
result = subprocess.run(cmd, cwd=REPO_ROOT, env=env, capture_output=True, text=True, check=False)
assert result.returncode == 0, (
f"Command failed (exit {result.returncode}).\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}"
)
generated_md5 = _md5(output_path)
assert generated_md5 == expected_md5, (
f"Unexpected output md5: {generated_md5} != {expected_md5}.\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}"
)
# ── T2V tests ──
@pytest.mark.advanced_model
@pytest.mark.diffusion
@pytest.mark.parallel
@pytest.mark.slow
@hardware_test(res={"cuda": "L4"}, num_cards=2)
def test_ltx2_t2v_cfg_parallel(tmp_path: Path):
"""T2V with CFG=4.0, cfg-parallel-size=2."""
output = tmp_path / "t2v_cfg4.mp4"
env = os.environ.copy()
env.setdefault("CUDA_VISIBLE_DEVICES", "0,1")
cmd = [
sys.executable,
str(T2V_EXAMPLE),
"--model",
_get_ltx2_model(),
"--prompt",
T2V_PROMPT,
"--negative-prompt",
T2V_NEGATIVE_PROMPT,
"--height",
"256",
"--width",
"256",
"--num-frames",
"145",
"--num-inference-steps",
"6",
"--guidance-scale",
"4.0",
"--frame-rate",
"24",
"--fps",
"24",
"--seed",
"42",
"--cfg-parallel-size",
"2",
"--enforce-eager",
"--output",
str(output),
]
_run_and_check(cmd, env, output, expected_md5="08e606b9c522fee4b6f30cee8b77db40")
@pytest.mark.advanced_model
@pytest.mark.diffusion
@pytest.mark.slow
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_ltx2_t2v_no_cfg(tmp_path: Path):
"""T2V with CFG=1.0 (no classifier-free guidance)."""
output = tmp_path / "t2v_nocfg.mp4"
env = os.environ.copy()
env.setdefault("CUDA_VISIBLE_DEVICES", "0")
cmd = [
sys.executable,
str(T2V_EXAMPLE),
"--model",
_get_ltx2_model(),
"--prompt",
T2V_PROMPT,
"--height",
"256",
"--width",
"256",
"--num-frames",
"145",
"--num-inference-steps",
"6",
"--guidance-scale",
"1.0",
"--frame-rate",
"24",
"--fps",
"24",
"--seed",
"42",
"--enforce-eager",
"--output",
str(output),
]
_run_and_check(cmd, env, output, expected_md5="a83994b94b6e67c54a524e0383c45ce8")
# ── I2V tests ──
@pytest.mark.advanced_model
@pytest.mark.diffusion
@pytest.mark.parallel
@pytest.mark.slow
@hardware_test(res={"cuda": "L4"}, num_cards=2)
def test_ltx2_i2v_cfg_parallel(tmp_path: Path):
"""I2V with CFG=4.0, cfg-parallel-size=2."""
test_image = tmp_path / "test_input.png"
_make_deterministic_test_image(test_image)
output = tmp_path / "i2v_cfg4.mp4"
env = os.environ.copy()
env.setdefault("CUDA_VISIBLE_DEVICES", "0,1")
cmd = [
sys.executable,
str(I2V_EXAMPLE),
"--model",
_get_ltx2_model(),
"--model-class-name",
"LTX2ImageToVideoPipeline",
"--image",
str(test_image),
"--prompt",
I2V_PROMPT,
"--negative-prompt",
I2V_NEGATIVE_PROMPT,
"--height",
"256",
"--width",
"256",
"--num-frames",
"73",
"--num-inference-steps",
"6",
"--guidance-scale",
"4.0",
"--frame-rate",
"24",
"--fps",
"24",
"--seed",
"42",
"--cfg-parallel-size",
"2",
"--enforce-eager",
"--output",
str(output),
]
_run_and_check(cmd, env, output, expected_md5="aed7e56084b36373244d8f839b16d115")
@pytest.mark.advanced_model
@pytest.mark.diffusion
@pytest.mark.slow
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_ltx2_i2v_no_cfg(tmp_path: Path):
"""I2V with CFG=1.0 (no classifier-free guidance)."""
test_image = tmp_path / "test_input.png"
_make_deterministic_test_image(test_image)
output = tmp_path / "i2v_nocfg.mp4"
env = os.environ.copy()
env.setdefault("CUDA_VISIBLE_DEVICES", "0")
cmd = [
sys.executable,
str(I2V_EXAMPLE),
"--model",
_get_ltx2_model(),
"--model-class-name",
"LTX2ImageToVideoPipeline",
"--image",
str(test_image),
"--prompt",
I2V_PROMPT,
"--height",
"256",
"--width",
"256",
"--num-frames",
"73",
"--num-inference-steps",
"6",
"--guidance-scale",
"1.0",
"--frame-rate",
"24",
"--fps",
"24",
"--seed",
"42",
"--enforce-eager",
"--output",
str(output),
]
_run_and_check(cmd, env, output, expected_md5="81b21ede12753e9e14a357a6c548b666")