anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

[Diffusion] Add LTX-2.5 Diffusion VAE decoder support (#6189)

Browse Source 
Signed-off-by: mglyn <1203789601@qq.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
Mu GuanLinandHongsheng Liu authored and GitHub committed 2026-08-27 23:54:40 +08:00
1 parent 262ff14693
commit 5e4daaf197
22 files changed
+1832 -61
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/diffusion/models/ltx2/test_ltx25_pipeline.py
+165 -5
View File
@@ -1,11 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Focused LTX-2.5 pipeline correctness tests."""
import json
from dataclasses import replace
from types import SimpleNamespace
from typing import Any
import pytest
import torch
@@ -170,7 +171,10 @@ def _resolve_request_inputs_for_test(
def test_ltx25_missing_gemma4_recommends_supported_transformers_range(monkeypatch):
pipe = SimpleNamespace(component_profile=replace(LTX25_FULL_COMPONENT_PROFILE, text_encoder_cls=None))
pipe = SimpleNamespace(
component_profile=replace(LTX25_FULL_COMPONENT_PROFILE, text_encoder_cls=None),
use_diffusion_decoder=True,
)
od_config = SimpleNamespace(model="Lightricks/LTX-2.5-Diffusers", dtype=torch.bfloat16)
monkeypatch.setattr(ltx2_components, "get_local_device", lambda: torch.device("cpu"))
@@ -187,7 +191,19 @@ def test_ltx25_missing_gemma4_recommends_supported_transformers_range(monkeypatc
def test_ltx_converted_component_loading_propagates_revision(monkeypatch):
revision = "pinned-revision"
calls = {"components": []}
calls: dict[str, Any] = {"components": []}
natten_processor = object()
class FakeDiffusionDecoder:
def set_attn_processor(self, processor):
calls["diffusion_decoder_processor"] = processor
def enable_tiling(self):
calls["diffusion_decoder_tiling"] = True
def set_parallel_size(self, parallel_size, mode):
calls["diffusion_decoder_parallel"] = (parallel_size, mode)
profile = replace(
LTX25_DISTILLED_COMPONENT_PROFILE,
text_encoder_cls=object,
@@ -196,12 +212,19 @@ def test_ltx_converted_component_loading_propagates_revision(monkeypatch):
vocoder_fallback_cls=None,
scheduler_use_dynamic_shifting=False,
)
pipeline = SimpleNamespace(component_profile=profile, pipeline_kind="distilled_two_stage")
pipeline = SimpleNamespace(
component_profile=profile,
pipeline_kind="distilled_two_stage",
use_diffusion_decoder=True,
)
od_config = SimpleNamespace(
model="org/converted-ltx25",
revision=revision,
dtype=torch.bfloat16,
quantization_config=None,
extras={},
vae_use_tiling=False,
parallel_config=SimpleNamespace(vae_patch_parallel_size=2, vae_parallel_mode="tile"),
)
monkeypatch.setattr(ltx2_components, "get_local_device", lambda: torch.device("cpu"))
@@ -227,6 +250,10 @@ def test_ltx_converted_component_loading_propagates_revision(monkeypatch):
)
return object()
def fake_native_diffusion_decoder(model, **kwargs):
calls["native_diffusion_decoder"] = (model, kwargs)
return FakeDiffusionDecoder()
def fake_transformer_config(model, subfolder, local_files_only, *, revision):
calls["transformer_config"] = (model, subfolder, local_files_only, revision)
return {"component": "transformer"}
@@ -234,7 +261,13 @@ def test_ltx_converted_component_loading_propagates_revision(monkeypatch):
monkeypatch.setattr(ltx2_components, "prefetch_subfolders", fake_prefetch)
monkeypatch.setattr(ltx2_components.AutoTokenizer, "from_pretrained", fake_tokenizer)
monkeypatch.setattr(ltx2_components, "_load_component", fake_component)
monkeypatch.setattr(ltx2_components, "_load_ltx25_native_diffusion_decoder", fake_native_diffusion_decoder)
monkeypatch.setattr(ltx2_components, "load_transformer_config", fake_transformer_config)
monkeypatch.setattr(
ltx2_components,
"LTX2VideoVaeNeighborhoodNattenProcessor",
lambda: natten_processor,
)
monkeypatch.setattr(
ltx2_components,
"create_transformer_from_config",
@@ -261,6 +294,14 @@ def test_ltx_converted_component_loading_propagates_revision(monkeypatch):
"latent_upsampler",
}
assert all(call[3]["revision"] == revision for call in calls["components"])
assert "diffusion_decoder" not in calls["prefetch"][1]
assert calls["native_diffusion_decoder"] == (
od_config.model,
{"local_files_only": False, "dtype": torch.bfloat16, "revision": revision},
)
assert calls["diffusion_decoder_processor"] is natten_processor
assert calls["diffusion_decoder_tiling"] is True
assert calls["diffusion_decoder_parallel"] == (2, "tile")
assert calls["transformer_config"] == (
od_config.model,
profile.transformer_subfolder,
@@ -270,6 +311,85 @@ def test_ltx_converted_component_loading_propagates_revision(monkeypatch):
assert calls["scheduler"][1]["revision"] == revision
def test_ltx25_native_diffusion_decoder_uses_diffusers_config_and_canonical_artifact(monkeypatch):
revision = "diffusers-revision"
config = {"decoder_stage_channels": [8, 8, 8, 8, 8]}
calls: dict[str, Any] = {}
class FakeDecoder:
def init_distributed(self):
calls["init_distributed"] = True
expected = FakeDecoder()
def fake_load_config(model, **kwargs):
calls["config"] = (model, kwargs)
return config
def fake_resolve(model, repo_id, filename, **kwargs):
calls["artifact"] = (model, repo_id, filename, kwargs)
return "/models/native-diffvae.safetensors"
def fake_load(cls, path, passed_config, dtype):
calls["load"] = (path, passed_config, dtype)
return expected
monkeypatch.setattr(
ltx2_components.DistributedLTX2VideoDiffusionDecoderModel,
"load_config",
staticmethod(fake_load_config),
)
monkeypatch.setattr(ltx2_components, "resolve_ltx_artifact", fake_resolve)
monkeypatch.setattr(
ltx2_components.DistributedLTX2VideoDiffusionDecoderModel,
"from_ltx25_native_checkpoint",
classmethod(fake_load),
)
actual = ltx2_components._load_ltx25_native_diffusion_decoder(
"Lightricks/LTX-2.5-Diffusers",
local_files_only=False,
dtype=torch.bfloat16,
revision=revision,
)
assert actual is expected
assert calls["config"] == (
"Lightricks/LTX-2.5-Diffusers",
{
"subfolder": "diffusion_decoder",
"local_files_only": False,
"revision": revision,
},
)
assert calls["artifact"] == (
"Lightricks/LTX-2.5-Diffusers",
"Lightricks/LTX-2.5",
"vae/ltx-2.5-video-vae-bf16.safetensors",
{
"model_revision": revision,
"artifact_revision": ltx2_components.LTX25_NATIVE_ARTIFACT_REVISION,
},
)
assert calls["load"] == ("/models/native-diffvae.safetensors", config, torch.bfloat16)
assert calls["init_distributed"] is True
def test_ltx25_natten_failure_has_actionable_omni_remedy(monkeypatch):
class MissingNattenProcessor:
def __init__(self):
raise ImportError("kernels is unavailable")
monkeypatch.setattr(
ltx2_components,
"LTX2VideoVaeNeighborhoodNattenProcessor",
MissingNattenProcessor,
)
with pytest.raises(RuntimeError, match=r"kernels==0\.14\.1.*supported GPU.*allow Hub access"):
ltx2_components._create_ltx25_natten_processor()
def test_ltx_checkpoint_explicit_version_precedes_structural_heuristics(tmp_path):
(tmp_path / "model_index.json").write_text(json.dumps({"model_version": "2.3.1"}))
(tmp_path / "transformer").mkdir()
@@ -411,6 +531,46 @@ def test_ltx25_four_public_pipeline_semantics_are_disjoint():
assert LTX2DistilledTwoStagePipeline.pipeline_kind == "distilled_two_stage"
@pytest.mark.parametrize(
"pipeline_cls",
[
LTX2Pipeline,
LTX2TwoStagePipeline,
LTX2DistilledOneStagePipeline,
LTX2DistilledTwoStagePipeline,
],
)
@pytest.mark.parametrize(
("extras", "use_diffusion_decoder"),
[({}, True), ({"ltx2_use_conv_vae": True}, False)],
)
def test_ltx25_all_public_pipelines_default_to_diffvae_with_conv_opt_in(
tmp_path, monkeypatch, pipeline_cls, extras, use_diffusion_decoder
):
(tmp_path / "model_index.json").write_text(
json.dumps({"text_encoder": ["transformers", "Gemma4UnifiedForConditionalGeneration"]})
)
def stub_components(pipe, od_config):
pipe.od_config = od_config
pipe.vae_spatial_compression_ratio = 32
monkeypatch.setattr(ltx2_runtime, "initialize_pipeline_components", stub_components)
monkeypatch.setattr(ltx2_runtime, "build_ltx_phase_adapter", lambda _pipe: None)
monkeypatch.setattr(LTXRuntime, "setup_diffusion_pipeline_profiler", lambda *_args, **_kwargs: None)
pipe = pipeline_cls(
od_config=SimpleNamespace(
model=str(tmp_path),
extras=extras,
enable_diffusion_pipeline_profiler=False,
)
)
assert pipe.use_diffusion_decoder is use_diffusion_decoder
assert pipe._vae_modules.count("diffusion_decoder") == int(use_diffusion_decoder)
@pytest.mark.parametrize(
("pipeline_kind", "expected"),
[
@@ -673,7 +833,7 @@ def test_ltx25_distilled_two_stage_executes_custom_phase_schedules():
max_sequence_length=16,
)
prompt_context = object()
phase_calls = []
phase_calls: list[Any] = []
def resolve_request_inputs(req, **kwargs):
return request_inputs
tests/diffusion/models/ltx2/test_ltx2_conditioning.py
+34 -1
View File
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Unit tests for LTX image-to-video input and conditioning behavior."""
@@ -258,6 +258,39 @@ class TestLTXImageToVideoConditioning:
assert LTX2Pipeline.support_image_input is True
def test_ltx25_i2v_encode_uses_diffusion_decoder_statistics(self, monkeypatch):
import vllm_omni.diffusion.models.ltx2.ltx2_conditioning as ltx2_conditioning
from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline
pipe = object.__new__(LTX2Pipeline)
torch.nn.Module.__init__(pipe)
pipe.use_diffusion_decoder = True
pipe.vae = SimpleNamespace(
encode=lambda image: image,
latents_mean=torch.full((2,), 100.0),
latents_std=torch.full((2,), 10.0),
config=SimpleNamespace(scaling_factor=1.0),
)
pipe.diffusion_decoder = SimpleNamespace(
latents_mean=torch.tensor([1.0, 3.0]),
latents_std=torch.tensor([2.0, 4.0]),
config=SimpleNamespace(scaling_factor=2.0),
)
monkeypatch.setattr(
ltx2_conditioning,
"retrieve_latents",
lambda *_args, **_kwargs: torch.tensor([[[[[3.0]]], [[[7.0]]]]]),
)
actual = pipe._encode_i2v_image_latents(
torch.zeros(1, 3, 1, 1),
batch_size=1,
generator=None,
dtype=torch.float32,
)
torch.testing.assert_close(actual, torch.full((1, 2, 1, 1, 1), 2.0))
def test_ltx23_i2v_rejects_multi_image_prompt_list(self):
from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline
tests/diffusion/models/ltx2/test_ltx2_latents.py
+36
View File
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from types import SimpleNamespace
import pytest
@@ -47,6 +50,39 @@ def test_prepare_video_latents_matches_official_values_and_token_major_layout():
assert actual.stride()[1:] == (1, actual.shape[1])
def test_prepare_video_latents_uses_diffusion_decoder_statistics():
pipeline = _make_pipeline(LTX2Pipeline)
pipeline.use_diffusion_decoder = True
pipeline.vae_spatial_compression_ratio = 1
pipeline.vae_temporal_compression_ratio = 1
pipeline.transformer_spatial_patch_size = 1
pipeline.transformer_temporal_patch_size = 1
pipeline.vae = SimpleNamespace(
latents_mean=torch.full((2,), 100.0),
latents_std=torch.full((2,), 10.0),
config=SimpleNamespace(scaling_factor=1.0),
)
pipeline.diffusion_decoder = SimpleNamespace(
latents_mean=torch.tensor([1.0, 3.0]),
latents_std=torch.tensor([2.0, 4.0]),
config=SimpleNamespace(scaling_factor=2.0),
)
actual = pipeline.prepare_latents(
batch_size=1,
num_channels_latents=2,
height=1,
width=1,
num_frames=1,
noise_scale=0.0,
dtype=torch.float32,
device=torch.device("cpu"),
latents=torch.tensor([[[[[3.0]]], [[[7.0]]]]]),
)
torch.testing.assert_close(actual, torch.full((1, 1, 2), 2.0))
def test_prepare_audio_latents_samples_directly_in_packed_token_space():
pipeline = _make_pipeline(LTX2Pipeline)
tests/diffusion/models/ltx2/test_ltx2_phase_adapter.py
+7 -3
View File
@@ -1,9 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Unit tests for the fixed LTX refinement-phase adapter."""
from types import SimpleNamespace
from typing import Any
import pytest
import torch
@@ -148,10 +149,13 @@ def test_ltx_phase_adapter_selects_execution_from_quantization(
):
pipeline = _pipeline(dtype=dtype, quantization_config=quantization_config)
manifest = object()
captured = {}
captured: dict[str, Any] = {}
def resolve_artifact(*_args, **kwargs):
assert not kwargs
assert kwargs == {
"model_revision": None,
"artifact_revision": None,
}
return "adapter"
def parse(transformer, path):
tests/diffusion/models/ltx2/test_ltx2_pipeline.py
+121 -8
View File
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Unit tests for the shared LTX pipeline runtime and public contracts."""
@@ -8,6 +8,7 @@ import os
import tempfile
from dataclasses import replace
from types import SimpleNamespace
from typing import Any
import pytest
import torch
@@ -165,7 +166,19 @@ def test_ltx_checkpoint_version_detection_uses_metadata(tmp_path):
assert detect_ltx_model_version(str(tmp_path)) == "2"
def test_ltx_artifact_falls_back_to_unpinned_hub_download(tmp_path, monkeypatch):
@pytest.mark.parametrize(
("model", "repo_id", "expected_revision"),
[
("Lightricks/test", "Lightricks/test", "model-revision"),
("Lightricks/test-Diffusers", "Lightricks/test", "artifact-revision"),
],
)
def test_ltx_artifact_uses_source_revision_and_hub_fallback(
monkeypatch,
model,
repo_id,
expected_revision,
):
filename = "ltx-sidecar.safetensors"
calls = []
@@ -176,9 +189,22 @@ def test_ltx_artifact_falls_back_to_unpinned_hub_download(tmp_path, monkeypatch)
monkeypatch.setattr(ltx2_components, "hf_hub_download", fake_download)
assert (
resolve_ltx_artifact(str(tmp_path / "model"), "Lightricks/test", filename) == "/cache/ltx-sidecar.safetensors"
resolve_ltx_artifact(
model,
repo_id,
filename,
model_revision="model-revision",
artifact_revision="artifact-revision",
)
== "/cache/ltx-sidecar.safetensors"
)
assert calls == [{"repo_id": "Lightricks/test", "filename": filename}]
assert calls == [
{
"repo_id": repo_id,
"filename": filename,
"revision": expected_revision,
}
]
def test_ltx_artifact_prefers_model_root(tmp_path, monkeypatch):
@@ -187,7 +213,41 @@ def test_ltx_artifact_prefers_model_root(tmp_path, monkeypatch):
expected.write_bytes(b"sidecar")
monkeypatch.setattr(ltx2_components, "hf_hub_download", lambda **_kwargs: pytest.fail("unexpected Hub lookup"))
assert resolve_ltx_artifact(str(tmp_path), "Lightricks/test", filename) == str(expected)
assert resolve_ltx_artifact(
str(tmp_path),
"Lightricks/test",
filename,
model_revision="unused-model-revision",
artifact_revision="unused-artifact-revision",
) == str(expected)
def test_ltx_artifact_local_model_missing_sidecar_falls_back_to_hub(tmp_path, monkeypatch):
calls = []
def fake_download(**kwargs):
calls.append(kwargs)
return "/cache/ltx-sidecar.safetensors"
monkeypatch.setattr(ltx2_components, "hf_hub_download", fake_download)
assert (
resolve_ltx_artifact(
str(tmp_path),
"Lightricks/test",
"ltx-sidecar.safetensors",
model_revision="local-model-revision",
artifact_revision="artifact-revision",
)
== "/cache/ltx-sidecar.safetensors"
)
assert calls == [
{
"repo_id": "Lightricks/test",
"filename": "ltx-sidecar.safetensors",
"revision": "artifact-revision",
}
]
@pytest.mark.parametrize(
@@ -900,7 +960,7 @@ def test_ltx_two_stage_executes_declarative_i2v_phase_plan(pipeline_cls):
max_sequence_length=16,
)
prompt_context = object()
phase_calls = []
phase_calls: list[Any] = []
def resolve_request_inputs(req, **kwargs):
return request_inputs
@@ -959,7 +1019,7 @@ def test_ltx_two_stage_executes_declarative_i2v_phase_plan(pipeline_cls):
pipeline.vae_spatial_compression_ratio = 32
pipeline.vae_temporal_compression_ratio = 8
pipeline.latent_upsampler = FakeUpsampler()
activated_slots = []
activated_slots: list[Any] = []
if pipeline_cls is LTX2TwoStagePipeline:
pipeline._phase_adapter = SimpleNamespace(activate=activated_slots.append)
object.__setattr__(pipeline, "_resolve_request_inputs", resolve_request_inputs)
@@ -1133,6 +1193,59 @@ def test_ltx_two_stage_recipe_rejects_sigmas():
)
def test_ltx_request_applies_diffvae_minimum_spatial_size_only_when_requested():
request = LTXRequestInputs(
prompt="prompt",
negative_prompt="",
height=128,
width=128,
num_frames=9,
frame_rate=24.0,
num_inference_steps=30,
guidance=LTX2_ONE_STAGE_RECIPE.request_guidance,
num_videos_per_prompt=1,
generator=None,
latents=None,
audio_latents=None,
prompt_embeds=None,
negative_prompt_embeds=None,
prompt_attention_mask=None,
negative_prompt_attention_mask=None,
decode_timestep=0.0,
decode_noise_scale=None,
output_type="np",
max_sequence_length=16,
)
with pytest.raises(ValueError, match="DiffVAE requires video latent spatial dimensions of at least 7x7"):
validate_pipeline_request(
request,
pipeline_recipe=LTX2_ONE_STAGE_RECIPE,
vae_spatial_compression_ratio=32,
vae_temporal_compression_ratio=8,
pipeline_name="LTX2Pipeline",
min_video_latent_spatial_size=(7, 7),
)
# ConvVAE does not use NATTEN, so its existing 128x128 boundary remains valid.
validate_pipeline_request(
request,
pipeline_recipe=LTX2_ONE_STAGE_RECIPE,
vae_spatial_compression_ratio=32,
vae_temporal_compression_ratio=8,
pipeline_name="LTX2Pipeline",
)
validate_pipeline_request(
replace(request, height=256, width=256),
pipeline_recipe=LTX2_ONE_STAGE_RECIPE,
vae_spatial_compression_ratio=32,
vae_temporal_compression_ratio=8,
pipeline_name="LTX2Pipeline",
min_video_latent_spatial_size=(7, 7),
)
@pytest.mark.parametrize(
("direct_kwargs", "sampling_kwargs", "error"),
[
@@ -1284,7 +1397,7 @@ def test_ltx_official_two_stage_recipes_only_vary_by_model_defaults(recipe, step
def test_ltx_two_stage_entries_select_the_official_adapter_slots():
phase_switches = []
phase_switches: list[Any] = []
ordinary_pipeline = object.__new__(LTX2TwoStagePipeline)
ordinary_pipeline._phase_adapter = SimpleNamespace(activate=phase_switches.append)
for phase in LTX2_TWO_STAGE_RECIPE.phases:
tests/diffusion/models/ltx2/test_ltx2_vae.py
+439 -3
View File
@@ -1,9 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Unit tests for LTX-2.3 VAE tiling and distributed decode behavior."""
"""Unit tests for LTX video VAE tiling and distributed decode behavior."""
from types import SimpleNamespace
from typing import Any
import pytest
import torch
@@ -36,6 +37,441 @@ def test_ltx_base_vocoder_keeps_native_dtype(monkeypatch):
assert output.dtype == torch.bfloat16
class TestLTXDiffusionDecoder:
@pytest.mark.parametrize("mode", ["spatial_shard_height", "spatial_shard_width"])
def test_distributed_diffusion_decoder_rejects_non_tile_parallel_modes(self, mode):
from vllm_omni.diffusion.models.ltx2.ltx2_diffusion_decoder_distributed import (
DistributedLTX2VideoDiffusionDecoderModel,
)
model = object.__new__(DistributedLTX2VideoDiffusionDecoderModel)
torch.nn.Module.__init__(model)
model.distributed_executor = SimpleNamespace(set_parallel_size=lambda *_args, **_kwargs: None)
with pytest.raises(ValueError, match="only supports vae_parallel_mode='tile'"):
model.set_parallel_size(2, mode=mode)
def test_distributed_diffusion_decoder_accepts_tile_parallel_mode(self):
from vllm_omni.diffusion.models.ltx2.ltx2_diffusion_decoder_distributed import (
DistributedLTX2VideoDiffusionDecoderModel,
)
calls = []
model = object.__new__(DistributedLTX2VideoDiffusionDecoderModel)
torch.nn.Module.__init__(model)
model.distributed_executor = SimpleNamespace(
set_parallel_size=lambda parallel_size, mode: calls.append((parallel_size, mode))
)
model.set_parallel_size(4, mode="tile")
assert calls == [(4, "tile")]
def test_short_clip_keeps_stage5_temporal_context_then_crops_output(self):
from vllm_omni.diffusion.models.ltx2.ltx2_diffusion_decoder import (
LTX2VideoDiffusionDecoder3d,
LTX2VideoDiffusionDecoderModel,
)
class IdentityAttention:
def build_block_mask(self, _hidden_states):
return None
class IdentityBlock(torch.nn.Module):
def __init__(self):
super().__init__()
self.attn = IdentityAttention()
def forward(self, hidden_states, _block_mask):
return hidden_states
class IdentityUpsample(torch.nn.Module):
def forward(self, hidden_states, *, drop_leading_frame):
assert drop_leading_frame
return hidden_states
decoder = object.__new__(LTX2VideoDiffusionDecoder3d)
torch.nn.Module.__init__(decoder)
decoder.det_stages = torch.nn.ModuleList([torch.nn.ModuleList([IdentityBlock()])])
decoder.upsamples = torch.nn.ModuleList([IdentityUpsample()])
decoder.trailing_pad_latent_frames = 2
decoder.temporal_compression_ratio = 8
decoder.stage5_kernel = (11, 3, 3)
# 17 frames contain one real frame followed by 16 propagated ghost
# frames. The reference keeps 11 for stage-5 NATTEN instead of
# cropping the tensor immediately back to one.
context = decoder.forward_stage_4(torch.zeros(1, 17, 3, 3, 4))
assert context.shape == (1, 11, 3, 3, 4)
class FakeDecoder(torch.nn.Module):
def forward(self, z, *, generator, num_inference_steps):
assert generator is None
assert num_inference_steps is None
return torch.zeros(z.shape[0], 3, 11, z.shape[3] * 2, z.shape[4] * 2)
model = object.__new__(LTX2VideoDiffusionDecoderModel)
torch.nn.Module.__init__(model)
model.decoder = FakeDecoder()
model.use_tiling = False
model.tile_sample_min_num_frames = 80
model.tile_sample_min_height = 768
model.tile_sample_min_width = 768
model.temporal_compression_ratio = 8
model.spatial_compression_ratio = 2
output = model.decode(torch.zeros(1, 4, 1, 2, 3), generator=None, return_dict=False)[0]
assert output.shape == (1, 3, 1, 4, 6)
@pytest.mark.parametrize(
("model_version", "extras", "expected"),
[
("2.5", {}, True),
("2.5", {"ltx2_use_conv_vae": True}, False),
("2.3", {}, False),
],
)
def test_ltx25_diffusion_decoder_is_default_with_conv_vae_opt_in(self, model_version, extras, expected):
from vllm_omni.diffusion.models.ltx2.ltx2_components import _ltx2_use_diffusion_decoder
assert _ltx2_use_diffusion_decoder(SimpleNamespace(extras=extras), model_version) is expected
def test_conv_vae_opt_in_rejects_non_boolean_model_extra(self):
from vllm_omni.diffusion.models.ltx2.ltx2_components import _ltx2_use_diffusion_decoder
with pytest.raises(TypeError, match="ltx2_use_conv_vae"):
_ltx2_use_diffusion_decoder(SimpleNamespace(extras={"ltx2_use_conv_vae": "true"}), "2.5")
def test_native_diffusion_decoder_conversion_splits_qkv_and_folds_gates(self):
from vllm_omni.diffusion.models.ltx2.ltx2_diffusion_decoder import (
convert_ltx25_native_diffusion_decoder_state_dict,
)
qkv_weight = torch.arange(24, dtype=torch.float32).reshape(6, 4)
qkv_bias = torch.arange(6, dtype=torch.float32)
projection_weight = torch.arange(4, dtype=torch.float32).reshape(2, 2)
projection_bias = torch.ones(2)
gate = torch.tensor([2.0, 3.0])
mean = torch.tensor([0.1, 0.2])
std = torch.tensor([0.3, 0.4])
native = {
"encoder.ignored": torch.ones(1),
"per_channel_statistics.mean-of-means": mean,
"vae.per_channel_statistics.std-of-means": std,
"decoder.type_emb": torch.ones(2),
"decoder.coarse_head.weight": torch.ones(1),
"decoder.det_stages.0.0.attn.qkv.weight": qkv_weight,
"decoder.det_stages.0.0.attn.qkv.bias": qkv_bias,
"decoder.det_stages.0.0.attn.q_norm.weight": torch.tensor([5.0, 6.0]),
"decoder.det_stages.0.0.attn.k_norm.weight": torch.tensor([7.0, 8.0]),
"decoder.diff_blocks.0.attn.proj.weight": projection_weight,
"decoder.diff_blocks.0.attn.proj.bias": projection_bias,
"decoder.diff_blocks.0.gate_msa": gate,
"vae.decoder.t_embedder.mlp.0.weight": torch.ones(2, 2),
}
converted = convert_ltx25_native_diffusion_decoder_state_dict(native)
torch.testing.assert_close(converted["latents_mean"], mean)
torch.testing.assert_close(converted["latents_std"], std)
torch.testing.assert_close(converted["decoder.det_stages.0.0.attn.to_q.weight"], qkv_weight[:2])
torch.testing.assert_close(converted["decoder.det_stages.0.0.attn.to_k.weight"], qkv_weight[2:4])
torch.testing.assert_close(converted["decoder.det_stages.0.0.attn.to_v.weight"], qkv_weight[4:])
torch.testing.assert_close(converted["decoder.det_stages.0.0.attn.to_q.bias"], qkv_bias[:2])
torch.testing.assert_close(converted["decoder.det_stages.0.0.attn.to_k.bias"], qkv_bias[2:4])
torch.testing.assert_close(converted["decoder.det_stages.0.0.attn.to_v.bias"], qkv_bias[4:])
torch.testing.assert_close(converted["decoder.det_stages.0.0.attn.norm_q.weight"], torch.tensor([5.0, 6.0]))
torch.testing.assert_close(converted["decoder.det_stages.0.0.attn.norm_k.weight"], torch.tensor([7.0, 8.0]))
torch.testing.assert_close(
converted["decoder.diff_blocks.0.attn.to_out.0.weight"], gate.unsqueeze(1) * projection_weight
)
torch.testing.assert_close(converted["decoder.diff_blocks.0.attn.to_out.0.bias"], gate * projection_bias)
assert "decoder.t_embedder.timestep_embedder.linear_1.weight" in converted
assert not any("type_emb" in key or "coarse" in key or "gate_msa" in key for key in converted)
def test_native_diffusion_decoder_conversion_rejects_invalid_fused_qkv(self):
from vllm_omni.diffusion.models.ltx2.ltx2_diffusion_decoder import (
convert_ltx25_native_diffusion_decoder_state_dict,
)
with pytest.raises(ValueError, match="not divisible by 3"):
convert_ltx25_native_diffusion_decoder_state_dict(
{"decoder.det_stages.0.0.attn.qkv.weight": torch.ones(5, 2)}
)
def test_decode_uses_diffusion_decoder_without_conv_vae_conditioning(self):
from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline
seen = {}
class ConvVae:
dtype = torch.float32
config = SimpleNamespace(timestep_conditioning=True)
def decode(self, *_args, **_kwargs):
raise AssertionError("the convolutional VAE decoder must not run")
class DiffusionDecoder:
dtype = torch.float32
def decode(self, latents, *, generator, return_dict):
seen["latents"] = latents
seen["generator"] = generator
seen["return_dict"] = return_dict
return (latents + 1,)
class AudioVae:
dtype = torch.float32
def decode(self, audio_latents, *, return_dict):
assert return_dict is False
return (audio_latents + 2,)
class VideoProcessor:
def postprocess_video(self, video, *, output_type):
seen["output_type"] = output_type
return video
pipe = object.__new__(LTX2Pipeline)
torch.nn.Module.__init__(pipe)
pipe.use_diffusion_decoder = True
pipe.vae = ConvVae()
pipe.diffusion_decoder = DiffusionDecoder()
pipe.audio_vae = AudioVae()
pipe.vocoder = torch.nn.Identity()
pipe.video_processor = VideoProcessor()
generator = torch.Generator().manual_seed(123)
latents = torch.ones(1, 1)
output = pipe._decode_output(
latents=latents,
audio_latents=torch.ones(1, 1),
output_type="pt",
connector_prompt_embeds=torch.ones(1, 1),
generator=generator,
device=torch.device("cpu"),
decode_timestep=0.7,
decode_noise_scale=0.5,
prompt_batch_size=1,
)
torch.testing.assert_close(seen["latents"], latents)
assert seen["generator"] is generator
assert seen["return_dict"] is False
assert seen["output_type"] == "pt"
torch.testing.assert_close(output.output[0], latents + 1)
torch.testing.assert_close(output.output[1], torch.full((1, 1), 3.0))
def test_diffusion_decode_denormalizes_with_decoder_statistics(self):
from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline
pipe = object.__new__(LTX2Pipeline)
torch.nn.Module.__init__(pipe)
pipe.use_diffusion_decoder = True
pipe.transformer_spatial_patch_size = 1
pipe.transformer_temporal_patch_size = 1
pipe.vae = SimpleNamespace(
latents_mean=torch.full((2,), 100.0),
latents_std=torch.full((2,), 10.0),
config=SimpleNamespace(scaling_factor=1.0),
)
pipe.diffusion_decoder = SimpleNamespace(
latents_mean=torch.tensor([10.0, 20.0]),
latents_std=torch.tensor([2.0, 4.0]),
config=SimpleNamespace(scaling_factor=2.0),
)
pipe.audio_vae = SimpleNamespace(
latents_mean=torch.tensor(0.0),
latents_std=torch.tensor(1.0),
)
forward_ctx = SimpleNamespace(
latent_num_frames=1,
latent_height=1,
latent_width=1,
original_audio_num_frames=1,
latent_mel_bins=2,
)
video, audio = pipe._unpack_and_denormalize_stage(
forward_ctx,
torch.tensor([[[1.0, 2.0]]]),
torch.zeros(1, 1, 2),
)
torch.testing.assert_close(video.flatten(), torch.tensor([11.0, 24.0]))
torch.testing.assert_close(audio, torch.zeros(1, 1, 1, 2))
def test_non_output_rank_does_not_enter_diffusion_decoder(self, monkeypatch):
import vllm_omni.diffusion.models.ltx2.ltx2_runtime as ltx_runtime
from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline
class ConvVae:
config = SimpleNamespace(timestep_conditioning=False)
def is_distributed_enabled(self):
raise AssertionError("DiffVAE decode must not inspect ConvVAE collectives")
class DiffusionDecoder:
dtype = torch.float32
def decode(self, *_args, **_kwargs):
raise AssertionError("only the output rank may run DiffVAE decode")
monkeypatch.setattr(ltx_runtime.torch.distributed, "is_initialized", lambda: True)
monkeypatch.setattr(ltx_runtime.torch.distributed, "get_rank", lambda: 1)
pipe = object.__new__(LTX2Pipeline)
torch.nn.Module.__init__(pipe)
pipe.use_diffusion_decoder = True
pipe.vae = ConvVae()
pipe.diffusion_decoder = DiffusionDecoder()
output = pipe._decode_output(
latents=torch.ones(1, 1),
audio_latents=torch.ones(1, 1),
output_type="pt",
connector_prompt_embeds=torch.ones(1, 1),
generator=None,
device=torch.device("cpu"),
decode_timestep=0.0,
decode_noise_scale=None,
prompt_batch_size=1,
)
assert output.output[0].numel() == 0
assert output.output[1].numel() == 0
def test_non_output_rank_enters_distributed_diffusion_decoder(self, monkeypatch):
import vllm_omni.diffusion.models.ltx2.ltx2_runtime as ltx_runtime
from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline
class ConvVae:
config = SimpleNamespace(timestep_conditioning=False)
def is_distributed_enabled(self):
raise AssertionError("DiffVAE decode must query the active decoder")
class DiffusionDecoder:
dtype = torch.float32
def __init__(self):
self.decode_calls = 0
def is_distributed_enabled(self):
return True
def decode(self, *_args, **_kwargs):
self.decode_calls += 1
return (torch.empty(0, 3, 0, 0, 0),)
monkeypatch.setattr(ltx_runtime.torch.distributed, "is_initialized", lambda: True)
monkeypatch.setattr(ltx_runtime.torch.distributed, "get_rank", lambda: 1)
pipe = object.__new__(LTX2Pipeline)
torch.nn.Module.__init__(pipe)
pipe.use_diffusion_decoder = True
pipe.vae = ConvVae()
pipe.diffusion_decoder = DiffusionDecoder()
output = pipe._decode_output(
latents=torch.ones(1, 1),
audio_latents=torch.ones(1, 1),
output_type="pt",
connector_prompt_embeds=torch.ones(1, 1),
generator=None,
device=torch.device("cpu"),
decode_timestep=0.0,
decode_noise_scale=None,
prompt_batch_size=1,
)
assert pipe.diffusion_decoder.decode_calls == 1
assert output.output[0].numel() == 0
assert output.output[1].numel() == 0
def test_diffusion_decoder_patch_parallel_size_one_uses_native_tiling(self, monkeypatch):
from vllm_omni.diffusion.models.ltx2.ltx2_diffusion_decoder import LTX2VideoDiffusionDecoderModel
from vllm_omni.diffusion.models.ltx2.ltx2_diffusion_decoder_distributed import (
DistributedLTX2VideoDiffusionDecoderModel,
)
expected = torch.ones(1, 3, 1, 2, 2)
seen = {}
def native_tiled_decode(self, z, generator=None, num_inference_steps=None):
seen["z"] = z
seen["generator"] = generator
seen["num_inference_steps"] = num_inference_steps
return expected
monkeypatch.setattr(LTX2VideoDiffusionDecoderModel, "tiled_decode", native_tiled_decode)
model = object.__new__(DistributedLTX2VideoDiffusionDecoderModel)
torch.nn.Module.__init__(model)
model.is_distributed_enabled = lambda: False
generator = torch.Generator().manual_seed(17)
z = torch.zeros(1, 1, 1, 1, 1)
result = DistributedLTX2VideoDiffusionDecoderModel.tiled_decode(
model,
z,
generator=generator,
num_inference_steps=1,
)
assert result is expected
assert seen["z"] is z
assert seen["generator"] is generator
assert seen["num_inference_steps"] == 1
def test_distributed_diffusion_tiles_preserve_serial_noise_order(self, monkeypatch):
from diffusers.utils.torch_utils import randn_tensor
from vllm_omni.diffusion.models.ltx2.ltx2_diffusion_decoder_distributed import (
DistributedLTX2VideoDiffusionDecoderModel,
)
model = DistributedLTX2VideoDiffusionDecoderModel(
out_channels=1,
latent_channels=1,
patch_size=1,
decoder_head_dim=8,
decoder_stage_channels=(8, 8, 8, 8, 8),
decoder_stage_depths=(1, 1, 1, 1, 1),
decoder_stage_kernels=((1, 1, 1),) * 4,
decoder_upsample_strides=((1, 1, 1),) * 4,
decoder_upsample_channel_reductions=(1, 1, 1, 1),
decoder_stage5_kernel=(1, 1, 1),
spatial_compression_ratio=1,
temporal_compression_ratio=1,
)
model.tile_sample_min_num_frames = 2
model.tile_sample_stride_num_frames = 1
model.tile_sample_min_height = 2
model.tile_sample_stride_height = 1
model.tile_sample_min_width = 2
model.tile_sample_stride_width = 1
model.distributed_executor = SimpleNamespace(group=None)
model.decoder.forward_stages_1_to_3 = lambda z: z.permute(0, 2, 3, 4, 1)
monkeypatch.setattr(torch.distributed, "broadcast", lambda *_args, **_kwargs: None)
z = torch.zeros(1, 1, 2, 3, 3)
distributed_generator = torch.Generator().manual_seed(123)
reference_generator = torch.Generator().manual_seed(123)
tasks, grid_spec = model._distributed_tile_split(z, distributed_generator, num_inference_steps=1)
assert grid_spec.grid_shape == (1, 3, 3)
assert grid_spec.split_dims == (2, 3, 4)
for task in tasks:
tile_shape = model._tiled_pixel_shape_from_features(
task.tensor,
drop_leading_frame=task.drop_leading_frame,
crop_trailing_ghost=task.crop_trailing_ghost,
)
expected = randn_tensor(tile_shape, generator=reference_generator, device=z.device, dtype=z.dtype)
actual = randn_tensor(tile_shape, generator=task.noise_generator, device=z.device, dtype=z.dtype)
torch.testing.assert_close(actual, expected)
assert torch.equal(distributed_generator.get_state(), reference_generator.get_state())
class TestLTXOutputRank:
@pytest.mark.parametrize(
("distributed_vae_state", "expected_decode_calls"),
@@ -138,7 +574,7 @@ class TestLTX23VaeDistributedDecode:
},
output_dtype=torch.float32,
)
seen = {}
seen: dict[str, Any] = {}
def exec_tile(task):
return torch.full(tile_output_shapes[task.tile_id], float(task.tile_id + 1))