Files
vllm-omni/tests/diffusion/models/ltx2/test_ltx2_pipeline.py
T
2026-08-27 23:54:40 +08:00

2162 lines
81 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Unit tests for the shared LTX pipeline runtime and public contracts."""
import json
import os
import tempfile
from dataclasses import replace
from types import SimpleNamespace
from typing import Any
import pytest
import torch
from vllm_omni.diffusion.data import DiffusionOutput
from vllm_omni.diffusion.models.ltx2 import (
ltx2_components,
ltx2_transformer,
)
from vllm_omni.diffusion.models.ltx2.ltx2_components import (
LTX2_COMPONENT_PROFILE,
LTX2_DISTILLED_COMPONENT_PROFILE,
LTX2_DISTILLED_ONE_STAGE_COMPONENT_PROFILE,
LTX2_TWO_STAGE_COMPONENT_PROFILE,
LTX23_COMPONENT_PROFILE,
LTX23_DISTILLED_COMPONENT_PROFILE,
LTX23_TWO_STAGE_COMPONENT_PROFILE,
_install_connector_attention,
detect_ltx_model_version,
resolve_ltx_artifact,
resolve_ltx_component_profile,
)
from vllm_omni.diffusion.models.ltx2.ltx2_conditioning import LTXI2VConditioningMixin
from vllm_omni.diffusion.models.ltx2.ltx2_denoise import (
LTXDenoiseExecutor,
LTXPhaseResult,
_official_ltx_sigmas,
prepare_scheduler_stage,
)
from vllm_omni.diffusion.models.ltx2.ltx2_guidance import (
LTX_GUIDANCE_EXECUTOR,
LTXGuidancePlan,
LTXGuidanceSpec,
LTXModalityGuidance,
_repeat_batch,
build_perturbation_kwargs,
combine_guided_x0,
)
from vllm_omni.diffusion.models.ltx2.ltx2_latents import LTXAVState
from vllm_omni.diffusion.models.ltx2.ltx2_recipes import (
LTX2_DISTILLED_ONE_STAGE_RECIPE,
LTX2_DISTILLED_TWO_STAGE_RECIPE,
LTX2_ONE_STAGE_RECIPE,
LTX2_TWO_STAGE_RECIPE,
LTX23_DISTILLED_TWO_STAGE_RECIPE,
LTX23_ONE_STAGE_RECIPE,
LTX23_TWO_STAGE_RECIPE,
LTX_DEFAULT_NEGATIVE_PROMPT,
LTX_POSITIVE_ONLY_RECIPE,
resolve_ltx_pipeline_recipe,
)
from vllm_omni.diffusion.models.ltx2.ltx2_request import (
LTXRequestInputs,
validate_ltx_checkpoint,
validate_pipeline_request,
)
from vllm_omni.diffusion.models.ltx2.ltx2_runtime import LTXRuntime
from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import (
LTX2DistilledOneStagePipeline,
LTX2I2VDMD2Pipeline,
LTX2Pipeline,
LTX2T2VDMD2Pipeline,
)
from vllm_omni.diffusion.models.ltx2.pipeline_ltx2_two_stage import (
LTX2DistilledPipeline,
LTX2DistilledTwoStagePipeline,
LTX2TwoStagePipeline,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _make_ltx_request_pipe(cls):
pipe = object.__new__(cls)
torch.nn.Module.__init__(pipe)
pipe.device = torch.device("cpu")
pipe.tokenizer_max_length = 99
pipe.vae_spatial_compression_ratio = 32
pipe.vae_temporal_compression_ratio = 8
return pipe
def _resolve_request_inputs_for_test(
pipe,
req,
*,
guidance_scale=4.0,
negative_prompt=None,
):
return pipe._resolve_request_inputs(
req,
prompt=None,
negative_prompt=negative_prompt,
height=None,
width=None,
num_frames=None,
frame_rate=None,
num_inference_steps=None,
guidance_scale=guidance_scale,
num_videos_per_prompt=1,
generator=None,
latents=None,
audio_latents=None,
prompt_embeds=None,
negative_prompt_embeds=None,
prompt_attention_mask=None,
negative_prompt_attention_mask=None,
decode_timestep=0.0,
decode_noise_scale=None,
output_type="np",
max_sequence_length=None,
)
def test_ltx_public_entries_share_runtime_and_keep_recipe_boundaries():
from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_ltx2 import (
DistributedAutoencoderKLLTX2Video,
)
assert issubclass(LTX2Pipeline, LTXRuntime)
assert issubclass(LTX2TwoStagePipeline, LTXRuntime)
assert issubclass(LTX2DistilledOneStagePipeline, LTXRuntime)
assert issubclass(LTX2DistilledTwoStagePipeline, LTXRuntime)
assert issubclass(LTX2DistilledPipeline, LTXRuntime)
assert LTX2DistilledPipeline is LTX2DistilledTwoStagePipeline
assert LTX2Pipeline.component_profile is LTX2_COMPONENT_PROFILE
assert LTX2Pipeline.pipeline_recipe is LTX2_ONE_STAGE_RECIPE
assert LTX2TwoStagePipeline.component_profile is LTX2_TWO_STAGE_COMPONENT_PROFILE
assert LTX2TwoStagePipeline.pipeline_recipe is LTX2_TWO_STAGE_RECIPE
assert LTX2DistilledOneStagePipeline.component_profile is LTX2_DISTILLED_ONE_STAGE_COMPONENT_PROFILE
assert LTX2DistilledOneStagePipeline.pipeline_recipe is LTX2_DISTILLED_ONE_STAGE_RECIPE
assert LTX2DistilledPipeline.component_profile is LTX2_DISTILLED_COMPONENT_PROFILE
for pipeline_cls, profile in (
(LTX2Pipeline, LTX2_COMPONENT_PROFILE),
(LTX2TwoStagePipeline, LTX2_TWO_STAGE_COMPONENT_PROFILE),
(LTX2DistilledOneStagePipeline, LTX2_DISTILLED_ONE_STAGE_COMPONENT_PROFILE),
(LTX2DistilledPipeline, LTX2_DISTILLED_COMPONENT_PROFILE),
):
assert pipeline_cls._dit_modules == list(profile.dit_modules)
assert pipeline_cls._encoder_modules == list(profile.encoder_modules)
assert pipeline_cls._vae_modules == list(profile.vae_modules)
assert pipeline_cls._resident_modules == list(profile.resident_modules)
assert LTX2_COMPONENT_PROFILE.video_vae_cls is DistributedAutoencoderKLLTX2Video
assert LTX23_COMPONENT_PROFILE.video_vae_cls is DistributedAutoencoderKLLTX2Video
def test_ltx_checkpoint_version_detection_uses_metadata(tmp_path):
(tmp_path / "model_index.json").write_text(json.dumps({"vocoder": ["ltx2", "LTX2VocoderWithBWE"]}))
assert detect_ltx_model_version(str(tmp_path)) == "2.3"
(tmp_path / "model_index.json").write_text(json.dumps({"model_version": "2.3.1"}))
assert detect_ltx_model_version(str(tmp_path)) == "2.3"
(tmp_path / "model_index.json").write_text(json.dumps({"vocoder": ["ltx2", "LTX2Vocoder"]}))
assert detect_ltx_model_version(str(tmp_path)) == "2"
@pytest.mark.parametrize(
("model", "repo_id", "expected_revision"),
[
("Lightricks/test", "Lightricks/test", "model-revision"),
("Lightricks/test-Diffusers", "Lightricks/test", "artifact-revision"),
],
)
def test_ltx_artifact_uses_source_revision_and_hub_fallback(
monkeypatch,
model,
repo_id,
expected_revision,
):
filename = "ltx-sidecar.safetensors"
calls = []
def fake_download(**kwargs):
calls.append(kwargs)
return "/cache/ltx-sidecar.safetensors"
monkeypatch.setattr(ltx2_components, "hf_hub_download", fake_download)
assert (
resolve_ltx_artifact(
model,
repo_id,
filename,
model_revision="model-revision",
artifact_revision="artifact-revision",
)
== "/cache/ltx-sidecar.safetensors"
)
assert calls == [
{
"repo_id": repo_id,
"filename": filename,
"revision": expected_revision,
}
]
def test_ltx_artifact_prefers_model_root(tmp_path, monkeypatch):
filename = "ltx-sidecar.safetensors"
expected = tmp_path / filename
expected.write_bytes(b"sidecar")
monkeypatch.setattr(ltx2_components, "hf_hub_download", lambda **_kwargs: pytest.fail("unexpected Hub lookup"))
assert resolve_ltx_artifact(
str(tmp_path),
"Lightricks/test",
filename,
model_revision="unused-model-revision",
artifact_revision="unused-artifact-revision",
) == str(expected)
def test_ltx_artifact_local_model_missing_sidecar_falls_back_to_hub(tmp_path, monkeypatch):
calls = []
def fake_download(**kwargs):
calls.append(kwargs)
return "/cache/ltx-sidecar.safetensors"
monkeypatch.setattr(ltx2_components, "hf_hub_download", fake_download)
assert (
resolve_ltx_artifact(
str(tmp_path),
"Lightricks/test",
"ltx-sidecar.safetensors",
model_revision="local-model-revision",
artifact_revision="artifact-revision",
)
== "/cache/ltx-sidecar.safetensors"
)
assert calls == [
{
"repo_id": "Lightricks/test",
"filename": "ltx-sidecar.safetensors",
"revision": "artifact-revision",
}
]
@pytest.mark.parametrize(
("expected_kind", "scheduler_config"),
[
("regular", {"use_dynamic_shifting": True, "shift_terminal": 0.1}),
("distilled", {"use_dynamic_shifting": False, "shift_terminal": None}),
],
)
def test_ltx_checkpoint_validation_accepts_matching_scheduler_metadata(expected_kind, scheduler_config):
validate_ltx_checkpoint(
scheduler_config,
expected_kind=expected_kind,
pipeline_name="TestLTXPipeline",
)
@pytest.mark.parametrize(
("expected_kind", "scheduler_config", "error"),
[
("regular", {"use_dynamic_shifting": False, "shift_terminal": None}, "regular non-distilled"),
("distilled", {"use_dynamic_shifting": True, "shift_terminal": 0.1}, "merged distilled"),
],
)
def test_ltx_checkpoint_validation_rejects_mismatched_scheduler_metadata(expected_kind, scheduler_config, error):
with pytest.raises(ValueError, match=error):
validate_ltx_checkpoint(
scheduler_config,
expected_kind=expected_kind,
pipeline_name="TestLTXPipeline",
)
def test_ltx_rejects_advanced_uaa_before_component_initialization():
od_config = SimpleNamespace(
model="unused",
parallel_config=SimpleNamespace(ulysses_mode="advanced_uaa"),
)
with pytest.raises(ValueError, match="does not support ulysses_mode='advanced_uaa'"):
LTX2Pipeline(od_config=od_config)
def test_ltx23_checkpoint_selects_version_specific_one_stage_profile(tmp_path, monkeypatch):
from vllm_omni.diffusion.models.ltx2 import ltx2_runtime
(tmp_path / "model_index.json").write_text(json.dumps({"vocoder": ["ltx2", "LTX2VocoderWithBWE"]}))
def stub_components(pipe, od_config):
pipe.od_config = od_config
pipe.vae_spatial_compression_ratio = 32
monkeypatch.setattr(ltx2_runtime, "initialize_pipeline_components", stub_components)
monkeypatch.setattr(LTXRuntime, "setup_diffusion_pipeline_profiler", lambda *_args, **_kwargs: None)
pipe = LTX2Pipeline(od_config=SimpleNamespace(model=str(tmp_path), enable_diffusion_pipeline_profiler=False))
assert pipe.model_version == "2.3"
assert pipe.component_profile is LTX23_COMPONENT_PROFILE
assert pipe.pipeline_recipe is LTX23_ONE_STAGE_RECIPE
assert pipe.preserve_sp_padded_audio_duration
assert pipe.reports_stage_durations
def test_ltx_multistage_rejects_cache_dit_before_component_initialization(tmp_path, monkeypatch):
from vllm_omni.diffusion.models.ltx2 import ltx2_runtime
(tmp_path / "model_index.json").write_text(json.dumps({"vocoder": ["ltx2", "LTX2Vocoder"]}))
initialized = False
def stub_components(*_args, **_kwargs):
nonlocal initialized
initialized = True
monkeypatch.setattr(ltx2_runtime, "initialize_pipeline_components", stub_components)
with pytest.raises(ValueError, match="not qualified for this LTX recipe"):
LTX2TwoStagePipeline(
od_config=SimpleNamespace(
model=str(tmp_path),
cache_backend="cache_dit",
)
)
assert not initialized
def test_ltx23_checkpoint_selects_version_specific_two_stage_profiles(tmp_path, monkeypatch):
from vllm_omni.diffusion.models.ltx2 import ltx2_runtime
(tmp_path / "model_index.json").write_text(json.dumps({"vocoder": ["ltx2", "LTX2VocoderWithBWE"]}))
def stub_components(pipe, od_config):
pipe.od_config = od_config
pipe.vae_spatial_compression_ratio = 32
monkeypatch.setattr(ltx2_runtime, "initialize_pipeline_components", stub_components)
monkeypatch.setattr(LTXRuntime, "setup_diffusion_pipeline_profiler", lambda *_args, **_kwargs: None)
monkeypatch.setattr(ltx2_runtime, "build_ltx_phase_adapter", lambda *_args: SimpleNamespace())
pipe = LTX2TwoStagePipeline(
od_config=SimpleNamespace(
model=str(tmp_path),
enable_diffusion_pipeline_profiler=False,
lora_path=None,
model_config={},
)
)
assert pipe.model_version == "2.3"
assert pipe.component_profile is LTX23_TWO_STAGE_COMPONENT_PROFILE
assert pipe.pipeline_recipe is LTX23_TWO_STAGE_RECIPE
def test_ltx23_checkpoint_selects_full_distilled_profile_and_recipe(tmp_path, monkeypatch):
from vllm_omni.diffusion.models.ltx2 import ltx2_runtime
(tmp_path / "model_index.json").write_text(json.dumps({"vocoder": ["ltx2", "LTX2VocoderWithBWE"]}))
def stub_components(pipe, od_config):
pipe.od_config = od_config
pipe.vae_spatial_compression_ratio = 32
monkeypatch.setattr(ltx2_runtime, "initialize_pipeline_components", stub_components)
monkeypatch.setattr(LTXRuntime, "setup_diffusion_pipeline_profiler", lambda *_args, **_kwargs: None)
pipe = LTX2DistilledPipeline(
od_config=SimpleNamespace(
model=str(tmp_path),
enable_diffusion_pipeline_profiler=False,
)
)
assert pipe.model_version == "2.3"
assert pipe.component_profile is LTX23_DISTILLED_COMPONENT_PROFILE
assert pipe.pipeline_recipe is LTX23_DISTILLED_TWO_STAGE_RECIPE
assert pipe._phase_adapter is None
assert pipe.preserve_sp_padded_audio_duration
assert pipe.reports_stage_durations
def test_ltx_entries_expose_batch_image_and_distributed_decode_contracts():
assert LTX2Pipeline.supports_request_batch
assert LTX2Pipeline.support_image_input
assert LTX2Pipeline.unified_text_image_entry
assert LTX2Pipeline.distributed_video_decode
assert not LTX2TwoStagePipeline.supports_request_batch
assert LTX2TwoStagePipeline.support_image_input
assert LTX2TwoStagePipeline.unified_text_image_entry
assert not LTX2DistilledPipeline.supports_request_batch
assert LTX2DistilledPipeline.support_image_input
assert LTX2DistilledPipeline.unified_text_image_entry
def test_ltx_two_stage_resolves_request_embedded_images():
pipe = object.__new__(LTX2TwoStagePipeline)
request_inputs = SimpleNamespace(latents=None)
image = torch.zeros(3, 8, 8)
req = SimpleNamespace(prompts=[{"prompt": "image conditioned", "multi_modal_data": {"image": image}}])
assert pipe._resolve_request_image(req, None, request_inputs) is image
def test_ltx_dmd2_entries_retain_their_task_boundaries():
assert not LTX2T2VDMD2Pipeline.support_image_input
assert LTX2I2VDMD2Pipeline.support_image_input
assert not LTX2I2VDMD2Pipeline.unified_text_image_entry
def test_ltx_request_batch_decode_splits_video_and_audio_per_request():
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
requests = [
OmniDiffusionRequest(
prompt=f"prompt-{index}",
sampling_params=OmniDiffusionSamplingParams(),
request_id=f"request-{index}",
)
for index in range(2)
]
request_batch = DiffusionRequestBatch(requests)
video = torch.tensor([[1.0], [2.0]])
audio = torch.tensor([[3.0], [4.0]])
pipe = object.__new__(LTX2Pipeline)
torch.nn.Module.__init__(pipe)
object.__setattr__(pipe, "_decode_output", lambda **_kwargs: DiffusionOutput(output=(video, audio)))
forward_ctx = SimpleNamespace(
req=request_batch,
request_inputs=SimpleNamespace(
output_type="np",
generator=None,
decode_timestep=0.0,
decode_noise_scale=None,
),
prompt_context=SimpleNamespace(connector_prompt_embeds=torch.empty(2, 1)),
device=torch.device("cpu"),
batch_size=2,
num_videos_per_prompt=1,
)
outputs = pipe.decode_phase(
LTXPhaseResult(
forward_context=forward_ctx,
video=video,
audio=audio,
)
)
assert len(outputs) == 2
torch.testing.assert_close(outputs[0].output[0], video[:1])
torch.testing.assert_close(outputs[0].output[1], audio[:1])
torch.testing.assert_close(outputs[1].output[0], video[1:])
torch.testing.assert_close(outputs[1].output[1], audio[1:])
def test_ltx_public_entries_share_runtime_forward_template():
assert "_forward_request" not in LTX2Pipeline.__dict__
assert "forward" not in LTX2Pipeline.__dict__
assert LTX2Pipeline._forward_request is LTXRuntime._forward_request
assert LTX2Pipeline._run_recipe is LTXRuntime._run_recipe
@pytest.mark.parametrize(
("kwargs", "field_name"),
[
({"timesteps": [1]}, "timesteps"),
({"guidance_rescale": 0.5}, "guidance_rescale"),
({"noise_scale": 1.0}, "noise_scale"),
({"return_dict": False}, "return_dict"),
({"attention_kwargs": {}}, "attention_kwargs"),
],
)
def test_ltx_public_forward_rejects_unsupported_generic_options(kwargs, field_name):
pipe = object.__new__(LTX2Pipeline)
with pytest.raises(ValueError, match=field_name):
pipe.forward(SimpleNamespace(), **kwargs)
def test_ltx2_canonical_forward_requires_keyword_arguments():
pipe = object.__new__(LTX2Pipeline)
with pytest.raises(TypeError):
pipe.forward(SimpleNamespace(), "prompt")
def test_ltx_versions_share_request_prompt_and_step_templates():
shared_methods = (
"_get_gemma_prompt_embeds",
"encode_prompt",
"check_inputs",
"_resolve_request_inputs",
"_prepare_prompt_context",
"_denoise_step",
)
for method_name in shared_methods:
base_method = getattr(LTXRuntime, method_name)
assert getattr(LTX2Pipeline, method_name) is base_method
def test_ltx_versions_select_guidance_without_overriding_control_flow():
assert LTX2Pipeline.guidance_executor is LTX_GUIDANCE_EXECUTOR
assert LTX2Pipeline._predict_noise_for_step is LTXRuntime._predict_noise_for_step
assert LTX2Pipeline.combine_cfg_noise is LTXRuntime.combine_cfg_noise
def test_ltx_official_recipes_build_all_guidance_passes():
ltx2_plan = LTXGuidancePlan.build(LTX2_ONE_STAGE_RECIPE.request_guidance)
ltx23_plan = LTXGuidancePlan.build(LTX23_ONE_STAGE_RECIPE.request_guidance)
assert ltx2_plan.names == ltx23_plan.names == ("cond", "uncond", "ptb", "mod")
assert LTX2_ONE_STAGE_RECIPE.request_guidance.video.stg_blocks == (29,)
assert LTX23_ONE_STAGE_RECIPE.request_guidance.video.stg_blocks == (28,)
assert LTX23_ONE_STAGE_RECIPE.request_guidance.video.cfg_scale == 3.0
assert LTX23_ONE_STAGE_RECIPE.request_guidance.audio.cfg_scale == 7.0
assert LTX2_ONE_STAGE_RECIPE.phases[0].use_official_sigma_schedule
assert LTX23_ONE_STAGE_RECIPE.phases[0].use_official_sigma_schedule
assert not LTX_POSITIVE_ONLY_RECIPE.phases[0].use_official_sigma_schedule
def test_ltx_guidance_combines_official_x0_deltas_in_fp32():
cond = torch.tensor([[1.0, 3.0]], dtype=torch.bfloat16)
uncond = torch.tensor([[0.0, 1.0]], dtype=torch.bfloat16)
perturbed = torch.tensor([[0.5, 2.0]], dtype=torch.bfloat16)
isolated = torch.tensor([[0.25, 2.5]], dtype=torch.bfloat16)
guidance = LTXModalityGuidance(
cfg_scale=3.0,
stg_scale=0.5,
modality_scale=2.0,
stg_blocks=(0,),
)
actual = combine_guided_x0(
cond=cond,
uncond_text=uncond,
uncond_perturbed=perturbed,
uncond_modality=isolated,
guidance=guidance,
)
expected = (
cond.float()
+ 2.0 * (cond.float() - uncond.float())
+ 0.5 * (cond.float() - perturbed.float())
+ (cond.float() - isolated.float())
).to(cond.dtype)
torch.testing.assert_close(actual, expected)
def test_ltx_guidance_skips_stg_without_perturbed_blocks():
cond = torch.tensor([[[1.0, 2.0], [3.0, 4.0]]])
guidance = LTXModalityGuidance(stg_scale=1.0, stg_blocks=())
plan = LTXGuidancePlan.build(LTXGuidanceSpec(video=guidance))
actual = combine_guided_x0(
cond=cond,
uncond_text=0.0,
uncond_perturbed=0.0,
uncond_modality=0.0,
guidance=guidance,
)
assert plan.names == ("cond",)
torch.testing.assert_close(actual, cond)
def test_ltx_guidance_uses_official_close_to_disabled_semantics():
guidance = LTXModalityGuidance(
cfg_scale=1.0 + 1e-10,
stg_scale=0.0,
modality_scale=1.0 + 1e-10,
stg_blocks=(0,),
)
assert LTXGuidancePlan.build(LTXGuidanceSpec(video=guidance)).names == ("cond",)
def test_ltx_positive_only_guidance_preserves_token_major_layout():
tensor = torch.arange(24).view(1, 3, 8).transpose(1, 2).contiguous().transpose(1, 2)
repeated = _repeat_batch(tensor, 1)
assert repeated is tensor
assert repeated.stride() == tensor.stride()
def test_ltx_guidance_rescale_is_invariant_to_request_batching():
cond = torch.tensor(
[
[[1.0, 2.0], [3.0, 4.0]],
[[10.0, 30.0], [50.0, 70.0]],
]
)
uncond = torch.stack((torch.zeros_like(cond[0]), cond[1]))
guidance = LTXModalityGuidance(cfg_scale=2.0, rescale_scale=1.0)
actual = combine_guided_x0(
cond=cond,
uncond_text=uncond,
uncond_perturbed=0.0,
uncond_modality=0.0,
guidance=guidance,
)
individually = torch.cat(
[
combine_guided_x0(
cond=cond[index : index + 1],
uncond_text=uncond[index : index + 1],
uncond_perturbed=0.0,
uncond_modality=0.0,
guidance=guidance,
)
for index in range(cond.shape[0])
]
)
# Fusing independent requests must not change either request's output.
torch.testing.assert_close(actual, individually)
def test_ltx_guidance_rescale_handles_zero_variance_prediction():
cond = torch.tensor([[[1.0, 2.0], [3.0, 4.0]]])
guidance = LTXModalityGuidance(cfg_scale=2.0, rescale_scale=1.0)
actual = combine_guided_x0(
cond=cond,
uncond_text=2 * cond,
uncond_perturbed=0.0,
uncond_modality=0.0,
guidance=guidance,
)
assert torch.isfinite(actual).all()
torch.testing.assert_close(actual, torch.zeros_like(cond))
def test_ltx_audio_guidance_rescale_ignores_sp_padding_tokens():
logical_cond = torch.tensor([[[1.0, 2.0], [3.0, 5.0]]])
logical_uncond = torch.tensor([[[0.5, -1.0], [2.0, 1.0]]])
guidance = LTXModalityGuidance(cfg_scale=3.0, rescale_scale=1.0)
expected = combine_guided_x0(
cond=logical_cond,
uncond_text=logical_uncond,
uncond_perturbed=0.0,
uncond_modality=0.0,
guidance=guidance,
)
padded_cond = torch.cat([logical_cond, torch.tensor([[[1000.0, -1000.0]]])], dim=1)
padded_uncond = torch.cat([logical_uncond, torch.tensor([[[-500.0, 500.0]]])], dim=1)
actual = combine_guided_x0(
cond=padded_cond,
uncond_text=padded_uncond,
uncond_perturbed=0.0,
uncond_modality=0.0,
guidance=guidance,
rescale_token_count=2,
)
torch.testing.assert_close(actual[:, :2], expected)
def test_ltx_guidance_builds_per_sample_stg_and_modality_masks():
plan = LTXGuidancePlan.build(LTX23_ONE_STAGE_RECIPE.request_guidance)
perturbations = build_perturbation_kwargs(plan, batch_size=2, reference=torch.ones(8, 1, 1))
assert perturbations["video_self_attention_blocks"] == (28,)
assert perturbations["audio_self_attention_blocks"] == (28,)
assert perturbations["video_self_attention_mask"].flatten().tolist() == [1, 1, 1, 1, 0, 0, 1, 1]
assert perturbations["a2v_cross_attention_mask"].flatten().tolist() == [1, 1, 1, 1, 1, 1, 0, 0]
torch.testing.assert_close(
perturbations["a2v_cross_attention_mask"],
perturbations["v2a_cross_attention_mask"],
)
def test_ltx_default_sigma_schedule_matches_official_shift_and_stretch():
scheduler = SimpleNamespace(
config={
"base_image_seq_len": 1024,
"max_image_seq_len": 4096,
"base_shift": 0.95,
"max_shift": 2.05,
"shift_terminal": 0.1,
}
)
sigmas = _official_ltx_sigmas(scheduler, steps=4, device=torch.device("cpu"))
torch.testing.assert_close(
sigmas,
torch.tensor([1.0, 0.8671, 0.6316, 0.1, 0.0]),
rtol=1e-4,
atol=1e-4,
)
def test_ltx_sigma_schedule_does_not_stretch_when_terminal_config_is_missing():
scheduler = SimpleNamespace(
config={
"base_image_seq_len": 1024,
"max_image_seq_len": 4096,
"base_shift": 0.95,
"max_shift": 2.05,
}
)
sigmas = _official_ltx_sigmas(scheduler, steps=4, device=torch.device("cpu"))
torch.testing.assert_close(
sigmas,
torch.tensor([1.0, 0.9589, 0.8859, 0.7214, 0.0]),
rtol=1e-4,
atol=1e-4,
)
def test_ltx_sigma_schedule_does_not_stretch_when_terminal_is_disabled():
scheduler = SimpleNamespace(
config={
"base_image_seq_len": 1024,
"max_image_seq_len": 4096,
"base_shift": 0.95,
"max_shift": 2.05,
"shift_terminal": None,
}
)
sigmas = _official_ltx_sigmas(scheduler, steps=4, device=torch.device("cpu"))
torch.testing.assert_close(
sigmas,
torch.tensor([1.0, 0.9589, 0.8859, 0.7214, 0.0]),
rtol=1e-4,
atol=1e-4,
)
def test_ltx_component_loader_installs_omni_connector_attention(monkeypatch):
installed = []
kernels = []
class OmniAttention:
def __init__(self, **kwargs):
kernels.append(kwargs)
class Attention:
heads = 32
head_dim = 128
inner_kv_dim = 4096
def set_processor(self, processor):
installed.append(processor)
monkeypatch.setattr(ltx2_components, "OmniAttention", OmniAttention)
connectors = SimpleNamespace(
video_connector=SimpleNamespace(
learnable_registers=object(),
transformer_blocks=[SimpleNamespace(attn1=Attention())],
),
audio_connector=SimpleNamespace(
learnable_registers=object(),
transformer_blocks=[SimpleNamespace(attn1=Attention())],
),
)
_install_connector_attention(connectors)
assert len(installed) == 2
assert all(processor.__class__.__name__ == "_LTXConnectorAttnProcessor" for processor in installed)
assert all(kernel["role"] == "ltx2.connector" for kernel in kernels)
assert all(kernel["role_category"] == "self" for kernel in kernels)
assert all(kernel["skip_sequence_parallel"] for kernel in kernels)
assert all(kernel["disable_kv_quant"] for kernel in kernels)
assert all(processor.has_learned_registers for processor in installed)
assert not any(processor.preserve_learned_register_mask for processor in installed)
def test_ltx25_connector_attention_preserves_learned_register_mask(monkeypatch):
installed = []
class OmniAttention:
def __init__(self, **_kwargs):
pass
class Attention:
heads = 32
head_dim = 128
inner_kv_dim = 4096
def set_processor(self, processor):
installed.append(processor)
monkeypatch.setattr(ltx2_components, "OmniAttention", OmniAttention)
connectors = SimpleNamespace(
video_connector=SimpleNamespace(
learnable_registers=object(),
transformer_blocks=[SimpleNamespace(attn1=Attention())],
),
audio_connector=SimpleNamespace(
learnable_registers=object(),
transformer_blocks=[SimpleNamespace(attn1=Attention())],
),
)
_install_connector_attention(connectors, preserve_learned_register_mask=True)
assert all(processor.has_learned_registers for processor in installed)
assert all(processor.preserve_learned_register_mask for processor in installed)
def test_ltx_connector_attention_dispatches_through_omni_kernel():
calls = []
class Backend:
@staticmethod
def get_name():
return "FLASH_ATTN"
class OmniAttention:
attn_backend = Backend
def __call__(self, query, key, value, metadata):
calls.append((query, key, value, metadata))
return query
class UpcastingNorm(torch.nn.Module):
def forward(self, tensor):
return tensor.float()
attention = SimpleNamespace(
heads=2,
head_dim=4,
inner_kv_dim=8,
norm_q=UpcastingNorm(),
norm_k=UpcastingNorm(),
to_q=torch.nn.Identity(),
to_k=torch.nn.Identity(),
to_v=torch.nn.Identity(),
to_out=(torch.nn.Identity(), torch.nn.Identity()),
to_gate_logits=None,
rope_type="split",
omni_attention=OmniAttention(),
)
hidden_states = torch.randn(2, 3, 8, dtype=torch.bfloat16)
additive_mask = torch.zeros(2, 1, 3, 3)
rotary = (
torch.randn(2, 2, 3, 2, dtype=torch.float32),
torch.randn(2, 2, 3, 2, dtype=torch.float32),
)
output = ltx2_components._LTXConnectorAttnProcessor()(
attention,
hidden_states,
attention_mask=additive_mask,
query_rotary_emb=rotary,
)
query, key, value, metadata = calls[0]
assert query.shape == key.shape == value.shape == (2, 3, 2, 4)
assert query.dtype == key.dtype == value.dtype == torch.bfloat16
assert metadata.attn_mask.shape == (2, 3)
assert metadata.attn_mask.dtype == torch.bool
assert metadata.attn_mask.all()
expected_rotary = tuple(component.to(torch.bfloat16) for component in rotary)
expected = ltx2_components.apply_split_rotary_emb(hidden_states, expected_rotary, head_dim=4)
torch.testing.assert_close(output, expected, rtol=0, atol=0)
def test_ltx2_distilled_pipeline_shares_recipe_runtime():
assert issubclass(LTX2DistilledPipeline, LTXI2VConditioningMixin)
assert not issubclass(LTX2DistilledPipeline, LTX2Pipeline)
assert LTX2DistilledPipeline._run_recipe is LTXRuntime._run_recipe
assert "_forward_request" not in LTX2DistilledPipeline.__dict__
def test_ltx_two_stage_entry_is_a_thin_pipeline_variant():
assert "_run_recipe" not in LTX2TwoStagePipeline.__dict__
assert "_forward_request" not in LTX2TwoStagePipeline.__dict__
assert "load_weights" not in LTX2TwoStagePipeline.__dict__
assert issubclass(LTX2TwoStagePipeline, LTXI2VConditioningMixin)
assert not issubclass(LTX2TwoStagePipeline, LTX2Pipeline)
def test_ltx_variants_share_denoise_loop_and_i2v_conditioning():
assert "_denoise_loop" not in LTX2Pipeline.__dict__
assert issubclass(LTX2Pipeline, LTXI2VConditioningMixin)
assert LTX2Pipeline.prepare_latents is LTXI2VConditioningMixin.prepare_latents
assert LTX2Pipeline.prepare_audio_latents is LTXRuntime.prepare_audio_latents
assert LTX2Pipeline.decode_phase is LTXRuntime.decode_phase
def test_denoise_executor_owns_progress_and_interrupt():
updates = []
class Progress:
def __enter__(self):
return self
def __exit__(self, exc_type, exc, traceback):
return False
def update(self):
updates.append(True)
pipeline = SimpleNamespace(
interrupt=False,
progress_bar=lambda total: Progress(),
)
seen: list[tuple[int, float]] = []
timesteps = torch.tensor([3.0, 2.0, 1.0])
initial_state = LTXAVState(video=torch.tensor(0.0), audio=torch.tensor(10.0))
def step(index, timestep, state):
seen.append((index, timestep.item()))
pipeline.interrupt = True
return LTXAVState(video=state.video + 1, audio=state.audio + 1)
state = LTXDenoiseExecutor.run(pipeline, initial_state, timesteps, step)
assert seen == [(0, 3.0)]
assert updates == [True]
torch.testing.assert_close(state.video, torch.tensor(1.0))
torch.testing.assert_close(state.audio, torch.tensor(11.0))
@pytest.mark.parametrize("pipeline_cls", [LTX2TwoStagePipeline, LTX2DistilledPipeline])
def test_ltx_two_stage_executes_declarative_i2v_phase_plan(pipeline_cls):
request_inputs = LTXRequestInputs(
prompt="prompt",
negative_prompt="",
height=64,
width=64,
num_frames=1,
frame_rate=24.0,
num_inference_steps=8,
guidance=LTXGuidanceSpec.positive_only(),
num_videos_per_prompt=1,
generator=None,
latents=None,
audio_latents=None,
prompt_embeds=None,
negative_prompt_embeds=None,
prompt_attention_mask=None,
negative_prompt_attention_mask=None,
decode_timestep=0.25,
decode_noise_scale=0.5,
output_type="np",
max_sequence_length=16,
)
prompt_context = object()
phase_calls: list[Any] = []
def resolve_request_inputs(req, **kwargs):
return request_inputs
def run_phase(req, inputs, *, prompt_context=None, **kwargs):
phase_recipe = kwargs["phase_recipe"]
assert kwargs["image"] is source_image
phase_calls.append((phase_recipe, inputs, prompt_context))
if len(phase_calls) == 1:
assert prompt_context is None
assert inputs.decode_timestep == 0.25
assert inputs.decode_noise_scale == 0.5
context = prompt_context_sentinel
assert phase_recipe.name == "generate_lowres"
assert (inputs.height, inputs.width) == (32, 32)
assert inputs.num_inference_steps == 8
assert kwargs["noise_scale"] == 1.0
video = torch.ones(1, 128, 1, 1, 1)
audio = torch.full((1, 8, 1, 2), 2.0)
audio_for_next_phase = torch.full((1, 8, 1, 2), 5.0)
else:
assert prompt_context is prompt_context_sentinel
assert phase_recipe.name == "refine"
assert (inputs.height, inputs.width) == (64, 64)
assert inputs.latents.shape == (1, 128, 1, 2, 2)
torch.testing.assert_close(inputs.audio_latents, torch.full((1, 8, 1, 2), 5.0))
assert inputs.audio_latents_normalized
assert inputs.guidance_scale == 1.0
assert inputs.num_inference_steps == 3
assert inputs.decode_timestep == 0.0
assert inputs.decode_noise_scale is None
context = prompt_context
video = torch.full((1, 128, 1, 2, 2), 3.0)
audio = torch.full((1, 8, 1, 2), 4.0)
audio_for_next_phase = None
return LTXPhaseResult(
forward_context=SimpleNamespace(prompt_context=context),
video=video,
audio=audio,
audio_for_next_phase=audio_for_next_phase,
)
def decode_phase(phase):
return DiffusionOutput(output=(phase.video, phase.audio))
class FakeUpsampler(torch.nn.Module):
dtype = torch.float32
def forward(self, latents):
return latents.repeat_interleave(2, dim=-2).repeat_interleave(2, dim=-1)
prompt_context_sentinel = prompt_context
pipeline = object.__new__(pipeline_cls)
torch.nn.Module.__init__(pipeline)
pipeline.device = torch.device("cpu")
pipeline.vae_spatial_compression_ratio = 32
pipeline.vae_temporal_compression_ratio = 8
pipeline.latent_upsampler = FakeUpsampler()
activated_slots: list[Any] = []
if pipeline_cls is LTX2TwoStagePipeline:
pipeline._phase_adapter = SimpleNamespace(activate=activated_slots.append)
object.__setattr__(pipeline, "_resolve_request_inputs", resolve_request_inputs)
object.__setattr__(pipeline, "run_phase", run_phase)
object.__setattr__(pipeline, "decode_phase", decode_phase)
source_image = object()
output = pipeline.forward(SimpleNamespace(sampling_params_list=[], prompts=[]), image=source_image)
assert len(phase_calls) == 2
assert phase_calls[1][2] is prompt_context_sentinel
if pipeline_cls is LTX2TwoStagePipeline:
assert activated_slots == [None, "ltx_distilled"]
torch.testing.assert_close(output.output[0], torch.full((1, 128, 1, 2, 2), 3.0))
expected_audio = 2.0 if pipeline_cls is LTX2TwoStagePipeline else 4.0
torch.testing.assert_close(output.output[1], torch.full((1, 8, 1, 2), expected_audio))
@pytest.mark.parametrize("pipeline_cls", [LTX2TwoStagePipeline, LTX2DistilledPipeline])
def test_ltx_two_stage_stage2_reapplies_final_resolution_i2v_conditioning(pipeline_cls):
from vllm_omni.diffusion.models.ltx2 import ltx2_latents
pipeline = object.__new__(pipeline_cls)
torch.nn.Module.__init__(pipeline)
pipeline.vae_spatial_compression_ratio = 32
pipeline.vae_temporal_compression_ratio = 8
pipeline.transformer_spatial_patch_size = 1
pipeline.transformer_temporal_patch_size = 1
pipeline.vae = SimpleNamespace(
latents_mean=torch.zeros(2),
latents_std=torch.ones(2),
config=SimpleNamespace(scaling_factor=1.0),
)
object.__setattr__(
pipeline,
"_encode_i2v_image_latents",
lambda image, **_kwargs: torch.full((image.shape[0], 2, 1, 2, 2), 7.0),
)
packed, conditioning_mask = pipeline.prepare_latents(
image=torch.zeros(1, 3, 64, 64),
batch_size=1,
num_channels_latents=2,
height=64,
width=64,
num_frames=9,
noise_scale=0.0,
dtype=torch.float32,
device=torch.device("cpu"),
latents=torch.ones(1, 2, 2, 2, 2),
)
unpacked = ltx2_latents.unpack_latents(packed, 2, 2, 2)
torch.testing.assert_close(unpacked[:, :, :1], torch.full((1, 2, 1, 2, 2), 7.0))
torch.testing.assert_close(unpacked[:, :, 1:], torch.ones(1, 2, 1, 2, 2))
assert conditioning_mask.shape == (1, 8)
assert conditioning_mask[:, :4].all()
assert not conditioning_mask[:, 4:].any()
def test_ltx2_distilled_two_stage_recipe_is_fixed_positive_only():
stage1, stage2 = LTX2_DISTILLED_TWO_STAGE_RECIPE.phases
assert stage1.spatial_downscale == 2
assert stage1.guidance == stage2.guidance == LTXGuidanceSpec.positive_only()
assert LTXGuidancePlan.build(stage1.guidance).names == ("cond",)
assert LTXGuidancePlan.build(stage2.guidance).names == ("cond",)
assert not stage1.allow_guidance_override
assert not stage2.allow_guidance_override
assert stage1.noise_scale == 1.0
assert stage1.sigmas == (1.0, 0.99375, 0.9875, 0.98125, 0.975, 0.909375, 0.725, 0.421875, 0.0)
assert stage2.sigmas == (0.909375, 0.725, 0.421875, 0.0)
assert not LTX2_DISTILLED_TWO_STAGE_RECIPE.allow_request_sigmas
assert not LTX2_DISTILLED_TWO_STAGE_RECIPE.allow_request_latents
assert not LTX2_DISTILLED_TWO_STAGE_RECIPE.allow_negative_prompt
assert LTX2_DISTILLED_TWO_STAGE_RECIPE.fixed_num_inference_steps
assert (LTX2_DISTILLED_TWO_STAGE_RECIPE.height, LTX2_DISTILLED_TWO_STAGE_RECIPE.width) == (1024, 1536)
def test_ltx23_full_distilled_route_uses_merged_weights_without_lora():
assert resolve_ltx_component_profile("distilled_two_stage", "2.3") is LTX23_DISTILLED_COMPONENT_PROFILE
assert resolve_ltx_pipeline_recipe("distilled_two_stage", "2.3") is LTX23_DISTILLED_TWO_STAGE_RECIPE
assert LTX23_DISTILLED_TWO_STAGE_RECIPE is LTX2_DISTILLED_TWO_STAGE_RECIPE
assert all(phase.adapter_slot is None for phase in LTX23_DISTILLED_TWO_STAGE_RECIPE.phases)
assert LTX23_DISTILLED_COMPONENT_PROFILE.distilled_lora_filename is None
assert LTX23_DISTILLED_COMPONENT_PROFILE.vocoder_cls is LTX23_COMPONENT_PROFILE.vocoder_cls
assert LTX23_DISTILLED_COMPONENT_PROFILE.latent_upsampler_filename == (
"ltx-2.3-spatial-upscaler-x2-1.1.safetensors"
)
@pytest.mark.parametrize(
("request_inputs", "error"),
[
(lambda request: replace(request, height=32), "divisible by 64"),
(lambda request: replace(request, num_frames=8), "8 \\* k \\+ 1"),
(lambda request: replace(request, num_inference_steps=7), "uses 8 fixed Stage 1"),
(lambda request: replace(request, latents=torch.empty(1)), "request-provided"),
(lambda request: replace(request, audio_latents=torch.empty(1)), "request-provided"),
(lambda request: replace(request, negative_prompt="negative"), "negative conditioning"),
(lambda request: replace(request, negative_prompt_embeds=torch.empty(1)), "negative conditioning"),
(lambda request: replace(request, guidance=LTX2_ONE_STAGE_RECIPE.request_guidance), "positive-only"),
],
)
def test_ltx_two_stage_recipe_rejects_unsupported_request_values(request_inputs, error):
request = LTXRequestInputs(
prompt="prompt",
negative_prompt="",
height=64,
width=64,
num_frames=1,
frame_rate=24.0,
num_inference_steps=8,
guidance=LTXGuidanceSpec.positive_only(),
num_videos_per_prompt=1,
generator=None,
latents=None,
audio_latents=None,
prompt_embeds=None,
negative_prompt_embeds=None,
prompt_attention_mask=None,
negative_prompt_attention_mask=None,
decode_timestep=0.0,
decode_noise_scale=None,
output_type="np",
max_sequence_length=16,
)
with pytest.raises(ValueError, match=error):
validate_pipeline_request(
request_inputs(request),
pipeline_recipe=LTX2_DISTILLED_TWO_STAGE_RECIPE,
vae_spatial_compression_ratio=32,
vae_temporal_compression_ratio=8,
pipeline_name="LTX2DistilledPipeline",
)
def test_ltx_two_stage_recipe_rejects_sigmas():
request = LTXRequestInputs(
prompt="prompt",
negative_prompt="",
height=64,
width=64,
num_frames=1,
frame_rate=24.0,
num_inference_steps=8,
guidance=LTXGuidanceSpec.positive_only(),
num_videos_per_prompt=1,
generator=None,
latents=None,
audio_latents=None,
prompt_embeds=None,
negative_prompt_embeds=None,
prompt_attention_mask=None,
negative_prompt_attention_mask=None,
decode_timestep=0.0,
decode_noise_scale=None,
output_type="np",
max_sequence_length=16,
)
with pytest.raises(ValueError, match="fixed phase sigma schedules"):
validate_pipeline_request(
request,
pipeline_recipe=LTX2_DISTILLED_TWO_STAGE_RECIPE,
vae_spatial_compression_ratio=32,
vae_temporal_compression_ratio=8,
pipeline_name="LTX2DistilledPipeline",
request_sigmas=[1.0, 0.0],
)
def test_ltx_request_applies_diffvae_minimum_spatial_size_only_when_requested():
request = LTXRequestInputs(
prompt="prompt",
negative_prompt="",
height=128,
width=128,
num_frames=9,
frame_rate=24.0,
num_inference_steps=30,
guidance=LTX2_ONE_STAGE_RECIPE.request_guidance,
num_videos_per_prompt=1,
generator=None,
latents=None,
audio_latents=None,
prompt_embeds=None,
negative_prompt_embeds=None,
prompt_attention_mask=None,
negative_prompt_attention_mask=None,
decode_timestep=0.0,
decode_noise_scale=None,
output_type="np",
max_sequence_length=16,
)
with pytest.raises(ValueError, match="DiffVAE requires video latent spatial dimensions of at least 7x7"):
validate_pipeline_request(
request,
pipeline_recipe=LTX2_ONE_STAGE_RECIPE,
vae_spatial_compression_ratio=32,
vae_temporal_compression_ratio=8,
pipeline_name="LTX2Pipeline",
min_video_latent_spatial_size=(7, 7),
)
# ConvVAE does not use NATTEN, so its existing 128x128 boundary remains valid.
validate_pipeline_request(
request,
pipeline_recipe=LTX2_ONE_STAGE_RECIPE,
vae_spatial_compression_ratio=32,
vae_temporal_compression_ratio=8,
pipeline_name="LTX2Pipeline",
)
validate_pipeline_request(
replace(request, height=256, width=256),
pipeline_recipe=LTX2_ONE_STAGE_RECIPE,
vae_spatial_compression_ratio=32,
vae_temporal_compression_ratio=8,
pipeline_name="LTX2Pipeline",
min_video_latent_spatial_size=(7, 7),
)
@pytest.mark.parametrize(
("direct_kwargs", "sampling_kwargs", "error"),
[
({"sigmas": [1.0, 0.0]}, {}, "fixed phase sigma schedules"),
({}, {"sigmas": [1.0, 0.0]}, "fixed phase sigma schedules"),
({"latents": torch.empty(1)}, {}, "request-provided video or audio latents"),
({}, {"latents": torch.empty(1)}, "request-provided video or audio latents"),
({"audio_latents": torch.empty(1)}, {}, "request-provided video or audio latents"),
(
{},
{"extra_args": {"audio_latents": torch.empty(1)}},
"request-provided video or audio latents",
),
({"negative_prompt": "negative"}, {}, "negative conditioning"),
({"negative_prompt_embeds": torch.empty(1)}, {}, "negative conditioning"),
({}, {"num_inference_steps": 7}, "uses 8 fixed Stage 1"),
({"guidance_scale": 4.0}, {}, "fixed positive-only guidance"),
({}, {"guidance_scale": 4.0}, "fixed positive-only guidance"),
],
ids=(
"direct-sigmas",
"request-sigmas",
"direct-video-latents",
"request-video-latents",
"direct-audio-latents",
"request-audio-latents",
"direct-negative-prompt",
"direct-negative-embeds",
"request-num-steps",
"direct-guidance",
"request-guidance",
),
)
def test_ltx_distilled_forward_rejects_fixed_recipe_overrides(direct_kwargs, sampling_kwargs, error):
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
sampling_values = {
"height": 64,
"width": 64,
"num_frames": 1,
"num_inference_steps": 8,
**sampling_kwargs,
}
sampling = OmniDiffusionSamplingParams(**sampling_values)
req = DiffusionRequestBatch(
[
OmniDiffusionRequest(
prompt="prompt",
sampling_params=sampling,
request_id="ltx-distilled-fixed-inputs",
)
]
)
pipeline = _make_ltx_request_pipe(LTX2DistilledPipeline)
with pytest.raises(ValueError, match=error):
pipeline.forward(req, **direct_kwargs)
def test_ltx_distilled_dummy_run_uses_fixed_recipe_values():
from vllm_omni.diffusion.request import DUMMY_DIFFUSION_REQUEST_ID, OmniDiffusionRequest
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
req = DiffusionRequestBatch(
[
OmniDiffusionRequest(
prompt="dummy run",
sampling_params=OmniDiffusionSamplingParams(
height=512,
width=512,
num_frames=LTX2DistilledPipeline.dummy_run_num_frames,
num_inference_steps=1,
guidance_scale=0.0,
),
request_id=DUMMY_DIFFUSION_REQUEST_ID,
)
]
)
pipeline = _make_ltx_request_pipe(LTX2DistilledPipeline)
seen = {}
def fake_run_recipe(_req, request_inputs, **_kwargs):
seen["request_inputs"] = request_inputs
return "dummy-output"
object.__setattr__(pipeline, "_run_recipe", fake_run_recipe)
assert pipeline.forward(req) == "dummy-output"
assert seen["request_inputs"].num_frames == 1
assert seen["request_inputs"].num_inference_steps == 8
assert seen["request_inputs"].guidance == LTXGuidanceSpec.positive_only()
def test_ltx_custom_sigmas_bypass_scheduler_shifting():
from diffusers import FlowMatchEulerDiscreteScheduler
scheduler = FlowMatchEulerDiscreteScheduler(
use_dynamic_shifting=True,
base_shift=0.95,
max_shift=2.05,
shift_terminal=0.1,
)
pipeline = SimpleNamespace(scheduler=scheduler)
sigmas = [1.0, 0.75, 0.25, 0.0]
audio_scheduler, _, timesteps = prepare_scheduler_stage(
pipeline,
SimpleNamespace(num_inference_steps=3),
device=torch.device("cpu"),
sigmas=sigmas,
timesteps=None,
latent_num_frames=1,
latent_height=1,
latent_width=1,
use_official_sigma_schedule=False,
)
expected = torch.tensor(sigmas, dtype=torch.float32)
torch.testing.assert_close(pipeline.scheduler.sigmas, expected)
torch.testing.assert_close(audio_scheduler.sigmas, expected)
torch.testing.assert_close(timesteps, expected[:-1] * scheduler.config.num_train_timesteps)
@pytest.mark.parametrize(
("recipe", "steps", "stg_block"),
[
(LTX2_TWO_STAGE_RECIPE, 40, 29),
(LTX23_TWO_STAGE_RECIPE, 30, 28),
],
)
def test_ltx_official_two_stage_recipes_only_vary_by_model_defaults(recipe, steps, stg_block):
stage1, stage2 = recipe.phases
assert (recipe.width, recipe.height) == (1536, 1024)
assert recipe.num_inference_steps == steps
assert stage1.spatial_downscale == 2
assert stage1.adapter_slot is None
assert stage1.guidance.video.stg_blocks == (stg_block,)
assert stage1.guidance.audio.stg_blocks == (stg_block,)
assert stage2.input_transform == "spatial_upsample"
assert stage2.adapter_slot == "ltx_distilled"
assert stage2.guidance == LTXGuidanceSpec.positive_only()
assert stage2.num_inference_steps == 3
assert stage2.sigmas == (0.909375, 0.725, 0.421875, 0.0)
# Official Stage 2 refines video only; its audio result is discarded.
assert (recipe.video_output_phase, recipe.audio_output_phase) == (1, 0)
def test_ltx_two_stage_entries_select_the_official_adapter_slots():
phase_switches: list[Any] = []
ordinary_pipeline = object.__new__(LTX2TwoStagePipeline)
ordinary_pipeline._phase_adapter = SimpleNamespace(activate=phase_switches.append)
for phase in LTX2_TWO_STAGE_RECIPE.phases:
ordinary_pipeline._enter_phase(phase)
assert phase_switches == [None, "ltx_distilled"]
distilled_pipeline = object.__new__(LTX2DistilledPipeline)
for phase in LTX2_DISTILLED_TWO_STAGE_RECIPE.phases:
distilled_pipeline._enter_phase(phase)
def test_ltx23_gate_projection_declares_its_global_column_parallel_layout(monkeypatch):
class FakeColumn(torch.nn.Module):
def __init__(
self,
input_size,
output_size,
*,
bias,
gather_output,
return_bias,
quant_config,
prefix,
):
super().__init__()
self.input_size = input_size
self.output_size = output_size
self.bias_enabled = bias
self.gather_output = gather_output
self.return_bias = return_bias
self.quant_config = quant_config
self.prefix = prefix
class FakeRow(torch.nn.Module):
def __init__(self, *args, **kwargs):
super().__init__()
class FakeAttention(torch.nn.Module):
def __init__(self, **kwargs):
super().__init__()
monkeypatch.setattr(ltx2_transformer, "ColumnParallelLinear", FakeColumn)
monkeypatch.setattr(ltx2_transformer, "RowParallelLinear", FakeRow)
monkeypatch.setattr(ltx2_transformer, "_LTX2ParallelAttention", FakeAttention)
monkeypatch.setattr(ltx2_transformer, "get_tensor_model_parallel_world_size", lambda: 2)
attention = ltx2_transformer.LTX2Attention(
query_dim=16,
heads=4,
dim_head=4,
qk_norm="rms_norm",
apply_gated_attention=True,
pack_qkv=False,
prefix="transformer_blocks.0.attn1",
)
gate = attention.to_gate_logits
assert isinstance(gate, FakeColumn)
assert gate.input_size == 16
assert gate.output_size == 4
assert gate.gather_output is False
assert gate.return_bias is False
assert gate.quant_config is None
assert gate.prefix == "transformer_blocks.0.attn1.to_gate_logits"
class TestLTXRequestParsing:
def test_request_resolves_official_and_explicit_negative_prompts_per_sample(self):
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
req = DiffusionRequestBatch(
[
OmniDiffusionRequest(
prompt=prompt,
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=2),
request_id=f"ltx-negative-prompt-{index}",
)
for index, prompt in enumerate(
(
"use the official default",
{"prompt": "disable the default", "negative_prompt": ""},
{"prompt": "custom negative", "negative_prompt": "custom"},
)
)
]
)
resolved = _resolve_request_inputs_for_test(_make_ltx_request_pipe(LTX2Pipeline), req)
assert resolved.negative_prompt == [LTX_DEFAULT_NEGATIVE_PROMPT, "", "custom"]
def test_forward_negative_prompt_fallback_is_not_discarded(self):
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
req = DiffusionRequestBatch(
[
OmniDiffusionRequest(
prompt="prompt",
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=2),
request_id="ltx-forward-negative-prompt",
)
]
)
resolved = _resolve_request_inputs_for_test(
_make_ltx_request_pipe(LTX2Pipeline),
req,
negative_prompt="forward fallback",
)
assert resolved.negative_prompt == ["forward fallback"]
def test_unified_t2v_i2v_entry_resolves_request_inputs(self):
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
prompt_embeds = torch.tensor([[1.0, 2.0]])
negative_prompt_embeds = torch.tensor([[3.0, 4.0]])
prompt_attention_mask = torch.tensor([True, False])
negative_attention_mask = torch.tensor([False, True])
video_latents = torch.ones(1, 2, 3)
audio_latents = torch.zeros(1, 2, 3)
generator = torch.Generator().manual_seed(123)
req = DiffusionRequestBatch(
[
OmniDiffusionRequest(
prompt={
"prompt": "shared prompt",
"negative_prompt": "shared negative",
"additional_information": {
"prompt_embeds": prompt_embeds,
"negative_prompt_embeds": negative_prompt_embeds,
"attention_mask": prompt_attention_mask,
"negative_attention_mask": negative_attention_mask,
},
},
sampling_params=OmniDiffusionSamplingParams(
height=384,
width=512,
num_frames=25,
frame_rate=12.5,
num_inference_steps=1,
num_outputs_per_prompt=2,
guidance_scale=6.0,
generator=generator,
latents=video_latents,
extra_args={"audio_latents": audio_latents},
decode_timestep=[0.1],
decode_noise_scale=[0.2],
output_type="latent",
max_sequence_length=17,
),
request_id="ltx23-shared-request-inputs",
)
]
)
resolved = _resolve_request_inputs_for_test(_make_ltx_request_pipe(LTX2Pipeline), req)
assert resolved.prompt is None
assert resolved.negative_prompt is None
assert resolved.height == 384
assert resolved.width == 512
assert resolved.num_frames == 25
assert resolved.frame_rate == 12.5
assert resolved.num_inference_steps == 2
assert resolved.guidance_scale == 6.0
assert resolved.num_videos_per_prompt == 2
assert isinstance(resolved.generator, list)
assert [gen.initial_seed() for gen in resolved.generator] == [123, 123]
assert resolved.decode_timestep == [0.1]
assert resolved.decode_noise_scale == [0.2]
assert resolved.output_type == "latent"
assert resolved.max_sequence_length == 17
torch.testing.assert_close(resolved.latents, video_latents)
torch.testing.assert_close(resolved.audio_latents, audio_latents)
torch.testing.assert_close(resolved.prompt_embeds, torch.stack([prompt_embeds]))
torch.testing.assert_close(resolved.negative_prompt_embeds, torch.stack([negative_prompt_embeds]))
torch.testing.assert_close(resolved.prompt_attention_mask, torch.stack([prompt_attention_mask]))
torch.testing.assert_close(resolved.negative_prompt_attention_mask, torch.stack([negative_attention_mask]))
def test_nonbatch_distilled_request_uses_precomputed_positive_embeddings(self):
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
prompt_embeds = torch.tensor([[1.0, 2.0]])
prompt_attention_mask = torch.tensor([True, False])
req = DiffusionRequestBatch(
[
OmniDiffusionRequest(
prompt={
"prompt": "must not be forwarded with embeddings",
"additional_information": {
"prompt_embeds": prompt_embeds,
"attention_mask": prompt_attention_mask,
},
},
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=8),
request_id="ltx-distilled-prompt-embeds",
)
]
)
resolved = _resolve_request_inputs_for_test(
_make_ltx_request_pipe(LTX2DistilledPipeline),
req,
guidance_scale=None,
)
assert resolved.prompt is None
assert resolved.negative_prompt == [""]
torch.testing.assert_close(resolved.prompt_embeds, torch.stack([prompt_embeds]))
torch.testing.assert_close(resolved.prompt_attention_mask, torch.stack([prompt_attention_mask]))
def test_request_resolves_per_modality_guidance_and_common_cfg_override(self):
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
req = DiffusionRequestBatch(
[
OmniDiffusionRequest(
prompt={"prompt": "prompt", "negative_prompt": "negative"},
sampling_params=OmniDiffusionSamplingParams(
num_inference_steps=2,
extra_args={
"video_cfg_scale": 4.0,
"audio_cfg_guidance_scale": 6.0,
"video_stg_guidance_scale": 0.5,
"audio_stg_scale": 0.25,
"a2v_guidance_scale": 2.0,
"v2a_guidance_scale": 4.0,
"video_rescale_scale": 0.2,
"audio_rescale_scale": 0.3,
"video_stg_blocks": [3, 4],
"audio_stg_blocks": 5,
},
),
request_id="ltx-guidance-overrides",
)
]
)
pipe = _make_ltx_request_pipe(LTX2Pipeline)
resolved = _resolve_request_inputs_for_test(pipe, req, guidance_scale=None)
assert resolved.guidance.video == LTXModalityGuidance(4.0, 0.5, 2.0, 0.2, (3, 4))
assert resolved.guidance.audio == LTXModalityGuidance(6.0, 0.25, 4.0, 0.3, (5,))
common = _resolve_request_inputs_for_test(pipe, req, guidance_scale=2.0)
assert common.guidance.video.cfg_scale == common.guidance.audio.cfg_scale == 2.0
# LTX exposes independent video/audio rescale controls, not the
# model-agnostic sampling field.
assert common.guidance.video.rescale_scale == 0.2
assert common.guidance.audio.rescale_scale == 0.3
def test_request_rejects_unsupported_generic_scheduler_and_rescale_controls(self):
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
cases = (
(OmniDiffusionSamplingParams(timesteps=torch.tensor([1.0])), "timesteps"),
(OmniDiffusionSamplingParams(guidance_rescale=0.5), "guidance_rescale"),
(OmniDiffusionSamplingParams(extra_args={"flow_shift": 3.0}), "flow_shift"),
)
pipe = _make_ltx_request_pipe(LTX2Pipeline)
for index, (sampling, field_name) in enumerate(cases):
req = DiffusionRequestBatch(
[
OmniDiffusionRequest(
prompt="prompt",
sampling_params=sampling,
request_id=f"ltx-unsupported-{index}",
)
]
)
with pytest.raises(ValueError, match=field_name):
_resolve_request_inputs_for_test(pipe, req)
def test_request_batch_rejects_mixed_guidance_specs(self):
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
req = DiffusionRequestBatch(
[
OmniDiffusionRequest(
prompt="prompt-0",
sampling_params=OmniDiffusionSamplingParams(
num_inference_steps=2,
extra_args={"video_cfg_scale": scale},
),
request_id=f"ltx-mixed-guidance-{index}",
)
for index, scale in enumerate((3.0, 4.0))
]
)
with pytest.raises(ValueError, match="identical video/audio guidance"):
_resolve_request_inputs_for_test(
_make_ltx_request_pipe(LTX2Pipeline),
req,
guidance_scale=None,
)
def test_request_input_resolution_rejects_mixed_precomputed_prompt_fields(self):
from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
pipe = _make_ltx_request_pipe(LTX2Pipeline)
req = DiffusionRequestBatch(
[
OmniDiffusionRequest(
prompt={
"prompt": "with embeds",
"additional_information": {"prompt_embeds": torch.tensor([[1.0]])},
},
sampling_params=OmniDiffusionSamplingParams(
height=384,
width=512,
num_frames=25,
num_inference_steps=2,
),
request_id="ltx23-mixed-precomputed-fields-0",
),
OmniDiffusionRequest(
prompt={"prompt": "without embeds"},
sampling_params=OmniDiffusionSamplingParams(
height=384,
width=512,
num_frames=25,
num_inference_steps=2,
),
request_id="ltx23-mixed-precomputed-fields-1",
),
]
)
with pytest.raises(ValueError, match="mix of provided and missing prompt_embeds"):
_resolve_request_inputs_for_test(pipe, req)
class TestLTXForwardStages:
def test_encode_prompt_batches_positive_and_negative_gemma_inputs(self):
pipe = _make_ltx_request_pipe(LTX2Pipeline)
calls = []
def fake_get_gemma_prompt_embeds(**kwargs):
calls.append(kwargs)
count = len(kwargs["prompt"])
embeds = torch.arange(count, dtype=torch.float32).view(count, 1, 1)
mask = torch.arange(count).view(count, 1)
return embeds, mask
object.__setattr__(pipe, "_get_gemma_prompt_embeds", fake_get_gemma_prompt_embeds)
prompt_embeds, prompt_mask, negative_embeds, negative_mask = pipe.encode_prompt(
prompt=["positive-0", "positive-1"],
negative_prompt=["negative-0", "negative-1"],
do_classifier_free_guidance=True,
num_videos_per_prompt=2,
device=torch.device("cpu"),
)
assert len(calls) == 1
assert calls[0]["prompt"] == ["positive-0", "positive-1", "negative-0", "negative-1"]
assert calls[0]["num_videos_per_prompt"] == 1
torch.testing.assert_close(prompt_embeds.flatten(), torch.tensor([0.0, 0.0, 1.0, 1.0]))
torch.testing.assert_close(negative_embeds.flatten(), torch.tensor([2.0, 2.0, 3.0, 3.0]))
torch.testing.assert_close(prompt_mask.flatten(), torch.tensor([0, 0, 1, 1]))
torch.testing.assert_close(negative_mask.flatten(), torch.tensor([2, 2, 3, 3]))
def test_encode_prompt_repeats_precomputed_embeds_for_num_outputs(self):
from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline
pipe = _make_ltx_request_pipe(LTX2Pipeline)
prompt_embeds = torch.tensor([[[1.0], [2.0]], [[3.0], [4.0]]])
negative_prompt_embeds = torch.tensor([[[-1.0], [-2.0]], [[-3.0], [-4.0]]])
prompt_attention_mask = torch.tensor([[True, False], [False, True]])
negative_prompt_attention_mask = torch.tensor([[False, False], [True, True]])
(
repeated_prompt_embeds,
repeated_prompt_attention_mask,
repeated_negative_prompt_embeds,
repeated_negative_prompt_attention_mask,
) = pipe.encode_prompt(
prompt=None,
do_classifier_free_guidance=True,
num_videos_per_prompt=2,
prompt_embeds=prompt_embeds,
negative_prompt_embeds=negative_prompt_embeds,
prompt_attention_mask=prompt_attention_mask,
negative_prompt_attention_mask=negative_prompt_attention_mask,
device=torch.device("cpu"),
)
torch.testing.assert_close(
repeated_prompt_embeds,
torch.tensor([[[1.0], [2.0]], [[1.0], [2.0]], [[3.0], [4.0]], [[3.0], [4.0]]]),
)
torch.testing.assert_close(
repeated_negative_prompt_embeds,
torch.tensor([[[-1.0], [-2.0]], [[-1.0], [-2.0]], [[-3.0], [-4.0]], [[-3.0], [-4.0]]]),
)
torch.testing.assert_close(
repeated_prompt_attention_mask,
torch.tensor([[True, False], [True, False], [False, True], [False, True]]),
)
torch.testing.assert_close(
repeated_negative_prompt_attention_mask,
torch.tensor([[False, False], [False, False], [True, True], [True, True]]),
)
def test_t2v_forward_delegates_to_shared_recipe_runtime(self):
from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
pipe = _make_ltx_request_pipe(LTX2Pipeline)
request_sigmas = [1.0, 0.4]
req = DiffusionRequestBatch(
[
OmniDiffusionRequest(
prompt="stage refactor prompt",
sampling_params=OmniDiffusionSamplingParams(
height=384,
width=512,
num_frames=25,
frame_rate=16,
num_inference_steps=3,
guidance_scale=4.5,
output_type="latent",
sigmas=request_sigmas,
),
request_id="ltx23-t2v-forward-stage-delegation",
)
]
)
fallback_sigmas = [1.0, 0.5]
seen = {}
def fake_run_recipe(req_arg, request_inputs, **kwargs):
seen["req"] = req_arg
seen["request_inputs"] = request_inputs
seen["kwargs"] = kwargs
return ["delegated"]
object.__setattr__(pipe, "_run_recipe", fake_run_recipe)
output = pipe.forward(req, sigmas=fallback_sigmas)
assert output == ["delegated"]
assert seen["req"] is req
assert seen["request_inputs"].height == 384
assert seen["request_inputs"].width == 512
assert seen["request_inputs"].num_frames == 25
assert seen["request_inputs"].frame_rate == 16.0
assert seen["request_inputs"].guidance_scale == 4.5
assert seen["request_inputs"].output_type == "latent"
assert seen["kwargs"] == {
"request_sigmas": request_sigmas,
"request_phase_sigmas": None,
"image": None,
}
def test_t2v_forward_prefers_request_sigmas(self):
from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
request_sigmas = [1.0, 0.25]
req = DiffusionRequestBatch(
[
OmniDiffusionRequest(
prompt="custom sigma schedule",
sampling_params=OmniDiffusionSamplingParams(
num_inference_steps=2,
sigmas=request_sigmas,
),
request_id="ltx23-request-sigmas",
)
]
)
pipe = _make_ltx_request_pipe(LTX2Pipeline)
seen = {}
def fake_run_recipe(req_arg, request_inputs, **kwargs):
del req_arg, request_inputs
seen.update(kwargs)
return ["delegated"]
object.__setattr__(pipe, "_run_recipe", fake_run_recipe)
output = pipe.forward(req, sigmas=[1.0, 0.5])
assert output == ["delegated"]
assert seen["request_sigmas"] == request_sigmas
class TestPipelineComponents:
def test_ltx_pipeline_declares_offload_components(self):
"""LTX2Pipeline exposes all one-stage modules to offload discovery."""
from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline
from vllm_omni.diffusion.offloader.module_collector import ModuleDiscovery
pipe = object.__new__(LTX2Pipeline)
torch.nn.Module.__init__(pipe)
pipe.transformer = torch.nn.Linear(1, 1)
pipe.text_encoder = torch.nn.Linear(1, 1)
pipe.connectors = torch.nn.Linear(1, 1)
pipe.vae = torch.nn.Linear(1, 1)
pipe.audio_vae = torch.nn.Linear(1, 1)
pipe.vocoder = torch.nn.Linear(1, 1)
modules = ModuleDiscovery.discover(pipe)
assert LTX2Pipeline._dit_modules == ["transformer"]
assert LTX2Pipeline._encoder_modules == ["text_encoder", "connectors"]
assert LTX2Pipeline._vae_modules == ["vae", "audio_vae"]
assert LTX2Pipeline._resident_modules == ["vocoder"]
assert modules.dit_names == ["transformer"]
assert modules.encoder_names == ["text_encoder", "connectors"]
assert modules.resident_names == ["vocoder"]
assert len(modules.vaes) == 2
def test_distilled_profile_keeps_upsampler_in_managed_resident_components(self):
assert LTX2_DISTILLED_COMPONENT_PROFILE.resident_modules == ("vocoder", "latent_upsampler")
assert LTX23_DISTILLED_COMPONENT_PROFILE.resident_modules == ("vocoder", "latent_upsampler")
class TestLTX23DecodeConditioning:
def test_decode_conditioning_expands_per_prompt_values_to_effective_batch(self):
from vllm_omni.diffusion.models.ltx2.ltx2_runtime import _expand_per_prompt_decode_value
assert _expand_per_prompt_decode_value(
[0.1, 0.2],
prompt_batch_size=2,
effective_batch_size=4,
field_name="decode_timestep",
) == [0.1, 0.1, 0.2, 0.2]
assert _expand_per_prompt_decode_value(
[0.3],
prompt_batch_size=2,
effective_batch_size=4,
field_name="decode_timestep",
) == [0.3, 0.3, 0.3, 0.3]
assert _expand_per_prompt_decode_value(
[0.1, 0.2, 0.3, 0.4],
prompt_batch_size=2,
effective_batch_size=4,
field_name="decode_timestep",
) == [0.1, 0.2, 0.3, 0.4]
def test_decode_conditioning_rejects_ambiguous_lengths(self):
from vllm_omni.diffusion.models.ltx2.ltx2_runtime import _expand_per_prompt_decode_value
with pytest.raises(ValueError, match="decode_timestep"):
_expand_per_prompt_decode_value(
[0.1, 0.2, 0.3],
prompt_batch_size=2,
effective_batch_size=4,
field_name="decode_timestep",
)
class TestRegistryIntegration:
"""Verify the unified one-stage and existing two-stage entries."""
def test_pipeline_module_paths(self):
"""Registry entries must point to the correct modules."""
from vllm_omni.diffusion.registry import _DIFFUSION_MODELS
assert _DIFFUSION_MODELS["LTX2Pipeline"] == ("ltx2", "pipeline_ltx2", "LTX2Pipeline")
assert _DIFFUSION_MODELS["LTX2TwoStagePipeline"] == (
"ltx2",
"pipeline_ltx2_two_stage",
"LTX2TwoStagePipeline",
)
assert _DIFFUSION_MODELS["LTX2DistilledOneStagePipeline"] == (
"ltx2",
"pipeline_ltx2",
"LTX2DistilledOneStagePipeline",
)
assert _DIFFUSION_MODELS["LTX2DistilledTwoStagePipeline"] == (
"ltx2",
"pipeline_ltx2_two_stage",
"LTX2DistilledTwoStagePipeline",
)
assert _DIFFUSION_MODELS["LTX2DistilledPipeline"] == (
"ltx2",
"pipeline_ltx2_two_stage",
"LTX2DistilledPipeline",
)
removed_entries = {
"LTX2ImageToVideoPipeline",
"LTX23Pipeline",
"LTX23ImageToVideoPipeline",
"LTX2TwoStagesPipeline",
"LTX2ImageToVideoTwoStagesPipeline",
"LTX2ImageToVideoTwoStagePipeline",
"LTX23TwoStagePipeline",
"LTX23ImageToVideoTwoStagePipeline",
}
assert removed_entries.isdisjoint(_DIFFUSION_MODELS)
def test_post_process_funcs_registered(self):
"""Pipeline variants must map to get_ltx2_post_process_func."""
from vllm_omni.diffusion.registry import _DIFFUSION_POST_PROCESS_FUNCS
expected = [
"LTX2Pipeline",
"LTX2TwoStagePipeline",
"LTX2DistilledOneStagePipeline",
"LTX2DistilledTwoStagePipeline",
"LTX2DistilledPipeline",
"LTX2T2VDMD2Pipeline",
"LTX2I2VDMD2Pipeline",
]
for name in expected:
assert name in _DIFFUSION_POST_PROCESS_FUNCS, f"{name} not in _DIFFUSION_POST_PROCESS_FUNCS"
assert _DIFFUSION_POST_PROCESS_FUNCS[name] == "get_ltx2_post_process_func"
@pytest.mark.parametrize(
"model_class_name",
[
"LTX2Pipeline",
"LTX2TwoStagePipeline",
"LTX2DistilledOneStagePipeline",
"LTX2DistilledTwoStagePipeline",
"LTX2DistilledPipeline",
"LTX2T2VDMD2Pipeline",
"LTX2I2VDMD2Pipeline",
],
)
def test_post_process_func_resolves_from_every_entry_module(self, model_class_name):
from vllm_omni.diffusion.registry import get_diffusion_post_process_func
with tempfile.TemporaryDirectory() as tmpdir:
vocoder_dir = os.path.join(tmpdir, "vocoder")
os.makedirs(vocoder_dir)
with open(os.path.join(vocoder_dir, "config.json"), "w") as config_file:
json.dump({"output_sampling_rate": 48000}, config_file)
post_process = get_diffusion_post_process_func(
SimpleNamespace(model_class_name=model_class_name, model=tmpdir)
)
result = post_process((torch.zeros(1), torch.zeros(1)))
assert result["audio_sample_rate"] == 48000
def test_cache_dit_for_ltx2_does_not_have_custom_enablers_registered(self):
"""Pipeline variants are *not* registered in CUSTOM_DIT_ENABLERS."""
from vllm_omni.diffusion.cache.cachedit import CUSTOM_DIT_ENABLERS
# NOTE: We used to have custom enablers for this model, but refactored to handle
# it more generically. Now we only need to ensure it has git cache adapter config.
expected = ["LTX2Pipeline"]
for name in expected:
assert name not in CUSTOM_DIT_ENABLERS, f"{name} not in CUSTOM_DIT_ENABLERS"
class TestVocoderSampleRateDetection:
"""Test _detect_vocoder_output_sample_rate logic."""
def test_detects_48khz_from_config(self):
"""Should detect output_sampling_rate=48000 from vocoder/config.json."""
from vllm_omni.diffusion.models.ltx2.ltx2_components import _detect_vocoder_output_sample_rate
with tempfile.TemporaryDirectory() as tmpdir:
vocoder_dir = os.path.join(tmpdir, "vocoder")
os.makedirs(vocoder_dir)
with open(os.path.join(vocoder_dir, "config.json"), "w") as f:
json.dump({"output_sampling_rate": 48000, "input_sampling_rate": 16000}, f)
result = _detect_vocoder_output_sample_rate(tmpdir)
assert result == 48000
def test_returns_none_for_no_output_sr(self):
"""Should return None if vocoder config has no output_sampling_rate."""
from vllm_omni.diffusion.models.ltx2.ltx2_components import _detect_vocoder_output_sample_rate
with tempfile.TemporaryDirectory() as tmpdir:
vocoder_dir = os.path.join(tmpdir, "vocoder")
os.makedirs(vocoder_dir)
with open(os.path.join(vocoder_dir, "config.json"), "w") as f:
json.dump({"sampling_rate": 16000}, f)
result = _detect_vocoder_output_sample_rate(tmpdir)
assert result is None
def test_returns_none_for_missing_directory(self):
"""Should return None if vocoder directory doesn't exist."""
from vllm_omni.diffusion.models.ltx2.ltx2_components import _detect_vocoder_output_sample_rate
result = _detect_vocoder_output_sample_rate("/nonexistent/path")
assert result is None
class TestPostProcessFunction:
"""Test the post-process function factory."""
def test_post_process_includes_audio_sample_rate(self):
"""Post-process func should include audio_sample_rate when detected."""
import torch
from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import get_ltx2_post_process_func
with tempfile.TemporaryDirectory() as tmpdir:
vocoder_dir = os.path.join(tmpdir, "vocoder")
os.makedirs(vocoder_dir)
with open(os.path.join(vocoder_dir, "config.json"), "w") as f:
json.dump({"output_sampling_rate": 48000}, f)
# Create a minimal od_config mock
class MockConfig:
model = tmpdir
func = get_ltx2_post_process_func(MockConfig())
video = torch.zeros(1, 3, 4, 64, 64)
audio = torch.zeros(1, 1, 48000)
result = func((video, audio))
assert isinstance(result, dict)
assert "video" in result
assert "audio" in result
assert result["audio_sample_rate"] == 48000
def test_post_process_without_vocoder_config(self):
"""Post-process func should work without vocoder config (no audio_sample_rate key)."""
import torch
from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import get_ltx2_post_process_func
class MockConfig:
model = "/nonexistent/path"
func = get_ltx2_post_process_func(MockConfig())
video = torch.zeros(1, 3, 4, 64, 64)
audio = torch.zeros(1, 1, 16000)
result = func((video, audio))
assert isinstance(result, dict)
assert "video" in result
assert "audio" in result
assert "audio_sample_rate" not in result