# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Unit tests for LTX image-to-video input and conditioning behavior.""" from types import SimpleNamespace import pytest import torch from PIL import Image pytestmark = [pytest.mark.core_model, pytest.mark.cpu] def _make_ltx23_request_pipe(cls): pipe = object.__new__(cls) torch.nn.Module.__init__(pipe) pipe.device = torch.device("cpu") pipe.tokenizer_max_length = 99 pipe.vae_spatial_compression_ratio = 32 pipe.vae_temporal_compression_ratio = 8 return pipe class TestLTXImageToVideoForwardStages: def test_i2v_pil_preprocessing_matches_official_center_crop(self): from vllm_omni.diffusion.models.ltx2.ltx2_conditioning import _preprocess_i2v_pil_images pixels = torch.zeros(4, 8, 3, dtype=torch.uint8) pixels[:, :, 0] = torch.arange(8, dtype=torch.uint8) image = Image.fromarray(pixels.numpy()) actual = _preprocess_i2v_pil_images(image, height=4, width=4) expected = pixels[:, 2:6].permute(2, 0, 1).unsqueeze(0).float() / 127.5 - 1.0 torch.testing.assert_close(actual, expected) def test_forward_resolves_request_image_and_delegates_to_shared_recipe_runtime(self): from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch from vllm_omni.inputs.data import OmniDiffusionSamplingParams pipe = _make_ltx23_request_pipe(LTX2Pipeline) image = torch.zeros(3, 8, 8) req = DiffusionRequestBatch( [ OmniDiffusionRequest( prompt={ "prompt": "make the image move", "negative_prompt": "jitter", "multi_modal_data": {"image": image}, }, sampling_params=OmniDiffusionSamplingParams( height=384, width=512, num_frames=25, num_inference_steps=2, ), request_id="ltx23-i2v-forward-stage-delegation", ) ] ) seen = {} def fake_run_recipe(req_arg, request_inputs, **kwargs): seen["req"] = req_arg seen["request_inputs"] = request_inputs seen["kwargs"] = kwargs return ["i2v-delegated"] object.__setattr__(pipe, "_run_recipe", fake_run_recipe) output = pipe.forward(req) assert output == ["i2v-delegated"] assert seen["req"] is req assert seen["request_inputs"].prompt == ["make the image move"] assert seen["request_inputs"].negative_prompt == ["jitter"] assert seen["kwargs"]["image"] is image assert seen["kwargs"]["request_sigmas"] is None def test_unified_entry_selects_t2v_without_images_and_rejects_mixed_batches(self): from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline pipe = object.__new__(LTX2Pipeline) request_inputs = SimpleNamespace(latents=None) t2v_req = SimpleNamespace(prompts=["text only", {"prompt": "also text only"}]) assert pipe._resolve_request_image(t2v_req, None, request_inputs) is None mixed_req = SimpleNamespace( prompts=[ "text only", {"prompt": "image conditioned", "multi_modal_data": {"image": torch.zeros(3, 8, 8)}}, ] ) with pytest.raises(ValueError, match="cannot mix text-to-video and image-to-video"): pipe._resolve_request_image(mixed_req, None, request_inputs) def test_denoise_timestep_kwargs_masks_video_only(self): from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline pipe = object.__new__(LTX2Pipeline) ts = torch.tensor([2.0, 4.0]) denoise_ctx = SimpleNamespace( conditioning_mask=torch.tensor([[1.0, 0.0]]), conditioning_mask_for_model=torch.tensor([[1.0, 0.0], [0.0, 1.0]]), ) kwargs = pipe._denoise_timestep_kwargs( ts, SimpleNamespace(guidance_parallel_ready=False), denoise_ctx, video_token_count=2, audio_token_count=1, ) torch.testing.assert_close(kwargs["timestep"], torch.tensor([[0.0, 2.0], [4.0, 0.0]])) torch.testing.assert_close(kwargs["audio_timestep"], ts[:, None]) torch.testing.assert_close(kwargs["sigma"], ts) class TestLTXImageToVideoConditioning: def test_ltx23_i2v_supports_image_input(self): from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline assert LTX2Pipeline.support_image_input is True def test_ltx23_i2v_rejects_multi_image_prompt_list(self): from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline image = object() assert LTX2Pipeline._resolve_single_prompt_image([image]) is image with pytest.raises(ValueError, match="exactly one image per prompt"): LTX2Pipeline._resolve_single_prompt_image([object(), object()]) def test_ltx23_i2v_additional_image_resolution_is_tensor_safe(self): from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline image = torch.zeros(1, 3, 4, 4) additional = { "preprocessed_image": None, "pixel_values": image, "image": torch.ones_like(image), } assert LTX2Pipeline._resolve_additional_image(additional) is image def test_ltx23_i2v_packed_latents_are_not_noised(self, monkeypatch): import vllm_omni.diffusion.models.ltx2.ltx2_conditioning as ltx2_conditioning import vllm_omni.diffusion.models.ltx2.ltx2_latents as ltx2_latents from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline pipe = object.__new__(LTX2Pipeline) torch.nn.Module.__init__(pipe) pipe.vae_spatial_compression_ratio = 1 pipe.vae_temporal_compression_ratio = 1 pipe.transformer_spatial_patch_size = 1 pipe.transformer_temporal_patch_size = 1 object.__setattr__( pipe, "_encode_i2v_image_latents", lambda *_args, **_kwargs: torch.tensor([[[[[40.0]]], [[[41.0]]]]]), ) def fake_randn_tensor(shape, generator=None, device=None, dtype=None): raise AssertionError("packed I2V latents should not be noised") monkeypatch.setattr(ltx2_conditioning, "randn_tensor", fake_randn_tensor) monkeypatch.setattr(ltx2_latents, "randn_tensor", fake_randn_tensor) latents = torch.tensor([[[10.0, 11.0], [20.0, 21.0], [30.0, 31.0]]]) out, conditioning_mask = pipe.prepare_latents( image=torch.zeros(1, 3, 1, 1), batch_size=1, num_channels_latents=2, height=1, width=1, num_frames=3, noise_scale=1.0, dtype=torch.float32, device=torch.device("cpu"), latents=latents, ) torch.testing.assert_close(conditioning_mask, torch.tensor([[1.0, 0.0, 0.0]])) torch.testing.assert_close(out, torch.tensor([[[40.0, 41.0], [20.0, 21.0], [30.0, 31.0]]])) def test_ltx23_i2v_5d_latents_noise_preserves_conditioning_frame(self, monkeypatch): import vllm_omni.diffusion.models.ltx2.ltx2_conditioning as ltx2_conditioning import vllm_omni.diffusion.models.ltx2.ltx2_latents as ltx2_latents from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline pipe = object.__new__(LTX2Pipeline) torch.nn.Module.__init__(pipe) pipe.vae_spatial_compression_ratio = 1 pipe.vae_temporal_compression_ratio = 1 pipe.transformer_spatial_patch_size = 1 pipe.transformer_temporal_patch_size = 1 pipe.vae = SimpleNamespace( latents_mean=torch.zeros(2), latents_std=torch.ones(2), config=SimpleNamespace(scaling_factor=1.0), ) object.__setattr__( pipe, "_encode_i2v_image_latents", lambda *_args, **_kwargs: torch.tensor([[[[[40.0]]], [[[41.0]]]]]), ) sampled_shapes = [] def fake_randn_tensor(shape, generator=None, device=None, dtype=None): sampled_shapes.append(tuple(shape)) return torch.ones(shape, device=device, dtype=dtype) monkeypatch.setattr(ltx2_conditioning, "randn_tensor", fake_randn_tensor) monkeypatch.setattr(ltx2_latents, "randn_tensor", fake_randn_tensor) latents = torch.tensor([[[[[10.0]], [[20.0]], [[30.0]]], [[[11.0]], [[21.0]], [[31.0]]]]]) out, conditioning_mask = pipe.prepare_latents( image=torch.zeros(1, 3, 1, 1), batch_size=1, num_channels_latents=2, height=1, width=1, num_frames=3, noise_scale=1.0, dtype=torch.float32, device=torch.device("cpu"), latents=latents, ) torch.testing.assert_close(conditioning_mask, torch.tensor([[1.0, 0.0, 0.0]])) torch.testing.assert_close(out, torch.tensor([[[40.0, 41.0], [1.0, 1.0], [1.0, 1.0]]])) assert sampled_shapes == [(1, 3, 2)] def test_ltx23_i2v_image_noise_is_sampled_after_packing(self, monkeypatch): import vllm_omni.diffusion.models.ltx2.ltx2_conditioning as ltx2_conditioning from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline pipe = object.__new__(LTX2Pipeline) torch.nn.Module.__init__(pipe) pipe.vae_spatial_compression_ratio = 1 pipe.vae_temporal_compression_ratio = 1 pipe.transformer_spatial_patch_size = 1 pipe.transformer_temporal_patch_size = 1 pipe.vae = SimpleNamespace( encode=lambda image: image, latents_mean=torch.zeros(2), latents_std=torch.ones(2), ) monkeypatch.setattr( ltx2_conditioning, "retrieve_latents", lambda *_args, **_kwargs: torch.tensor([[[[[10.0]]], [[[11.0]]]]]), ) sampled_shapes = [] def fake_randn_tensor(shape, generator=None, device=None, dtype=None): sampled_shapes.append(tuple(shape)) return torch.ones(shape, device=device, dtype=dtype) monkeypatch.setattr(ltx2_conditioning, "randn_tensor", fake_randn_tensor) out, conditioning_mask = pipe.prepare_latents( image=torch.zeros(1, 3, 1, 1), batch_size=1, num_channels_latents=2, height=1, width=1, num_frames=3, noise_scale=1.0, dtype=torch.float32, device=torch.device("cpu"), ) assert sampled_shapes == [(1, 3, 2)] torch.testing.assert_close(conditioning_mask, torch.tensor([[1.0, 0.0, 0.0]])) torch.testing.assert_close(out, torch.tensor([[[10.0, 11.0], [1.0, 1.0], [1.0, 1.0]]])) def test_ltx23_i2v_video_step_preserves_conditioning_frame(self): from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline pipe = object.__new__(LTX2Pipeline) torch.nn.Module.__init__(pipe) pipe.transformer_spatial_patch_size = 1 pipe.transformer_temporal_patch_size = 1 class FakeScheduler: sigmas = torch.tensor([1.0, 0.5]) pipe.scheduler = FakeScheduler() latents = torch.tensor([[[1.0], [2.0], [3.0]]]) noise_pred = torch.full_like(latents, 10.0) out = pipe._step_video_latents_i2v( noise_pred, latents, 0, latent_num_frames=3, latent_height=1, latent_width=1, ) torch.testing.assert_close(out[:, :1], latents[:, :1]) torch.testing.assert_close(out[:, 1:], latents[:, 1:] - 0.5 * noise_pred[:, 1:]) def test_shared_step_adapter_uses_official_euler_for_t2v(self): from vllm_omni.diffusion.models.ltx2.ltx2_denoise import LTXVideoAudioStepAdapter pipeline = SimpleNamespace(scheduler=SimpleNamespace(sigmas=torch.tensor([1.0, 0.5]))) adapter = LTXVideoAudioStepAdapter( pipeline, SimpleNamespace(sigmas=torch.tensor([1.0, 0.25])), latent_num_frames=1, latent_height=1, latent_width=1, image_conditioned=False, ) video = torch.tensor([[[2.0]]]) audio = torch.tensor([[[4.0]]]) ((video, audio),) = adapter.step( (torch.tensor([[[2.0]]]), torch.tensor([[[4.0]]])), (torch.tensor(1.0), torch.tensor(1.0)), (video, audio), ) torch.testing.assert_close(video, torch.tensor([[[1.0]]])) torch.testing.assert_close(audio, torch.tensor([[[1.0]]])) def test_shared_step_adapter_uses_official_euler_for_i2v_audio(self): from vllm_omni.diffusion.models.ltx2.ltx2_denoise import LTXVideoAudioStepAdapter class FakePipeline: scheduler = object() def __init__(self): self.step_indices = [] def _step_video_latents_i2v( self, noise_pred, latents, step_index, latent_num_frames, latent_height, latent_width, ): del noise_pred, latent_num_frames, latent_height, latent_width self.step_indices.append(step_index) return latents pipeline = FakePipeline() adapter = LTXVideoAudioStepAdapter( pipeline, SimpleNamespace(sigmas=torch.tensor([1.0, 0.5, 0.0])), latent_num_frames=1, latent_height=1, latent_width=1, image_conditioned=True, ) video = torch.tensor([[[1.0]]]) audio = torch.tensor([[[2.0]]]) video_velocity = torch.zeros_like(video) audio_velocity = torch.tensor([[[4.0]]]) ((video, audio),) = adapter.step( (video_velocity, audio_velocity), (torch.tensor(1.0), torch.tensor(1.0)), (video, audio), ) torch.testing.assert_close(audio, torch.tensor([[[0.0]]])) ((video, audio),) = adapter.step( (video_velocity, audio_velocity), (torch.tensor(0.5), torch.tensor(0.5)), (video, audio), ) torch.testing.assert_close(audio, torch.tensor([[[-2.0]]])) assert pipeline.step_indices == [0, 1] def test_i2v_guidance_uses_zero_sigma_for_conditioned_tokens(self): from vllm_omni.diffusion.models.ltx2.ltx2_denoise import LTXDenoiseContext from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline pipe = object.__new__(LTX2Pipeline) denoise_ctx = LTXDenoiseContext( latents=torch.empty(1, 3, 1), audio_latents=torch.empty(1, 1, 1), video_coords=torch.empty(1), audio_coords=torch.empty(1), conditioning_mask=torch.tensor([[1.0, 0.0, 0.0]]), ) actual = pipe._video_guidance_model_sigma(torch.tensor(0.75), denoise_ctx) torch.testing.assert_close(actual, torch.tensor([[[0.0], [0.75], [0.75]]]))