vllm-omni/tests/diffusion/models/ltx2/test_ltx2_latents.py

from types import SimpleNamespace

import pytest
import torch

from vllm_omni.diffusion.models.ltx2 import ltx2_latents as latent_ops
from vllm_omni.diffusion.models.ltx2.pipeline_ltx2 import LTX2Pipeline

pytestmark = [pytest.mark.core_model, pytest.mark.cpu]


def _make_pipeline(pipeline_cls, sequence_parallel_size: int = 1):
    pipeline = object.__new__(pipeline_cls)
    torch.nn.Module.__init__(pipeline)
    pipeline.audio_vae_temporal_compression_ratio = 4
    pipeline.audio_vae_mel_compression_ratio = 4
    pipeline.od_config = SimpleNamespace(parallel_config=SimpleNamespace(sequence_parallel_size=sequence_parallel_size))
    # Mock audio_vae with identity normalization (mean=0, std=1).
    pipeline.audio_vae = SimpleNamespace(
        latents_mean=torch.tensor(0.0),
        latents_std=torch.tensor(1.0),
    )
    return pipeline


def test_prepare_video_latents_samples_directly_in_packed_token_space():
    pipeline = _make_pipeline(LTX2Pipeline)
    pipeline.vae_spatial_compression_ratio = 8
    pipeline.vae_temporal_compression_ratio = 8
    pipeline.transformer_spatial_patch_size = 2
    pipeline.transformer_temporal_patch_size = 1

    expected_generator = torch.Generator().manual_seed(42)
    expected = torch.randn((1, 32, 16), generator=expected_generator)
    actual = pipeline.prepare_latents(
        batch_size=1,
        num_channels_latents=4,
        height=64,
        width=64,
        num_frames=9,
        dtype=torch.float32,
        device=torch.device("cpu"),
        generator=torch.Generator().manual_seed(42),
    )

    torch.testing.assert_close(actual, expected, rtol=0, atol=0)


def test_prepare_audio_latents_samples_directly_in_packed_token_space():
    pipeline = _make_pipeline(LTX2Pipeline)

    expected_generator = torch.Generator().manual_seed(42)
    expected = torch.randn((1, 3, 32), generator=expected_generator)
    actual, original_num_frames, padded_num_frames = pipeline.prepare_audio_latents(
        batch_size=1,
        num_channels_latents=2,
        num_mel_bins=64,
        audio_latent_length=3,
        dtype=torch.float32,
        device=torch.device("cpu"),
        generator=torch.Generator().manual_seed(42),
    )

    assert original_num_frames == 3
    assert padded_num_frames == 3
    torch.testing.assert_close(actual, expected, rtol=0, atol=0)


def test_prepare_audio_latents_pads_generated_dummy_length_for_sp():
    pipeline = _make_pipeline(LTX2Pipeline, sequence_parallel_size=2)

    latents, original_num_frames, padded_num_frames = pipeline.prepare_audio_latents(
        batch_size=1,
        num_channels_latents=8,
        num_mel_bins=64,
        audio_latent_length=1,
        dtype=torch.float32,
        device=torch.device("cpu"),
    )

    assert original_num_frames == 1
    assert padded_num_frames == 2
    assert latents.shape == (1, 2, 128)


def test_prepare_audio_latents_pads_packed_sequence_dim_for_provided_latents():
    pipeline = _make_pipeline(LTX2Pipeline, sequence_parallel_size=4)
    latents = torch.arange(40, dtype=torch.float32).view(1, 10, 4)

    padded, original_num_frames, padded_num_frames = pipeline.prepare_audio_latents(
        batch_size=1,
        num_channels_latents=2,
        num_mel_bins=8,
        audio_latent_length=10,
        dtype=torch.float32,
        device=torch.device("cpu"),
        latents=latents,
    )

    assert original_num_frames == 10
    assert padded_num_frames == 12
    assert padded.shape == (1, 12, 4)
    torch.testing.assert_close(padded[:, :10], latents)
    torch.testing.assert_close(padded[:, 10:], torch.zeros(1, 2, 4))


def test_unpad_audio_latents_restores_original_frames_before_unpack():
    original = torch.arange(40, dtype=torch.float32).view(1, 10, 4)
    padded = torch.cat([original, torch.full((1, 2, 4), 999.0)], dim=1)

    unpadded = latent_ops.unpad_audio_latents(padded, 10)
    unpacked = latent_ops.unpack_audio_latents(unpadded, num_mel_bins=2)
    expected = latent_ops.unpack_audio_latents(original, num_mel_bins=2)

    assert unpacked.shape == (1, 2, 10, 2)
    assert not (unpacked == 999.0).any()
    torch.testing.assert_close(unpacked, expected)


def test_prepare_audio_latents_accepts_already_padded_4d_latents_for_sp():
    pipeline = _make_pipeline(LTX2Pipeline, sequence_parallel_size=4)
    pipeline.preserve_sp_padded_audio_duration = True
    latents = torch.arange(96, dtype=torch.float32).view(1, 2, 12, 4)

    audio_latent_length = pipeline._resolve_audio_latent_length(10, latents)
    padded, original_num_frames, padded_num_frames = pipeline.prepare_audio_latents(
        batch_size=1,
        num_channels_latents=2,
        num_mel_bins=16,
        audio_latent_length=audio_latent_length,
        dtype=torch.float32,
        device=torch.device("cpu"),
        latents=latents,
    )

    assert audio_latent_length == 10
    assert original_num_frames == 10
    assert padded_num_frames == 12
    assert padded.shape == (1, 12, 8)
    torch.testing.assert_close(padded, latent_ops.pack_audio_latents(latents))


def test_resolve_audio_latent_length_preserves_legacy_4d_shape_inference():
    pipeline = _make_pipeline(LTX2Pipeline, sequence_parallel_size=4)
    latents = torch.zeros(1, 2, 13, 4)

    audio_latent_length = pipeline._resolve_audio_latent_length(10, latents)

    assert audio_latent_length == 13


def test_prepare_audio_latents_rejects_incompatible_provided_length():
    pipeline = _make_pipeline(LTX2Pipeline, sequence_parallel_size=4)
    latents = torch.zeros(1, 11, 4)

    with pytest.raises(ValueError, match="incompatible audio frame count"):
        pipeline.prepare_audio_latents(
            batch_size=1,
            num_channels_latents=2,
            num_mel_bins=8,
            audio_latent_length=10,
            dtype=torch.float32,
            device=torch.device("cpu"),
            latents=latents,
        )