vllm-omni/tests/diffusion/models/lance/test_pipeline_lance.py

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

from types import SimpleNamespace

import numpy as np
import pytest
from PIL import Image

from vllm_omni.diffusion.models.lance.lance_transformer import LanceBagel
from vllm_omni.diffusion.models.lance.pipeline_lance import LancePipeline
from vllm_omni.model_executor.models.utils import normalize_decoded_video_frames

pytestmark = [pytest.mark.core_model, pytest.mark.cpu]


class FramesWithMetadata(list[Image.Image]):
    def __init__(self, frames: list[Image.Image], **metadata: object) -> None:
        super().__init__(frames)
        for name, value in metadata.items():
            setattr(self, name, value)


class StopAfterVideoPreprocessError(Exception):
    pass


def test_normalize_decoded_video_frames_preserves_rgb_pixels_and_fps():
    red = Image.new("RGB", (3, 2), color=(255, 0, 0))
    rgba = Image.new("RGBA", (3, 2), color=(0, 255, 0, 64))
    rgba_bytes = rgba.tobytes()

    video, fps = normalize_decoded_video_frames(FramesWithMetadata([red, rgba], fps=8), default_fps=24.0)

    assert video.shape == (2, 2, 3, 3)
    assert video.dtype == np.uint8
    assert video.flags.c_contiguous
    np.testing.assert_array_equal(video[0, 0, 0], [255, 0, 0])
    np.testing.assert_array_equal(video[1, 0, 0], [0, 255, 0])
    assert fps == 8.0
    assert rgba.mode == "RGBA"
    assert rgba.tobytes() == rgba_bytes


def test_normalize_decoded_video_frames_uses_frame_rate_when_fps_is_missing():
    frames = FramesWithMetadata([Image.new("RGB", (2, 2))] * 2, frame_rate=12)

    _, fps = normalize_decoded_video_frames(frames, default_fps=24.0)

    assert fps == 12.0


def test_normalize_decoded_video_frames_prefers_valid_fps_over_frame_rate():
    frames = FramesWithMetadata([Image.new("RGB", (2, 2))] * 2, fps=8, frame_rate=12)

    _, fps = normalize_decoded_video_frames(frames, default_fps=24.0)

    assert fps == 8.0


def test_normalize_decoded_video_frames_uses_default_fps_for_plain_list_and_tuple():
    frames = [Image.new("RGB", (2, 2)), Image.new("RGB", (2, 2))]

    assert normalize_decoded_video_frames(frames, default_fps=24.0)[1] == 24.0
    assert normalize_decoded_video_frames(tuple(frames), default_fps=24.0)[1] == 24.0


def test_normalize_decoded_video_frames_accepts_numeric_string_fps():
    frames = FramesWithMetadata([Image.new("RGB", (2, 2))] * 2, fps="12.5")

    _, frame_rate = normalize_decoded_video_frames(frames, default_fps=24.0)

    assert frame_rate == 12.5


@pytest.mark.parametrize(
    ("fps", "frame_rate", "expected"),
    [
        (float("nan"), 12, 12.0),
        (0, 15, 15.0),
        (-1, 8, 8.0),
        (float("inf"), 10, 10.0),
        ("invalid", "6.5", 6.5),
    ],
)
def test_normalize_decoded_video_frames_uses_valid_frame_rate_after_invalid_fps(fps, frame_rate, expected):
    frames = FramesWithMetadata([Image.new("RGB", (2, 2))] * 2, fps=fps, frame_rate=frame_rate)

    _, frame_rate = normalize_decoded_video_frames(frames, default_fps=24.0)

    assert frame_rate == expected


def test_normalize_decoded_video_frames_uses_default_when_both_metadata_values_are_invalid():
    frames = FramesWithMetadata([Image.new("RGB", (2, 2))] * 2, fps="invalid", frame_rate=0)

    _, frame_rate = normalize_decoded_video_frames(frames, default_fps=24.0)

    assert frame_rate == 24.0


@pytest.mark.parametrize(
    ("frames", "match"),
    [
        ([], "empty decoded video frame sequence"),
        ([Image.new("RGB", (2, 2)), object()], "must be a PIL.Image.Image"),
        (
            [Image.new("RGB", (2, 2)), Image.new("RGB", (3, 2))],
            "must have identical dimensions",
        ),
    ],
)
def test_normalize_decoded_video_frames_rejects_invalid_sequences(frames, match):
    with pytest.raises(ValueError, match=match):
        normalize_decoded_video_frames(frames, default_fps=24.0)


@pytest.mark.parametrize(
    ("frames", "match"),
    [
        ([Image.new("RGB", (0, 16))], r"index 0.*0x16"),
        ([Image.new("RGB", (16, 0))], r"index 0.*16x0"),
        ([Image.new("RGB", (0, 0))], r"index 0.*0x0"),
        (
            [Image.new("RGB", (16, 16)), Image.new("RGB", (0, 16))],
            r"index 1.*0x16",
        ),
    ],
)
def test_normalize_decoded_video_frames_rejects_zero_dimensions(frames, match):
    with pytest.raises(ValueError, match=match):
        normalize_decoded_video_frames(frames, default_fps=24.0)


@pytest.mark.parametrize(
    ("video_input", "expected_fps"),
    [
        (FramesWithMetadata([Image.new("RGB", (3, 2))] * 2, fps=8), 8.0),
        (tuple([Image.new("RGBA", (3, 2))] * 2), 12.0),
    ],
)
def test_lance_video_edit_normalizes_decoded_frame_sequences(monkeypatch, video_input, expected_fps):
    captured = {}

    def capture_preprocessed_input(video, fps):
        captured["video"] = video
        captured["fps"] = fps
        raise StopAfterVideoPreprocessError

    monkeypatch.setattr(LanceBagel, "_lance_video_preprocess", staticmethod(capture_preprocessed_input))
    request = SimpleNamespace(
        prompts=[
            {
                "modalities": ["video"],
                "user_text": "edit the video",
                "multi_modal_data": {"video": video_input},
                "extra_args": {"origin_fps": 12.0},
            }
        ],
        sampling_params=SimpleNamespace(extra_args={}, num_inference_steps=1),
    )

    with pytest.raises(StopAfterVideoPreprocessError):
        object.__new__(LancePipeline)._forward_video_edit(request)

    video = captured["video"]
    assert isinstance(video, np.ndarray)
    assert video.shape == (2, 2, 3, 3)
    assert video.dtype == np.uint8
    assert video.flags.c_contiguous
    assert captured["fps"] == expected_fps