vllm-omni/tests/diffusion/models/seedvr2/test_seedvr2_e2e.py

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""End-to-end native video restoration and the HTTP long-video route.

Set VLLM_TEST_SEEDVR2_MODEL_DIR to the directory containing the DiT, VAE,
conditioning tensor, and model_index.json to run the complete video test.
Each rank exits normally; the launcher does not terminate workers.
"""

from __future__ import annotations

import json
import os
import subprocess
import time
from fractions import Fraction
from pathlib import Path

import pytest

pytestmark = [pytest.mark.local_model, pytest.mark.cuda, pytest.mark.diffusion, pytest.mark.parallel]
MODEL_DIR_ENV = "VLLM_TEST_SEEDVR2_MODEL_DIR"
# Two model windows with one four-frame seam, at the smallest legal frame size.
LONG_FRAMES = 20
LONG_SIZE = 64
FPS = 24


@pytest.mark.skipif(not os.environ.get(MODEL_DIR_ENV), reason=f"set {MODEL_DIR_ENV} to an authorized model directory")
@pytest.mark.parametrize("method", ["lab", "wavelet", "adain", "none"])
def test_seedvr2_native_video_e2e(method: str) -> None:
    import numpy as np
    import torch

    from vllm_omni.entrypoints.omni import Omni
    from vllm_omni.inputs.data import OmniDiffusionSamplingParams

    model_dir = Path(os.environ[MODEL_DIR_ENV]).resolve()
    for name in ("seedvr2_ema_3b_fp16.safetensors", "ema_vae_fp16.safetensors", "pos_emb.pt", "model_index.json"):
        assert (model_dir / name).is_file(), f"Missing SeedVR2 model file: {name}"
    frames = torch.rand(5, 3, 64, 112, generator=torch.Generator().manual_seed(7723))
    engine = Omni(
        model=str(model_dir),
        model_class_name="SeedVR2Pipeline",
        dtype="float16",
        enforce_eager=True,
        vae_use_tiling=True,
    )
    try:
        output = engine.generate(
            {"prompt": " ", "multi_modal_data": {"video": frames}},
            OmniDiffusionSamplingParams(
                height=128,
                width=224,
                num_frames=5,
                fps=24,
                num_inference_steps=1,
                guidance_scale=1.0,
                seed=7723,
                output_type="np",
                extra_args={"color_correction_method": method},
            ),
            use_tqdm=False,
        )
    finally:
        engine.close()
    assert len(output) == 1
    restored = np.asarray(output[0].images[0])
    assert restored.shape == (1, 5, 128, 224, 3)
    assert np.isfinite(restored).all()


def _write_source(path: Path, frames: int) -> None:
    """A constant-rate source with audio, which the route requires."""
    import imageio_ffmpeg

    subprocess.run(
        [
            imageio_ffmpeg.get_ffmpeg_exe(),
            "-nostdin",
            "-y",
            "-hide_banner",
            "-loglevel",
            "error",
            "-f",
            "lavfi",
            "-i",
            f"testsrc=size={LONG_SIZE}x{LONG_SIZE}:rate={FPS}",
            "-f",
            "lavfi",
            "-i",
            "sine=frequency=440",
            # The generators are endless, so bound the video by frame count and
            # the audio by a slightly longer wall time.
            "-frames:v",
            str(frames),
            "-t",
            str((frames + 1) / FPS),
            "-c:v",
            "libx264",
            "-pix_fmt",
            "yuv420p",
            "-c:a",
            "aac",
            str(path),
        ],
        check=True,
    )


def _await_status(base: str, job_id: str, timeout: float) -> dict:
    """Poll until the job leaves the queued/running states."""
    import requests

    deadline = time.monotonic() + timeout
    while time.monotonic() < deadline:
        record = requests.get(f"{base}/{job_id}", timeout=30).json()
        if record["status"] not in {"queued", "running"}:
            return record
        time.sleep(2)
    raise AssertionError(f"SeedVR2 long-video job stayed {record['status']} for {timeout}s")


@pytest.mark.skipif(not os.environ.get(MODEL_DIR_ENV), reason=f"set {MODEL_DIR_ENV} to an authorized model directory")
@pytest.mark.parametrize("degree", [1, 8])
def test_seedvr2_long_video_route_e2e(tmp_path: Path, degree: int) -> None:
    """Restore a clip spanning two model windows, then cancel a longer job."""
    import av
    import requests
    import torch

    if torch.accelerator.device_count() < degree:
        pytest.skip(f"SeedVR2 USP={degree} requires {degree} CUDA devices")

    from tests.helpers.runtime import OmniServer
    from vllm_omni.diffusion.models.seedvr2.long_video import MAX_FRAMES

    model_dir = Path(os.environ[MODEL_DIR_ENV]).resolve()
    source = tmp_path / "input.mp4"
    _write_source(source, LONG_FRAMES)
    server = OmniServer(
        model=str(model_dir),
        serve_args=[
            "--model-class-name",
            "SeedVR2Pipeline",
            "--dtype",
            "float16",
            "--enforce-eager",
            "--num-gpus",
            str(degree),
            "--vae-use-tiling",
            "--stage-overrides",
            json.dumps(
                {
                    "0": {
                        "ulysses_degree": degree,
                        "vae_patch_parallel_size": degree,
                        "vae_parallel_mode": "spatial_shard_height",
                    }
                }
            ),
            "--api-server-count",
            "1",
        ],
        env_dict={
            "VLLM_OMNI_SEEDVR2_LONG_OUTPUT_DIR": str(tmp_path / "jobs"),
            "VLLM_OMNI_SEEDVR2_LONG_MAX_WINDOW": "13",
        },
    )

    with server:
        base = f"http://{server.host}:{server.port}/v1/seedvr2/restore-long"

        def submit(**overrides: str) -> requests.Response:
            with source.open("rb") as upload:
                return requests.post(
                    base,
                    data={
                        "prompt": " ",
                        "size": f"{LONG_SIZE}x{LONG_SIZE}",
                        "num_frames": str(LONG_FRAMES),
                        "seed": "7723",
                        **overrides,
                    },
                    files={"input_references": ("input.mp4", upload, "video/mp4")},
                    timeout=120,
                )

        assert submit(num_frames=str(MAX_FRAMES + 1)).status_code == 400

        # Validation must happen before a job is admitted, through model-owned
        # extra_params on both the long route and the common video endpoint.
        for invalid in ("{", "[]", '{"color_correction_method":"invalid"}'):
            assert submit(extra_params=invalid).status_code == 400
        with source.open("rb") as upload:
            invalid = requests.post(
                f"http://{server.host}:{server.port}/v1/videos/sync",
                data={
                    "prompt": " ",
                    "size": f"{LONG_SIZE}x{LONG_SIZE}",
                    "num_inference_steps": "1",
                    "guidance_scale": "1",
                    "extra_params": '{"color_correction_method":"invalid"}',
                },
                files={"input_references": ("input.mp4", upload, "video/mp4")},
                timeout=120,
            )
        assert invalid.status_code == 400, invalid.text
        accepted = submit(extra_params='{"color_correction_method":"wavelet"}')
        assert accepted.status_code == 202, accepted.text
        job_id = accepted.json()["id"]
        record = _await_status(base, job_id, timeout=900)
        assert record["status"] == "completed", record["error"]
        assert record["frames"] == LONG_FRAMES

        restored = tmp_path / "restored.mp4"
        content = requests.get(f"{base}/{job_id}/content", timeout=300)
        assert content.status_code == 200
        restored.write_bytes(content.content)
        with av.open(str(restored)) as container:
            video = container.streams.video[0]
            timestamps = [frame.pts * frame.time_base for frame in container.decode(video=0)]
            assert (video.width, video.height) == (LONG_SIZE, LONG_SIZE)
            assert video.average_rate == Fraction(FPS)
            assert timestamps == [Fraction(index, FPS) for index in range(LONG_FRAMES)]
            assert container.streams.audio, "the route must carry the source audio through"

        # Many windows, so the cancel lands well before the job could finish.
        running = submit(num_frames="600", loop_input="true")
        assert running.status_code == 202, running.text
        cancelled_id = running.json()["id"]
        assert submit().status_code == 409, "the route runs one job per server"
        assert requests.delete(f"{base}/{cancelled_id}", timeout=30).status_code == 202
        settled = _await_status(base, cancelled_id, timeout=900)
        assert settled["status"] == "cancelled", settled
        assert settled["frames"] < 600
        assert requests.get(f"{base}/{cancelled_id}/content", timeout=30).status_code == 404