vllm-omni/vllm_omni/diffusion/models/seedvr2/long_video.py

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""One asynchronous SeedVR2 request, bounded model windows, one MP4 writer."""

import asyncio
import hashlib
import io
import itertools
import json
import logging
import os
import shutil
import subprocess
import tempfile
import threading
import time
from collections import deque
from collections.abc import Iterator
from concurrent.futures import Future, ThreadPoolExecutor
from fractions import Fraction
from pathlib import Path
from uuid import uuid4

import av
import imageio_ffmpeg
import numpy as np
import regex as re
import requests
from fastapi import APIRouter, FastAPI, File, Form, HTTPException, Request, UploadFile
from fastapi.responses import FileResponse

from vllm_omni.diffusion.models.seedvr2 import config as envs
from vllm_omni.diffusion.models.seedvr2.config import COLOR_CORRECTION_METHODS, DEFAULT_COLOR_CORRECTION_METHOD
from vllm_omni.diffusion.models.seedvr2.pipeline_seedvr2 import max_frames, sharded_budget

logger = logging.getLogger(__name__)
router = APIRouter()
MAX_FRAMES = 7200
FPS = 24
OVERLAP = 4
# Windows cross the HTTP boundary losslessly (x264 qp 0, which is also its
# fastest mode); only the final MP4 is compressed.
LOSSLESS = {"preset": "ultrafast", "qp": "0"}
UPLOAD_CHUNK = 1 << 20
JOB_ID = re.compile(r"[0-9a-f]{32}")
active_task: asyncio.Task[None] | None = None
job_lock = asyncio.Lock()


class JobError(Exception):
    """An error whose message is safe to return to the client."""


class JobCancelledError(JobError):
    """The client asked the job to stop."""


def _jobs_root() -> Path:
    return Path(envs.VLLM_OMNI_SEEDVR2_LONG_OUTPUT_DIR) / "seedvr2-long"


def _job_dir(job_id: str) -> Path:
    """Resolve a client-supplied job id, which must never shape the path."""
    if not JOB_ID.fullmatch(job_id):
        raise HTTPException(404, "SeedVR2 long-video job not found")
    return _jobs_root() / job_id


def _sweep_expired_jobs() -> None:
    """Drop settled job directories so repeated submissions cannot fill the disk."""
    deadline = time.time() - envs.VLLM_OMNI_SEEDVR2_LONG_JOB_TTL_SECONDS
    for job in _jobs_root().glob("*"):
        status = job / "status.json"
        if not job.is_dir() or not JOB_ID.fullmatch(job.name) or not status.exists():
            continue
        record = json.loads(status.read_text())
        # A job owned by a dead process is settled too, or a restart would leak it.
        settled = record["status"] not in {"queued", "running"} or record.get("pid") != os.getpid()
        if settled and status.stat().st_mtime <= deadline:
            shutil.rmtree(job, ignore_errors=True)


def _store_upload(upload: UploadFile, destination: Path) -> None:
    """Copy the upload under a size cap so one client cannot fill the disk."""
    limit = envs.VLLM_OMNI_SEEDVR2_LONG_MAX_UPLOAD_BYTES
    written = 0
    with destination.open("wb") as target:
        while chunk := upload.file.read(UPLOAD_CHUNK):
            written += len(chunk)
            if written > limit:
                raise HTTPException(413, f"SeedVR2 long-video upload exceeds {limit} bytes")
            target.write(chunk)
    if not written:
        raise HTTPException(400, "SeedVR2 long-video upload is empty")


def _window(width: int, height: int) -> int:
    """Longest window the whole-clip budget admits at this output size.

    Every window pays a fixed cost for its round trip through the serving
    stack and throws away its overlap, so the route uses the largest window the
    model accepts. The model pads clips to 4n+1 frames, so the window has that
    length too; 0 means not even one frame fits.
    """
    frame_pixels, clip_pixels = sharded_budget()
    if width * height > frame_pixels:
        return 0
    fit = min(clip_pixels // (width * height), max_frames(), envs.VLLM_OMNI_SEEDVR2_LONG_MAX_WINDOW)
    return fit - (fit - 1) % 4


def _status(job: Path, stage: str, frames: int, error: str = "") -> None:
    # The owning PID lets a poller detect a job that a server restart abandoned.
    record = {"status": stage, "frames": frames, "error": error, "pid": os.getpid()}
    pending = job / "status.json.tmp"
    pending.write_text(json.dumps(record) + "\n")
    pending.replace(job / "status.json")


def _frames(source: Path, width: int, height: int, loop: bool) -> Iterator[av.VideoFrame]:
    while True:
        count = 0
        with av.open(str(source)) as container:
            video = container.streams.video[0]
            if video.average_rate != Fraction(FPS):
                raise JobError("SeedVR2 long video requires 24 FPS input")
            for frame in container.decode(video=0):
                if frame.pts is None or frame.time_base is None or frame.pts * frame.time_base != Fraction(count, FPS):
                    raise JobError("SeedVR2 long video requires constant 24 FPS timestamps starting at zero")
                count += 1
                # The model resizes to the output on the device, so windows carry
                # source pixels; only a larger source shrinks here to stay in budget.
                if frame.width * frame.height > width * height:
                    yield frame.reformat(width=width, height=height, format="yuv420p", interpolation="BICUBIC")
                else:
                    yield frame.reformat(width=frame.width & ~1, height=frame.height & ~1, format="yuv420p")
        if count == 0:
            raise JobError("Input video has no frames")
        if not loop:
            return


# Consecutive windows share their overlap frames and encoding stamps each
# frame's pts, so two in-flight windows must not encode at the same time.
_segment_lock = threading.Lock()


def _segment(frames: list[av.VideoFrame]) -> bytes:
    buffer = io.BytesIO()
    with _segment_lock, av.open(buffer, "w", format="matroska") as container:
        stream = container.add_stream("libx264", rate=FPS)
        stream.width, stream.height, stream.pix_fmt = frames[0].width, frames[0].height, "yuv420p"
        stream.options = LOSSLESS
        for index, frame in enumerate(frames):
            frame.pts, frame.time_base = index, Fraction(1, FPS)
            for packet in stream.encode(frame):
                container.mux(packet)
        for packet in stream.encode():
            container.mux(packet)
    return buffer.getvalue()


def _restore(
    frames: list[av.VideoFrame],
    width: int,
    height: int,
    seed: int,
    color_correction_method: str,
    port: int,
    authorization: str,
) -> list[np.ndarray]:
    response = requests.post(
        f"http://127.0.0.1:{port}/v1/videos/sync",
        data={
            "prompt": " ",
            "size": f"{width}x{height}",
            "num_frames": str(len(frames)),
            "num_inference_steps": "1",
            "guidance_scale": "1",
            "seed": str(seed),
            "extra_params": json.dumps(
                {"video_codec_options": LOSSLESS, "color_correction_method": color_correction_method}
            ),
        },
        files={"input_references": ("window.mkv", _segment(frames), "video/x-matroska")},
        headers={"Authorization": authorization} if authorization else {},
        timeout=900,
    )
    try:
        response.raise_for_status()
    except requests.HTTPError as error:
        raise JobError("SeedVR2 window restoration failed") from error
    with av.open(io.BytesIO(response.content)) as container:
        video = container.streams.video[0]
        decoded = list(container.decode(video=0))
        if (video.width, video.height, video.average_rate, len(decoded)) != (width, height, Fraction(FPS), len(frames)):
            raise JobError("SeedVR2 window restoration returned an unexpected geometry or frame count")
        if [frame.pts * frame.time_base for frame in decoded] != [Fraction(index, FPS) for index in range(len(frames))]:
            raise JobError("SeedVR2 window restoration returned invalid timestamps")
        return [frame.to_ndarray(format="rgb24") for frame in decoded]


def _run(
    job: Path,
    width: int,
    height: int,
    target: int,
    loop: bool,
    seed: int,
    color_correction_method: str,
    port: int,
    authorization: str,
) -> None:
    window = _window(width, height)
    if window <= OVERLAP:
        raise JobError("SeedVR2 long video output exceeds the configured SeedVR2 clip budget")
    with tempfile.TemporaryDirectory(prefix="seedvr2-long-") as scratch:
        _restore_to(
            job, Path(scratch), window, width, height, target, loop, seed, color_correction_method, port, authorization
        )


def _restore_to(
    job: Path,
    scratch: Path,
    window: int,
    width: int,
    height: int,
    target: int,
    loop: bool,
    seed: int,
    color_correction_method: str,
    port: int,
    authorization: str,
) -> None:
    """Restore into local scratch files, then publish the result into ``job``."""
    source = job / "input.mp4"
    video_path = scratch / "video.mp4"
    output = scratch / "output.mp4"
    source_frames = _frames(source, width, height, loop)

    def windows() -> Iterator[list[av.VideoFrame]]:
        batch = list(itertools.islice(source_frames, min(window, target)))
        start = 0
        while True:
            if len(batch) != min(window, target - start):
                raise JobError("Input video has fewer frames than requested; set loop_input=true to repeat it")
            yield batch
            start += len(batch) - OVERLAP
            if start + OVERLAP >= target:
                return
            batch = batch[-OVERLAP:] + list(itertools.islice(source_frames, min(window, target - start) - OVERLAP))

    def restore(frames: list[av.VideoFrame]) -> Future[list[np.ndarray]]:
        if (job / "cancel").exists():
            raise JobCancelledError("SeedVR2 long-video job was cancelled")
        return pool.submit(_restore, frames, width, height, seed, color_correction_method, port, authorization)

    written = 0
    pending: list[np.ndarray] = []
    # Two windows stay in flight, so the engine always has the next one queued
    # while this thread blends and encodes, and the other prepares its upload.
    with ThreadPoolExecutor(max_workers=2) as pool, av.open(str(video_path), "w", format="mp4") as container:
        video = container.add_stream("libx264", rate=FPS)
        video.width, video.height, video.pix_fmt = width, height, "yuv420p"
        video.options = {"preset": "veryfast", "crf": "18", "tune": "zerolatency", "bf": "0"}

        def write(array: np.ndarray) -> None:
            nonlocal written
            frame = av.VideoFrame.from_ndarray(array, format="rgb24")
            frame.pts, frame.time_base = written, Fraction(1, FPS)
            for packet in video.encode(frame):
                container.mux(packet)
            written += 1

        batches = windows()
        in_flight = deque(restore(batch) for batch in itertools.islice(batches, 2))
        while in_flight:
            restored = in_flight.popleft().result()
            following = next(batches, None)
            if following is not None:
                in_flight.append(restore(following))
            last = not in_flight
            if not pending:
                for frame in restored if last else restored[:-OVERLAP]:
                    write(frame)
            else:
                for offset in range(OVERLAP):
                    newer = (offset + 1) / (OVERLAP + 1)
                    write(np.rint(pending[offset] * (1 - newer) + restored[offset] * newer).astype(np.uint8))
                for frame in restored[OVERLAP:] if last else restored[OVERLAP:-OVERLAP]:
                    write(frame)
            _status(job, "running", written)
            pending = restored[-OVERLAP:]
        for packet in video.encode():
            container.mux(packet)
    if written != target:
        raise JobError(f"SeedVR2 restored {written} of {target} requested frames")

    with av.open(str(source)) as container:
        has_audio = bool(container.streams.audio)
    if has_audio:
        subprocess.run(
            [
                imageio_ffmpeg.get_ffmpeg_exe(),
                "-nostdin",
                "-y",
                "-hide_banner",
                "-loglevel",
                "error",
                "-stream_loop",
                "-1" if loop else "0",
                "-i",
                str(source),
                "-i",
                str(video_path),
                "-map",
                "1:v:0",
                "-map",
                "0:a:0",
                "-frames:v",
                str(target),
                "-t",
                str(target / FPS),
                "-c:v",
                "copy",
                "-c:a",
                "aac",
                "-b:a",
                "160k",
                "-movflags",
                "+faststart",
                str(output),
            ],
            check=True,
        )
    else:
        video_path.replace(output)
    with av.open(str(output)) as container:
        video = container.streams.video[0]
        decoded = 0
        for decoded, frame in enumerate(container.decode(video=0), start=1):
            if (
                frame.pts is None
                or frame.time_base is None
                or frame.pts * frame.time_base != Fraction(decoded - 1, FPS)
            ):
                raise JobError("SeedVR2 output encoding produced invalid timestamps")
        if (video.width, video.height, video.average_rate, decoded) != (width, height, Fraction(FPS), target):
            raise JobError("SeedVR2 output encoding failed frame or geometry validation")
        if has_audio and not container.streams.audio:
            raise JobError("SeedVR2 output encoding lost the audio track")
    if has_audio:
        with av.open(str(output)) as container:
            audio = container.streams.audio[0]
            samples = sum(frame.samples for frame in container.decode(audio=0))
            if samples < (target / FPS - 1) * audio.rate:
                raise JobError("SeedVR2 output audio ends before the video")
    with output.open("rb") as content:
        hasher = hashlib.sha256()
        for chunk in iter(lambda: content.read(UPLOAD_CHUNK), b""):
            hasher.update(chunk)
        digest = hasher.hexdigest()
    # One sequential copy is the only write to the job directory, which may sit
    # on network storage where the encoder's and muxer's small writes are slow.
    staged = job / "output.mp4.tmp"
    shutil.move(output, staged)
    staged.replace(job / "output.mp4")
    (job / "result.json").write_text(
        json.dumps({"frames": target, "width": width, "height": height, "fps": FPS, "sha256": digest}) + "\n"
    )
    _status(job, "completed", target)


def _background(
    job: Path,
    width: int,
    height: int,
    target: int,
    loop: bool,
    seed: int,
    color_correction_method: str,
    port: int,
    authorization: str,
) -> None:
    try:
        _run(job, width, height, target, loop, seed, color_correction_method, port, authorization)
    except Exception as error:
        logger.exception("SeedVR2 long-video job %s failed", job.name)
        current = json.loads((job / "status.json").read_text())
        detail = str(error) if isinstance(error, JobError) else "SeedVR2 long-video restoration failed"
        _status(job, "cancelled" if isinstance(error, JobCancelledError) else "failed", current["frames"], detail)


@router.post("/v1/seedvr2/restore-long", status_code=202)
async def create_long_video(
    raw_request: Request,
    input_references: UploadFile = File(...),
    num_frames: int = Form(...),
    size: str = Form(...),
    loop_input: bool = Form(False),
    prompt: str = Form(" "),
    seed: int = Form(7723),
    extra_params: str | None = Form(None),
) -> dict[str, str | int]:
    global active_task
    if raw_request.app.state.api_server_count != 1:
        raise HTTPException(409, "SeedVR2 long-video jobs require one API server")
    try:
        width, height = (int(value) for value in size.lower().split("x"))
    except ValueError as error:
        raise HTTPException(400, "size must be WIDTHxHEIGHT") from error
    if not 1 <= num_frames <= MAX_FRAMES or min(width, height) < 16 or width % 16 or height % 16:
        raise HTTPException(400, "SeedVR2 long-video frame count or dimensions are invalid")
    if prompt.strip():
        raise HTTPException(400, "SeedVR2 long video requires a blank prompt")
    if _window(width, height) <= OVERLAP:
        raise HTTPException(400, "SeedVR2 long video output exceeds the configured SeedVR2 frame or clip budget")
    if not 0 <= seed <= 2**32 - 1:
        raise HTTPException(400, "Seed must be a 32-bit unsigned integer")
    try:
        options = json.loads(extra_params) if extra_params is not None else {}
    except (TypeError, ValueError) as error:
        raise HTTPException(400, "extra_params must be a JSON object") from error
    if not isinstance(options, dict):
        raise HTTPException(400, "extra_params must be a JSON object")
    unknown = options.keys() - {"color_correction_method"}
    if unknown:
        raise HTTPException(400, f"Unsupported SeedVR2 long-video extra_params: {sorted(unknown)}")
    color_correction_method = options.get("color_correction_method", DEFAULT_COLOR_CORRECTION_METHOD)
    if color_correction_method is None:
        color_correction_method = DEFAULT_COLOR_CORRECTION_METHOD
    if color_correction_method not in COLOR_CORRECTION_METHODS:
        raise HTTPException(400, f"color_correction_method must be one of {list(COLOR_CORRECTION_METHODS)}")
    async with job_lock:
        if active_task is not None and not active_task.done():
            raise HTTPException(409, "A SeedVR2 long-video job is already running")
        await asyncio.to_thread(_sweep_expired_jobs)
        job_id = uuid4().hex
        job = _jobs_root() / job_id
        job.mkdir(parents=True)
        try:
            await asyncio.to_thread(_store_upload, input_references, job / "input.mp4")
        except HTTPException:
            shutil.rmtree(job, ignore_errors=True)
            raise
        _status(job, "queued", 0)
        active_task = asyncio.create_task(
            asyncio.to_thread(
                _background,
                job,
                width,
                height,
                num_frames,
                loop_input,
                seed,
                color_correction_method,
                raw_request.app.state.seedvr2_long_port,
                raw_request.headers.get("authorization", ""),
            )
        )
    return {"id": job_id, "status": "queued", "max_frames": MAX_FRAMES}


@router.get("/v1/seedvr2/restore-long/{job_id}")
async def get_long_video(job_id: str) -> dict[str, str | int]:
    status = _job_dir(job_id) / "status.json"
    if not status.exists():
        raise HTTPException(404, "SeedVR2 long-video job not found")
    record = json.loads(status.read_text())
    if record["status"] in {"queued", "running"} and record.get("pid") != os.getpid():
        # Only this process runs jobs, so another owner means a restart lost it.
        record["status"], record["error"] = "failed", "SeedVR2 long-video job was interrupted by a server restart"
    return record


@router.get("/v1/seedvr2/restore-long/{job_id}/content")
async def download_long_video(job_id: str) -> FileResponse:
    job = _job_dir(job_id)
    if not (job / "result.json").exists():
        raise HTTPException(404, "SeedVR2 long-video output is not ready")
    return FileResponse(job / "output.mp4", media_type="video/mp4", filename=f"{job_id}.mp4")


@router.delete("/v1/seedvr2/restore-long/{job_id}", status_code=202)
async def cancel_long_video(job_id: str) -> dict[str, str]:
    """Ask the running job to stop; it settles at the next window boundary."""
    job = _job_dir(job_id)
    status = job / "status.json"
    if not status.exists():
        raise HTTPException(404, "SeedVR2 long-video job not found")
    record = json.loads(status.read_text())
    if record["status"] not in {"queued", "running"}:
        raise HTTPException(409, f"SeedVR2 long-video job already {record['status']}")
    (job / "cancel").touch()
    return {"id": job_id, "status": "cancelling"}


def register_routes(app: FastAPI, port: int) -> None:
    """Attach this model's optional long-video API to its serving process."""
    app.state.seedvr2_long_port = port
    app.include_router(router)