vllm-omni/vllm_omni/diffusion/postprocess/device_reduction.py

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Device-side reduction of decoded video tensors to uint8 frames."""

from __future__ import annotations

from dataclasses import replace

import torch
from vllm.logger import init_logger

from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.diffusion.media import (
    DiffusionMediaOutput,
    FloatVideoConsumer,
    VideoMediaOutput,
    VideoTensorEncoding,
    VideoTensorLayout,
    VideoTensorSpec,
    VideoTransportConstraints,
    VideoValueRange,
    ensure_request_owned_tensor,
)
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.platforms import current_omni_platform

# VaeImageProcessor.denormalize: (x * 0.5 + 0.5).clamp(0, 1)
_DENORM_SCALE = 0.5
_DENORM_SHIFT = 0.5

logger = init_logger(__name__)


def _request_float_consumers(
    sampling_params: OmniDiffusionSamplingParams | None,
) -> frozenset[FloatVideoConsumer]:
    if sampling_params is not None and sampling_params.enable_frame_interpolation:
        return frozenset({FloatVideoConsumer.FRAME_INTERPOLATION})
    return frozenset()


def _request_supports_uint8_frames(sampling_params: OmniDiffusionSamplingParams | None) -> bool:
    return sampling_params is None or (sampling_params.output_type or "np") == "np"


def _prepare_float_media_for_transport(
    media: DiffusionMediaOutput,
    video: VideoMediaOutput,
) -> DiffusionMediaOutput:
    prepared = replace(
        media,
        video=video.with_tensor(ensure_request_owned_tensor(video.tensor)),
        prepared_for_transport=True,
    )
    prepared.validate()
    return prepared


def prepare_diffusion_media_for_transport(
    media: DiffusionMediaOutput,
    *,
    od_config: OmniDiffusionConfig,
    sampling_params: OmniDiffusionSamplingParams | None = None,
) -> DiffusionMediaOutput:
    """Validate and prepare one request-local media output before D2H."""
    media.validate()
    if media.prepared_for_transport:
        # Only the model runner prepares media, exactly once, on media the
        # pipeline emits unprepared. Accepting pre-prepared media here would let
        # a pipeline forge runner-owned state and skip request policy (e.g. a
        # forged uint8 payload silently bypasses frame interpolation).
        raise ValueError(
            "Media reached transport preparation already prepared; pipelines must "
            "emit unprepared media so request policy is applied by the runner"
        )
    if media.video.spec.encoding is not VideoTensorEncoding.NORMALIZED_FLOAT:
        raise ValueError("Pipelines must emit unprepared media as NORMALIZED_FLOAT video")

    video = media.video
    constraints = VideoTransportConstraints(
        pending_float_consumers=(video.constraints.pending_float_consumers | _request_float_consumers(sampling_params))
    )
    constrained_video = replace(video, constraints=constraints)

    enabled = od_config.video_output_transport.enable_device_postprocess
    output_supported = _request_supports_uint8_frames(sampling_params)
    if not enabled or constraints.pending_float_consumers or not output_supported:
        if not enabled:
            reason = "disabled"
        elif constraints.pending_float_consumers:
            reason = ",".join(sorted(consumer.value for consumer in constraints.pending_float_consumers))
        else:
            reason = "unsupported_presentation"
        logger.debug("Device video preparation kept normalized float: reason=%s", reason)
        return _prepare_float_media_for_transport(media, constrained_video)

    try:
        frames = reduce_video_to_uint8_frames(
            constrained_video.tensor,
            do_denormalize=constrained_video.spec.value_range is VideoValueRange.NEGATIVE_ONE_TO_ONE,
        )
    except torch.OutOfMemoryError:
        logger.warning("Device video preparation ran out of memory; using normalized float transport")
    else:
        prepared_video = replace(
            constrained_video,
            tensor=frames,
            spec=VideoTensorSpec(
                layout=VideoTensorLayout.BTHWC,
                encoding=VideoTensorEncoding.UINT8_FRAMES,
                value_range=VideoValueRange.ZERO_TO_255,
            ),
            constraints=VideoTransportConstraints(),
        )
        prepared = replace(media, video=prepared_video, prepared_for_transport=True)
        prepared.validate()
        logger.debug(
            "Device video preparation converted normalized float to uint8: shape=%s",
            tuple(frames.shape),
        )
        return prepared

    # Leave the exception handler before allocator cleanup and fallback
    # construction. A late conversion OOM otherwise keeps the helper traceback
    # (and its live float32 intermediates) reachable while the fallback may need
    # to clone a non-compact request view.
    current_omni_platform.empty_cache()
    current_omni_platform.synchronize()
    return _prepare_float_media_for_transport(media, constrained_video)


def reduce_video_to_uint8_frames(video: torch.Tensor, *, do_denormalize: bool = True) -> torch.Tensor:
    """Reduce a decoded ``[B, C, F, H, W]`` video to uint8 ``[B, F, H, W, C]`` frames.

    Runs denormalize/clamp/permute/round on the input's device so the following
    D2H copy carries uint8 instead of float. The result matches
    ``VideoProcessor.postprocess_video(output_type="np")`` then the ``*255``
    rounding done in the API server. Pass ``do_denormalize=False`` for VAEs that
    already emit ``[0, 1]``.
    """
    if video.dim() != 5:
        raise ValueError(f"expected a [B, C, F, H, W] video tensor, got shape {tuple(video.shape)}")

    # Match the numpy path, which promotes to float before scaling.
    frames = video.to(torch.float32)
    if do_denormalize:
        frames = frames.mul(_DENORM_SCALE).add(_DENORM_SHIFT).clamp_(0.0, 1.0)
    else:
        frames = frames.clamp(0.0, 1.0)
    frames = frames.permute(0, 2, 3, 4, 1)
    frames = frames.mul_(255.0).round_().clamp_(0.0, 255.0).to(torch.uint8)
    return frames.contiguous()