vllm-omni/vllm_omni/diffusion/models/diffusers_adapter/pipeline_utils.py

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project

from typing import Any

from diffusers.pipelines.pipeline_utils import DiffusionPipeline

from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.diffusion.request import resolve_video_num_frames
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
from vllm_omni.inputs.data import OmniDiffusionSamplingParams


class BasePipelineUtils:
    """No-op hooks for pipeline-specific diffusers adapter behavior."""

    def resolve_pipeline_class(
        self,
        od_config: OmniDiffusionConfig,
        configured_pipeline_class: type[DiffusionPipeline] | None,
    ) -> type[DiffusionPipeline] | None:
        """Resolve the concrete Diffusers class used to load a checkpoint."""
        return configured_pipeline_class

    def update_load_kwargs(self, od_config: OmniDiffusionConfig, load_kwargs: dict[str, Any]) -> None:
        pass

    def apply_post_load_updates(self, pipeline: DiffusionPipeline, od_config: OmniDiffusionConfig) -> None:
        pass

    def validate_runtime_sampling_params(self, sampling: OmniDiffusionSamplingParams) -> None:
        if sampling.enable_frame_interpolation:
            raise ValueError(
                "Frame interpolation is not supported with the Diffusers backend because "
                "Diffusers returns already postprocessed frames."
            )

    def update_call_kwargs(
        self,
        req: DiffusionRequestBatch,
        sampling: OmniDiffusionSamplingParams,
        accepted_call_kwargs: set[str] | None,
        call_kwargs: dict[str, Any],
    ) -> None:
        pass


class SanaVideoPipelineUtils(BasePipelineUtils):
    def resolve_pipeline_class(
        self,
        od_config: OmniDiffusionConfig,
        configured_pipeline_class: type[DiffusionPipeline] | None,
    ) -> type[DiffusionPipeline] | None:
        # Both released SANA-Video checkpoints declare SanaVideoPipeline in
        # model_index.json. Honour an explicit Omni I2V class selection by
        # loading the same components into Diffusers' I2V pipeline instead.
        if od_config.model_class_name == "SanaImageToVideoPipeline":
            from diffusers import SanaImageToVideoPipeline

            return SanaImageToVideoPipeline
        return configured_pipeline_class

    def update_call_kwargs(
        self,
        req: DiffusionRequestBatch,
        sampling: OmniDiffusionSamplingParams,
        accepted_call_kwargs: set[str] | None,
        call_kwargs: dict[str, Any],
    ) -> None:
        # Diffusers SANA-Video calls the shared request's `num_frames`
        # parameter `frames`.
        if (
            accepted_call_kwargs is not None
            and "frames" in accepted_call_kwargs
            and "num_frames" not in accepted_call_kwargs
        ):
            configured_default = call_kwargs.get("frames", 81)
            if configured_default is None:
                configured_default = 81
            call_kwargs["frames"] = resolve_video_num_frames(
                sampling.num_frames,
                default_num_frames=configured_default,
                is_dummy_run=req.is_dummy_run(),
            )

        # SANA's startup dummy dimensions can map to a resolution bucket that
        # conflicts with the upstream pipeline's divisibility validation.
        if req.is_dummy_run() and accepted_call_kwargs is not None and "use_resolution_binning" in accepted_call_kwargs:
            call_kwargs["use_resolution_binning"] = False


class WanPipelineUtils(BasePipelineUtils):
    def update_load_kwargs(self, od_config: OmniDiffusionConfig, load_kwargs: dict[str, Any]) -> None:
        if od_config.boundary_ratio is not None:
            load_kwargs["boundary_ratio"] = od_config.boundary_ratio

    def apply_post_load_updates(self, pipeline: DiffusionPipeline, od_config: OmniDiffusionConfig) -> None:
        if od_config.flow_shift is not None:
            from diffusers.schedulers.scheduling_unipc_multistep import UniPCMultistepScheduler

            pipeline.scheduler = UniPCMultistepScheduler.from_config(
                pipeline.scheduler.config, flow_shift=od_config.flow_shift
            )

    def validate_runtime_sampling_params(self, sampling: OmniDiffusionSamplingParams) -> None:
        super().validate_runtime_sampling_params(sampling)
        if sampling.boundary_ratio is not None:
            raise ValueError(
                "Boundary ratio is not supported at runtime with the diffusers backend for Wan models. Please set "
                "it at model loading time using the `boundary_ratio` kwarg or `--diffusers-load-kwargs` JSON."
            )
        if sampling.extra_args.get("flow_shift") is not None:
            raise ValueError(
                "Flow shift is not supported at runtime with the diffusers backend for Wan models. Please set "
                "it at model loading time using the `flow_shift` kwarg."
            )


PIPELINE_UTILS_REGISTRY: dict[str, type[BasePipelineUtils]] = {
    "SanaVideoPipeline": SanaVideoPipelineUtils,
    "SanaImageToVideoPipeline": SanaVideoPipelineUtils,
    "WanPipeline": WanPipelineUtils,
    "WanImageToVideoPipeline": WanPipelineUtils,
    "WanVACEPipeline": WanPipelineUtils,
    "WanVideoToVideoPipeline": WanPipelineUtils,
    "WanAnimatePipeline": WanPipelineUtils,
}


def get_pipeline_utils(pipeline_class_name: str | None) -> BasePipelineUtils:
    if pipeline_class_name is None:
        return BasePipelineUtils()
    pipeline_utils_cls = PIPELINE_UTILS_REGISTRY.get(pipeline_class_name, BasePipelineUtils)
    return pipeline_utils_cls()


def get_pipeline_utils_for_config(od_config: OmniDiffusionConfig) -> BasePipelineUtils:
    """Return model-specific adapter hooks for the requested or configured class."""
    requested_class_name = od_config.model_class_name
    if requested_class_name in PIPELINE_UTILS_REGISTRY:
        return get_pipeline_utils(requested_class_name)

    configured_pipeline_class = od_config.diffusers_pipeline_cls
    configured_class_name = configured_pipeline_class.__name__ if configured_pipeline_class is not None else None
    return get_pipeline_utils(configured_class_name)


def resolve_diffusers_pipeline_class(od_config: OmniDiffusionConfig) -> type[DiffusionPipeline] | None:
    """Resolve the concrete Diffusers pipeline used for loading and capability checks."""
    pipeline_utils = get_pipeline_utils_for_config(od_config)
    return pipeline_utils.resolve_pipeline_class(od_config, od_config.diffusers_pipeline_cls)