[Model] Add native MAGI-2 Preview diffusion support (#5918)

Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com>
Signed-off-by: hsliuustc0106 <liuhongsheng4@huawei.com>

vllm_omni/entrypoints/openai/api_server.py

@@ -3348,7 +3348,7 @@ async def _parse_video_form(
     width: int | None = Form(default=None),
     height: int | None = Form(default=None),
     num_frames: int | None = Form(default=None),
-    fps: int | None = Form(default=None),
+    fps: float | None = Form(default=None, ge=1, allow_inf_nan=False),
     aspect_ratio: str | None = Form(default=None),
     short_edge: int | None = Form(default=None, ge=1),
     num_outputs_per_prompt: int = Form(default=1, ge=1, le=10),

vllm_omni/entrypoints/openai/protocol/videos.py

@@ -55,7 +55,13 @@ class VideoParams(BaseModel):
     width: int | None = Field(default=None, ge=1, le=_INT64_MAX, description="Video width in pixels")
     height: int | None = Field(default=None, ge=1, le=_INT64_MAX, description="Video height in pixels")
     num_frames: int | None = Field(default=None, ge=1, le=_INT64_MAX, description="Number of frames")
-    fps: int | None = Field(default=None, ge=1, le=_INT64_MAX, description="Frames per second for output video")
+    fps: float | None = Field(
+        default=None,
+        ge=1,
+        le=_INT64_MAX,
+        allow_inf_nan=False,
+        description="Frames per second for output video",
+    )
 
     @property
     def size(self) -> str | None:
@@ -146,7 +152,13 @@ class VideoGenerationRequest(BaseModel):
     # Video-specific fields (top-level for OpenAI-style compatibility)
     width: int | None = Field(default=None, ge=1, le=_INT64_MAX, description="Video width in pixels")
     height: int | None = Field(default=None, ge=1, le=_INT64_MAX, description="Video height in pixels")
-    fps: int | None = Field(default=None, ge=1, le=_INT64_MAX, description="Frames per second for output video")
+    fps: float | None = Field(
+        default=None,
+        ge=1,
+        le=_INT64_MAX,
+        allow_inf_nan=False,
+        description="Frames per second for output video",
+    )
     num_frames: int | None = Field(default=None, ge=1, le=_INT64_MAX, description="Number of frames to generate")
     aspect_ratio: str | None = Field(
         default=None,
@@ -279,7 +291,17 @@ class VideoGenerationRequest(BaseModel):
         description=("Optional model-specific parameters passed directly to the model's extra_args. "),
     )
 
-    def resolve_video_params(self) -> VideoParams:
+    def resolve_video_params(
+        self,
+        *,
+        default_fps: float | None = DEFAULT_FPS,
+        default_num_frames: int | None = None,
+    ) -> VideoParams:
+        """Resolve explicit fields with optional model-owned defaults.
+
+        Callers that know the active diffusion pipeline can supply its native
+        frame contract. Other callers retain the generic 24 fps behavior.
+        """
         vp = VideoParams(width=self.width, height=self.height, fps=self.fps, num_frames=self.num_frames)
 
         if self.video_params is not None:
@@ -292,10 +314,13 @@ class VideoGenerationRequest(BaseModel):
             vp.width, vp.height = parse_size(self.size)
 
         if vp.fps is None:
-            vp.fps = DEFAULT_FPS
+            vp.fps = default_fps
 
-        if vp.num_frames is None and self.seconds is not None:
-            vp.num_frames = int(self.seconds) * int(vp.fps)
+        if vp.num_frames is None:
+            if default_num_frames is not None:
+                vp.num_frames = default_num_frames
+            elif self.seconds is not None and vp.fps is not None:
+                vp.num_frames = int(float(self.seconds) * float(vp.fps))
 
         return vp
 

vllm_omni/entrypoints/openai/serving_video.py

@@ -1,9 +1,10 @@
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 
 from __future__ import annotations
 
 import copy
+import math
 import time
 from collections.abc import Mapping
 from dataclasses import dataclass
@@ -36,7 +37,8 @@ from vllm_omni.entrypoints.openai.video_api_utils import (
     encode_video_base64,
 )
 from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniTextPrompt
-from vllm_omni.model_extras import should_preserve_reference_image_size
+from vllm_omni.model_extras import get_video_generation_defaults, should_preserve_reference_image_size
+from vllm_omni.model_extras.video_generation import VideoGenerationDefaults
 from vllm_omni.outputs.output_metadata import (
     DiffusionMetadataMapping,
     DiffusionMultimodalOutput,
@@ -82,7 +84,7 @@ class VideoGenerationArtifacts:
     audios: list[DiffusionPayloadValue | None]
     actions: list[VideoAction | None]
     audio_sample_rate: int
-    output_fps: int
+    output_fps: float
     stage_durations: dict[str, float]
     peak_memory_mb: float
 
@@ -111,6 +113,21 @@ class OmniOpenAIServingVideo:
             return get_od_config()
         return getattr(self._engine_client, "od_config", None)
 
+    def _resolve_video_generation_defaults(
+        self,
+        request: VideoGenerationRequest,
+    ) -> VideoGenerationDefaults | None:
+        """Resolve defaults owned by the active diffusion pipeline."""
+        od_config = self._resolve_diffusion_od_config()
+        model_class_name = None if od_config is None else getattr(od_config, "model_class_name", None)
+        try:
+            return get_video_generation_defaults(model_class_name, request.extra_params)
+        except (TypeError, ValueError) as exc:
+            raise HTTPException(
+                status_code=HTTPStatus.BAD_REQUEST.value,
+                detail=str(exc),
+            ) from exc
+
     @property
     def model_name(self) -> str | None:
         return self._model_name
@@ -206,7 +223,49 @@ class OmniOpenAIServingVideo:
             )
         provided_fields = request.model_fields_set
         fps_provided = self._request_fps_provided(request)
-        vp = request.resolve_video_params()
+        num_frames_provided = self._request_num_frames_provided(request)
+        video_defaults = self._resolve_video_generation_defaults(request)
+        if video_defaults is None:
+            vp = request.resolve_video_params()
+        else:
+            vp = request.resolve_video_params(
+                default_fps=video_defaults.fps,
+                default_num_frames=video_defaults.num_frames,
+            )
+            vp.width = vp.width or video_defaults.width
+            vp.height = vp.height or video_defaults.height
+            if num_frames_provided and vp.num_frames != video_defaults.num_frames:
+                raise HTTPException(
+                    status_code=HTTPStatus.BAD_REQUEST.value,
+                    detail=(f"This diffusion model requires {video_defaults.num_frames} frames; got {vp.num_frames}."),
+                )
+            if "num_inference_steps" not in provided_fields and gen_params.num_inference_steps is None:
+                gen_params.num_inference_steps = video_defaults.num_inference_steps
+
+        # Some native pipelines have a fixed duration. Validate both the
+        # OpenAI top-level field and model-specific aliases before dispatching
+        # the request; otherwise ``seconds`` would reach MAGI-2 as a duration
+        # override and fail only after the worker has started generation.
+        if video_defaults is not None and video_defaults.duration_seconds is not None:
+            requested_durations: list[float] = []
+            try:
+                if request.seconds is not None:
+                    requested_durations.append(float(request.seconds))
+                if request.extra_params is not None:
+                    for key in ("seconds", "duration"):
+                        if key in request.extra_params:
+                            requested_durations.append(float(request.extra_params[key]))
+            except (TypeError, ValueError) as exc:
+                raise HTTPException(
+                    status_code=HTTPStatus.BAD_REQUEST.value,
+                    detail="The requested video duration must be a finite number of seconds.",
+                ) from exc
+            expected_duration = video_defaults.duration_seconds
+            if any(not math.isfinite(duration) or duration != expected_duration for duration in requested_durations):
+                raise HTTPException(
+                    status_code=HTTPStatus.BAD_REQUEST.value,
+                    detail=f"This diffusion model supports {expected_duration:g}-second clips only.",
+                )
         if (
             input_image is not None
             and vp.width is not None
@@ -236,15 +295,17 @@ class OmniOpenAIServingVideo:
             gen_params.num_frames = vp.num_frames
         gen_params.num_outputs_per_prompt = request.num_outputs_per_prompt
         if request.seconds is not None:
-            gen_params.extra_args.setdefault("duration", float(request.seconds))
+            if video_defaults is None or video_defaults.duration_seconds is None:
+                gen_params.extra_args.setdefault("duration", float(request.seconds))
         if request.aspect_ratio is not None:
             gen_params.extra_args["aspect_ratio"] = request.aspect_ratio
         if request.short_edge is not None:
             gen_params.extra_args["short_edge"] = request.short_edge
         if request.start_time_seconds is not None:
             gen_params.extra_args["start_time_seconds"] = request.start_time_seconds
-        # Leave fps/frame_rate as None when the user did not provide fps.
-        if fps_provided and vp.fps is not None:
+        # Model-owned defaults are part of the serving contract. Other models
+        # preserve their engine defaults when the user did not provide fps.
+        if (fps_provided or video_defaults is not None) and vp.fps is not None:
             gen_params.fps = vp.fps
             gen_params.frame_rate = float(vp.fps)
         if "enable_frame_interpolation" in provided_fields:
@@ -464,6 +525,15 @@ class OmniOpenAIServingVideo:
             return False
         return "fps" in video_params.model_fields_set and video_params.fps is not None
 
+    @staticmethod
+    def _request_num_frames_provided(request: VideoGenerationRequest) -> bool:
+        if "num_frames" in request.model_fields_set and request.num_frames is not None:
+            return True
+        video_params = request.video_params
+        if video_params is None or "video_params" not in request.model_fields_set:
+            return False
+        return "num_frames" in video_params.model_fields_set and video_params.num_frames is not None
+
     def _resolve_default_sampling_params(self) -> OmniDiffusionSamplingParams:
         default_sampling_params_list = getattr(self._engine_client, "default_sampling_params_list", None)
         if default_sampling_params_list:
@@ -747,15 +817,15 @@ class OmniOpenAIServingVideo:
             return None
 
     @staticmethod
-    def _resolve_fps(result: object) -> int | None:
+    def _resolve_fps(result: object) -> float | None:
         """Extract fps from multimodal_output if the model reported it."""
         multimodal_output = getattr(result, "multimodal_output", None)
         if isinstance(multimodal_output, Mapping):
             fps = multimodal_output.get("fps")
             if fps is not None:
                 try:
-                    fps_val = fps.item() if hasattr(fps, "item") else int(fps)
-                    if fps_val > 0:
+                    fps_val = float(fps.item() if hasattr(fps, "item") else fps)
+                    if math.isfinite(fps_val) and fps_val > 0:
                         return fps_val
                 except (TypeError, ValueError):
                     pass
@@ -767,8 +837,8 @@ class OmniOpenAIServingVideo:
                 fps = mm.get("fps")
                 if fps is not None:
                     try:
-                        fps_val = fps.item() if hasattr(fps, "item") else int(fps)
-                        if fps_val > 0:
+                        fps_val = float(fps.item() if hasattr(fps, "item") else fps)
+                        if math.isfinite(fps_val) and fps_val > 0:
                             return fps_val
                     except (TypeError, ValueError):
                         pass
@@ -778,8 +848,8 @@ class OmniOpenAIServingVideo:
                 fps = mm.get("fps")
                 if fps is not None:
                     try:
-                        fps_val = fps.item() if hasattr(fps, "item") else int(fps)
-                        if fps_val > 0:
+                        fps_val = float(fps.item() if hasattr(fps, "item") else fps)
+                        if math.isfinite(fps_val) and fps_val > 0:
                             return fps_val
                     except (TypeError, ValueError):
                         pass

vllm_omni/entrypoints/openai/video_api_utils.py

@@ -786,7 +786,7 @@ def _encode_video_bytes_legacy(
 
 def _encode_video_bytes(
     video: Any,
-    fps: int,
+    fps: int | float,
     audio: Any | None = None,
     audio_sample_rate: int | None = None,
     video_codec_options: dict[str, str] | None = None,
@@ -908,7 +908,7 @@ def create_streaming_video_encoder(
 
 def encode_video_base64(
     video: Any,
-    fps: int,
+    fps: int | float,
     audio: Any | None = None,
     audio_sample_rate: int | None = None,
     video_codec_options: dict[str, str] | None = None,

vllm_omni/entrypoints/utils.py

@@ -26,6 +26,7 @@ from vllm_omni.config.yaml_util import create_config, load_yaml_config
 from vllm_omni.diffusion.utils.hf_utils import (
     _looks_like_dreamzero,
     get_diffusion_model_index,
+    resolve_native_diffusion_model_class,
 )
 from vllm_omni.entrypoints.stage_utils import _to_dict
 from vllm_omni.inputs.data import OmniSamplingParams
@@ -294,7 +295,10 @@ def resolve_model_config_path(model: str) -> str | None:
         model_type = hf_config.model_type
     except (ValueError, Exception):
         # If standard transformers format fails, try diffusers format
-        if get_diffusion_model_index(config_source) is not None:
+        native_model_class = resolve_native_diffusion_model_class(config_source)
+        if native_model_class is not None:
+            model_type = native_model_class
+        elif get_diffusion_model_index(config_source) is not None:
             model_type = _try_get_class_name_from_diffusers_config(config_source)
             if model_type is None:
                 raise ValueError(