[Model] Add native MAGI-2 Preview diffusion support (#5918)

Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com>
Signed-off-by: hsliuustc0106 <liuhongsheng4@huawei.com>
This commit is contained in:
Hongsheng Liu authored and GitHub committed 2026-09-05 15:32:30 +08:00
1 parent 13c617a140
commit d4096e01fe
78 files changed
+9896 -459

No files matched your search

+84 -14
View File
@@ -1,9 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
import copy
import math
import time
from collections.abc import Mapping
from dataclasses import dataclass
@@ -36,7 +37,8 @@ from vllm_omni.entrypoints.openai.video_api_utils import (
encode_video_base64,
)
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniTextPrompt
from vllm_omni.model_extras import should_preserve_reference_image_size
from vllm_omni.model_extras import get_video_generation_defaults, should_preserve_reference_image_size
from vllm_omni.model_extras.video_generation import VideoGenerationDefaults
from vllm_omni.outputs.output_metadata import (
DiffusionMetadataMapping,
DiffusionMultimodalOutput,
@@ -82,7 +84,7 @@ class VideoGenerationArtifacts:
audios: list[DiffusionPayloadValue | None]
actions: list[VideoAction | None]
audio_sample_rate: int
output_fps: int
output_fps: float
stage_durations: dict[str, float]
peak_memory_mb: float
@@ -111,6 +113,21 @@ class OmniOpenAIServingVideo:
return get_od_config()
return getattr(self._engine_client, "od_config", None)
def _resolve_video_generation_defaults(
self,
request: VideoGenerationRequest,
) -> VideoGenerationDefaults | None:
"""Resolve defaults owned by the active diffusion pipeline."""
od_config = self._resolve_diffusion_od_config()
model_class_name = None if od_config is None else getattr(od_config, "model_class_name", None)
try:
return get_video_generation_defaults(model_class_name, request.extra_params)
except (TypeError, ValueError) as exc:
raise HTTPException(
status_code=HTTPStatus.BAD_REQUEST.value,
detail=str(exc),
) from exc
@property
def model_name(self) -> str | None:
return self._model_name
@@ -206,7 +223,49 @@ class OmniOpenAIServingVideo:
)
provided_fields = request.model_fields_set
fps_provided = self._request_fps_provided(request)
vp = request.resolve_video_params()
num_frames_provided = self._request_num_frames_provided(request)
video_defaults = self._resolve_video_generation_defaults(request)
if video_defaults is None:
vp = request.resolve_video_params()
else:
vp = request.resolve_video_params(
default_fps=video_defaults.fps,
default_num_frames=video_defaults.num_frames,
)
vp.width = vp.width or video_defaults.width
vp.height = vp.height or video_defaults.height
if num_frames_provided and vp.num_frames != video_defaults.num_frames:
raise HTTPException(
status_code=HTTPStatus.BAD_REQUEST.value,
detail=(f"This diffusion model requires {video_defaults.num_frames} frames; got {vp.num_frames}."),
)
if "num_inference_steps" not in provided_fields and gen_params.num_inference_steps is None:
gen_params.num_inference_steps = video_defaults.num_inference_steps
# Some native pipelines have a fixed duration. Validate both the
# OpenAI top-level field and model-specific aliases before dispatching
# the request; otherwise ``seconds`` would reach MAGI-2 as a duration
# override and fail only after the worker has started generation.
if video_defaults is not None and video_defaults.duration_seconds is not None:
requested_durations: list[float] = []
try:
if request.seconds is not None:
requested_durations.append(float(request.seconds))
if request.extra_params is not None:
for key in ("seconds", "duration"):
if key in request.extra_params:
requested_durations.append(float(request.extra_params[key]))
except (TypeError, ValueError) as exc:
raise HTTPException(
status_code=HTTPStatus.BAD_REQUEST.value,
detail="The requested video duration must be a finite number of seconds.",
) from exc
expected_duration = video_defaults.duration_seconds
if any(not math.isfinite(duration) or duration != expected_duration for duration in requested_durations):
raise HTTPException(
status_code=HTTPStatus.BAD_REQUEST.value,
detail=f"This diffusion model supports {expected_duration:g}-second clips only.",
)
if (
input_image is not None
and vp.width is not None
@@ -236,15 +295,17 @@ class OmniOpenAIServingVideo:
gen_params.num_frames = vp.num_frames
gen_params.num_outputs_per_prompt = request.num_outputs_per_prompt
if request.seconds is not None:
gen_params.extra_args.setdefault("duration", float(request.seconds))
if video_defaults is None or video_defaults.duration_seconds is None:
gen_params.extra_args.setdefault("duration", float(request.seconds))
if request.aspect_ratio is not None:
gen_params.extra_args["aspect_ratio"] = request.aspect_ratio
if request.short_edge is not None:
gen_params.extra_args["short_edge"] = request.short_edge
if request.start_time_seconds is not None:
gen_params.extra_args["start_time_seconds"] = request.start_time_seconds
# Leave fps/frame_rate as None when the user did not provide fps.
if fps_provided and vp.fps is not None:
# Model-owned defaults are part of the serving contract. Other models
# preserve their engine defaults when the user did not provide fps.
if (fps_provided or video_defaults is not None) and vp.fps is not None:
gen_params.fps = vp.fps
gen_params.frame_rate = float(vp.fps)
if "enable_frame_interpolation" in provided_fields:
@@ -464,6 +525,15 @@ class OmniOpenAIServingVideo:
return False
return "fps" in video_params.model_fields_set and video_params.fps is not None
@staticmethod
def _request_num_frames_provided(request: VideoGenerationRequest) -> bool:
if "num_frames" in request.model_fields_set and request.num_frames is not None:
return True
video_params = request.video_params
if video_params is None or "video_params" not in request.model_fields_set:
return False
return "num_frames" in video_params.model_fields_set and video_params.num_frames is not None
def _resolve_default_sampling_params(self) -> OmniDiffusionSamplingParams:
default_sampling_params_list = getattr(self._engine_client, "default_sampling_params_list", None)
if default_sampling_params_list:
@@ -747,15 +817,15 @@ class OmniOpenAIServingVideo:
return None
@staticmethod
def _resolve_fps(result: object) -> int | None:
def _resolve_fps(result: object) -> float | None:
"""Extract fps from multimodal_output if the model reported it."""
multimodal_output = getattr(result, "multimodal_output", None)
if isinstance(multimodal_output, Mapping):
fps = multimodal_output.get("fps")
if fps is not None:
try:
fps_val = fps.item() if hasattr(fps, "item") else int(fps)
if fps_val > 0:
fps_val = float(fps.item() if hasattr(fps, "item") else fps)
if math.isfinite(fps_val) and fps_val > 0:
return fps_val
except (TypeError, ValueError):
pass
@@ -767,8 +837,8 @@ class OmniOpenAIServingVideo:
fps = mm.get("fps")
if fps is not None:
try:
fps_val = fps.item() if hasattr(fps, "item") else int(fps)
if fps_val > 0:
fps_val = float(fps.item() if hasattr(fps, "item") else fps)
if math.isfinite(fps_val) and fps_val > 0:
return fps_val
except (TypeError, ValueError):
pass
@@ -778,8 +848,8 @@ class OmniOpenAIServingVideo:
fps = mm.get("fps")
if fps is not None:
try:
fps_val = fps.item() if hasattr(fps, "item") else int(fps)
if fps_val > 0:
fps_val = float(fps.item() if hasattr(fps, "item") else fps)
if math.isfinite(fps_val) and fps_val > 0:
return fps_val
except (TypeError, ValueError):
pass