[Model] Add native MAGI-2 Preview diffusion support (#5918)
Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com> Signed-off-by: hsliuustc0106 <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
13c617a140
commit
d4096e01fe
78 files changed
+9896
-459
No files matched your search
@@ -1,9 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import math
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
@@ -36,7 +37,8 @@ from vllm_omni.entrypoints.openai.video_api_utils import (
|
||||
encode_video_base64,
|
||||
)
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniTextPrompt
|
||||
from vllm_omni.model_extras import should_preserve_reference_image_size
|
||||
from vllm_omni.model_extras import get_video_generation_defaults, should_preserve_reference_image_size
|
||||
from vllm_omni.model_extras.video_generation import VideoGenerationDefaults
|
||||
from vllm_omni.outputs.output_metadata import (
|
||||
DiffusionMetadataMapping,
|
||||
DiffusionMultimodalOutput,
|
||||
@@ -82,7 +84,7 @@ class VideoGenerationArtifacts:
|
||||
audios: list[DiffusionPayloadValue | None]
|
||||
actions: list[VideoAction | None]
|
||||
audio_sample_rate: int
|
||||
output_fps: int
|
||||
output_fps: float
|
||||
stage_durations: dict[str, float]
|
||||
peak_memory_mb: float
|
||||
|
||||
@@ -111,6 +113,21 @@ class OmniOpenAIServingVideo:
|
||||
return get_od_config()
|
||||
return getattr(self._engine_client, "od_config", None)
|
||||
|
||||
def _resolve_video_generation_defaults(
|
||||
self,
|
||||
request: VideoGenerationRequest,
|
||||
) -> VideoGenerationDefaults | None:
|
||||
"""Resolve defaults owned by the active diffusion pipeline."""
|
||||
od_config = self._resolve_diffusion_od_config()
|
||||
model_class_name = None if od_config is None else getattr(od_config, "model_class_name", None)
|
||||
try:
|
||||
return get_video_generation_defaults(model_class_name, request.extra_params)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.BAD_REQUEST.value,
|
||||
detail=str(exc),
|
||||
) from exc
|
||||
|
||||
@property
|
||||
def model_name(self) -> str | None:
|
||||
return self._model_name
|
||||
@@ -206,7 +223,49 @@ class OmniOpenAIServingVideo:
|
||||
)
|
||||
provided_fields = request.model_fields_set
|
||||
fps_provided = self._request_fps_provided(request)
|
||||
vp = request.resolve_video_params()
|
||||
num_frames_provided = self._request_num_frames_provided(request)
|
||||
video_defaults = self._resolve_video_generation_defaults(request)
|
||||
if video_defaults is None:
|
||||
vp = request.resolve_video_params()
|
||||
else:
|
||||
vp = request.resolve_video_params(
|
||||
default_fps=video_defaults.fps,
|
||||
default_num_frames=video_defaults.num_frames,
|
||||
)
|
||||
vp.width = vp.width or video_defaults.width
|
||||
vp.height = vp.height or video_defaults.height
|
||||
if num_frames_provided and vp.num_frames != video_defaults.num_frames:
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.BAD_REQUEST.value,
|
||||
detail=(f"This diffusion model requires {video_defaults.num_frames} frames; got {vp.num_frames}."),
|
||||
)
|
||||
if "num_inference_steps" not in provided_fields and gen_params.num_inference_steps is None:
|
||||
gen_params.num_inference_steps = video_defaults.num_inference_steps
|
||||
|
||||
# Some native pipelines have a fixed duration. Validate both the
|
||||
# OpenAI top-level field and model-specific aliases before dispatching
|
||||
# the request; otherwise ``seconds`` would reach MAGI-2 as a duration
|
||||
# override and fail only after the worker has started generation.
|
||||
if video_defaults is not None and video_defaults.duration_seconds is not None:
|
||||
requested_durations: list[float] = []
|
||||
try:
|
||||
if request.seconds is not None:
|
||||
requested_durations.append(float(request.seconds))
|
||||
if request.extra_params is not None:
|
||||
for key in ("seconds", "duration"):
|
||||
if key in request.extra_params:
|
||||
requested_durations.append(float(request.extra_params[key]))
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.BAD_REQUEST.value,
|
||||
detail="The requested video duration must be a finite number of seconds.",
|
||||
) from exc
|
||||
expected_duration = video_defaults.duration_seconds
|
||||
if any(not math.isfinite(duration) or duration != expected_duration for duration in requested_durations):
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.BAD_REQUEST.value,
|
||||
detail=f"This diffusion model supports {expected_duration:g}-second clips only.",
|
||||
)
|
||||
if (
|
||||
input_image is not None
|
||||
and vp.width is not None
|
||||
@@ -236,15 +295,17 @@ class OmniOpenAIServingVideo:
|
||||
gen_params.num_frames = vp.num_frames
|
||||
gen_params.num_outputs_per_prompt = request.num_outputs_per_prompt
|
||||
if request.seconds is not None:
|
||||
gen_params.extra_args.setdefault("duration", float(request.seconds))
|
||||
if video_defaults is None or video_defaults.duration_seconds is None:
|
||||
gen_params.extra_args.setdefault("duration", float(request.seconds))
|
||||
if request.aspect_ratio is not None:
|
||||
gen_params.extra_args["aspect_ratio"] = request.aspect_ratio
|
||||
if request.short_edge is not None:
|
||||
gen_params.extra_args["short_edge"] = request.short_edge
|
||||
if request.start_time_seconds is not None:
|
||||
gen_params.extra_args["start_time_seconds"] = request.start_time_seconds
|
||||
# Leave fps/frame_rate as None when the user did not provide fps.
|
||||
if fps_provided and vp.fps is not None:
|
||||
# Model-owned defaults are part of the serving contract. Other models
|
||||
# preserve their engine defaults when the user did not provide fps.
|
||||
if (fps_provided or video_defaults is not None) and vp.fps is not None:
|
||||
gen_params.fps = vp.fps
|
||||
gen_params.frame_rate = float(vp.fps)
|
||||
if "enable_frame_interpolation" in provided_fields:
|
||||
@@ -464,6 +525,15 @@ class OmniOpenAIServingVideo:
|
||||
return False
|
||||
return "fps" in video_params.model_fields_set and video_params.fps is not None
|
||||
|
||||
@staticmethod
|
||||
def _request_num_frames_provided(request: VideoGenerationRequest) -> bool:
|
||||
if "num_frames" in request.model_fields_set and request.num_frames is not None:
|
||||
return True
|
||||
video_params = request.video_params
|
||||
if video_params is None or "video_params" not in request.model_fields_set:
|
||||
return False
|
||||
return "num_frames" in video_params.model_fields_set and video_params.num_frames is not None
|
||||
|
||||
def _resolve_default_sampling_params(self) -> OmniDiffusionSamplingParams:
|
||||
default_sampling_params_list = getattr(self._engine_client, "default_sampling_params_list", None)
|
||||
if default_sampling_params_list:
|
||||
@@ -747,15 +817,15 @@ class OmniOpenAIServingVideo:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _resolve_fps(result: object) -> int | None:
|
||||
def _resolve_fps(result: object) -> float | None:
|
||||
"""Extract fps from multimodal_output if the model reported it."""
|
||||
multimodal_output = getattr(result, "multimodal_output", None)
|
||||
if isinstance(multimodal_output, Mapping):
|
||||
fps = multimodal_output.get("fps")
|
||||
if fps is not None:
|
||||
try:
|
||||
fps_val = fps.item() if hasattr(fps, "item") else int(fps)
|
||||
if fps_val > 0:
|
||||
fps_val = float(fps.item() if hasattr(fps, "item") else fps)
|
||||
if math.isfinite(fps_val) and fps_val > 0:
|
||||
return fps_val
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
@@ -767,8 +837,8 @@ class OmniOpenAIServingVideo:
|
||||
fps = mm.get("fps")
|
||||
if fps is not None:
|
||||
try:
|
||||
fps_val = fps.item() if hasattr(fps, "item") else int(fps)
|
||||
if fps_val > 0:
|
||||
fps_val = float(fps.item() if hasattr(fps, "item") else fps)
|
||||
if math.isfinite(fps_val) and fps_val > 0:
|
||||
return fps_val
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
@@ -778,8 +848,8 @@ class OmniOpenAIServingVideo:
|
||||
fps = mm.get("fps")
|
||||
if fps is not None:
|
||||
try:
|
||||
fps_val = fps.item() if hasattr(fps, "item") else int(fps)
|
||||
if fps_val > 0:
|
||||
fps_val = float(fps.item() if hasattr(fps, "item") else fps)
|
||||
if math.isfinite(fps_val) and fps_val > 0:
|
||||
return fps_val
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
Reference in new issue
Block a user