[Model] Add native MAGI-2 Preview diffusion support (#5918)

Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com>
Signed-off-by: hsliuustc0106 <liuhongsheng4@huawei.com>
This commit is contained in:
Hongsheng Liu authored and GitHub committed 2026-09-05 15:32:30 +08:00
1 parent 13c617a140
commit d4096e01fe
78 files changed
+9896 -459

No files matched your search

+3 -1
View File
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from vllm_omni.model_extras.registry import (
build_image_to_image_prompt,
@@ -11,6 +11,7 @@ from vllm_omni.model_extras.registry import (
get_model_class_name,
get_output_tensor_range,
get_transformer_config_subfolder,
get_video_generation_defaults,
get_x_to_text_model_family,
should_init_extra_args_for_non_diffusion_stages,
should_preserve_reference_image_size,
@@ -26,6 +27,7 @@ __all__ = [
"get_model_class_name",
"get_output_tensor_range",
"get_transformer_config_subfolder",
"get_video_generation_defaults",
"get_x_to_text_model_family",
"should_init_extra_args_for_non_diffusion_stages",
"should_preserve_reference_image_size",
+55
View File
@@ -0,0 +1,55 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Request fields supported by the MAGI-2 Preview pipeline."""
from __future__ import annotations
from collections.abc import Mapping
from typing import Any
from vllm_omni.model_extras.video_generation import VideoGenerationDefaults
MAGI2_EXTRA_BODY_PARAMS = frozenset(
{
"seconds",
"resolution",
"image_path",
"output_width",
"output_height",
"deterministic",
}
)
MAGI2_EXTRA_OUTPUT_PARAMS: frozenset[str] = frozenset()
def magi2_preserves_reference_image_size(
*,
model: str | None,
revision: str | None = None,
) -> bool:
"""MAGI-2 owns checkpoint-aligned reference resize and padding."""
del model, revision
return True
def get_magi2_video_generation_defaults(
extra_body: Mapping[str, Any] | None = None,
) -> VideoGenerationDefaults:
resolution = str((extra_body or {}).get("resolution", "540p")).lower()
if resolution == "272p":
width, height = 448, 256
elif resolution == "540p":
width, height = 896, 512
else:
raise ValueError("MAGI-2 Preview resolution must be '272p' or '540p'.")
return VideoGenerationDefaults(
width=width,
height=height,
num_frames=125,
num_inference_steps=100,
fps=12.5,
duration_seconds=10.0,
output="magi2_output.mp4",
default_negative_prompt=None,
)
+26 -1
View File
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
@@ -36,6 +36,12 @@ from vllm_omni.model_extras.ltx2 import (
ltx_preserves_reference_image_size,
ltx_transformer_config_subfolder,
)
from vllm_omni.model_extras.magi2 import (
MAGI2_EXTRA_BODY_PARAMS,
MAGI2_EXTRA_OUTPUT_PARAMS,
get_magi2_video_generation_defaults,
magi2_preserves_reference_image_size,
)
from vllm_omni.model_extras.mammothmodal2_preview import (
MAMMOTHMODA2_PREVIEW_EXTRA_BODY_PARAMS,
MAMMOTHMODA2_PREVIEW_EXTRA_OUTPUT_PARAMS,
@@ -70,6 +76,7 @@ from vllm_omni.model_extras.vace import (
from vllm_omni.model_extras.vace import (
build_image_to_video_prompt as build_vace_image_to_video_prompt,
)
from vllm_omni.model_extras.video_generation import VideoGenerationDefaults
TextToImagePromptBuilder = Callable[
[str, str | None, int | None, int | None],
@@ -192,6 +199,12 @@ _EXTRA_SPECS: dict[str, dict[str, Any]] = {
"extra_body_params": COSMOS3_EXTRA_BODY_PARAMS,
"extra_output_params": COSMOS3_EXTRA_OUTPUT_PARAMS,
},
"Magi2Pipeline": {
"extra_body_params": MAGI2_EXTRA_BODY_PARAMS,
"extra_output_params": MAGI2_EXTRA_OUTPUT_PARAMS,
"video_generation_defaults_builder": get_magi2_video_generation_defaults,
"reference_image_size_resolver": magi2_preserves_reference_image_size,
},
"HeliosPipeline": {
"extra_body_params": HELIOS_EXTRA_BODY_PARAMS,
"extra_output_params": HELIOS_EXTRA_OUTPUT_PARAMS,
@@ -286,6 +299,18 @@ def get_extra_output_params(model_class_name: str | None) -> frozenset[str]:
return spec.get("extra_output_params", frozenset()) if spec is not None else frozenset()
def get_video_generation_defaults(
model_class_name: str | None,
extra_body: Mapping[str, Any] | None = None,
) -> VideoGenerationDefaults | None:
"""Return model-owned defaults for the shared video examples, if declared."""
spec = _get_spec(model_class_name)
if spec is None:
return None
builder = spec.get("video_generation_defaults_builder")
return builder(extra_body) if builder is not None else None
def get_output_tensor_range(model_class_name: str | None) -> OutputTensorRange:
"""Return the declared range for floating-point tensor outputs.
@@ -0,0 +1,44 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
from dataclasses import dataclass
from typing import Any
@dataclass(frozen=True)
class VideoGenerationDefaults:
"""Model-owned defaults consumed by shared video examples."""
width: int
height: int
num_frames: int
num_inference_steps: int
fps: float
output: str | None = None
guidance_scale: float | None = None
flow_shift: float | None = None
dimension_multiple: int = 16
default_negative_prompt: str | None = ""
duration_seconds: float | None = None
@property
def max_area(self) -> int:
return self.width * self.height
def cli_defaults(self) -> dict[str, Any]:
defaults: dict[str, Any] = {
"height": self.height,
"width": self.width,
"num_frames": self.num_frames,
"num_inference_steps": self.num_inference_steps,
"fps": self.fps,
}
if self.output is not None:
defaults["output"] = self.output
if self.guidance_scale is not None:
defaults["guidance_scale"] = self.guidance_scale
if self.flow_shift is not None:
defaults["flow_shift"] = self.flow_shift
return defaults