[Model] Add native MAGI-2 Preview diffusion support (#5918)
Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com> Signed-off-by: hsliuustc0106 <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
13c617a140
commit
d4096e01fe
78 files changed
+9896
-459
No files matched your search
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from vllm_omni.model_extras.registry import (
|
||||
build_image_to_image_prompt,
|
||||
@@ -11,6 +11,7 @@ from vllm_omni.model_extras.registry import (
|
||||
get_model_class_name,
|
||||
get_output_tensor_range,
|
||||
get_transformer_config_subfolder,
|
||||
get_video_generation_defaults,
|
||||
get_x_to_text_model_family,
|
||||
should_init_extra_args_for_non_diffusion_stages,
|
||||
should_preserve_reference_image_size,
|
||||
@@ -26,6 +27,7 @@ __all__ = [
|
||||
"get_model_class_name",
|
||||
"get_output_tensor_range",
|
||||
"get_transformer_config_subfolder",
|
||||
"get_video_generation_defaults",
|
||||
"get_x_to_text_model_family",
|
||||
"should_init_extra_args_for_non_diffusion_stages",
|
||||
"should_preserve_reference_image_size",
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Request fields supported by the MAGI-2 Preview pipeline."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
from vllm_omni.model_extras.video_generation import VideoGenerationDefaults
|
||||
|
||||
MAGI2_EXTRA_BODY_PARAMS = frozenset(
|
||||
{
|
||||
"seconds",
|
||||
"resolution",
|
||||
"image_path",
|
||||
"output_width",
|
||||
"output_height",
|
||||
"deterministic",
|
||||
}
|
||||
)
|
||||
MAGI2_EXTRA_OUTPUT_PARAMS: frozenset[str] = frozenset()
|
||||
|
||||
|
||||
def magi2_preserves_reference_image_size(
|
||||
*,
|
||||
model: str | None,
|
||||
revision: str | None = None,
|
||||
) -> bool:
|
||||
"""MAGI-2 owns checkpoint-aligned reference resize and padding."""
|
||||
del model, revision
|
||||
return True
|
||||
|
||||
|
||||
def get_magi2_video_generation_defaults(
|
||||
extra_body: Mapping[str, Any] | None = None,
|
||||
) -> VideoGenerationDefaults:
|
||||
resolution = str((extra_body or {}).get("resolution", "540p")).lower()
|
||||
if resolution == "272p":
|
||||
width, height = 448, 256
|
||||
elif resolution == "540p":
|
||||
width, height = 896, 512
|
||||
else:
|
||||
raise ValueError("MAGI-2 Preview resolution must be '272p' or '540p'.")
|
||||
return VideoGenerationDefaults(
|
||||
width=width,
|
||||
height=height,
|
||||
num_frames=125,
|
||||
num_inference_steps=100,
|
||||
fps=12.5,
|
||||
duration_seconds=10.0,
|
||||
output="magi2_output.mp4",
|
||||
default_negative_prompt=None,
|
||||
)
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -36,6 +36,12 @@ from vllm_omni.model_extras.ltx2 import (
|
||||
ltx_preserves_reference_image_size,
|
||||
ltx_transformer_config_subfolder,
|
||||
)
|
||||
from vllm_omni.model_extras.magi2 import (
|
||||
MAGI2_EXTRA_BODY_PARAMS,
|
||||
MAGI2_EXTRA_OUTPUT_PARAMS,
|
||||
get_magi2_video_generation_defaults,
|
||||
magi2_preserves_reference_image_size,
|
||||
)
|
||||
from vllm_omni.model_extras.mammothmodal2_preview import (
|
||||
MAMMOTHMODA2_PREVIEW_EXTRA_BODY_PARAMS,
|
||||
MAMMOTHMODA2_PREVIEW_EXTRA_OUTPUT_PARAMS,
|
||||
@@ -70,6 +76,7 @@ from vllm_omni.model_extras.vace import (
|
||||
from vllm_omni.model_extras.vace import (
|
||||
build_image_to_video_prompt as build_vace_image_to_video_prompt,
|
||||
)
|
||||
from vllm_omni.model_extras.video_generation import VideoGenerationDefaults
|
||||
|
||||
TextToImagePromptBuilder = Callable[
|
||||
[str, str | None, int | None, int | None],
|
||||
@@ -192,6 +199,12 @@ _EXTRA_SPECS: dict[str, dict[str, Any]] = {
|
||||
"extra_body_params": COSMOS3_EXTRA_BODY_PARAMS,
|
||||
"extra_output_params": COSMOS3_EXTRA_OUTPUT_PARAMS,
|
||||
},
|
||||
"Magi2Pipeline": {
|
||||
"extra_body_params": MAGI2_EXTRA_BODY_PARAMS,
|
||||
"extra_output_params": MAGI2_EXTRA_OUTPUT_PARAMS,
|
||||
"video_generation_defaults_builder": get_magi2_video_generation_defaults,
|
||||
"reference_image_size_resolver": magi2_preserves_reference_image_size,
|
||||
},
|
||||
"HeliosPipeline": {
|
||||
"extra_body_params": HELIOS_EXTRA_BODY_PARAMS,
|
||||
"extra_output_params": HELIOS_EXTRA_OUTPUT_PARAMS,
|
||||
@@ -286,6 +299,18 @@ def get_extra_output_params(model_class_name: str | None) -> frozenset[str]:
|
||||
return spec.get("extra_output_params", frozenset()) if spec is not None else frozenset()
|
||||
|
||||
|
||||
def get_video_generation_defaults(
|
||||
model_class_name: str | None,
|
||||
extra_body: Mapping[str, Any] | None = None,
|
||||
) -> VideoGenerationDefaults | None:
|
||||
"""Return model-owned defaults for the shared video examples, if declared."""
|
||||
spec = _get_spec(model_class_name)
|
||||
if spec is None:
|
||||
return None
|
||||
builder = spec.get("video_generation_defaults_builder")
|
||||
return builder(extra_body) if builder is not None else None
|
||||
|
||||
|
||||
def get_output_tensor_range(model_class_name: str | None) -> OutputTensorRange:
|
||||
"""Return the declared range for floating-point tensor outputs.
|
||||
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class VideoGenerationDefaults:
|
||||
"""Model-owned defaults consumed by shared video examples."""
|
||||
|
||||
width: int
|
||||
height: int
|
||||
num_frames: int
|
||||
num_inference_steps: int
|
||||
fps: float
|
||||
output: str | None = None
|
||||
guidance_scale: float | None = None
|
||||
flow_shift: float | None = None
|
||||
dimension_multiple: int = 16
|
||||
default_negative_prompt: str | None = ""
|
||||
duration_seconds: float | None = None
|
||||
|
||||
@property
|
||||
def max_area(self) -> int:
|
||||
return self.width * self.height
|
||||
|
||||
def cli_defaults(self) -> dict[str, Any]:
|
||||
defaults: dict[str, Any] = {
|
||||
"height": self.height,
|
||||
"width": self.width,
|
||||
"num_frames": self.num_frames,
|
||||
"num_inference_steps": self.num_inference_steps,
|
||||
"fps": self.fps,
|
||||
}
|
||||
if self.output is not None:
|
||||
defaults["output"] = self.output
|
||||
if self.guidance_scale is not None:
|
||||
defaults["guidance_scale"] = self.guidance_scale
|
||||
if self.flow_shift is not None:
|
||||
defaults["flow_shift"] = self.flow_shift
|
||||
return defaults
|
||||
Reference in new issue
Block a user