[Model] Migrate MammothModa2 DiT to the shared diffusion runtime (#7134)

Signed-off-by: levius <2114377220@qq.com>
Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
authored and GitHub committed 2026-09-18 13:29:18 -07:00
1 parent ca3ccfc9a3
commit 7b4fc7ce2e
16 files changed
+1046 -427

No files matched your search

@@ -4,8 +4,9 @@
"""
End-to-end test for MammothModa2 text-to-image generation.
Verifies that the AR->DiT pipeline produces an image tensor whose pixel values
match a golden reference.
Verifies that the AR->DiT pipeline produces a valid PIL image. When an
optional golden fixture is present, the test also compares fixed pixel samples;
the portable golden/stability oracle remains tracked in #7090.
Model Hub repo id: ``bytedance-research/MammothModa2-Preview``.
Deploy config: ``get_deploy_config_path("mammoth_moda2.yaml")`` -> ``vllm_omni/deploy/mammoth_moda2.yaml``
@@ -20,13 +21,17 @@ import json
import os
from pathlib import Path
import numpy as np
import pytest
import torch
from PIL import Image
from vllm.sampling_params import SamplingParams
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from tests.helpers.stage_config import get_deploy_config_path
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.transformers_utils.repo_utils import hf_api
# ---------------------------------------------------------------------------
@@ -43,7 +48,7 @@ T2I_DEPLOY_CONFIG = get_deploy_config_path("mammoth_moda2.yaml")
_OMNI_RUNNER_PARAM = (MODEL_PATH, T2I_DEPLOY_CONFIG)
# Golden pixel reference file. Set UPDATE_GOLDEN=1 to regenerate.
# Optional golden pixel reference file. Set UPDATE_GOLDEN=1 to regenerate.
_GOLDEN_T2I_PATH = Path(__file__).parent / "fixtures" / "mammoth_moda2_t2i_golden.json"
# Fixed sampling coordinates: (channel, row_fraction, col_fraction)
# Covers corners, centre, and mid-edges across all 3 channels.
@@ -98,17 +103,33 @@ def _sample_pixels(img_tensor: torch.Tensor) -> list[float]:
return values
# ---------------------------------------------------------------------------
# End-to-end test
# ---------------------------------------------------------------------------
pytestmark = [
pytest.mark.slow,
pytest.mark.diffusion,
pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True),
]
def _iter_images(outputs: list[object]):
"""Yield images from shared diffusion ``OmniRequestOutput`` objects."""
for out in outputs:
ro_list = out if isinstance(out, list) else [out]
for ro in ro_list:
images = getattr(ro, "images", None)
if isinstance(images, list):
yield from images
@pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/7718")
def _pil_to_tensor(image: Image.Image) -> torch.Tensor:
array = np.asarray(image, dtype=np.float32) / 255.0
return torch.from_numpy(array).permute(2, 0, 1)
@pytest.mark.cpu
def test_diffusion_output_exposes_images_at_top_level():
image = Image.new("RGB", (16, 16), "black")
output = OmniRequestOutput.from_diffusion(request_id="diffusion-test", images=[image])
assert output.outputs == []
assert list(_iter_images([output])) == [image]
@pytest.mark.slow
@pytest.mark.diffusion
@pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True)
@hardware_test(res={"cuda": "H100"})
def test_mammothmoda2_t2i_e2e(omni_runner: OmniRunner):
"""
@@ -116,9 +137,9 @@ def test_mammothmoda2_t2i_e2e(omni_runner: OmniRunner):
Verifies:
- Omni pipeline initialises with the two-stage YAML config.
- DiT stage outputs an image tensor with the correct shape.
- A fixed set of pixel values matches a golden reference
(regenerate with ``UPDATE_GOLDEN=1``).
- DiT stage outputs one RGB PIL image at the requested size.
- When the optional fixture exists, fixed pixel samples match its golden
reference (regenerate with ``UPDATE_GOLDEN=1``).
"""
gen_cfg = _load_t2i_gen_config(MODEL_PATH)
eol_token_id = int(gen_cfg["eol_token_id"])
@@ -139,7 +160,14 @@ def test_mammothmoda2_t2i_e2e(omni_runner: OmniRunner):
max_tokens=max(1, expected_grid_tokens + 1),
detokenize=False,
)
dit_sampling = SamplingParams(temperature=0.0, max_tokens=1, detokenize=False)
dit_sampling = OmniDiffusionSamplingParams(
height=height,
width=width,
seed=42,
guidance_scale=1.0,
num_inference_steps=2,
extra_args={"cfg_range": [0.0, 1.0]},
)
outputs = list(
omni.generate(
@@ -155,9 +183,6 @@ def test_mammothmoda2_t2i_e2e(omni_runner: OmniRunner):
"visual_token_end_id": [visual_end],
"image_height": [height],
"image_width": [width],
"num_inference_steps": [2],
"text_guidance_scale": [1.0],
"cfg_range": [0.0, 1.0],
"visual_ids": [
_IMAGE_TOKEN_ID,
_VIDEO_TOKEN_ID,
@@ -173,35 +198,21 @@ def test_mammothmoda2_t2i_e2e(omni_runner: OmniRunner):
assert len(outputs) > 0, "Pipeline produced no outputs"
found_image = False
for out in outputs:
ro_list = out
if not isinstance(ro_list, list):
ro_list = [ro_list]
for ro in ro_list:
completion_outputs = getattr(ro, "outputs", None)
if not isinstance(completion_outputs, list):
continue
for completion in completion_outputs:
mm = getattr(completion, "multimodal_output", None)
if not (isinstance(mm, dict) and "image" in mm):
continue
img_list = mm["image"] if isinstance(mm["image"], list) else [mm["image"]]
for img_tensor in img_list:
assert isinstance(img_tensor, torch.Tensor), f"Expected image tensor, got {type(img_tensor)}"
assert img_tensor.ndim in (3, 4), f"Expected 3D or 4D image tensor, got {img_tensor.ndim}D"
images = list(_iter_images(outputs))
assert len(images) == 1, f"Expected exactly one image, got {len(images)}"
sampled = _sample_pixels(img_tensor)
image = images[0]
assert isinstance(image, Image.Image), f"Expected PIL image, got {type(image)}"
assert image.mode == "RGB"
assert image.size == (width, height)
if os.environ.get("UPDATE_GOLDEN"):
_GOLDEN_T2I_PATH.parent.mkdir(parents=True, exist_ok=True)
_GOLDEN_T2I_PATH.write_text(json.dumps({"pixels": sampled}, indent=2))
print(f"\nGolden file written to {_GOLDEN_T2I_PATH}")
elif _GOLDEN_T2I_PATH.exists():
golden = json.loads(_GOLDEN_T2I_PATH.read_text())["pixels"]
for i, (got, exp) in enumerate(zip(sampled, golden)):
assert abs(got - exp) < 1e-4, f"Pixel {i} mismatch: got {got}, expected {exp}"
sampled = _sample_pixels(_pil_to_tensor(image))
found_image = True
assert found_image, "No image tensor found in pipeline output"
if os.environ.get("UPDATE_GOLDEN"):
_GOLDEN_T2I_PATH.parent.mkdir(parents=True, exist_ok=True)
_GOLDEN_T2I_PATH.write_text(json.dumps({"pixels": sampled}, indent=2))
print(f"\nGolden file written to {_GOLDEN_T2I_PATH}")
elif _GOLDEN_T2I_PATH.exists():
golden = json.loads(_GOLDEN_T2I_PATH.read_text())["pixels"]
for i, (got, exp) in enumerate(zip(sampled, golden)):
assert abs(got - exp) < 1e-4, f"Pixel {i} mismatch: got {got}, expected {exp}"