[Model] Migrate MammothModa2 DiT to the shared diffusion runtime (#7134)

Signed-off-by: levius <2114377220@qq.com>
Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
authored and GitHub committed 2026-09-18 13:29:18 -07:00
1 parent ca3ccfc9a3
commit 7b4fc7ce2e
16 files changed
+1046 -427

No files matched your search

+27 -1
View File
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Tests for endpoint restrictions logic."""
import pytest
@@ -44,6 +44,32 @@ def test_restricted_completions_returns_400():
assert body["error"]["type"] == "BadRequestError"
def test_restricted_image_edits_returns_400():
"""Ensure pipelines can reject image edits independently of generation."""
app = FastAPI()
@app.post("/v1/images/edits")
async def existing_handler():
return {"ok": True}
@app.post("/v1/images/generations")
async def generations_handler():
return {"ok": True}
restrictions = (EndpointRestriction(OmniServingCapability.IMAGE_EDITS, REJECTION_REASON),)
shutdown_unsupported_routes(app, restrictions)
client = TestClient(app)
resp = client.post("/v1/images/edits")
assert resp.status_code == 400
assert resp.json()["error"]["message"] == REJECTION_REASON
# The restriction is endpoint-specific and must not remove generations.
resp = client.post("/v1/images/generations")
assert resp.status_code == 200
assert resp.json() == {"ok": True}
def test_unrestricted_completions_not_blocked():
"""Ensure that if we don't shutdown any routes, the route stays."""
app = _make_app_with_server_route()
@@ -0,0 +1,60 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import pytest
from tests.helpers.stage_config import get_deploy_config_path
from vllm_omni.config.endpoint_policy import OmniServingCapability
from vllm_omni.config.stage_config import StageExecutionType, load_deploy_config
from vllm_omni.diffusion.registry import _DIFFUSION_MODELS
from vllm_omni.model_executor.models.mammoth_moda2.pipeline import (
MAMMOTH_MODA2_AR_PIPELINE,
MAMMOTH_MODA2_PIPELINE,
)
from vllm_omni.model_executor.models.registry import _OMNI_MODELS
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_mammothmoda2_generation_stage_uses_shared_diffusion_runtime() -> None:
ar_stage, dit_stage = MAMMOTH_MODA2_PIPELINE.stages
assert ar_stage.execution_type is StageExecutionType.LLM_AR
assert dit_stage.execution_type is StageExecutionType.DIFFUSION
assert dit_stage.model_arch == "MammothModa2DiTPipeline"
assert dit_stage.custom_process_input_func.endswith(".ar2diffusion")
assert dit_stage.omni_kv_config == {"need_recv_cache": False}
assert dit_stage.input_sources == (0,)
assert dit_stage.final_output is True
assert dit_stage.final_output_type == "image"
def test_mammothmoda2_dit_is_registered_only_with_diffusion_runtime() -> None:
assert _DIFFUSION_MODELS["MammothModa2DiTPipeline"] == (
"mammoth_moda2",
"pipeline_mammothmoda2_dit",
"MammothModa2DiTPipeline",
)
assert "MammothModa2DiTPipeline" not in _OMNI_MODELS
def test_mammothmoda2_deploy_does_not_shadow_request_sampling_fields() -> None:
deploy = load_deploy_config(get_deploy_config_path("mammoth_moda2.yaml"))
assert deploy.stages[1].default_sampling_params is None
def test_mammothmoda2_rejects_unsupported_image_edits() -> None:
restrictions = {
restriction.capability: restriction.reason for restriction in MAMMOTH_MODA2_PIPELINE.endpoint_restrictions
}
assert set(restrictions) == {OmniServingCapability.IMAGE_EDITS}
assert "img2img" in restrictions[OmniServingCapability.IMAGE_EDITS]
assert MAMMOTH_MODA2_AR_PIPELINE.endpoint_restrictions == ()
def test_mammothmoda2_ar_only_topology_is_unchanged() -> None:
assert len(MAMMOTH_MODA2_AR_PIPELINE.stages) == 1
stage = MAMMOTH_MODA2_AR_PIPELINE.stages[0]
assert stage.execution_type is StageExecutionType.LLM_AR
assert stage.final_output_type == "text"
+14
View File
@@ -95,6 +95,20 @@ def _from_pipeline_key(
)
def test_mammothmoda2_diffusion_stage_projects_native_backend_config() -> None:
config = _from_pipeline_key(
"mammoth_moda2",
cli_overrides={"model": "/models/MammothModa2-Preview"},
)
stage = config.stage_by_id(1)
assert isinstance(stage, VllmOmniDiffusionStageConfig)
assert stage.diffusion_config.model_class_name == "MammothModa2DiTPipeline"
assert stage.diffusion_config.model == "/models/MammothModa2-Preview"
assert stage.diffusion_config.step_execution is False
assert stage.scheduler_config.max_num_seqs == 1
assert stage.connector_config.omni_kv_config == {"need_recv_cache": False}
def test_minimax_h3_text_encoder_tp_targets_only_structured_stage_zero() -> None:
config = _from_pipeline_key(
"minimax_h3_disaggregated",
@@ -0,0 +1,392 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from dataclasses import dataclass
import pytest
import torch
from torch import nn
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig, TransformerConfig
from vllm_omni.diffusion.models.mammoth_moda2 import pipeline_mammothmoda2_dit
from vllm_omni.diffusion.models.mammoth_moda2.pipeline_mammothmoda2_dit import (
MammothModa2DiTPipeline,
_build_mammoth_config,
_root_weight_source,
)
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _raw_config() -> dict:
return {
"model_type": "mammothmoda2",
"llm_config": {
"model_type": "mammothmoda2_qwen2_5_vl",
"text_config": {
"model_type": "mammothmoda2_qwen2_5_vl_text",
"hidden_size": 8,
"gen_vocab_start_index": 100,
},
},
"gen_vae_config": {"block_out_channels": [8, 8]},
"gen_dit_config": {"hidden_size": 8, "in_channels": 4},
}
def _od_config() -> OmniDiffusionConfig:
return OmniDiffusionConfig(
model="/models/MammothModa2-Preview",
model_class_name="MammothModa2DiTPipeline",
tf_model_config=TransformerConfig.from_dict(_raw_config()),
)
def test_build_mammoth_config_uses_shared_transformer_projection() -> None:
config = _build_mammoth_config(_od_config())
assert config.model_type == "mammothmoda2"
assert config.get_text_config().hidden_size == 8
assert config.gen_dit_config == {"hidden_size": 8, "in_channels": 4}
def test_build_mammoth_config_rejects_empty_shared_projection() -> None:
config = _od_config()
config.tf_model_config = TransformerConfig()
with pytest.raises(ValueError, match="root checkpoint config"):
_build_mammoth_config(config)
def test_root_weight_source_loads_combined_checkpoint_once() -> None:
source = _root_weight_source(_od_config())
assert source.model_or_path == "/models/MammothModa2-Preview"
assert source.subfolder is None
assert source.prefix == ""
assert source.fall_back_to_pt is True
def test_root_weight_source_forwards_revision() -> None:
config = _od_config()
config.revision = "rev-7"
assert _root_weight_source(config).revision == "rev-7"
def test_pipeline_declares_native_components_and_single_request_mode_only() -> None:
assert MammothModa2DiTPipeline._dit_modules == ["gen_transformer"]
assert MammothModa2DiTPipeline._encoder_modules == ["gen_image_condition_refiner"]
assert MammothModa2DiTPipeline._vae_modules == ["gen_vae"]
assert MammothModa2DiTPipeline.supports_request_batch is False
assert MammothModa2DiTPipeline.supports_step_execution is False
def test_mammoth_postprocess_denormalizes_nonnegative_raw_vae_output() -> None:
factory = getattr(pipeline_mammothmoda2_dit, "get_mammoth_moda2_post_process_func", None)
assert factory is not None
images = factory(_od_config())(torch.zeros(1, 3, 2, 2))
assert len(images) == 1
assert images[0].getpixel((0, 0)) == (128, 128, 128)
def test_mammoth_postprocess_is_registered() -> None:
from vllm_omni.diffusion.registry import _DIFFUSION_POST_PROCESS_FUNCS
assert _DIFFUSION_POST_PROCESS_FUNCS["MammothModa2DiTPipeline"] == "get_mammoth_moda2_post_process_func"
def test_root_weight_source_rejects_missing_model_path() -> None:
config = _od_config()
config.model = None
with pytest.raises(ValueError, match="model path"):
_root_weight_source(config)
def _pipeline_shell() -> MammothModa2DiTPipeline:
pipeline = object.__new__(MammothModa2DiTPipeline)
nn.Module.__init__(pipeline)
pipeline.device = torch.device("cpu")
pipeline.config = _build_mammoth_config(_od_config())
pipeline._llm_hidden_size = 8
return pipeline
def _batch(
*,
request_id: str = "req-7",
prompt: object | None = None,
sampling: OmniDiffusionSamplingParams | None = None,
) -> DiffusionRequestBatch:
if prompt is None:
prompt = {
"prompt": "",
"height": 32,
"width": 48,
"additional_information": {
"full_hidden_states": torch.arange(32, dtype=torch.float32).reshape(4, 8),
"full_token_ids": [10, 11, 100, 101],
"answer_start_index": 2,
},
}
if sampling is None:
sampling = OmniDiffusionSamplingParams(
height=32,
width=48,
seed=42,
guidance_scale=4.0,
num_inference_steps=7,
extra_args={"cfg_range": [0.2, 0.8]},
)
return DiffusionRequestBatch([OmniDiffusionRequest(prompt=prompt, sampling_params=sampling, request_id=request_id)])
def test_parse_request_uses_standard_sampling_fields() -> None:
parsed = _pipeline_shell()._parse_request(_batch())
assert parsed.request_id == "req-7"
assert (parsed.height, parsed.width) == (32, 48)
assert parsed.num_inference_steps == 7
assert parsed.text_guidance_scale == 4.0
assert parsed.cfg_range == (0.2, 0.8)
assert parsed.seed == 42
assert parsed.answer_start_index == 2
def test_parse_request_prefers_legacy_sampling_overrides() -> None:
sampling = OmniDiffusionSamplingParams(
guidance_scale=3.0,
num_inference_steps=5,
seed=9,
extra_args={"text_guidance_scale": 6.0, "num_inference_steps": 11, "cfg_range": [0.0, 0.5]},
)
parsed = _pipeline_shell()._parse_request(_batch(sampling=sampling))
assert parsed.text_guidance_scale == 6.0
assert parsed.num_inference_steps == 11
assert parsed.cfg_range == (0.0, 0.5)
def test_parse_request_falls_back_to_request_level_sampling_values() -> None:
prompt = _batch().prompts[0]
prompt["additional_information"].update(
text_guidance_scale=[1.5],
num_inference_steps=[3],
cfg_range=[0.25, 0.75],
)
parsed = _pipeline_shell()._parse_request(_batch(prompt=prompt, sampling=OmniDiffusionSamplingParams()))
assert parsed.text_guidance_scale == 1.5
assert parsed.num_inference_steps == 3
assert parsed.cfg_range == (0.25, 0.75)
def test_parse_request_standard_fields_precede_request_level_fallbacks() -> None:
prompt = _batch().prompts[0]
prompt["additional_information"].update(
text_guidance_scale=[1.5],
num_inference_steps=[3],
)
sampling = OmniDiffusionSamplingParams(guidance_scale=4.0, num_inference_steps=7)
parsed = _pipeline_shell()._parse_request(_batch(prompt=prompt, sampling=sampling))
assert parsed.text_guidance_scale == 4.0
assert parsed.num_inference_steps == 7
def test_parse_request_rejects_multiple_requests_and_outputs() -> None:
pipeline = _pipeline_shell()
batch = _batch()
batch.requests.append(_batch(request_id="req-8").requests[0])
with pytest.raises(ValueError, match="exactly one request"):
pipeline._parse_request(batch)
with pytest.raises(ValueError, match="num_outputs_per_prompt=1"):
pipeline._parse_request(_batch(sampling=OmniDiffusionSamplingParams(num_outputs_per_prompt=2)))
@pytest.mark.parametrize(
("cfg_range", "message"),
[
([0.5], "two values"),
([-0.1, 0.5], "0 <= start <= end <= 1"),
([0.7, 0.2], "0 <= start <= end <= 1"),
([0.2, 1.1], "0 <= start <= end <= 1"),
],
)
def test_parse_request_rejects_invalid_cfg_range(cfg_range, message) -> None:
batch = _batch(sampling=OmniDiffusionSamplingParams(extra_args={"cfg_range": cfg_range}))
with pytest.raises(ValueError, match=message):
_pipeline_shell()._parse_request(batch)
def test_parse_request_requires_real_ar_conditions() -> None:
with pytest.raises(ValueError, match="req-missing"):
_pipeline_shell()._parse_request(_batch(request_id="req-missing", prompt={"prompt": "draw a cat"}))
def test_parse_request_rejects_hidden_state_token_count_mismatch() -> None:
batch = _batch()
batch.prompts[0]["additional_information"]["full_hidden_states"] = torch.zeros(3, 8)
with pytest.raises(ValueError, match="hidden-state/token-count mismatch"):
_pipeline_shell()._parse_request(batch)
@pytest.mark.parametrize(
("height", "width", "message"),
[
(0, 32, "Invalid image size"),
(32, -1, "Invalid image size"),
(30, 32, "multiples of 16"),
(32, 31, "multiples of 16"),
],
)
def test_parse_request_rejects_invalid_dimensions(height, width, message) -> None:
batch = _batch()
batch.prompts[0].update(height=height, width=width)
with pytest.raises(ValueError, match=message):
_pipeline_shell()._parse_request(batch)
def test_parse_request_rejects_explicit_zero_steps() -> None:
with pytest.raises(ValueError, match="num_inference_steps must be positive"):
_pipeline_shell()._parse_request(_batch(sampling=OmniDiffusionSamplingParams(num_inference_steps=0)))
@pytest.mark.parametrize(("standard_steps", "expected_steps"), [(7, 7), (None, 50)])
def test_parse_request_falls_through_null_legacy_step_count(standard_steps, expected_steps) -> None:
sampling = OmniDiffusionSamplingParams(num_inference_steps=standard_steps, extra_args={"num_inference_steps": None})
parsed = _pipeline_shell()._parse_request(_batch(sampling=sampling))
assert parsed.num_inference_steps == expected_steps
@pytest.mark.parametrize(("standard_guidance", "expected_guidance"), [(4.0, 4.0), (None, 9.0)])
def test_parse_request_falls_through_null_legacy_guidance(standard_guidance, expected_guidance) -> None:
sampling = OmniDiffusionSamplingParams(guidance_scale=standard_guidance, extra_args={"text_guidance_scale": None})
parsed = _pipeline_shell()._parse_request(_batch(sampling=sampling))
assert parsed.text_guidance_scale == expected_guidance
def test_parse_request_defaults_null_cfg_range() -> None:
sampling = OmniDiffusionSamplingParams(extra_args={"cfg_range": None})
parsed = _pipeline_shell()._parse_request(_batch(sampling=sampling))
assert parsed.cfg_range == (0.0, 1.0)
@pytest.mark.parametrize("generators", [[], [torch.Generator(), torch.Generator()]])
def test_parse_request_rejects_invalid_generator_list_cardinality(generators) -> None:
sampling = OmniDiffusionSamplingParams(generator=generators)
with pytest.raises(ValueError, match="exactly one generator.*req-generators"):
_pipeline_shell()._parse_request(_batch(request_id="req-generators", sampling=sampling))
def test_parse_request_synthesizes_dummy_ar_conditions() -> None:
batch = _batch(
request_id="dummy_req_id",
prompt={"prompt": "dummy run"},
sampling=OmniDiffusionSamplingParams(height=512, width=512, seed=1, guidance_scale=0.0, num_inference_steps=2),
)
parsed = _pipeline_shell()._parse_request(batch)
assert parsed.full_hidden_states.shape == (2, 8)
assert parsed.full_token_ids == [0, 100]
assert parsed.answer_start_index == 1
@dataclass
class _FakeTransformerConfig:
in_channels: int = 4
@dataclass
class _FakeTimeCaptionEmbed:
image_embedder: nn.Module | None = None
class _FakeTransformer(nn.Module):
def __init__(self) -> None:
super().__init__()
self.anchor = nn.Parameter(torch.zeros(1))
self.config = _FakeTransformerConfig()
self.time_caption_embed = _FakeTimeCaptionEmbed()
self.calls = 0
def forward(self, *, hidden_states, **kwargs):
self.calls += 1
return torch.zeros_like(hidden_states)
@dataclass
class _FakeVaeConfig:
scaling_factor: float | None = None
shift_factor: float | None = None
class _FakeVae(nn.Module):
def __init__(self) -> None:
super().__init__()
self.anchor = nn.Parameter(torch.zeros(1))
self.config = _FakeVaeConfig()
def decode(self, latents, return_dict=False):
assert return_dict is False
return (torch.zeros(1, 3, 32, 48, dtype=latents.dtype),)
class _FakeScheduler:
def __init__(self) -> None:
self.timesteps = torch.tensor([])
self.requested_steps = None
def set_timesteps(self, *, num_inference_steps, device, num_tokens):
self.requested_steps = num_inference_steps
self.timesteps = torch.arange(num_inference_steps, device=device, dtype=torch.float32)
def step(self, model_pred, timestep, latents, return_dict=False):
assert return_dict is False
return (latents - model_pred,)
def test_forward_returns_diffusion_output_with_request_sampling(mocker) -> None:
pipeline = _pipeline_shell()
pipeline.gen_transformer = _FakeTransformer()
pipeline.gen_image_condition_refiner = None
pipeline.gen_vae = _FakeVae()
pipeline.gen_freqs_cis = torch.zeros(1)
scheduler = _FakeScheduler()
captured = {}
def fake_randn_tensor(shape, *, generator, device, dtype):
captured["shape"] = shape
captured["seed"] = generator.initial_seed()
return torch.zeros(shape, device=device, dtype=dtype)
module = "vllm_omni.diffusion.models.mammoth_moda2.pipeline_mammothmoda2_dit"
mocker.patch(f"{module}.FlowMatchEulerDiscreteScheduler", return_value=scheduler)
mocker.patch(f"{module}.randn_tensor", side_effect=fake_randn_tensor)
result = pipeline.forward(
_batch(
sampling=OmniDiffusionSamplingParams(
height=32, width=48, seed=42, guidance_scale=1.0, num_inference_steps=2
)
)
)
assert isinstance(result, DiffusionOutput)
assert result.output.shape == (1, 3, 32, 48)
assert captured["shape"] == (1, 4, 4, 6)
assert captured["seed"] == 42
assert scheduler.requested_steps == 2
assert pipeline.gen_transformer.calls == 2
def test_forward_rejects_missing_visual_tokens_before_model_access() -> None:
prompt = {
"prompt": "",
"additional_information": {
"full_hidden_states": torch.zeros(3, 8),
"full_token_ids": [10, 11, 12],
"answer_start_index": 2,
},
}
with pytest.raises(ValueError, match="no visual-token hidden states.*req-empty"):
_pipeline_shell().forward(_batch(request_id="req-empty", prompt=prompt))
@@ -4,8 +4,9 @@
"""
End-to-end test for MammothModa2 text-to-image generation.
Verifies that the AR->DiT pipeline produces an image tensor whose pixel values
match a golden reference.
Verifies that the AR->DiT pipeline produces a valid PIL image. When an
optional golden fixture is present, the test also compares fixed pixel samples;
the portable golden/stability oracle remains tracked in #7090.
Model Hub repo id: ``bytedance-research/MammothModa2-Preview``.
Deploy config: ``get_deploy_config_path("mammoth_moda2.yaml")`` -> ``vllm_omni/deploy/mammoth_moda2.yaml``
@@ -20,13 +21,17 @@ import json
import os
from pathlib import Path
import numpy as np
import pytest
import torch
from PIL import Image
from vllm.sampling_params import SamplingParams
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from tests.helpers.stage_config import get_deploy_config_path
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
from vllm_omni.transformers_utils.repo_utils import hf_api
# ---------------------------------------------------------------------------
@@ -43,7 +48,7 @@ T2I_DEPLOY_CONFIG = get_deploy_config_path("mammoth_moda2.yaml")
_OMNI_RUNNER_PARAM = (MODEL_PATH, T2I_DEPLOY_CONFIG)
# Golden pixel reference file. Set UPDATE_GOLDEN=1 to regenerate.
# Optional golden pixel reference file. Set UPDATE_GOLDEN=1 to regenerate.
_GOLDEN_T2I_PATH = Path(__file__).parent / "fixtures" / "mammoth_moda2_t2i_golden.json"
# Fixed sampling coordinates: (channel, row_fraction, col_fraction)
# Covers corners, centre, and mid-edges across all 3 channels.
@@ -98,17 +103,33 @@ def _sample_pixels(img_tensor: torch.Tensor) -> list[float]:
return values
# ---------------------------------------------------------------------------
# End-to-end test
# ---------------------------------------------------------------------------
pytestmark = [
pytest.mark.slow,
pytest.mark.diffusion,
pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True),
]
def _iter_images(outputs: list[object]):
"""Yield images from shared diffusion ``OmniRequestOutput`` objects."""
for out in outputs:
ro_list = out if isinstance(out, list) else [out]
for ro in ro_list:
images = getattr(ro, "images", None)
if isinstance(images, list):
yield from images
@pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/7718")
def _pil_to_tensor(image: Image.Image) -> torch.Tensor:
array = np.asarray(image, dtype=np.float32) / 255.0
return torch.from_numpy(array).permute(2, 0, 1)
@pytest.mark.cpu
def test_diffusion_output_exposes_images_at_top_level():
image = Image.new("RGB", (16, 16), "black")
output = OmniRequestOutput.from_diffusion(request_id="diffusion-test", images=[image])
assert output.outputs == []
assert list(_iter_images([output])) == [image]
@pytest.mark.slow
@pytest.mark.diffusion
@pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True)
@hardware_test(res={"cuda": "H100"})
def test_mammothmoda2_t2i_e2e(omni_runner: OmniRunner):
"""
@@ -116,9 +137,9 @@ def test_mammothmoda2_t2i_e2e(omni_runner: OmniRunner):
Verifies:
- Omni pipeline initialises with the two-stage YAML config.
- DiT stage outputs an image tensor with the correct shape.
- A fixed set of pixel values matches a golden reference
(regenerate with ``UPDATE_GOLDEN=1``).
- DiT stage outputs one RGB PIL image at the requested size.
- When the optional fixture exists, fixed pixel samples match its golden
reference (regenerate with ``UPDATE_GOLDEN=1``).
"""
gen_cfg = _load_t2i_gen_config(MODEL_PATH)
eol_token_id = int(gen_cfg["eol_token_id"])
@@ -139,7 +160,14 @@ def test_mammothmoda2_t2i_e2e(omni_runner: OmniRunner):
max_tokens=max(1, expected_grid_tokens + 1),
detokenize=False,
)
dit_sampling = SamplingParams(temperature=0.0, max_tokens=1, detokenize=False)
dit_sampling = OmniDiffusionSamplingParams(
height=height,
width=width,
seed=42,
guidance_scale=1.0,
num_inference_steps=2,
extra_args={"cfg_range": [0.0, 1.0]},
)
outputs = list(
omni.generate(
@@ -155,9 +183,6 @@ def test_mammothmoda2_t2i_e2e(omni_runner: OmniRunner):
"visual_token_end_id": [visual_end],
"image_height": [height],
"image_width": [width],
"num_inference_steps": [2],
"text_guidance_scale": [1.0],
"cfg_range": [0.0, 1.0],
"visual_ids": [
_IMAGE_TOKEN_ID,
_VIDEO_TOKEN_ID,
@@ -173,35 +198,21 @@ def test_mammothmoda2_t2i_e2e(omni_runner: OmniRunner):
assert len(outputs) > 0, "Pipeline produced no outputs"
found_image = False
for out in outputs:
ro_list = out
if not isinstance(ro_list, list):
ro_list = [ro_list]
for ro in ro_list:
completion_outputs = getattr(ro, "outputs", None)
if not isinstance(completion_outputs, list):
continue
for completion in completion_outputs:
mm = getattr(completion, "multimodal_output", None)
if not (isinstance(mm, dict) and "image" in mm):
continue
img_list = mm["image"] if isinstance(mm["image"], list) else [mm["image"]]
for img_tensor in img_list:
assert isinstance(img_tensor, torch.Tensor), f"Expected image tensor, got {type(img_tensor)}"
assert img_tensor.ndim in (3, 4), f"Expected 3D or 4D image tensor, got {img_tensor.ndim}D"
images = list(_iter_images(outputs))
assert len(images) == 1, f"Expected exactly one image, got {len(images)}"
sampled = _sample_pixels(img_tensor)
image = images[0]
assert isinstance(image, Image.Image), f"Expected PIL image, got {type(image)}"
assert image.mode == "RGB"
assert image.size == (width, height)
if os.environ.get("UPDATE_GOLDEN"):
_GOLDEN_T2I_PATH.parent.mkdir(parents=True, exist_ok=True)
_GOLDEN_T2I_PATH.write_text(json.dumps({"pixels": sampled}, indent=2))
print(f"\nGolden file written to {_GOLDEN_T2I_PATH}")
elif _GOLDEN_T2I_PATH.exists():
golden = json.loads(_GOLDEN_T2I_PATH.read_text())["pixels"]
for i, (got, exp) in enumerate(zip(sampled, golden)):
assert abs(got - exp) < 1e-4, f"Pixel {i} mismatch: got {got}, expected {exp}"
sampled = _sample_pixels(_pil_to_tensor(image))
found_image = True
assert found_image, "No image tensor found in pipeline output"
if os.environ.get("UPDATE_GOLDEN"):
_GOLDEN_T2I_PATH.parent.mkdir(parents=True, exist_ok=True)
_GOLDEN_T2I_PATH.write_text(json.dumps({"pixels": sampled}, indent=2))
print(f"\nGolden file written to {_GOLDEN_T2I_PATH}")
elif _GOLDEN_T2I_PATH.exists():
golden = json.loads(_GOLDEN_T2I_PATH.read_text())["pixels"]
for i, (got, exp) in enumerate(zip(sampled, golden)):
assert abs(got - exp) < 1e-4, f"Pixel {i} mismatch: got {got}, expected {exp}"
@@ -1,8 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""CPU regression tests for MammothModa2's completed-AR to DiT bridge."""
"""CPU regressions for MammothModa2's completed-AR to diffusion bridge."""
from types import SimpleNamespace
from dataclasses import dataclass
import pytest
import torch
@@ -16,90 +16,129 @@ from vllm_omni.engine.serialization import (
)
from vllm_omni.engine.stage_engine_core_client import StageEngineCoreClient
from vllm_omni.model_executor.models.mammoth_moda2.pipeline import MAMMOTH_MODA2_PIPELINE
from vllm_omni.model_executor.stage_input_processors.mammoth_moda2 import ar2dit
from vllm_omni.model_executor.stage_input_processors.mammoth_moda2 import ar2diffusion
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
_PROMPT_TOKEN_IDS = [7, 8, 9]
_GENERATED_TOKEN_IDS = [101, 102, 103]
@dataclass
class _CompletionOutputStub:
cumulative_token_ids: list[int]
multimodal_output: dict[str, torch.Tensor]
def _hidden_states(rows: int = 5, dtype: torch.dtype = torch.bfloat16) -> torch.Tensor:
return torch.arange(rows * 4, dtype=dtype).reshape(rows, 4)
@dataclass
class _AROutputStub:
request_id: str
prompt_token_ids: list[int]
outputs: list[_CompletionOutputStub]
def _ar_output(hidden_states: torch.Tensor | None = None) -> SimpleNamespace:
multimodal_output = {} if hidden_states is None else {"latent": hidden_states}
return SimpleNamespace(
request_id="mammoth-request",
prompt_token_ids=list(_PROMPT_TOKEN_IDS),
outputs=[
SimpleNamespace(
cumulative_token_ids=list(_GENERATED_TOKEN_IDS),
multimodal_output=multimodal_output,
)
],
@dataclass
class _TextConfigStub:
gen_vocab_start_index: int
@dataclass
class _MammothConfigStub:
llm_config: _TextConfigStub
image_token_id: int
video_token_id: int
vision_start_token_id: int
vision_end_token_id: int
def _source_output(*, include_latent: bool = True) -> _AROutputStub:
multimodal_output = {"latent": torch.arange(32, dtype=torch.float32).reshape(4, 8)} if include_latent else {}
completion = _CompletionOutputStub(
cumulative_token_ids=[100, 101, 102],
multimodal_output=multimodal_output,
)
return _AROutputStub(
request_id="req-7",
prompt_token_ids=[10, 11],
outputs=[completion],
)
def _prompt(
*,
target_h: int | None = None,
target_w: int | None = None,
) -> dict[str, object]:
mm_processor_kwargs: dict[str, int] = {}
if target_h is not None:
mm_processor_kwargs["target_h"] = target_h
if target_w is not None:
mm_processor_kwargs["target_w"] = target_w
return {
"additional_information": {
"image_height": [384],
"image_width": [640],
"text_guidance_scale": [4.5],
"cfg_range": [0.1, 0.9],
"num_inference_steps": [20],
def test_ar2diffusion_builds_one_prompt_with_raw_ar_conditions() -> None:
result = ar2diffusion(
[_source_output()],
{"prompt": "a cat", "mm_processor_kwargs": {"target_h": 512, "target_w": 768}},
)
assert not isinstance(result, list)
assert result["prompt"] == ""
assert result["height"] == 512
assert result["width"] == 768
info = result["additional_information"]
assert info["full_token_ids"] == [10, 11, 100, 101]
assert info["answer_start_index"] == 2
torch.testing.assert_close(
info["full_hidden_states"],
torch.arange(32, dtype=torch.float32).reshape(4, 8),
)
assert info["full_hidden_states"].is_contiguous()
def test_ar2diffusion_uses_prompt_dimension_fallbacks() -> None:
result = ar2diffusion(
[_source_output()],
{"additional_information": {"image_height": [256], "image_width": [384]}},
)
assert (result["height"], result["width"]) == (256, 384)
def test_ar2diffusion_preserves_request_level_sampling_fallbacks() -> None:
result = ar2diffusion(
[_source_output()],
{
"additional_information": {
"text_guidance_scale": [1.5],
"num_inference_steps": [3],
"cfg_range": [0.25, 0.75],
}
},
"mm_processor_kwargs": mm_processor_kwargs,
}
)
info = result["additional_information"]
assert info["text_guidance_scale"] == [1.5]
assert info["num_inference_steps"] == [3]
assert info["cfg_range"] == [0.25, 0.75]
def _assert_complete_dit_input(dit_inputs: list[dict[str, object]], hidden_states: torch.Tensor) -> None:
assert len(dit_inputs) == 1
dit_input = dit_inputs[0]
assert dit_input["prompt_token_ids"] == [0]
additional_information = dit_input["additional_information"]
assert isinstance(additional_information, dict)
full_hidden_states = additional_information["full_hidden_states"]
assert isinstance(full_hidden_states, torch.Tensor)
assert full_hidden_states.dtype == torch.bfloat16
assert full_hidden_states.is_contiguous()
assert torch.equal(full_hidden_states, hidden_states)
assert additional_information["full_token_ids"] == [7, 8, 9, 101, 102]
assert additional_information["answer_start_index"] == [3]
assert additional_information["image_height"] == [384]
assert additional_information["image_width"] == [640]
assert additional_information["text_guidance_scale"] == [4.5]
assert additional_information["cfg_range"] == [0.1, 0.9]
assert additional_information["num_inference_steps"] == [20]
def test_ar2diffusion_unwraps_the_orchestrator_prompt_list() -> None:
result = ar2diffusion(
[_source_output()],
[{"mm_processor_kwargs": {"target_h": 640, "target_w": 960}}],
)
assert (result["height"], result["width"]) == (640, 960)
def test_ar2dit_builds_one_complete_dit_input() -> None:
hidden_states = _hidden_states()
def test_ar2diffusion_rejects_multiple_source_requests() -> None:
with pytest.raises(ValueError, match="exactly one AR output"):
ar2diffusion([_source_output(), _source_output()], {})
dit_inputs = ar2dit([_ar_output(hidden_states)], _prompt())
_assert_complete_dit_input(dit_inputs, hidden_states)
def test_ar2diffusion_reports_missing_latent_with_request_id() -> None:
with pytest.raises(ValueError, match="req-7"):
ar2diffusion([_source_output(include_latent=False)], {})
def test_ar2diffusion_rejects_hidden_state_length_mismatch() -> None:
source = _source_output()
source.outputs[0].multimodal_output["latent"] = torch.zeros(3, 8)
with pytest.raises(ValueError, match="Hidden states length mismatch"):
ar2diffusion([source], {})
@pytest.mark.parametrize("dtype", (torch.float16, torch.bfloat16))
def test_ar2dit_preserves_low_precision_through_engine_core_payload(dtype: torch.dtype) -> None:
hidden_states = _hidden_states(dtype=dtype)
dit_input = ar2dit([_ar_output(hidden_states)], _prompt())[0]
def test_ar2diffusion_preserves_low_precision_through_engine_core_payload(dtype: torch.dtype) -> None:
source = _source_output()
hidden_states = torch.arange(32, dtype=dtype).reshape(4, 8)
source.outputs[0].multimodal_output["latent"] = hidden_states
diffusion_input = ar2diffusion([source], {})
wire_payload = serialize_additional_information(dit_input["additional_information"])
wire_payload = serialize_additional_information(diffusion_input["additional_information"])
assert wire_payload is not None
restored = deserialize_additional_information(wire_payload)
restored_hidden_states = restored["full_hidden_states"]
@@ -112,21 +151,19 @@ def test_ar2dit_preserves_low_precision_through_engine_core_payload(dtype: torch
@pytest.mark.parametrize("dtype", (torch.float16, torch.bfloat16))
def test_dit_condition_split_preserves_compact_transfer_dtype(dtype: torch.dtype) -> None:
# Construct only the config read by _split_ar_conditions. This keeps the
# test CPU-only and avoids loading the DiT/VAE weights.
pipeline = object.__new__(MammothModa2DiTPipeline)
object.__setattr__(
pipeline,
"config",
SimpleNamespace(
llm_config=SimpleNamespace(gen_vocab_start_index=100),
_MammothConfigStub(
llm_config=_TextConfigStub(gen_vocab_start_index=100),
image_token_id=20,
video_token_id=21,
vision_start_token_id=22,
vision_end_token_id=23,
),
)
hidden_states = _hidden_states(dtype=dtype)
hidden_states = torch.arange(20, dtype=dtype).reshape(5, 4)
text_cond, image_cond = pipeline._split_ar_conditions(
full_hidden_states=hidden_states,
@@ -142,44 +179,29 @@ def test_dit_condition_split_preserves_compact_transfer_dtype(dtype: torch.dtype
assert torch.equal(image_cond, hidden_states[[3, 4]])
def test_ar2dit_prefers_processor_image_dimensions() -> None:
hidden_states = _hidden_states()
dit_inputs = ar2dit([_ar_output(hidden_states)], _prompt(target_h=512, target_w=768))
additional_information = dit_inputs[0]["additional_information"]
assert additional_information["image_height"] == [512]
assert additional_information["image_width"] == [768]
def test_ar2dit_rejects_missing_latent_output() -> None:
with pytest.raises(ValueError, match="missing latent multimodal output"):
ar2dit([_ar_output()], _prompt())
def test_ar2dit_rejects_hidden_token_row_mismatch() -> None:
with pytest.raises(AssertionError, match="Hidden states length mismatch"):
ar2dit([_ar_output(_hidden_states(rows=4))], _prompt())
def test_mammoth_pipeline_uses_standard_completed_ar_forwarding() -> None:
stage0, stage1 = MAMMOTH_MODA2_PIPELINE.stages
assert stage0.custom_process_next_stage_input_func is None
assert stage1.custom_process_input_func.endswith(".ar2dit")
assert stage1.custom_process_input_func.endswith(".ar2diffusion")
assert stage1.sync_process_input_func is None
assert stage1.requires_full_payload_input is False
def test_stage_client_forwards_completed_ar_output_to_mammoth_adapter() -> None:
hidden_states = _hidden_states()
source = _source_output()
client = object.__new__(StageEngineCoreClient)
client.custom_process_input_func = ar2dit
client.custom_process_input_func = ar2diffusion
client.requires_multimodal_data = False
dit_inputs = client.process_engine_inputs(
[_ar_output(hidden_states)],
_prompt(),
diffusion_input = client.process_engine_inputs(
[source],
{"mm_processor_kwargs": {"target_h": 512, "target_w": 768}},
)
_assert_complete_dit_input(dit_inputs, hidden_states)
assert diffusion_input["height"] == 512
assert diffusion_input["width"] == 768
info = diffusion_input["additional_information"]
assert info["full_token_ids"] == [10, 11, 100, 101]
assert info["answer_start_index"] == 2
assert torch.equal(info["full_hidden_states"], source.outputs[0].multimodal_output["latent"])
@@ -28,6 +28,9 @@ EXCLUDED_MODELS = [
"GlmImagePipeline",
"ZImagePipeline",
"OvisImagePipeline",
# Requires upstream AR hidden states and token boundaries; covered by
# dedicated MammothModa2 bridge/pipeline tests and real-checkpoint E2E.
"MammothModa2DiTPipeline",
"WanPipeline",
"WanDMDPipeline",
"WanVACEPipeline",
+1 -1
View File
@@ -397,7 +397,7 @@ class TestLoadCustomFuncSelection(unittest.TestCase):
def test_skips_non_payload_stage_input_processors_for_full_payload_mode(self):
incompatible_paths = [
"vllm_omni.model_executor.stage_input_processors.mimo_audio.llm2code2wav",
"vllm_omni.model_executor.stage_input_processors.mammoth_moda2.ar2dit",
"vllm_omni.model_executor.stage_input_processors.mammoth_moda2.ar2diffusion",
"vllm_omni.model_executor.stage_input_processors.cosyvoice3.text2flow",
"vllm_omni.model_executor.stage_input_processors.glm_image.ar2diffusion",
]