[Model] Migrate MammothModa2 DiT to the shared diffusion runtime (#7134)
Signed-off-by: levius <2114377220@qq.com> Signed-off-by: Yueqian Lin <linyueqian@outlook.com> Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com> Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
16 files changed
+1046
-427
No files matched your search
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Tests for endpoint restrictions logic."""
|
||||
|
||||
import pytest
|
||||
@@ -44,6 +44,32 @@ def test_restricted_completions_returns_400():
|
||||
assert body["error"]["type"] == "BadRequestError"
|
||||
|
||||
|
||||
def test_restricted_image_edits_returns_400():
|
||||
"""Ensure pipelines can reject image edits independently of generation."""
|
||||
app = FastAPI()
|
||||
|
||||
@app.post("/v1/images/edits")
|
||||
async def existing_handler():
|
||||
return {"ok": True}
|
||||
|
||||
@app.post("/v1/images/generations")
|
||||
async def generations_handler():
|
||||
return {"ok": True}
|
||||
|
||||
restrictions = (EndpointRestriction(OmniServingCapability.IMAGE_EDITS, REJECTION_REASON),)
|
||||
shutdown_unsupported_routes(app, restrictions)
|
||||
|
||||
client = TestClient(app)
|
||||
resp = client.post("/v1/images/edits")
|
||||
assert resp.status_code == 400
|
||||
assert resp.json()["error"]["message"] == REJECTION_REASON
|
||||
|
||||
# The restriction is endpoint-specific and must not remove generations.
|
||||
resp = client.post("/v1/images/generations")
|
||||
assert resp.status_code == 200
|
||||
assert resp.json() == {"ok": True}
|
||||
|
||||
|
||||
def test_unrestricted_completions_not_blocked():
|
||||
"""Ensure that if we don't shutdown any routes, the route stays."""
|
||||
app = _make_app_with_server_route()
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
from vllm_omni.config.endpoint_policy import OmniServingCapability
|
||||
from vllm_omni.config.stage_config import StageExecutionType, load_deploy_config
|
||||
from vllm_omni.diffusion.registry import _DIFFUSION_MODELS
|
||||
from vllm_omni.model_executor.models.mammoth_moda2.pipeline import (
|
||||
MAMMOTH_MODA2_AR_PIPELINE,
|
||||
MAMMOTH_MODA2_PIPELINE,
|
||||
)
|
||||
from vllm_omni.model_executor.models.registry import _OMNI_MODELS
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_mammothmoda2_generation_stage_uses_shared_diffusion_runtime() -> None:
|
||||
ar_stage, dit_stage = MAMMOTH_MODA2_PIPELINE.stages
|
||||
assert ar_stage.execution_type is StageExecutionType.LLM_AR
|
||||
assert dit_stage.execution_type is StageExecutionType.DIFFUSION
|
||||
assert dit_stage.model_arch == "MammothModa2DiTPipeline"
|
||||
assert dit_stage.custom_process_input_func.endswith(".ar2diffusion")
|
||||
assert dit_stage.omni_kv_config == {"need_recv_cache": False}
|
||||
assert dit_stage.input_sources == (0,)
|
||||
assert dit_stage.final_output is True
|
||||
assert dit_stage.final_output_type == "image"
|
||||
|
||||
|
||||
def test_mammothmoda2_dit_is_registered_only_with_diffusion_runtime() -> None:
|
||||
assert _DIFFUSION_MODELS["MammothModa2DiTPipeline"] == (
|
||||
"mammoth_moda2",
|
||||
"pipeline_mammothmoda2_dit",
|
||||
"MammothModa2DiTPipeline",
|
||||
)
|
||||
assert "MammothModa2DiTPipeline" not in _OMNI_MODELS
|
||||
|
||||
|
||||
def test_mammothmoda2_deploy_does_not_shadow_request_sampling_fields() -> None:
|
||||
deploy = load_deploy_config(get_deploy_config_path("mammoth_moda2.yaml"))
|
||||
|
||||
assert deploy.stages[1].default_sampling_params is None
|
||||
|
||||
|
||||
def test_mammothmoda2_rejects_unsupported_image_edits() -> None:
|
||||
restrictions = {
|
||||
restriction.capability: restriction.reason for restriction in MAMMOTH_MODA2_PIPELINE.endpoint_restrictions
|
||||
}
|
||||
|
||||
assert set(restrictions) == {OmniServingCapability.IMAGE_EDITS}
|
||||
assert "img2img" in restrictions[OmniServingCapability.IMAGE_EDITS]
|
||||
assert MAMMOTH_MODA2_AR_PIPELINE.endpoint_restrictions == ()
|
||||
|
||||
|
||||
def test_mammothmoda2_ar_only_topology_is_unchanged() -> None:
|
||||
assert len(MAMMOTH_MODA2_AR_PIPELINE.stages) == 1
|
||||
stage = MAMMOTH_MODA2_AR_PIPELINE.stages[0]
|
||||
assert stage.execution_type is StageExecutionType.LLM_AR
|
||||
assert stage.final_output_type == "text"
|
||||
@@ -95,6 +95,20 @@ def _from_pipeline_key(
|
||||
)
|
||||
|
||||
|
||||
def test_mammothmoda2_diffusion_stage_projects_native_backend_config() -> None:
|
||||
config = _from_pipeline_key(
|
||||
"mammoth_moda2",
|
||||
cli_overrides={"model": "/models/MammothModa2-Preview"},
|
||||
)
|
||||
stage = config.stage_by_id(1)
|
||||
assert isinstance(stage, VllmOmniDiffusionStageConfig)
|
||||
assert stage.diffusion_config.model_class_name == "MammothModa2DiTPipeline"
|
||||
assert stage.diffusion_config.model == "/models/MammothModa2-Preview"
|
||||
assert stage.diffusion_config.step_execution is False
|
||||
assert stage.scheduler_config.max_num_seqs == 1
|
||||
assert stage.connector_config.omni_kv_config == {"need_recv_cache": False}
|
||||
|
||||
|
||||
def test_minimax_h3_text_encoder_tp_targets_only_structured_stage_zero() -> None:
|
||||
config = _from_pipeline_key(
|
||||
"minimax_h3_disaggregated",
|
||||
|
||||
@@ -0,0 +1,392 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig, TransformerConfig
|
||||
from vllm_omni.diffusion.models.mammoth_moda2 import pipeline_mammothmoda2_dit
|
||||
from vllm_omni.diffusion.models.mammoth_moda2.pipeline_mammothmoda2_dit import (
|
||||
MammothModa2DiTPipeline,
|
||||
_build_mammoth_config,
|
||||
_root_weight_source,
|
||||
)
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _raw_config() -> dict:
|
||||
return {
|
||||
"model_type": "mammothmoda2",
|
||||
"llm_config": {
|
||||
"model_type": "mammothmoda2_qwen2_5_vl",
|
||||
"text_config": {
|
||||
"model_type": "mammothmoda2_qwen2_5_vl_text",
|
||||
"hidden_size": 8,
|
||||
"gen_vocab_start_index": 100,
|
||||
},
|
||||
},
|
||||
"gen_vae_config": {"block_out_channels": [8, 8]},
|
||||
"gen_dit_config": {"hidden_size": 8, "in_channels": 4},
|
||||
}
|
||||
|
||||
|
||||
def _od_config() -> OmniDiffusionConfig:
|
||||
return OmniDiffusionConfig(
|
||||
model="/models/MammothModa2-Preview",
|
||||
model_class_name="MammothModa2DiTPipeline",
|
||||
tf_model_config=TransformerConfig.from_dict(_raw_config()),
|
||||
)
|
||||
|
||||
|
||||
def test_build_mammoth_config_uses_shared_transformer_projection() -> None:
|
||||
config = _build_mammoth_config(_od_config())
|
||||
assert config.model_type == "mammothmoda2"
|
||||
assert config.get_text_config().hidden_size == 8
|
||||
assert config.gen_dit_config == {"hidden_size": 8, "in_channels": 4}
|
||||
|
||||
|
||||
def test_build_mammoth_config_rejects_empty_shared_projection() -> None:
|
||||
config = _od_config()
|
||||
config.tf_model_config = TransformerConfig()
|
||||
with pytest.raises(ValueError, match="root checkpoint config"):
|
||||
_build_mammoth_config(config)
|
||||
|
||||
|
||||
def test_root_weight_source_loads_combined_checkpoint_once() -> None:
|
||||
source = _root_weight_source(_od_config())
|
||||
assert source.model_or_path == "/models/MammothModa2-Preview"
|
||||
assert source.subfolder is None
|
||||
assert source.prefix == ""
|
||||
assert source.fall_back_to_pt is True
|
||||
|
||||
|
||||
def test_root_weight_source_forwards_revision() -> None:
|
||||
config = _od_config()
|
||||
config.revision = "rev-7"
|
||||
assert _root_weight_source(config).revision == "rev-7"
|
||||
|
||||
|
||||
def test_pipeline_declares_native_components_and_single_request_mode_only() -> None:
|
||||
assert MammothModa2DiTPipeline._dit_modules == ["gen_transformer"]
|
||||
assert MammothModa2DiTPipeline._encoder_modules == ["gen_image_condition_refiner"]
|
||||
assert MammothModa2DiTPipeline._vae_modules == ["gen_vae"]
|
||||
assert MammothModa2DiTPipeline.supports_request_batch is False
|
||||
assert MammothModa2DiTPipeline.supports_step_execution is False
|
||||
|
||||
|
||||
def test_mammoth_postprocess_denormalizes_nonnegative_raw_vae_output() -> None:
|
||||
factory = getattr(pipeline_mammothmoda2_dit, "get_mammoth_moda2_post_process_func", None)
|
||||
assert factory is not None
|
||||
|
||||
images = factory(_od_config())(torch.zeros(1, 3, 2, 2))
|
||||
|
||||
assert len(images) == 1
|
||||
assert images[0].getpixel((0, 0)) == (128, 128, 128)
|
||||
|
||||
|
||||
def test_mammoth_postprocess_is_registered() -> None:
|
||||
from vllm_omni.diffusion.registry import _DIFFUSION_POST_PROCESS_FUNCS
|
||||
|
||||
assert _DIFFUSION_POST_PROCESS_FUNCS["MammothModa2DiTPipeline"] == "get_mammoth_moda2_post_process_func"
|
||||
|
||||
|
||||
def test_root_weight_source_rejects_missing_model_path() -> None:
|
||||
config = _od_config()
|
||||
config.model = None
|
||||
with pytest.raises(ValueError, match="model path"):
|
||||
_root_weight_source(config)
|
||||
|
||||
|
||||
def _pipeline_shell() -> MammothModa2DiTPipeline:
|
||||
pipeline = object.__new__(MammothModa2DiTPipeline)
|
||||
nn.Module.__init__(pipeline)
|
||||
pipeline.device = torch.device("cpu")
|
||||
pipeline.config = _build_mammoth_config(_od_config())
|
||||
pipeline._llm_hidden_size = 8
|
||||
return pipeline
|
||||
|
||||
|
||||
def _batch(
|
||||
*,
|
||||
request_id: str = "req-7",
|
||||
prompt: object | None = None,
|
||||
sampling: OmniDiffusionSamplingParams | None = None,
|
||||
) -> DiffusionRequestBatch:
|
||||
if prompt is None:
|
||||
prompt = {
|
||||
"prompt": "",
|
||||
"height": 32,
|
||||
"width": 48,
|
||||
"additional_information": {
|
||||
"full_hidden_states": torch.arange(32, dtype=torch.float32).reshape(4, 8),
|
||||
"full_token_ids": [10, 11, 100, 101],
|
||||
"answer_start_index": 2,
|
||||
},
|
||||
}
|
||||
if sampling is None:
|
||||
sampling = OmniDiffusionSamplingParams(
|
||||
height=32,
|
||||
width=48,
|
||||
seed=42,
|
||||
guidance_scale=4.0,
|
||||
num_inference_steps=7,
|
||||
extra_args={"cfg_range": [0.2, 0.8]},
|
||||
)
|
||||
return DiffusionRequestBatch([OmniDiffusionRequest(prompt=prompt, sampling_params=sampling, request_id=request_id)])
|
||||
|
||||
|
||||
def test_parse_request_uses_standard_sampling_fields() -> None:
|
||||
parsed = _pipeline_shell()._parse_request(_batch())
|
||||
assert parsed.request_id == "req-7"
|
||||
assert (parsed.height, parsed.width) == (32, 48)
|
||||
assert parsed.num_inference_steps == 7
|
||||
assert parsed.text_guidance_scale == 4.0
|
||||
assert parsed.cfg_range == (0.2, 0.8)
|
||||
assert parsed.seed == 42
|
||||
assert parsed.answer_start_index == 2
|
||||
|
||||
|
||||
def test_parse_request_prefers_legacy_sampling_overrides() -> None:
|
||||
sampling = OmniDiffusionSamplingParams(
|
||||
guidance_scale=3.0,
|
||||
num_inference_steps=5,
|
||||
seed=9,
|
||||
extra_args={"text_guidance_scale": 6.0, "num_inference_steps": 11, "cfg_range": [0.0, 0.5]},
|
||||
)
|
||||
parsed = _pipeline_shell()._parse_request(_batch(sampling=sampling))
|
||||
assert parsed.text_guidance_scale == 6.0
|
||||
assert parsed.num_inference_steps == 11
|
||||
assert parsed.cfg_range == (0.0, 0.5)
|
||||
|
||||
|
||||
def test_parse_request_falls_back_to_request_level_sampling_values() -> None:
|
||||
prompt = _batch().prompts[0]
|
||||
prompt["additional_information"].update(
|
||||
text_guidance_scale=[1.5],
|
||||
num_inference_steps=[3],
|
||||
cfg_range=[0.25, 0.75],
|
||||
)
|
||||
|
||||
parsed = _pipeline_shell()._parse_request(_batch(prompt=prompt, sampling=OmniDiffusionSamplingParams()))
|
||||
|
||||
assert parsed.text_guidance_scale == 1.5
|
||||
assert parsed.num_inference_steps == 3
|
||||
assert parsed.cfg_range == (0.25, 0.75)
|
||||
|
||||
|
||||
def test_parse_request_standard_fields_precede_request_level_fallbacks() -> None:
|
||||
prompt = _batch().prompts[0]
|
||||
prompt["additional_information"].update(
|
||||
text_guidance_scale=[1.5],
|
||||
num_inference_steps=[3],
|
||||
)
|
||||
sampling = OmniDiffusionSamplingParams(guidance_scale=4.0, num_inference_steps=7)
|
||||
|
||||
parsed = _pipeline_shell()._parse_request(_batch(prompt=prompt, sampling=sampling))
|
||||
|
||||
assert parsed.text_guidance_scale == 4.0
|
||||
assert parsed.num_inference_steps == 7
|
||||
|
||||
|
||||
def test_parse_request_rejects_multiple_requests_and_outputs() -> None:
|
||||
pipeline = _pipeline_shell()
|
||||
batch = _batch()
|
||||
batch.requests.append(_batch(request_id="req-8").requests[0])
|
||||
with pytest.raises(ValueError, match="exactly one request"):
|
||||
pipeline._parse_request(batch)
|
||||
with pytest.raises(ValueError, match="num_outputs_per_prompt=1"):
|
||||
pipeline._parse_request(_batch(sampling=OmniDiffusionSamplingParams(num_outputs_per_prompt=2)))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("cfg_range", "message"),
|
||||
[
|
||||
([0.5], "two values"),
|
||||
([-0.1, 0.5], "0 <= start <= end <= 1"),
|
||||
([0.7, 0.2], "0 <= start <= end <= 1"),
|
||||
([0.2, 1.1], "0 <= start <= end <= 1"),
|
||||
],
|
||||
)
|
||||
def test_parse_request_rejects_invalid_cfg_range(cfg_range, message) -> None:
|
||||
batch = _batch(sampling=OmniDiffusionSamplingParams(extra_args={"cfg_range": cfg_range}))
|
||||
with pytest.raises(ValueError, match=message):
|
||||
_pipeline_shell()._parse_request(batch)
|
||||
|
||||
|
||||
def test_parse_request_requires_real_ar_conditions() -> None:
|
||||
with pytest.raises(ValueError, match="req-missing"):
|
||||
_pipeline_shell()._parse_request(_batch(request_id="req-missing", prompt={"prompt": "draw a cat"}))
|
||||
|
||||
|
||||
def test_parse_request_rejects_hidden_state_token_count_mismatch() -> None:
|
||||
batch = _batch()
|
||||
batch.prompts[0]["additional_information"]["full_hidden_states"] = torch.zeros(3, 8)
|
||||
with pytest.raises(ValueError, match="hidden-state/token-count mismatch"):
|
||||
_pipeline_shell()._parse_request(batch)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("height", "width", "message"),
|
||||
[
|
||||
(0, 32, "Invalid image size"),
|
||||
(32, -1, "Invalid image size"),
|
||||
(30, 32, "multiples of 16"),
|
||||
(32, 31, "multiples of 16"),
|
||||
],
|
||||
)
|
||||
def test_parse_request_rejects_invalid_dimensions(height, width, message) -> None:
|
||||
batch = _batch()
|
||||
batch.prompts[0].update(height=height, width=width)
|
||||
with pytest.raises(ValueError, match=message):
|
||||
_pipeline_shell()._parse_request(batch)
|
||||
|
||||
|
||||
def test_parse_request_rejects_explicit_zero_steps() -> None:
|
||||
with pytest.raises(ValueError, match="num_inference_steps must be positive"):
|
||||
_pipeline_shell()._parse_request(_batch(sampling=OmniDiffusionSamplingParams(num_inference_steps=0)))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("standard_steps", "expected_steps"), [(7, 7), (None, 50)])
|
||||
def test_parse_request_falls_through_null_legacy_step_count(standard_steps, expected_steps) -> None:
|
||||
sampling = OmniDiffusionSamplingParams(num_inference_steps=standard_steps, extra_args={"num_inference_steps": None})
|
||||
parsed = _pipeline_shell()._parse_request(_batch(sampling=sampling))
|
||||
assert parsed.num_inference_steps == expected_steps
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("standard_guidance", "expected_guidance"), [(4.0, 4.0), (None, 9.0)])
|
||||
def test_parse_request_falls_through_null_legacy_guidance(standard_guidance, expected_guidance) -> None:
|
||||
sampling = OmniDiffusionSamplingParams(guidance_scale=standard_guidance, extra_args={"text_guidance_scale": None})
|
||||
parsed = _pipeline_shell()._parse_request(_batch(sampling=sampling))
|
||||
assert parsed.text_guidance_scale == expected_guidance
|
||||
|
||||
|
||||
def test_parse_request_defaults_null_cfg_range() -> None:
|
||||
sampling = OmniDiffusionSamplingParams(extra_args={"cfg_range": None})
|
||||
parsed = _pipeline_shell()._parse_request(_batch(sampling=sampling))
|
||||
assert parsed.cfg_range == (0.0, 1.0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("generators", [[], [torch.Generator(), torch.Generator()]])
|
||||
def test_parse_request_rejects_invalid_generator_list_cardinality(generators) -> None:
|
||||
sampling = OmniDiffusionSamplingParams(generator=generators)
|
||||
with pytest.raises(ValueError, match="exactly one generator.*req-generators"):
|
||||
_pipeline_shell()._parse_request(_batch(request_id="req-generators", sampling=sampling))
|
||||
|
||||
|
||||
def test_parse_request_synthesizes_dummy_ar_conditions() -> None:
|
||||
batch = _batch(
|
||||
request_id="dummy_req_id",
|
||||
prompt={"prompt": "dummy run"},
|
||||
sampling=OmniDiffusionSamplingParams(height=512, width=512, seed=1, guidance_scale=0.0, num_inference_steps=2),
|
||||
)
|
||||
parsed = _pipeline_shell()._parse_request(batch)
|
||||
assert parsed.full_hidden_states.shape == (2, 8)
|
||||
assert parsed.full_token_ids == [0, 100]
|
||||
assert parsed.answer_start_index == 1
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeTransformerConfig:
|
||||
in_channels: int = 4
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeTimeCaptionEmbed:
|
||||
image_embedder: nn.Module | None = None
|
||||
|
||||
|
||||
class _FakeTransformer(nn.Module):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.anchor = nn.Parameter(torch.zeros(1))
|
||||
self.config = _FakeTransformerConfig()
|
||||
self.time_caption_embed = _FakeTimeCaptionEmbed()
|
||||
self.calls = 0
|
||||
|
||||
def forward(self, *, hidden_states, **kwargs):
|
||||
self.calls += 1
|
||||
return torch.zeros_like(hidden_states)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeVaeConfig:
|
||||
scaling_factor: float | None = None
|
||||
shift_factor: float | None = None
|
||||
|
||||
|
||||
class _FakeVae(nn.Module):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.anchor = nn.Parameter(torch.zeros(1))
|
||||
self.config = _FakeVaeConfig()
|
||||
|
||||
def decode(self, latents, return_dict=False):
|
||||
assert return_dict is False
|
||||
return (torch.zeros(1, 3, 32, 48, dtype=latents.dtype),)
|
||||
|
||||
|
||||
class _FakeScheduler:
|
||||
def __init__(self) -> None:
|
||||
self.timesteps = torch.tensor([])
|
||||
self.requested_steps = None
|
||||
|
||||
def set_timesteps(self, *, num_inference_steps, device, num_tokens):
|
||||
self.requested_steps = num_inference_steps
|
||||
self.timesteps = torch.arange(num_inference_steps, device=device, dtype=torch.float32)
|
||||
|
||||
def step(self, model_pred, timestep, latents, return_dict=False):
|
||||
assert return_dict is False
|
||||
return (latents - model_pred,)
|
||||
|
||||
|
||||
def test_forward_returns_diffusion_output_with_request_sampling(mocker) -> None:
|
||||
pipeline = _pipeline_shell()
|
||||
pipeline.gen_transformer = _FakeTransformer()
|
||||
pipeline.gen_image_condition_refiner = None
|
||||
pipeline.gen_vae = _FakeVae()
|
||||
pipeline.gen_freqs_cis = torch.zeros(1)
|
||||
scheduler = _FakeScheduler()
|
||||
captured = {}
|
||||
|
||||
def fake_randn_tensor(shape, *, generator, device, dtype):
|
||||
captured["shape"] = shape
|
||||
captured["seed"] = generator.initial_seed()
|
||||
return torch.zeros(shape, device=device, dtype=dtype)
|
||||
|
||||
module = "vllm_omni.diffusion.models.mammoth_moda2.pipeline_mammothmoda2_dit"
|
||||
mocker.patch(f"{module}.FlowMatchEulerDiscreteScheduler", return_value=scheduler)
|
||||
mocker.patch(f"{module}.randn_tensor", side_effect=fake_randn_tensor)
|
||||
result = pipeline.forward(
|
||||
_batch(
|
||||
sampling=OmniDiffusionSamplingParams(
|
||||
height=32, width=48, seed=42, guidance_scale=1.0, num_inference_steps=2
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
assert isinstance(result, DiffusionOutput)
|
||||
assert result.output.shape == (1, 3, 32, 48)
|
||||
assert captured["shape"] == (1, 4, 4, 6)
|
||||
assert captured["seed"] == 42
|
||||
assert scheduler.requested_steps == 2
|
||||
assert pipeline.gen_transformer.calls == 2
|
||||
|
||||
|
||||
def test_forward_rejects_missing_visual_tokens_before_model_access() -> None:
|
||||
prompt = {
|
||||
"prompt": "",
|
||||
"additional_information": {
|
||||
"full_hidden_states": torch.zeros(3, 8),
|
||||
"full_token_ids": [10, 11, 12],
|
||||
"answer_start_index": 2,
|
||||
},
|
||||
}
|
||||
with pytest.raises(ValueError, match="no visual-token hidden states.*req-empty"):
|
||||
_pipeline_shell().forward(_batch(request_id="req-empty", prompt=prompt))
|
||||
@@ -4,8 +4,9 @@
|
||||
"""
|
||||
End-to-end test for MammothModa2 text-to-image generation.
|
||||
|
||||
Verifies that the AR->DiT pipeline produces an image tensor whose pixel values
|
||||
match a golden reference.
|
||||
Verifies that the AR->DiT pipeline produces a valid PIL image. When an
|
||||
optional golden fixture is present, the test also compares fixed pixel samples;
|
||||
the portable golden/stability oracle remains tracked in #7090.
|
||||
|
||||
Model Hub repo id: ``bytedance-research/MammothModa2-Preview``.
|
||||
Deploy config: ``get_deploy_config_path("mammoth_moda2.yaml")`` -> ``vllm_omni/deploy/mammoth_moda2.yaml``
|
||||
@@ -20,13 +21,17 @@ import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.transformers_utils.repo_utils import hf_api
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -43,7 +48,7 @@ T2I_DEPLOY_CONFIG = get_deploy_config_path("mammoth_moda2.yaml")
|
||||
|
||||
_OMNI_RUNNER_PARAM = (MODEL_PATH, T2I_DEPLOY_CONFIG)
|
||||
|
||||
# Golden pixel reference file. Set UPDATE_GOLDEN=1 to regenerate.
|
||||
# Optional golden pixel reference file. Set UPDATE_GOLDEN=1 to regenerate.
|
||||
_GOLDEN_T2I_PATH = Path(__file__).parent / "fixtures" / "mammoth_moda2_t2i_golden.json"
|
||||
# Fixed sampling coordinates: (channel, row_fraction, col_fraction)
|
||||
# Covers corners, centre, and mid-edges across all 3 channels.
|
||||
@@ -98,17 +103,33 @@ def _sample_pixels(img_tensor: torch.Tensor) -> list[float]:
|
||||
return values
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# End-to-end test
|
||||
# ---------------------------------------------------------------------------
|
||||
pytestmark = [
|
||||
pytest.mark.slow,
|
||||
pytest.mark.diffusion,
|
||||
pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True),
|
||||
]
|
||||
def _iter_images(outputs: list[object]):
|
||||
"""Yield images from shared diffusion ``OmniRequestOutput`` objects."""
|
||||
for out in outputs:
|
||||
ro_list = out if isinstance(out, list) else [out]
|
||||
for ro in ro_list:
|
||||
images = getattr(ro, "images", None)
|
||||
if isinstance(images, list):
|
||||
yield from images
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/7718")
|
||||
def _pil_to_tensor(image: Image.Image) -> torch.Tensor:
|
||||
array = np.asarray(image, dtype=np.float32) / 255.0
|
||||
return torch.from_numpy(array).permute(2, 0, 1)
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_diffusion_output_exposes_images_at_top_level():
|
||||
image = Image.new("RGB", (16, 16), "black")
|
||||
output = OmniRequestOutput.from_diffusion(request_id="diffusion-test", images=[image])
|
||||
|
||||
assert output.outputs == []
|
||||
assert list(_iter_images([output])) == [image]
|
||||
|
||||
|
||||
@pytest.mark.slow
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True)
|
||||
@hardware_test(res={"cuda": "H100"})
|
||||
def test_mammothmoda2_t2i_e2e(omni_runner: OmniRunner):
|
||||
"""
|
||||
@@ -116,9 +137,9 @@ def test_mammothmoda2_t2i_e2e(omni_runner: OmniRunner):
|
||||
|
||||
Verifies:
|
||||
- Omni pipeline initialises with the two-stage YAML config.
|
||||
- DiT stage outputs an image tensor with the correct shape.
|
||||
- A fixed set of pixel values matches a golden reference
|
||||
(regenerate with ``UPDATE_GOLDEN=1``).
|
||||
- DiT stage outputs one RGB PIL image at the requested size.
|
||||
- When the optional fixture exists, fixed pixel samples match its golden
|
||||
reference (regenerate with ``UPDATE_GOLDEN=1``).
|
||||
"""
|
||||
gen_cfg = _load_t2i_gen_config(MODEL_PATH)
|
||||
eol_token_id = int(gen_cfg["eol_token_id"])
|
||||
@@ -139,7 +160,14 @@ def test_mammothmoda2_t2i_e2e(omni_runner: OmniRunner):
|
||||
max_tokens=max(1, expected_grid_tokens + 1),
|
||||
detokenize=False,
|
||||
)
|
||||
dit_sampling = SamplingParams(temperature=0.0, max_tokens=1, detokenize=False)
|
||||
dit_sampling = OmniDiffusionSamplingParams(
|
||||
height=height,
|
||||
width=width,
|
||||
seed=42,
|
||||
guidance_scale=1.0,
|
||||
num_inference_steps=2,
|
||||
extra_args={"cfg_range": [0.0, 1.0]},
|
||||
)
|
||||
|
||||
outputs = list(
|
||||
omni.generate(
|
||||
@@ -155,9 +183,6 @@ def test_mammothmoda2_t2i_e2e(omni_runner: OmniRunner):
|
||||
"visual_token_end_id": [visual_end],
|
||||
"image_height": [height],
|
||||
"image_width": [width],
|
||||
"num_inference_steps": [2],
|
||||
"text_guidance_scale": [1.0],
|
||||
"cfg_range": [0.0, 1.0],
|
||||
"visual_ids": [
|
||||
_IMAGE_TOKEN_ID,
|
||||
_VIDEO_TOKEN_ID,
|
||||
@@ -173,35 +198,21 @@ def test_mammothmoda2_t2i_e2e(omni_runner: OmniRunner):
|
||||
|
||||
assert len(outputs) > 0, "Pipeline produced no outputs"
|
||||
|
||||
found_image = False
|
||||
for out in outputs:
|
||||
ro_list = out
|
||||
if not isinstance(ro_list, list):
|
||||
ro_list = [ro_list]
|
||||
for ro in ro_list:
|
||||
completion_outputs = getattr(ro, "outputs", None)
|
||||
if not isinstance(completion_outputs, list):
|
||||
continue
|
||||
for completion in completion_outputs:
|
||||
mm = getattr(completion, "multimodal_output", None)
|
||||
if not (isinstance(mm, dict) and "image" in mm):
|
||||
continue
|
||||
img_list = mm["image"] if isinstance(mm["image"], list) else [mm["image"]]
|
||||
for img_tensor in img_list:
|
||||
assert isinstance(img_tensor, torch.Tensor), f"Expected image tensor, got {type(img_tensor)}"
|
||||
assert img_tensor.ndim in (3, 4), f"Expected 3D or 4D image tensor, got {img_tensor.ndim}D"
|
||||
images = list(_iter_images(outputs))
|
||||
assert len(images) == 1, f"Expected exactly one image, got {len(images)}"
|
||||
|
||||
sampled = _sample_pixels(img_tensor)
|
||||
image = images[0]
|
||||
assert isinstance(image, Image.Image), f"Expected PIL image, got {type(image)}"
|
||||
assert image.mode == "RGB"
|
||||
assert image.size == (width, height)
|
||||
|
||||
if os.environ.get("UPDATE_GOLDEN"):
|
||||
_GOLDEN_T2I_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
_GOLDEN_T2I_PATH.write_text(json.dumps({"pixels": sampled}, indent=2))
|
||||
print(f"\nGolden file written to {_GOLDEN_T2I_PATH}")
|
||||
elif _GOLDEN_T2I_PATH.exists():
|
||||
golden = json.loads(_GOLDEN_T2I_PATH.read_text())["pixels"]
|
||||
for i, (got, exp) in enumerate(zip(sampled, golden)):
|
||||
assert abs(got - exp) < 1e-4, f"Pixel {i} mismatch: got {got}, expected {exp}"
|
||||
sampled = _sample_pixels(_pil_to_tensor(image))
|
||||
|
||||
found_image = True
|
||||
|
||||
assert found_image, "No image tensor found in pipeline output"
|
||||
if os.environ.get("UPDATE_GOLDEN"):
|
||||
_GOLDEN_T2I_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
_GOLDEN_T2I_PATH.write_text(json.dumps({"pixels": sampled}, indent=2))
|
||||
print(f"\nGolden file written to {_GOLDEN_T2I_PATH}")
|
||||
elif _GOLDEN_T2I_PATH.exists():
|
||||
golden = json.loads(_GOLDEN_T2I_PATH.read_text())["pixels"]
|
||||
for i, (got, exp) in enumerate(zip(sampled, golden)):
|
||||
assert abs(got - exp) < 1e-4, f"Pixel {i} mismatch: got {got}, expected {exp}"
|
||||
@@ -1,8 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""CPU regression tests for MammothModa2's completed-AR to DiT bridge."""
|
||||
"""CPU regressions for MammothModa2's completed-AR to diffusion bridge."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -16,90 +16,129 @@ from vllm_omni.engine.serialization import (
|
||||
)
|
||||
from vllm_omni.engine.stage_engine_core_client import StageEngineCoreClient
|
||||
from vllm_omni.model_executor.models.mammoth_moda2.pipeline import MAMMOTH_MODA2_PIPELINE
|
||||
from vllm_omni.model_executor.stage_input_processors.mammoth_moda2 import ar2dit
|
||||
from vllm_omni.model_executor.stage_input_processors.mammoth_moda2 import ar2diffusion
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
_PROMPT_TOKEN_IDS = [7, 8, 9]
|
||||
_GENERATED_TOKEN_IDS = [101, 102, 103]
|
||||
@dataclass
|
||||
class _CompletionOutputStub:
|
||||
cumulative_token_ids: list[int]
|
||||
multimodal_output: dict[str, torch.Tensor]
|
||||
|
||||
|
||||
def _hidden_states(rows: int = 5, dtype: torch.dtype = torch.bfloat16) -> torch.Tensor:
|
||||
return torch.arange(rows * 4, dtype=dtype).reshape(rows, 4)
|
||||
@dataclass
|
||||
class _AROutputStub:
|
||||
request_id: str
|
||||
prompt_token_ids: list[int]
|
||||
outputs: list[_CompletionOutputStub]
|
||||
|
||||
|
||||
def _ar_output(hidden_states: torch.Tensor | None = None) -> SimpleNamespace:
|
||||
multimodal_output = {} if hidden_states is None else {"latent": hidden_states}
|
||||
return SimpleNamespace(
|
||||
request_id="mammoth-request",
|
||||
prompt_token_ids=list(_PROMPT_TOKEN_IDS),
|
||||
outputs=[
|
||||
SimpleNamespace(
|
||||
cumulative_token_ids=list(_GENERATED_TOKEN_IDS),
|
||||
multimodal_output=multimodal_output,
|
||||
)
|
||||
],
|
||||
@dataclass
|
||||
class _TextConfigStub:
|
||||
gen_vocab_start_index: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class _MammothConfigStub:
|
||||
llm_config: _TextConfigStub
|
||||
image_token_id: int
|
||||
video_token_id: int
|
||||
vision_start_token_id: int
|
||||
vision_end_token_id: int
|
||||
|
||||
|
||||
def _source_output(*, include_latent: bool = True) -> _AROutputStub:
|
||||
multimodal_output = {"latent": torch.arange(32, dtype=torch.float32).reshape(4, 8)} if include_latent else {}
|
||||
completion = _CompletionOutputStub(
|
||||
cumulative_token_ids=[100, 101, 102],
|
||||
multimodal_output=multimodal_output,
|
||||
)
|
||||
return _AROutputStub(
|
||||
request_id="req-7",
|
||||
prompt_token_ids=[10, 11],
|
||||
outputs=[completion],
|
||||
)
|
||||
|
||||
|
||||
def _prompt(
|
||||
*,
|
||||
target_h: int | None = None,
|
||||
target_w: int | None = None,
|
||||
) -> dict[str, object]:
|
||||
mm_processor_kwargs: dict[str, int] = {}
|
||||
if target_h is not None:
|
||||
mm_processor_kwargs["target_h"] = target_h
|
||||
if target_w is not None:
|
||||
mm_processor_kwargs["target_w"] = target_w
|
||||
return {
|
||||
"additional_information": {
|
||||
"image_height": [384],
|
||||
"image_width": [640],
|
||||
"text_guidance_scale": [4.5],
|
||||
"cfg_range": [0.1, 0.9],
|
||||
"num_inference_steps": [20],
|
||||
def test_ar2diffusion_builds_one_prompt_with_raw_ar_conditions() -> None:
|
||||
result = ar2diffusion(
|
||||
[_source_output()],
|
||||
{"prompt": "a cat", "mm_processor_kwargs": {"target_h": 512, "target_w": 768}},
|
||||
)
|
||||
assert not isinstance(result, list)
|
||||
assert result["prompt"] == ""
|
||||
assert result["height"] == 512
|
||||
assert result["width"] == 768
|
||||
info = result["additional_information"]
|
||||
assert info["full_token_ids"] == [10, 11, 100, 101]
|
||||
assert info["answer_start_index"] == 2
|
||||
torch.testing.assert_close(
|
||||
info["full_hidden_states"],
|
||||
torch.arange(32, dtype=torch.float32).reshape(4, 8),
|
||||
)
|
||||
assert info["full_hidden_states"].is_contiguous()
|
||||
|
||||
|
||||
def test_ar2diffusion_uses_prompt_dimension_fallbacks() -> None:
|
||||
result = ar2diffusion(
|
||||
[_source_output()],
|
||||
{"additional_information": {"image_height": [256], "image_width": [384]}},
|
||||
)
|
||||
assert (result["height"], result["width"]) == (256, 384)
|
||||
|
||||
|
||||
def test_ar2diffusion_preserves_request_level_sampling_fallbacks() -> None:
|
||||
result = ar2diffusion(
|
||||
[_source_output()],
|
||||
{
|
||||
"additional_information": {
|
||||
"text_guidance_scale": [1.5],
|
||||
"num_inference_steps": [3],
|
||||
"cfg_range": [0.25, 0.75],
|
||||
}
|
||||
},
|
||||
"mm_processor_kwargs": mm_processor_kwargs,
|
||||
}
|
||||
)
|
||||
|
||||
info = result["additional_information"]
|
||||
assert info["text_guidance_scale"] == [1.5]
|
||||
assert info["num_inference_steps"] == [3]
|
||||
assert info["cfg_range"] == [0.25, 0.75]
|
||||
|
||||
|
||||
def _assert_complete_dit_input(dit_inputs: list[dict[str, object]], hidden_states: torch.Tensor) -> None:
|
||||
assert len(dit_inputs) == 1
|
||||
dit_input = dit_inputs[0]
|
||||
assert dit_input["prompt_token_ids"] == [0]
|
||||
|
||||
additional_information = dit_input["additional_information"]
|
||||
assert isinstance(additional_information, dict)
|
||||
full_hidden_states = additional_information["full_hidden_states"]
|
||||
assert isinstance(full_hidden_states, torch.Tensor)
|
||||
assert full_hidden_states.dtype == torch.bfloat16
|
||||
assert full_hidden_states.is_contiguous()
|
||||
assert torch.equal(full_hidden_states, hidden_states)
|
||||
assert additional_information["full_token_ids"] == [7, 8, 9, 101, 102]
|
||||
assert additional_information["answer_start_index"] == [3]
|
||||
assert additional_information["image_height"] == [384]
|
||||
assert additional_information["image_width"] == [640]
|
||||
assert additional_information["text_guidance_scale"] == [4.5]
|
||||
assert additional_information["cfg_range"] == [0.1, 0.9]
|
||||
assert additional_information["num_inference_steps"] == [20]
|
||||
def test_ar2diffusion_unwraps_the_orchestrator_prompt_list() -> None:
|
||||
result = ar2diffusion(
|
||||
[_source_output()],
|
||||
[{"mm_processor_kwargs": {"target_h": 640, "target_w": 960}}],
|
||||
)
|
||||
assert (result["height"], result["width"]) == (640, 960)
|
||||
|
||||
|
||||
def test_ar2dit_builds_one_complete_dit_input() -> None:
|
||||
hidden_states = _hidden_states()
|
||||
def test_ar2diffusion_rejects_multiple_source_requests() -> None:
|
||||
with pytest.raises(ValueError, match="exactly one AR output"):
|
||||
ar2diffusion([_source_output(), _source_output()], {})
|
||||
|
||||
dit_inputs = ar2dit([_ar_output(hidden_states)], _prompt())
|
||||
|
||||
_assert_complete_dit_input(dit_inputs, hidden_states)
|
||||
def test_ar2diffusion_reports_missing_latent_with_request_id() -> None:
|
||||
with pytest.raises(ValueError, match="req-7"):
|
||||
ar2diffusion([_source_output(include_latent=False)], {})
|
||||
|
||||
|
||||
def test_ar2diffusion_rejects_hidden_state_length_mismatch() -> None:
|
||||
source = _source_output()
|
||||
source.outputs[0].multimodal_output["latent"] = torch.zeros(3, 8)
|
||||
with pytest.raises(ValueError, match="Hidden states length mismatch"):
|
||||
ar2diffusion([source], {})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", (torch.float16, torch.bfloat16))
|
||||
def test_ar2dit_preserves_low_precision_through_engine_core_payload(dtype: torch.dtype) -> None:
|
||||
hidden_states = _hidden_states(dtype=dtype)
|
||||
dit_input = ar2dit([_ar_output(hidden_states)], _prompt())[0]
|
||||
def test_ar2diffusion_preserves_low_precision_through_engine_core_payload(dtype: torch.dtype) -> None:
|
||||
source = _source_output()
|
||||
hidden_states = torch.arange(32, dtype=dtype).reshape(4, 8)
|
||||
source.outputs[0].multimodal_output["latent"] = hidden_states
|
||||
diffusion_input = ar2diffusion([source], {})
|
||||
|
||||
wire_payload = serialize_additional_information(dit_input["additional_information"])
|
||||
wire_payload = serialize_additional_information(diffusion_input["additional_information"])
|
||||
assert wire_payload is not None
|
||||
restored = deserialize_additional_information(wire_payload)
|
||||
restored_hidden_states = restored["full_hidden_states"]
|
||||
@@ -112,21 +151,19 @@ def test_ar2dit_preserves_low_precision_through_engine_core_payload(dtype: torch
|
||||
|
||||
@pytest.mark.parametrize("dtype", (torch.float16, torch.bfloat16))
|
||||
def test_dit_condition_split_preserves_compact_transfer_dtype(dtype: torch.dtype) -> None:
|
||||
# Construct only the config read by _split_ar_conditions. This keeps the
|
||||
# test CPU-only and avoids loading the DiT/VAE weights.
|
||||
pipeline = object.__new__(MammothModa2DiTPipeline)
|
||||
object.__setattr__(
|
||||
pipeline,
|
||||
"config",
|
||||
SimpleNamespace(
|
||||
llm_config=SimpleNamespace(gen_vocab_start_index=100),
|
||||
_MammothConfigStub(
|
||||
llm_config=_TextConfigStub(gen_vocab_start_index=100),
|
||||
image_token_id=20,
|
||||
video_token_id=21,
|
||||
vision_start_token_id=22,
|
||||
vision_end_token_id=23,
|
||||
),
|
||||
)
|
||||
hidden_states = _hidden_states(dtype=dtype)
|
||||
hidden_states = torch.arange(20, dtype=dtype).reshape(5, 4)
|
||||
|
||||
text_cond, image_cond = pipeline._split_ar_conditions(
|
||||
full_hidden_states=hidden_states,
|
||||
@@ -142,44 +179,29 @@ def test_dit_condition_split_preserves_compact_transfer_dtype(dtype: torch.dtype
|
||||
assert torch.equal(image_cond, hidden_states[[3, 4]])
|
||||
|
||||
|
||||
def test_ar2dit_prefers_processor_image_dimensions() -> None:
|
||||
hidden_states = _hidden_states()
|
||||
|
||||
dit_inputs = ar2dit([_ar_output(hidden_states)], _prompt(target_h=512, target_w=768))
|
||||
|
||||
additional_information = dit_inputs[0]["additional_information"]
|
||||
assert additional_information["image_height"] == [512]
|
||||
assert additional_information["image_width"] == [768]
|
||||
|
||||
|
||||
def test_ar2dit_rejects_missing_latent_output() -> None:
|
||||
with pytest.raises(ValueError, match="missing latent multimodal output"):
|
||||
ar2dit([_ar_output()], _prompt())
|
||||
|
||||
|
||||
def test_ar2dit_rejects_hidden_token_row_mismatch() -> None:
|
||||
with pytest.raises(AssertionError, match="Hidden states length mismatch"):
|
||||
ar2dit([_ar_output(_hidden_states(rows=4))], _prompt())
|
||||
|
||||
|
||||
def test_mammoth_pipeline_uses_standard_completed_ar_forwarding() -> None:
|
||||
stage0, stage1 = MAMMOTH_MODA2_PIPELINE.stages
|
||||
|
||||
assert stage0.custom_process_next_stage_input_func is None
|
||||
assert stage1.custom_process_input_func.endswith(".ar2dit")
|
||||
assert stage1.custom_process_input_func.endswith(".ar2diffusion")
|
||||
assert stage1.sync_process_input_func is None
|
||||
assert stage1.requires_full_payload_input is False
|
||||
|
||||
|
||||
def test_stage_client_forwards_completed_ar_output_to_mammoth_adapter() -> None:
|
||||
hidden_states = _hidden_states()
|
||||
source = _source_output()
|
||||
client = object.__new__(StageEngineCoreClient)
|
||||
client.custom_process_input_func = ar2dit
|
||||
client.custom_process_input_func = ar2diffusion
|
||||
client.requires_multimodal_data = False
|
||||
|
||||
dit_inputs = client.process_engine_inputs(
|
||||
[_ar_output(hidden_states)],
|
||||
_prompt(),
|
||||
diffusion_input = client.process_engine_inputs(
|
||||
[source],
|
||||
{"mm_processor_kwargs": {"target_h": 512, "target_w": 768}},
|
||||
)
|
||||
|
||||
_assert_complete_dit_input(dit_inputs, hidden_states)
|
||||
assert diffusion_input["height"] == 512
|
||||
assert diffusion_input["width"] == 768
|
||||
info = diffusion_input["additional_information"]
|
||||
assert info["full_token_ids"] == [10, 11, 100, 101]
|
||||
assert info["answer_start_index"] == 2
|
||||
assert torch.equal(info["full_hidden_states"], source.outputs[0].multimodal_output["latent"])
|
||||
@@ -28,6 +28,9 @@ EXCLUDED_MODELS = [
|
||||
"GlmImagePipeline",
|
||||
"ZImagePipeline",
|
||||
"OvisImagePipeline",
|
||||
# Requires upstream AR hidden states and token boundaries; covered by
|
||||
# dedicated MammothModa2 bridge/pipeline tests and real-checkpoint E2E.
|
||||
"MammothModa2DiTPipeline",
|
||||
"WanPipeline",
|
||||
"WanDMDPipeline",
|
||||
"WanVACEPipeline",
|
||||
|
||||
@@ -397,7 +397,7 @@ class TestLoadCustomFuncSelection(unittest.TestCase):
|
||||
def test_skips_non_payload_stage_input_processors_for_full_payload_mode(self):
|
||||
incompatible_paths = [
|
||||
"vllm_omni.model_executor.stage_input_processors.mimo_audio.llm2code2wav",
|
||||
"vllm_omni.model_executor.stage_input_processors.mammoth_moda2.ar2dit",
|
||||
"vllm_omni.model_executor.stage_input_processors.mammoth_moda2.ar2diffusion",
|
||||
"vllm_omni.model_executor.stage_input_processors.cosyvoice3.text2flow",
|
||||
"vllm_omni.model_executor.stage_input_processors.glm_image.ar2diffusion",
|
||||
]
|
||||
|
||||
Reference in new issue
Block a user