diff --git a/tests/config/test_config_factory.py b/tests/config/test_config_factory.py index b04df0bf4..3723e5523 100644 --- a/tests/config/test_config_factory.py +++ b/tests/config/test_config_factory.py @@ -1475,6 +1475,11 @@ stages: deploy = load_deploy_config(Path(get_deploy_config_path("minimax_h3_disaggregated.yaml"))) stages = merge_pipeline_deploy(pipeline, deploy) + assert stages[0].yaml_engine_args["model_arch"] == "MiniMaxH3TextEncoder" + assert stages[1].yaml_engine_args["model_arch"] == "MiniMaxH3Pipeline" + assert stages[0].yaml_runtime["num_replicas"] == 1 + assert stages[1].yaml_runtime["num_replicas"] == 1 + assert stages[1].yaml_engine_args["model_loaded"] == {"text_encoder": False} assert stages[0].yaml_engine_args["max_num_seqs"] == 1 assert stages[0].yaml_engine_args["model_path_resolver"].endswith(".resolve_minimax_h3_model_root") assert stages[1].yaml_engine_args["model_path_resolver"].endswith(".resolve_minimax_h3_diffusion_model_path") diff --git a/tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py b/tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py index b11473498..cb12ce9e5 100644 --- a/tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py +++ b/tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py @@ -277,14 +277,16 @@ def test_combined_task_inference_and_transformer_routing(): "component_partition", "source_partitions", "expected_tasks", + "load_text_encoder", ), [ - (None, "combined", 2, "FL2VA", ["FL2VA", "Ref2VA"], {"t2va", "fl2va", "ref2va"}), - ("fl2va", "fl2va", 1, "FL2VA", ["FL2VA"], {"t2va", "fl2va"}), - ("ref2va", "ref2va", 1, "Ref2VA", ["Ref2VA"], {"ref2va"}), + (None, "combined", 2, "FL2VA", ["FL2VA", "Ref2VA"], {"t2va", "fl2va", "ref2va"}, True), + ("fl2va", "fl2va", 1, "FL2VA", ["FL2VA"], {"t2va", "fl2va"}, True), + ("ref2va", "ref2va", 1, "Ref2VA", ["Ref2VA"], {"ref2va"}, True), + ("fl2va", "fl2va", 1, "FL2VA", ["FL2VA"], {"t2va", "fl2va"}, False), ], ) -def test_pipeline_loads_task_selected_dits_and_shared_components_once( +def test_pipeline_loads_task_selected_components_with_encoder_ownership( monkeypatch, tmp_path, task_type, @@ -293,6 +295,7 @@ def test_pipeline_loads_task_selected_dits_and_shared_components_once( component_partition, source_partitions, expected_tasks, + load_text_encoder, ): from vllm_omni.diffusion.data import ( DiffusionParallelConfig, @@ -404,6 +407,11 @@ def test_pipeline_loads_task_selected_dits_and_shared_components_once( revision=None, task_type=task_type, quantization_config=None, + model_loaded={ + "transformer": True, + "vae": True, + "text_encoder": load_text_encoder, + }, parallel_config=DiffusionParallelConfig( cfg_parallel_size=1, text_encoder_tp_size=1, @@ -415,34 +423,54 @@ def test_pipeline_loads_task_selected_dits_and_shared_components_once( assert pipeline.supported_tasks == expected_tasks assert len(created["dit"]) == expected_dits component_path = tmp_path / component_partition - assert created["text_encoder"] == [str(component_path / "text_encoder")] assert created["video_vae"] == [str(component_path / "video_vae")] assert created["audio_vae"] == [str(component_path / "audio_vae")] - assert len(tokenizer_calls) == 1 - assert len(processor_calls) == 1 expected_dit_modules = ["transformer", "transformers_ref"] if expected_dits == 2 else ["transformer"] assert pipeline._dit_modules == expected_dit_modules assert hasattr(pipeline, "transformers_ref") is (expected_dits == 2) + if load_text_encoder: + assert created["text_encoder"] == [str(component_path / "text_encoder")] + assert len(tokenizer_calls) == 1 + assert len(processor_calls) == 1 + assert pipeline.text_encoder is not None + assert pipeline.tokenizer is not None + assert pipeline.processor is not None + assert pipeline._encoder_modules == ["text_encoder"] + assert pipeline.text_encoder_tp_size == 1 + else: + assert created["text_encoder"] == [] + assert tokenizer_calls == [] + assert processor_calls == [] + assert pipeline.text_encoder is None + assert pipeline.tokenizer is None + assert pipeline.processor is None + assert pipeline._encoder_modules == [] + assert pipeline.text_encoder_tp_size == 0 + assert pipeline.text_encoder_group is None if expected_partition == "ref2va": assert pipeline._transformer_for_task("ref2va") is pipeline.transformer assert [source.model_or_path for source in pipeline.weights_sources] == [ *(str(tmp_path / partition_name) for partition_name in source_partitions), - str(component_path), + *((str(component_path),) if load_text_encoder else ()), ] assert [source.subfolder for source in pipeline.weights_sources] == [ *("transformer" for _ in source_partitions), - "text_encoder", + *(("text_encoder",) if load_text_encoder else ()), ] expected_prefixes = ["transformer."] if expected_dits == 2: expected_prefixes.append("transformers_ref.") - expected_prefixes.append("text_encoder.") + if load_text_encoder: + expected_prefixes.append("text_encoder.") assert [source.prefix for source in pipeline.weights_sources] == expected_prefixes - expected_patterns = ( - pipeline_module.MINIMAX_H3_DOWNLOAD_PATTERNS - if expected_partition == "combined" - else pipeline_module.MINIMAX_H3_TASK_DOWNLOAD_PATTERNS[expected_partition] - ) + if load_text_encoder: + expected_patterns = ( + pipeline_module.MINIMAX_H3_DOWNLOAD_PATTERNS + if expected_partition == "combined" + else pipeline_module.MINIMAX_H3_TASK_DOWNLOAD_PATTERNS[expected_partition] + ) + else: + expected_patterns = pipeline_module.MINIMAX_H3_DIFFUSION_DOWNLOAD_PATTERNS[expected_partition] assert download_calls == [ { "model_name_or_path": "MiniMaxAI/MiniMax-H3", diff --git a/tests/model_executor/models/test_minimax_h3_token_tags.py b/tests/model_executor/models/test_minimax_h3_token_tags.py index 69ee7cb84..80d22133a 100644 --- a/tests/model_executor/models/test_minimax_h3_token_tags.py +++ b/tests/model_executor/models/test_minimax_h3_token_tags.py @@ -8,6 +8,7 @@ import pytest import torch from vllm.model_executor.models.qwen3_vl import Qwen3VLForConditionalGeneration +from vllm_omni.model_executor.models.minimax_h3.conditioning import MiniMaxH3TextConditioning from vllm_omni.model_executor.models.minimax_h3.text_encoder import MiniMaxH3TextEncoder pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -51,12 +52,15 @@ def test_invalid_output_length_consumes_token_tags(): def test_make_omni_output_uses_shared_stage_payload_fields(): model = _model_without_weights() model._token_tags = torch.tensor([1, 0]) - hidden = torch.zeros(2, 5120) + hidden = torch.zeros(2, 5120, dtype=torch.bfloat16) output = model.make_omni_output(hidden) assert output.multimodal_outputs is not None + conditioning = MiniMaxH3TextConditioning.from_omni_payload(output.multimodal_outputs) assert torch.equal(output.multimodal_outputs["hidden_states"]["output"], hidden) assert torch.equal(output.multimodal_outputs["meta"]["token_role_ids"], torch.tensor([[1], [0]])) + assert torch.equal(conditioning.hidden_states, hidden) + assert torch.equal(conditioning.token_tags, torch.tensor([1, 0])) assert "encoder_hidden_states" not in output.multimodal_outputs assert "token_tags" not in output.multimodal_outputs diff --git a/tests/model_executor/stage_input_processors/test_minimax_h3.py b/tests/model_executor/stage_input_processors/test_minimax_h3.py index 599be81f2..9fda89953 100644 --- a/tests/model_executor/stage_input_processors/test_minimax_h3.py +++ b/tests/model_executor/stage_input_processors/test_minimax_h3.py @@ -12,6 +12,7 @@ import pytest import torch from PIL import Image +from vllm_omni.data_entry_keys import OmniPayload, deserialize_payload, serialize_payload from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import ( _load_audio, resolve_minimax_h3_diffusion_model_path, @@ -24,6 +25,8 @@ from vllm_omni.model_executor.models.minimax_h3.checkpoint import ( from vllm_omni.model_executor.models.minimax_h3.conditioning import ( MINIMAX_H3_CONDITION_LABELS_KEY, MINIMAX_H3_PRESENTATION_TASK_KEY, + MINIMAX_H3_TEXT_CONDITIONING_SCHEMA, + MiniMaxH3TextConditioning, ) from vllm_omni.model_executor.models.minimax_h3.preprocessing import ( minimax_h3_ref2va_presentation, @@ -238,19 +241,24 @@ def _diffusion_prompt() -> dict: } +def _valid_text_conditioning_payload() -> OmniPayload: + return { + "hidden_states": { + "output": torch.zeros(4, 5120, dtype=torch.bfloat16), + }, + "meta": { + "token_role_ids": torch.tensor([[1], [1], [0], [0]], dtype=torch.int64), + }, + } + + def test_text_encoder2diffusion_reads_hidden_states_output_and_token_role_ids() -> None: - hidden = torch.randn(4, 5120) - token_role_ids = torch.tensor([[1], [1], [0], [0]]) + payload = _valid_text_conditioning_payload() + hidden = payload["hidden_states"]["output"] + token_role_ids = payload["meta"]["token_role_ids"] result = text_encoder2diffusion( - [ - _source_output( - { - "hidden_states": {"output": hidden}, - "meta": {"token_role_ids": token_role_ids}, - } - ) - ], + [_source_output(payload)], _diffusion_prompt(), ) @@ -259,6 +267,29 @@ def test_text_encoder2diffusion_reads_hidden_states_output_and_token_role_ids() torch.testing.assert_close(conditioning["token_tags"], token_role_ids.squeeze(-1)) +def test_text_conditioning_artifact_round_trips_through_omni_payload_wire_format() -> None: + payload = _valid_text_conditioning_payload() + + wire = serialize_payload(payload) + + assert wire is not None + restored = deserialize_payload(wire) + conditioning = MiniMaxH3TextConditioning.from_omni_payload(restored) + assert MINIMAX_H3_TEXT_CONDITIONING_SCHEMA == "minimax_h3.text_conditioning/v1" + torch.testing.assert_close( + conditioning.hidden_states, + payload["hidden_states"]["output"], + ) + torch.testing.assert_close( + conditioning.token_tags, + payload["meta"]["token_role_ids"].squeeze(-1), + ) + assert conditioning.hidden_states.dtype == torch.bfloat16 + assert conditioning.hidden_states.is_contiguous() + assert conditioning.token_tags.dtype == torch.int64 + assert conditioning.token_tags.is_contiguous() + + @pytest.mark.parametrize( ("payload", "message"), [ @@ -280,6 +311,54 @@ def test_text_encoder2diffusion_rejects_invalid_conditioning_payload(payload: di text_encoder2diffusion([_source_output(payload)], _diffusion_prompt()) +@pytest.mark.parametrize( + ("case", "message"), + [ + ("hidden_width", r"hidden_states must have shape \[tokens, 5120\]"), + ("hidden_dtype", "hidden_states must have dtype torch.bfloat16"), + ("hidden_noncontiguous", "hidden_states must use contiguous strided layout"), + ("hidden_non_strided", "hidden_states must use contiguous strided layout"), + ("token_dtype", "stage-wire token_role_ids must have dtype torch.int64"), + ("token_layout", "stage-wire token_role_ids must use contiguous strided layout"), + ("token_count", "token_tags must align with hidden_states"), + ("token_value", "token_tags must contain only 0 and 1"), + ], +) +def test_text_encoder2diffusion_rejects_h3_v1_contract_mismatch(case: str, message: str) -> None: + payload = _valid_text_conditioning_payload() + + if case == "hidden_width": + payload["hidden_states"]["output"] = torch.zeros(4, 5119, dtype=torch.bfloat16) + elif case == "hidden_dtype": + payload["hidden_states"]["output"] = torch.zeros(4, 5120, dtype=torch.float32) + elif case == "hidden_noncontiguous": + payload["hidden_states"]["output"] = torch.empty(5120, 4, dtype=torch.bfloat16).t() + elif case == "hidden_non_strided": + payload["hidden_states"]["output"] = torch.empty( + (4, 5120), + dtype=torch.bfloat16, + layout=torch.sparse_coo, + ) + elif case == "token_dtype": + payload["meta"]["token_role_ids"] = torch.tensor([[1], [1], [0], [0]], dtype=torch.int32) + elif case == "token_layout": + payload["meta"]["token_role_ids"] = torch.tensor( + [[1], [0], [1], [0], [1], [0], [1], [0]], + dtype=torch.int64, + )[::2] + elif case == "token_count": + payload["meta"]["token_role_ids"] = torch.tensor([[1], [0], [0]], dtype=torch.int64) + elif case == "token_value": + payload["meta"]["token_role_ids"] = torch.tensor([[1], [1], [2], [0]], dtype=torch.int64) + else: # pragma: no cover - parameterization is exhaustive + raise AssertionError(f"unknown test case: {case}") + + with pytest.raises(RuntimeError, match=message) as exc_info: + text_encoder2diffusion([_source_output(payload)], _diffusion_prompt()) + + assert MINIMAX_H3_TEXT_CONDITIONING_SCHEMA in str(exc_info.value) + + def test_ref2va_one_image_tokens_and_tags_match_fused_presentation(): tokenizer = _SegmentTokenizer() labels = [("image", 1), ("audio", 1)] diff --git a/vllm_omni/model_executor/models/minimax_h3/conditioning.py b/vllm_omni/model_executor/models/minimax_h3/conditioning.py index 8a22c7737..efea25771 100644 --- a/vllm_omni/model_executor/models/minimax_h3/conditioning.py +++ b/vllm_omni/model_executor/models/minimax_h3/conditioning.py @@ -1,4 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """MiniMax H3 text-conditioning contract.""" from __future__ import annotations @@ -9,14 +10,27 @@ from typing import Any import torch +MINIMAX_H3_TEXT_CONDITIONING_SCHEMA = "minimax_h3.text_conditioning/v1" MINIMAX_H3_TEXT_HIDDEN_SIZE = 5120 MINIMAX_H3_PRESENTATION_TASK_KEY = "_minimax_h3_presentation_task" MINIMAX_H3_CONDITION_LABELS_KEY = "_minimax_h3_condition_labels" +def _validate_contiguous_strided_layout(name: str, tensor: torch.Tensor) -> None: + if tensor.layout != torch.strided: + raise ValueError( + f"{MINIMAX_H3_TEXT_CONDITIONING_SCHEMA}: {name} must use contiguous strided layout, got {tensor.layout}" + ) + if not tensor.is_contiguous(): + raise ValueError( + f"{MINIMAX_H3_TEXT_CONDITIONING_SCHEMA}: {name} must use contiguous " + f"strided layout, got stride={tuple(tensor.stride())}" + ) + + @dataclass(frozen=True) class MiniMaxH3TextConditioning: - """Layer-50 Qwen3-VL hidden states and their H3 token roles.""" + """``minimax_h3.text_conditioning/v1`` semantic payload.""" hidden_states: torch.Tensor token_tags: torch.Tensor @@ -26,24 +40,80 @@ class MiniMaxH3TextConditioning: cls, payload: Mapping[str, Any], ) -> MiniMaxH3TextConditioning: + """Validate the semantic payload consumed by the diffusion stage.""" hidden_states = payload.get("hidden_states") token_tags = payload.get("token_tags") if not isinstance(hidden_states, torch.Tensor) or not isinstance(token_tags, torch.Tensor): - raise ValueError("MiniMax H3 conditioning requires hidden_states and token_tags tensors") + raise ValueError( + f"{MINIMAX_H3_TEXT_CONDITIONING_SCHEMA}: conditioning requires hidden_states and token_tags tensors" + ) if hidden_states.ndim != 2 or hidden_states.shape[-1] != MINIMAX_H3_TEXT_HIDDEN_SIZE: raise ValueError( - "MiniMax H3 hidden_states must have shape " + f"{MINIMAX_H3_TEXT_CONDITIONING_SCHEMA}: hidden_states must have shape " f"[tokens, {MINIMAX_H3_TEXT_HIDDEN_SIZE}], got {tuple(hidden_states.shape)}" ) + if hidden_states.dtype != torch.bfloat16: + raise ValueError( + f"{MINIMAX_H3_TEXT_CONDITIONING_SCHEMA}: hidden_states must have dtype " + f"torch.bfloat16, got {hidden_states.dtype}" + ) + _validate_contiguous_strided_layout("hidden_states", hidden_states) if token_tags.ndim != 1 or token_tags.shape[0] != hidden_states.shape[0]: raise ValueError( - "MiniMax H3 token_tags must align with hidden_states, got " + f"{MINIMAX_H3_TEXT_CONDITIONING_SCHEMA}: token_tags must align with hidden_states, got " f"token_tags={tuple(token_tags.shape)} and hidden_states={tuple(hidden_states.shape)}" ) + if token_tags.dtype != torch.int64: + raise ValueError( + f"{MINIMAX_H3_TEXT_CONDITIONING_SCHEMA}: token_tags must have dtype torch.int64, got {token_tags.dtype}" + ) + _validate_contiguous_strided_layout("token_tags", token_tags) if not torch.all((token_tags == 0) | (token_tags == 1)): - raise ValueError("MiniMax H3 text-encoder token_tags must contain only 0 and 1") + raise ValueError( + f"{MINIMAX_H3_TEXT_CONDITIONING_SCHEMA}: text-encoder token_tags must contain only 0 and 1" + ) return cls(hidden_states=hidden_states, token_tags=token_tags) + @classmethod + def from_omni_payload( + cls, + payload: Mapping[str, Any], + ) -> MiniMaxH3TextConditioning: + """Validate and adapt the existing ``OmniPayload`` stage-wire view.""" + hidden_states = payload.get("hidden_states") + if not isinstance(hidden_states, Mapping): + raise ValueError(f"{MINIMAX_H3_TEXT_CONDITIONING_SCHEMA}: text encoder returned no hidden_states payload") + hidden = hidden_states.get("output") + if not isinstance(hidden, torch.Tensor): + raise ValueError( + f"{MINIMAX_H3_TEXT_CONDITIONING_SCHEMA}: text encoder returned no hidden_states.output tensor" + ) + + meta = payload.get("meta") + if not isinstance(meta, Mapping): + raise ValueError(f"{MINIMAX_H3_TEXT_CONDITIONING_SCHEMA}: text encoder returned no conditioning metadata") + token_role_ids = meta.get("token_role_ids") + if not isinstance(token_role_ids, torch.Tensor): + raise ValueError(f"{MINIMAX_H3_TEXT_CONDITIONING_SCHEMA}: text encoder returned no token_role_ids tensor") + if token_role_ids.ndim != 2 or token_role_ids.shape[-1] != 1: + raise ValueError( + f"{MINIMAX_H3_TEXT_CONDITIONING_SCHEMA}: stage-wire token_role_ids must have shape " + f"[tokens, 1], got {tuple(token_role_ids.shape)}" + ) + if token_role_ids.dtype != torch.int64: + raise ValueError( + f"{MINIMAX_H3_TEXT_CONDITIONING_SCHEMA}: stage-wire token_role_ids must have dtype " + f"torch.int64, got {token_role_ids.dtype}" + ) + _validate_contiguous_strided_layout("stage-wire token_role_ids", token_role_ids) + + return cls.from_payload( + { + "hidden_states": hidden, + "token_tags": token_role_ids.squeeze(-1), + } + ) + def to_payload(self) -> dict[str, torch.Tensor]: return { "hidden_states": self.hidden_states, diff --git a/vllm_omni/model_executor/stage_input_processors/minimax_h3.py b/vllm_omni/model_executor/stage_input_processors/minimax_h3.py index 372d675b6..87808adc3 100644 --- a/vllm_omni/model_executor/stage_input_processors/minimax_h3.py +++ b/vllm_omni/model_executor/stage_input_processors/minimax_h3.py @@ -11,7 +11,6 @@ from collections.abc import Mapping, Sequence from typing import Any import numpy as np -import torch from PIL import Image from vllm_omni.data_entry_keys import REQUEST_ARTIFACT_DIRS_KEY @@ -332,29 +331,8 @@ def text_encoder2diffusion( payload = completion.multimodal_output if not isinstance(payload, Mapping): raise RuntimeError("MiniMax H3 text encoder returned no conditioning payload") - hidden_states = payload.get("hidden_states") - if not isinstance(hidden_states, Mapping): - raise RuntimeError("MiniMax H3 text encoder returned no hidden_states payload") - hidden = hidden_states.get("output") - if not isinstance(hidden, torch.Tensor): - raise RuntimeError("MiniMax H3 text encoder returned no hidden_states.output tensor") - meta = payload.get("meta") - if not isinstance(meta, Mapping): - raise RuntimeError("MiniMax H3 text encoder returned no conditioning metadata") - token_tags = meta.get("token_role_ids") - if not isinstance(token_tags, torch.Tensor): - raise RuntimeError("MiniMax H3 text encoder returned no token_role_ids tensor") - if token_tags.ndim != 2 or token_tags.shape[-1] != 1: - raise RuntimeError( - f"MiniMax H3 stage-wire token_tags must have shape [tokens, 1], got {tuple(token_tags.shape)}" - ) try: - conditioning = MiniMaxH3TextConditioning.from_payload( - { - "hidden_states": hidden, - "token_tags": token_tags.squeeze(-1), - } - ) + conditioning = MiniMaxH3TextConditioning.from_omni_payload(payload) except ValueError as exc: raise RuntimeError(str(exc)) from exc