[Model] MiniMax-H3: validate versioned text-conditioning handoff (#6720)

Signed-off-by: hsliuustc0106 <liuhongsheng4@huawei.com>
This commit is contained in:
Hongsheng Liu authored and GitHub committed 2026-09-06 21:45:10 +08:00
1 parent 14d2d00536
commit 715b8b8746
6 files changed
+218 -54

No files matched your search

+5
View File
@@ -1475,6 +1475,11 @@ stages:
deploy = load_deploy_config(Path(get_deploy_config_path("minimax_h3_disaggregated.yaml")))
stages = merge_pipeline_deploy(pipeline, deploy)
assert stages[0].yaml_engine_args["model_arch"] == "MiniMaxH3TextEncoder"
assert stages[1].yaml_engine_args["model_arch"] == "MiniMaxH3Pipeline"
assert stages[0].yaml_runtime["num_replicas"] == 1
assert stages[1].yaml_runtime["num_replicas"] == 1
assert stages[1].yaml_engine_args["model_loaded"] == {"text_encoder": False}
assert stages[0].yaml_engine_args["max_num_seqs"] == 1
assert stages[0].yaml_engine_args["model_path_resolver"].endswith(".resolve_minimax_h3_model_root")
assert stages[1].yaml_engine_args["model_path_resolver"].endswith(".resolve_minimax_h3_diffusion_model_path")
@@ -277,14 +277,16 @@ def test_combined_task_inference_and_transformer_routing():
"component_partition",
"source_partitions",
"expected_tasks",
"load_text_encoder",
),
[
(None, "combined", 2, "FL2VA", ["FL2VA", "Ref2VA"], {"t2va", "fl2va", "ref2va"}),
("fl2va", "fl2va", 1, "FL2VA", ["FL2VA"], {"t2va", "fl2va"}),
("ref2va", "ref2va", 1, "Ref2VA", ["Ref2VA"], {"ref2va"}),
(None, "combined", 2, "FL2VA", ["FL2VA", "Ref2VA"], {"t2va", "fl2va", "ref2va"}, True),
("fl2va", "fl2va", 1, "FL2VA", ["FL2VA"], {"t2va", "fl2va"}, True),
("ref2va", "ref2va", 1, "Ref2VA", ["Ref2VA"], {"ref2va"}, True),
("fl2va", "fl2va", 1, "FL2VA", ["FL2VA"], {"t2va", "fl2va"}, False),
],
)
def test_pipeline_loads_task_selected_dits_and_shared_components_once(
def test_pipeline_loads_task_selected_components_with_encoder_ownership(
monkeypatch,
tmp_path,
task_type,
@@ -293,6 +295,7 @@ def test_pipeline_loads_task_selected_dits_and_shared_components_once(
component_partition,
source_partitions,
expected_tasks,
load_text_encoder,
):
from vllm_omni.diffusion.data import (
DiffusionParallelConfig,
@@ -404,6 +407,11 @@ def test_pipeline_loads_task_selected_dits_and_shared_components_once(
revision=None,
task_type=task_type,
quantization_config=None,
model_loaded={
"transformer": True,
"vae": True,
"text_encoder": load_text_encoder,
},
parallel_config=DiffusionParallelConfig(
cfg_parallel_size=1,
text_encoder_tp_size=1,
@@ -415,34 +423,54 @@ def test_pipeline_loads_task_selected_dits_and_shared_components_once(
assert pipeline.supported_tasks == expected_tasks
assert len(created["dit"]) == expected_dits
component_path = tmp_path / component_partition
assert created["text_encoder"] == [str(component_path / "text_encoder")]
assert created["video_vae"] == [str(component_path / "video_vae")]
assert created["audio_vae"] == [str(component_path / "audio_vae")]
assert len(tokenizer_calls) == 1
assert len(processor_calls) == 1
expected_dit_modules = ["transformer", "transformers_ref"] if expected_dits == 2 else ["transformer"]
assert pipeline._dit_modules == expected_dit_modules
assert hasattr(pipeline, "transformers_ref") is (expected_dits == 2)
if load_text_encoder:
assert created["text_encoder"] == [str(component_path / "text_encoder")]
assert len(tokenizer_calls) == 1
assert len(processor_calls) == 1
assert pipeline.text_encoder is not None
assert pipeline.tokenizer is not None
assert pipeline.processor is not None
assert pipeline._encoder_modules == ["text_encoder"]
assert pipeline.text_encoder_tp_size == 1
else:
assert created["text_encoder"] == []
assert tokenizer_calls == []
assert processor_calls == []
assert pipeline.text_encoder is None
assert pipeline.tokenizer is None
assert pipeline.processor is None
assert pipeline._encoder_modules == []
assert pipeline.text_encoder_tp_size == 0
assert pipeline.text_encoder_group is None
if expected_partition == "ref2va":
assert pipeline._transformer_for_task("ref2va") is pipeline.transformer
assert [source.model_or_path for source in pipeline.weights_sources] == [
*(str(tmp_path / partition_name) for partition_name in source_partitions),
str(component_path),
*((str(component_path),) if load_text_encoder else ()),
]
assert [source.subfolder for source in pipeline.weights_sources] == [
*("transformer" for _ in source_partitions),
"text_encoder",
*(("text_encoder",) if load_text_encoder else ()),
]
expected_prefixes = ["transformer."]
if expected_dits == 2:
expected_prefixes.append("transformers_ref.")
expected_prefixes.append("text_encoder.")
if load_text_encoder:
expected_prefixes.append("text_encoder.")
assert [source.prefix for source in pipeline.weights_sources] == expected_prefixes
expected_patterns = (
pipeline_module.MINIMAX_H3_DOWNLOAD_PATTERNS
if expected_partition == "combined"
else pipeline_module.MINIMAX_H3_TASK_DOWNLOAD_PATTERNS[expected_partition]
)
if load_text_encoder:
expected_patterns = (
pipeline_module.MINIMAX_H3_DOWNLOAD_PATTERNS
if expected_partition == "combined"
else pipeline_module.MINIMAX_H3_TASK_DOWNLOAD_PATTERNS[expected_partition]
)
else:
expected_patterns = pipeline_module.MINIMAX_H3_DIFFUSION_DOWNLOAD_PATTERNS[expected_partition]
assert download_calls == [
{
"model_name_or_path": "MiniMaxAI/MiniMax-H3",
@@ -8,6 +8,7 @@ import pytest
import torch
from vllm.model_executor.models.qwen3_vl import Qwen3VLForConditionalGeneration
from vllm_omni.model_executor.models.minimax_h3.conditioning import MiniMaxH3TextConditioning
from vllm_omni.model_executor.models.minimax_h3.text_encoder import MiniMaxH3TextEncoder
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -51,12 +52,15 @@ def test_invalid_output_length_consumes_token_tags():
def test_make_omni_output_uses_shared_stage_payload_fields():
model = _model_without_weights()
model._token_tags = torch.tensor([1, 0])
hidden = torch.zeros(2, 5120)
hidden = torch.zeros(2, 5120, dtype=torch.bfloat16)
output = model.make_omni_output(hidden)
assert output.multimodal_outputs is not None
conditioning = MiniMaxH3TextConditioning.from_omni_payload(output.multimodal_outputs)
assert torch.equal(output.multimodal_outputs["hidden_states"]["output"], hidden)
assert torch.equal(output.multimodal_outputs["meta"]["token_role_ids"], torch.tensor([[1], [0]]))
assert torch.equal(conditioning.hidden_states, hidden)
assert torch.equal(conditioning.token_tags, torch.tensor([1, 0]))
assert "encoder_hidden_states" not in output.multimodal_outputs
assert "token_tags" not in output.multimodal_outputs
@@ -12,6 +12,7 @@ import pytest
import torch
from PIL import Image
from vllm_omni.data_entry_keys import OmniPayload, deserialize_payload, serialize_payload
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import (
_load_audio,
resolve_minimax_h3_diffusion_model_path,
@@ -24,6 +25,8 @@ from vllm_omni.model_executor.models.minimax_h3.checkpoint import (
from vllm_omni.model_executor.models.minimax_h3.conditioning import (
MINIMAX_H3_CONDITION_LABELS_KEY,
MINIMAX_H3_PRESENTATION_TASK_KEY,
MINIMAX_H3_TEXT_CONDITIONING_SCHEMA,
MiniMaxH3TextConditioning,
)
from vllm_omni.model_executor.models.minimax_h3.preprocessing import (
minimax_h3_ref2va_presentation,
@@ -238,19 +241,24 @@ def _diffusion_prompt() -> dict:
}
def _valid_text_conditioning_payload() -> OmniPayload:
return {
"hidden_states": {
"output": torch.zeros(4, 5120, dtype=torch.bfloat16),
},
"meta": {
"token_role_ids": torch.tensor([[1], [1], [0], [0]], dtype=torch.int64),
},
}
def test_text_encoder2diffusion_reads_hidden_states_output_and_token_role_ids() -> None:
hidden = torch.randn(4, 5120)
token_role_ids = torch.tensor([[1], [1], [0], [0]])
payload = _valid_text_conditioning_payload()
hidden = payload["hidden_states"]["output"]
token_role_ids = payload["meta"]["token_role_ids"]
result = text_encoder2diffusion(
[
_source_output(
{
"hidden_states": {"output": hidden},
"meta": {"token_role_ids": token_role_ids},
}
)
],
[_source_output(payload)],
_diffusion_prompt(),
)
@@ -259,6 +267,29 @@ def test_text_encoder2diffusion_reads_hidden_states_output_and_token_role_ids()
torch.testing.assert_close(conditioning["token_tags"], token_role_ids.squeeze(-1))
def test_text_conditioning_artifact_round_trips_through_omni_payload_wire_format() -> None:
payload = _valid_text_conditioning_payload()
wire = serialize_payload(payload)
assert wire is not None
restored = deserialize_payload(wire)
conditioning = MiniMaxH3TextConditioning.from_omni_payload(restored)
assert MINIMAX_H3_TEXT_CONDITIONING_SCHEMA == "minimax_h3.text_conditioning/v1"
torch.testing.assert_close(
conditioning.hidden_states,
payload["hidden_states"]["output"],
)
torch.testing.assert_close(
conditioning.token_tags,
payload["meta"]["token_role_ids"].squeeze(-1),
)
assert conditioning.hidden_states.dtype == torch.bfloat16
assert conditioning.hidden_states.is_contiguous()
assert conditioning.token_tags.dtype == torch.int64
assert conditioning.token_tags.is_contiguous()
@pytest.mark.parametrize(
("payload", "message"),
[
@@ -280,6 +311,54 @@ def test_text_encoder2diffusion_rejects_invalid_conditioning_payload(payload: di
text_encoder2diffusion([_source_output(payload)], _diffusion_prompt())
@pytest.mark.parametrize(
("case", "message"),
[
("hidden_width", r"hidden_states must have shape \[tokens, 5120\]"),
("hidden_dtype", "hidden_states must have dtype torch.bfloat16"),
("hidden_noncontiguous", "hidden_states must use contiguous strided layout"),
("hidden_non_strided", "hidden_states must use contiguous strided layout"),
("token_dtype", "stage-wire token_role_ids must have dtype torch.int64"),
("token_layout", "stage-wire token_role_ids must use contiguous strided layout"),
("token_count", "token_tags must align with hidden_states"),
("token_value", "token_tags must contain only 0 and 1"),
],
)
def test_text_encoder2diffusion_rejects_h3_v1_contract_mismatch(case: str, message: str) -> None:
payload = _valid_text_conditioning_payload()
if case == "hidden_width":
payload["hidden_states"]["output"] = torch.zeros(4, 5119, dtype=torch.bfloat16)
elif case == "hidden_dtype":
payload["hidden_states"]["output"] = torch.zeros(4, 5120, dtype=torch.float32)
elif case == "hidden_noncontiguous":
payload["hidden_states"]["output"] = torch.empty(5120, 4, dtype=torch.bfloat16).t()
elif case == "hidden_non_strided":
payload["hidden_states"]["output"] = torch.empty(
(4, 5120),
dtype=torch.bfloat16,
layout=torch.sparse_coo,
)
elif case == "token_dtype":
payload["meta"]["token_role_ids"] = torch.tensor([[1], [1], [0], [0]], dtype=torch.int32)
elif case == "token_layout":
payload["meta"]["token_role_ids"] = torch.tensor(
[[1], [0], [1], [0], [1], [0], [1], [0]],
dtype=torch.int64,
)[::2]
elif case == "token_count":
payload["meta"]["token_role_ids"] = torch.tensor([[1], [0], [0]], dtype=torch.int64)
elif case == "token_value":
payload["meta"]["token_role_ids"] = torch.tensor([[1], [1], [2], [0]], dtype=torch.int64)
else: # pragma: no cover - parameterization is exhaustive
raise AssertionError(f"unknown test case: {case}")
with pytest.raises(RuntimeError, match=message) as exc_info:
text_encoder2diffusion([_source_output(payload)], _diffusion_prompt())
assert MINIMAX_H3_TEXT_CONDITIONING_SCHEMA in str(exc_info.value)
def test_ref2va_one_image_tokens_and_tags_match_fused_presentation():
tokenizer = _SegmentTokenizer()
labels = [("image", 1), ("audio", 1)]
@@ -1,4 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""MiniMax H3 text-conditioning contract."""
from __future__ import annotations
@@ -9,14 +10,27 @@ from typing import Any
import torch
MINIMAX_H3_TEXT_CONDITIONING_SCHEMA = "minimax_h3.text_conditioning/v1"
MINIMAX_H3_TEXT_HIDDEN_SIZE = 5120
MINIMAX_H3_PRESENTATION_TASK_KEY = "_minimax_h3_presentation_task"
MINIMAX_H3_CONDITION_LABELS_KEY = "_minimax_h3_condition_labels"
def _validate_contiguous_strided_layout(name: str, tensor: torch.Tensor) -> None:
if tensor.layout != torch.strided:
raise ValueError(
f"{MINIMAX_H3_TEXT_CONDITIONING_SCHEMA}: {name} must use contiguous strided layout, got {tensor.layout}"
)
if not tensor.is_contiguous():
raise ValueError(
f"{MINIMAX_H3_TEXT_CONDITIONING_SCHEMA}: {name} must use contiguous "
f"strided layout, got stride={tuple(tensor.stride())}"
)
@dataclass(frozen=True)
class MiniMaxH3TextConditioning:
"""Layer-50 Qwen3-VL hidden states and their H3 token roles."""
"""``minimax_h3.text_conditioning/v1`` semantic payload."""
hidden_states: torch.Tensor
token_tags: torch.Tensor
@@ -26,24 +40,80 @@ class MiniMaxH3TextConditioning:
cls,
payload: Mapping[str, Any],
) -> MiniMaxH3TextConditioning:
"""Validate the semantic payload consumed by the diffusion stage."""
hidden_states = payload.get("hidden_states")
token_tags = payload.get("token_tags")
if not isinstance(hidden_states, torch.Tensor) or not isinstance(token_tags, torch.Tensor):
raise ValueError("MiniMax H3 conditioning requires hidden_states and token_tags tensors")
raise ValueError(
f"{MINIMAX_H3_TEXT_CONDITIONING_SCHEMA}: conditioning requires hidden_states and token_tags tensors"
)
if hidden_states.ndim != 2 or hidden_states.shape[-1] != MINIMAX_H3_TEXT_HIDDEN_SIZE:
raise ValueError(
"MiniMax H3 hidden_states must have shape "
f"{MINIMAX_H3_TEXT_CONDITIONING_SCHEMA}: hidden_states must have shape "
f"[tokens, {MINIMAX_H3_TEXT_HIDDEN_SIZE}], got {tuple(hidden_states.shape)}"
)
if hidden_states.dtype != torch.bfloat16:
raise ValueError(
f"{MINIMAX_H3_TEXT_CONDITIONING_SCHEMA}: hidden_states must have dtype "
f"torch.bfloat16, got {hidden_states.dtype}"
)
_validate_contiguous_strided_layout("hidden_states", hidden_states)
if token_tags.ndim != 1 or token_tags.shape[0] != hidden_states.shape[0]:
raise ValueError(
"MiniMax H3 token_tags must align with hidden_states, got "
f"{MINIMAX_H3_TEXT_CONDITIONING_SCHEMA}: token_tags must align with hidden_states, got "
f"token_tags={tuple(token_tags.shape)} and hidden_states={tuple(hidden_states.shape)}"
)
if token_tags.dtype != torch.int64:
raise ValueError(
f"{MINIMAX_H3_TEXT_CONDITIONING_SCHEMA}: token_tags must have dtype torch.int64, got {token_tags.dtype}"
)
_validate_contiguous_strided_layout("token_tags", token_tags)
if not torch.all((token_tags == 0) | (token_tags == 1)):
raise ValueError("MiniMax H3 text-encoder token_tags must contain only 0 and 1")
raise ValueError(
f"{MINIMAX_H3_TEXT_CONDITIONING_SCHEMA}: text-encoder token_tags must contain only 0 and 1"
)
return cls(hidden_states=hidden_states, token_tags=token_tags)
@classmethod
def from_omni_payload(
cls,
payload: Mapping[str, Any],
) -> MiniMaxH3TextConditioning:
"""Validate and adapt the existing ``OmniPayload`` stage-wire view."""
hidden_states = payload.get("hidden_states")
if not isinstance(hidden_states, Mapping):
raise ValueError(f"{MINIMAX_H3_TEXT_CONDITIONING_SCHEMA}: text encoder returned no hidden_states payload")
hidden = hidden_states.get("output")
if not isinstance(hidden, torch.Tensor):
raise ValueError(
f"{MINIMAX_H3_TEXT_CONDITIONING_SCHEMA}: text encoder returned no hidden_states.output tensor"
)
meta = payload.get("meta")
if not isinstance(meta, Mapping):
raise ValueError(f"{MINIMAX_H3_TEXT_CONDITIONING_SCHEMA}: text encoder returned no conditioning metadata")
token_role_ids = meta.get("token_role_ids")
if not isinstance(token_role_ids, torch.Tensor):
raise ValueError(f"{MINIMAX_H3_TEXT_CONDITIONING_SCHEMA}: text encoder returned no token_role_ids tensor")
if token_role_ids.ndim != 2 or token_role_ids.shape[-1] != 1:
raise ValueError(
f"{MINIMAX_H3_TEXT_CONDITIONING_SCHEMA}: stage-wire token_role_ids must have shape "
f"[tokens, 1], got {tuple(token_role_ids.shape)}"
)
if token_role_ids.dtype != torch.int64:
raise ValueError(
f"{MINIMAX_H3_TEXT_CONDITIONING_SCHEMA}: stage-wire token_role_ids must have dtype "
f"torch.int64, got {token_role_ids.dtype}"
)
_validate_contiguous_strided_layout("stage-wire token_role_ids", token_role_ids)
return cls.from_payload(
{
"hidden_states": hidden,
"token_tags": token_role_ids.squeeze(-1),
}
)
def to_payload(self) -> dict[str, torch.Tensor]:
return {
"hidden_states": self.hidden_states,
@@ -11,7 +11,6 @@ from collections.abc import Mapping, Sequence
from typing import Any
import numpy as np
import torch
from PIL import Image
from vllm_omni.data_entry_keys import REQUEST_ARTIFACT_DIRS_KEY
@@ -332,29 +331,8 @@ def text_encoder2diffusion(
payload = completion.multimodal_output
if not isinstance(payload, Mapping):
raise RuntimeError("MiniMax H3 text encoder returned no conditioning payload")
hidden_states = payload.get("hidden_states")
if not isinstance(hidden_states, Mapping):
raise RuntimeError("MiniMax H3 text encoder returned no hidden_states payload")
hidden = hidden_states.get("output")
if not isinstance(hidden, torch.Tensor):
raise RuntimeError("MiniMax H3 text encoder returned no hidden_states.output tensor")
meta = payload.get("meta")
if not isinstance(meta, Mapping):
raise RuntimeError("MiniMax H3 text encoder returned no conditioning metadata")
token_tags = meta.get("token_role_ids")
if not isinstance(token_tags, torch.Tensor):
raise RuntimeError("MiniMax H3 text encoder returned no token_role_ids tensor")
if token_tags.ndim != 2 or token_tags.shape[-1] != 1:
raise RuntimeError(
f"MiniMax H3 stage-wire token_tags must have shape [tokens, 1], got {tuple(token_tags.shape)}"
)
try:
conditioning = MiniMaxH3TextConditioning.from_payload(
{
"hidden_states": hidden,
"token_tags": token_tags.squeeze(-1),
}
)
conditioning = MiniMaxH3TextConditioning.from_omni_payload(payload)
except ValueError as exc:
raise RuntimeError(str(exc)) from exc