[feat] Extend disaggregated MiniMax-H3 stage with VAE encoders (#6939)

Signed-off-by: asukaqaq-s <1311722138@qq.com>
Signed-off-by: Jader <yjader@foxmail.com>
Co-authored-by: Jader <yjader@foxmail.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
authored and GitHub committed 2026-09-16 16:52:35 +08:00
1 parent e78a5d01f7
commit d554e6b67e
27 files changed
+3663 -2399

No files matched your search

+17 -3
View File
@@ -1562,7 +1562,10 @@ stages:
monkeypatch.setattr(stage_init_utils, "resolve_worker_cls", lambda _engine_args: None)
deploy = load_deploy_config(get_deploy_config_path("minimax_h3_disaggregated.yaml"))
assert deploy.stages[1].engine_extras["model_loaded"] == {"text_encoder": False}
assert deploy.stages[1].engine_extras["model_loaded"] == {
"text_encoder": False,
"vae_encoder": False,
}
stages = merge_pipeline_deploy(OMNI_PIPELINES["minimax_h3_disaggregated"], deploy)
resolved = [stage_init_utils.build_engine_args_dict(stage.to_omegaconf(), str(model_root)) for stage in stages]
@@ -1587,12 +1590,20 @@ stages:
deploy = load_deploy_config(Path(get_deploy_config_path("minimax_h3_disaggregated.yaml")))
stages = merge_pipeline_deploy(pipeline, deploy)
assert stages[0].yaml_engine_args["model_arch"] == "MiniMaxH3TextEncoder"
assert stages[0].yaml_engine_args["model_arch"] == "MiniMaxH3Encoder"
assert stages[1].yaml_engine_args["model_arch"] == "MiniMaxH3Pipeline"
assert stages[0].yaml_runtime["num_replicas"] == 1
assert stages[1].yaml_runtime["num_replicas"] == 1
assert stages[1].yaml_engine_args["model_loaded"] == {"text_encoder": False}
assert stages[1].yaml_engine_args["model_loaded"] == {
"text_encoder": False,
"vae_encoder": False,
}
assert stages[0].yaml_engine_args["max_num_seqs"] == 1
assert stages[0].yaml_engine_args["hf_overrides"]["minimax_h3_encoder_components"] == {
"text_encoder": {"parallel_mode": "tp"},
"video_vae": {"parallel_mode": "patch"},
"audio_vae": {"parallel_mode": "leader"},
}
assert stages[0].yaml_engine_args["model_path_resolver"].endswith(".resolve_minimax_h3_model_root")
assert stages[1].yaml_engine_args["model_path_resolver"].endswith(".resolve_minimax_h3_diffusion_model_path")
parallel = stages[1].yaml_engine_args["parallel_config"]
@@ -1604,6 +1615,9 @@ stages:
turbo = load_deploy_config(Path(get_deploy_config_path("minimax_h3_disaggregated_turbo.yaml")))
turbo_stages = merge_pipeline_deploy(pipeline, turbo)
assert turbo_stages[0].yaml_engine_args["model_arch"] == stages[0].yaml_engine_args["model_arch"]
assert turbo_stages[1].yaml_engine_args["model_loaded"] == stages[1].yaml_engine_args["model_loaded"]
assert turbo_stages[0].yaml_engine_args["hf_overrides"] == stages[0].yaml_engine_args["hf_overrides"]
turbo_sampling = turbo_stages[1].yaml_extras["default_sampling_params"]
assert turbo_sampling["num_inference_steps"] == 5
assert turbo_sampling["extra_args"] == {"flow_shift": 6.0, "audio_flow_shift": 3.0}
@@ -0,0 +1,157 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import os
import time
from datetime import timedelta
from types import SimpleNamespace
import pytest
import torch
import torch.distributed as dist
import torch.multiprocessing as mp
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion]
def _collective_worker(rank, rendezvous, backend):
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as module
from vllm_omni.errors import OmniClientError
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.model_executor.models.minimax_h3.conditioning import MiniMaxH3EncoderMediaInput
from vllm_omni.platforms import current_omni_platform
torch.set_num_threads(1)
device = torch.device("cuda", rank) if backend == "nccl" else torch.device("cpu")
if backend == "nccl":
current_omni_platform.set_device(device)
dist.init_process_group(
backend, init_method=f"file://{rendezvous}", rank=rank, world_size=2, timeout=timedelta(seconds=45)
)
try:
module._dit_rank_world = lambda: (dist.group.WORLD, rank, 2)
class VisualCodec:
fail = False
def is_distributed_enabled(self):
return True
def collective(self):
value = torch.ones(1, device=device)
dist.all_reduce(value)
assert value.item() == 2
def encode_image(self, image):
self.collective()
assert image.size == (32, 32)
return torch.ones(1, 96)
def encode_video(self, frames):
self.collective()
assert frames.shape == (4, 32, 32, 3)
if self.fail and rank == 1:
raise ValueError("codec failed on rank 1")
return torch.full((1, 96), 2.0), (1, 2, 2)
class AudioCodec:
def encode_waveform(self, waveform, sample_rate):
assert rank == 0
assert sample_rate == 32000
assert waveform.shape == (64000,)
return torch.full((160, 32), 3.0), 80
model = object.__new__(module.MiniMaxH3Pipeline)
torch.nn.Module.__init__(model)
model.device = device
model.od_config = SimpleNamespace(enable_layerwise_offload=False)
model.video_vae = VisualCodec()
model.audio_vae = AudioCodec()
model.partition = "combined"
model.supported_tasks = frozenset({"t2va", "fl2va", "ref2va"})
model.text_encoder_tp_size = 2
model.text_encoder_group = SimpleNamespace(
world_size=2, ranks=[0, 1], rank_in_group=rank, device_group=dist.group.WORLD
)
model.processor = SimpleNamespace(image_processor=SimpleNamespace(merge_size=2))
model.tokenizer = lambda text, **kwargs: {"input_ids": [1, 2, 3]}
def encode_text(ids, kwargs):
assert ids.tolist() == [1, 2, 3]
assert not kwargs
hidden = torch.ones(3, 5120, device=device, dtype=torch.bfloat16)
dist.all_reduce(hidden)
return hidden
model._encode_text_hidden = encode_text
sampling = OmniDiffusionSamplingParams(
height=32, width=32, num_frames=96, extra_args={"task": "t2va", "aspect_ratio": "1:1"}
)
text_conditioning = model._prepare_local_conditioning({"prompt": "test"}, sampling)
torch.testing.assert_close(text_conditioning.hidden_states, torch.full_like(text_conditioning.hidden_states, 2))
assert text_conditioning.visual_condition is None
media = (
MiniMaxH3EncoderMediaInput(
task="ref2va",
height=32,
width=32,
num_frames=107,
latent_t=32,
audio_t=178,
images=(torch.zeros(32, 32, 3, dtype=torch.uint8),),
videos=(torch.zeros(4, 32, 32, 3, dtype=torch.uint8),),
video_audios=(None,),
audios=((torch.zeros(64000), 32000),),
)
if rank == 0
else None
)
conditioning = model._encode_local_media(media)
assert [block["kind"] for block in conditioning.ref_blocks] == ["image", "video", "audio"]
torch.testing.assert_close(
conditioning.visual_condition.cpu(), torch.cat([torch.ones(1, 96), torch.full((1, 96), 2.0)])
)
torch.testing.assert_close(conditioning.audio_condition.cpu(), torch.full((160, 32), 3.0))
assert conditioning.audio_condition_lengths == (80,)
model.video_vae.fail = True
with pytest.raises(OmniClientError, match="codec failed on rank 1"):
model._encode_local_media(media)
with pytest.raises(OmniClientError, match="non-empty prompt"):
model._prepare_local_conditioning({"prompt": ""}, sampling)
model.video_vae.fail = False
assert model._encode_local_media(media).visual_condition_shapes == ((1, 2, 2), (1, 2, 2))
finally:
dist.destroy_process_group()
def _run_collectives(tmp_path, backend):
context = mp.spawn(_collective_worker, args=(str(tmp_path / "rendezvous"), backend), nprocs=2, join=False)
deadline = time.monotonic() + 150
try:
while not context.join(timeout=1):
if time.monotonic() > deadline:
pytest.fail("MiniMax H3 collectives did not finish within 150 seconds")
finally:
for process in context.processes:
if process.is_alive():
process.terminate()
process.join(timeout=5)
@pytest.mark.cpu
def test_two_rank_conditioning_and_errors_gloo(tmp_path):
_run_collectives(tmp_path, "gloo")
@pytest.mark.cuda
@pytest.mark.parallel
@pytest.mark.skipif(
os.environ.get("VLLM_TEST_MINIMAX_H3_COLLECTIVES") != "1",
reason="set VLLM_TEST_MINIMAX_H3_COLLECTIVES=1 with two reserved GPUs",
)
def test_two_rank_conditioning_and_errors_nccl(tmp_path):
if torch.accelerator.device_count() < 2:
pytest.skip("requires two GPUs")
_run_collectives(tmp_path, "nccl")
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,133 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from types import SimpleNamespace
from typing import Any
from unittest.mock import Mock
import pytest
import torch
from PIL import Image
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
@pytest.fixture
def pipeline():
model = object.__new__(MiniMaxH3Pipeline)
torch.nn.Module.__init__(model)
model.load_text_encoder = True
model.load_vae_encoder = True
model.device = torch.device("cpu")
model.partition = "combined"
model.supported_tasks = frozenset({"t2va", "fl2va", "ref2va"})
model.default_video_shift = 12.0
model.default_audio_shift = 3.0
model.od_config = SimpleNamespace(enable_layerwise_offload=False)
model._quality_policy = SimpleNamespace(resolve=Mock(return_value=SimpleNamespace(cache_dit=None)))
model._cache_dit_runtime = SimpleNamespace(prepare=Mock())
model.encode_prompt = Mock(
return_value=(torch.ones(2, 5120, dtype=torch.bfloat16), torch.ones(2, dtype=torch.long))
)
model.video_vae = Mock()
model.video_vae.is_distributed_enabled.return_value = False
model.video_vae.encode_image.side_effect = lambda image: torch.ones((image.height // 32) * (image.width // 32), 96)
model.audio_vae = Mock()
model.transformer = torch.nn.Identity()
model.diffuse = Mock(return_value=(torch.zeros(1), torch.zeros(1)))
model.decode = Mock(return_value=(torch.zeros(1, 3, 1, 32, 32), torch.zeros(1)))
return model
def _request(task):
prompt: dict[str, Any] = {"prompt": "A person waves."}
if task != "t2va":
prompt["multi_modal_data"] = {"image": Image.new("RGB", (256, 256))}
sampling = OmniDiffusionSamplingParams(
height=32, width=32, num_frames=96, num_inference_steps=2, extra_args={"task": task, "aspect_ratio": "1:1"}
)
return SimpleNamespace(prompts=[prompt], sampling_params=sampling)
@pytest.mark.parametrize("task", ["t2va", "fl2va", "ref2va"])
def test_single_stage_encodes_without_stage_zero(pipeline, task):
output = pipeline.forward(_request(task))
pipeline.encode_prompt.assert_called_once()
assert pipeline.video_vae.encode_image.call_count == int(task != "t2va")
pipeline.diffuse.assert_called_once()
assert pipeline.diffuse.call_args.kwargs["task"] == task
video, audio = output.output
torch.testing.assert_close(video, torch.zeros(1, 1, 32, 32, 3, dtype=torch.uint8))
torch.testing.assert_close(audio, pipeline.decode.return_value[1])
def test_mixed_reference_conditioning_matches_stage_zero_handoff(pipeline, monkeypatch):
import numpy as np
from vllm_omni.data_entry_keys import flatten_payload
from vllm_omni.model_executor.models.minimax_h3 import encoder_processing as processing
from vllm_omni.model_executor.models.minimax_h3.conditioning import (
MiniMaxH3EncoderConditioning,
MiniMaxH3TextConditioning,
)
from vllm_omni.model_executor.models.minimax_h3.encoder import MiniMaxH3Encoder
from vllm_omni.model_executor.stage_input_processors.minimax_h3 import encoder2diffusion, prepare_encoder_prompt
frames = np.zeros((48, 32, 32, 3), dtype=np.uint8)
decode_frames = Mock(return_value=frames)
sampler = Mock(wraps=processing.sample_reference_video_frames)
monkeypatch.setattr(processing, "load_video_frames", decode_frames)
monkeypatch.setattr(processing, "sample_reference_video_frames", sampler)
monkeypatch.setattr(
processing,
"prepare_reference_videos",
Mock(
return_value=[{"prepared_path": "prepared.mp4", "original_path": "original.mp4", "input_has_audio": True}]
),
)
monkeypatch.setattr(processing, "load_video_audio", Mock(return_value=(torch.full((64000,), 3.0), 32000)))
pipeline.video_vae.encode_video.return_value = (torch.full((1, 96), 2.0), (1, 2, 2))
def encode_audio(waveform, sample_rate):
length = round(waveform.shape[-1] / sample_rate * 40)
return torch.full((length * 2, 32), float(waveform[0])), length
pipeline.audio_vae.encode_waveform.side_effect = encode_audio
request = _request("ref2va")
request.prompts[0]["multi_modal_data"].update(video="original.mp4", audio=(torch.full((192000,), 4.0), 32000))
pipeline.forward(request)
local_kwargs = pipeline.diffuse.call_args.kwargs
transformed = prepare_encoder_prompt(request.prompts[0], [request.sampling_params])
stage_zero = object.__new__(MiniMaxH3Encoder)
torch.nn.Module.__init__(stage_zero)
stage_zero._component_leader = True
stage_zero.video_vae = pipeline.video_vae
stage_zero.audio_vae = pipeline.audio_vae
media = stage_zero._encode_media(stage_zero._media_input(transformed["additional_information"]))
text = MiniMaxH3TextConditioning(*pipeline.encode_prompt.return_value)
payload = MiniMaxH3EncoderConditioning.from_components(text, media).to_omni_payload()
source = SimpleNamespace(finished=True, outputs=[SimpleNamespace(multimodal_output=flatten_payload(payload))])
bridged = encoder2diffusion([source], request.prompts[0])
pipeline.load_text_encoder = False
pipeline.load_vae_encoder = False
pipeline.encode_prompt.reset_mock()
pipeline.forward(SimpleNamespace(prompts=[bridged], sampling_params=request.sampling_params))
external_kwargs = pipeline.diffuse.call_args.kwargs
for key, value in local_kwargs.items():
if isinstance(value, torch.Tensor):
torch.testing.assert_close(value, external_kwargs[key])
else:
assert value == external_kwargs[key]
pipeline.encode_prompt.assert_not_called()
assert decode_frames.call_count == sampler.call_count == 2
assert all(call.kwargs["decoded_frames"] is frames for call in sampler.call_args_list)
assert [block["kind"] for block in local_kwargs["ref_blocks"]] == ["image", "video_audio", "audio"]
assert local_kwargs["audio_condition_lengths"] == [80, round(107 / 24 * 40)]
assert all(
call.args[0].shape[-1] == round(107 / 24 * 32000)
for call in pipeline.audio_vae.encode_waveform.call_args_list[1::2]
)
@@ -9,72 +9,25 @@ from unittest.mock import Mock
import pytest
import torch
from PIL import Image
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
def test_h3_profiler_targets_cover_reference_encoders_and_vae_decoders():
def test_h3_full_pipeline_profiler_includes_local_encoding():
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
targets = MiniMaxH3Pipeline._PROFILER_TARGETS
assert "_encode_visual_conditions" in targets
assert "_encode_reference_audio_conditions" in targets
assert "video_vae.decode_latent" in targets
assert "audio_vae.decode_latent" in targets
assert "_encode_video_conditions" not in targets
assert "_encode_video_audio_conditions" not in targets
assert "_encode_audio_conditions" not in targets
@pytest.mark.parametrize("distributed", [False, True])
def test_manual_component_offload_wins_over_requested_model_offload(distributed):
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
pipeline = object.__new__(MiniMaxH3Pipeline)
torch.nn.Module.__init__(pipeline)
pipeline.od_config = SimpleNamespace(
enable_cpu_offload=True,
enable_layerwise_offload=not distributed,
enable_distributed_layerwise_offload=distributed,
)
pipeline._model_cpu_offload_modules = []
pipeline.text_encoder = Mock()
expected = torch.ones(2, 3)
pipeline.text_encoder.encode_ids.return_value = expected
component = Mock()
actual = pipeline._encode_text_hidden(torch.tensor([1, 2]), {})
with pipeline._component_on_device(component):
component.load_to_device.assert_called_once_with()
component.offload_to_cpu.assert_not_called()
assert actual is expected
pipeline.text_encoder.load_to_device.assert_called_once_with()
pipeline.text_encoder.offload_to_cpu.assert_called_once_with()
component.offload_to_cpu.assert_called_once_with()
def test_requested_model_offload_without_backend_uses_resident_components():
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
pipeline = object.__new__(MiniMaxH3Pipeline)
torch.nn.Module.__init__(pipeline)
pipeline.od_config = SimpleNamespace(
enable_cpu_offload=True,
enable_layerwise_offload=False,
enable_distributed_layerwise_offload=False,
)
pipeline._model_cpu_offload_modules = []
pipeline.text_encoder = Mock()
expected = torch.ones(2, 3)
pipeline.text_encoder.encode_ids.return_value = expected
actual = pipeline._encode_text_hidden(torch.tensor([1, 2]), {})
assert actual is expected
pipeline.text_encoder.load_to_device.assert_called_once_with()
pipeline.text_encoder.encode_ids.assert_called_once()
assert targets == [
"encode_prompt",
"_encode_local_media",
"diffuse",
"decode",
"video_vae.decode_latent",
"audio_vae.decode_latent",
"prepare_encode",
"denoise_step",
"post_decode",
]
def test_decoded_output_is_unchanged_without_active_model_offload():
@@ -194,7 +147,7 @@ def test_forward_releases_video_after_quantization_and_before_next_seed(monkeypa
_extract_prompt=lambda raw: ("a prompt", {}),
_extract_text_conditioning=lambda raw: None,
_extract_prepared_reference_videos=lambda raw: None,
_prepare_request_inputs=lambda **kwargs: context,
_prepare_request_inputs=lambda *args, **kwargs: context,
_denoise_kwargs=lambda ctx: {},
diffuse=diffuse,
decode=decode,
@@ -243,7 +196,7 @@ def test_forward_drops_the_decoded_video_before_the_next_seed_diffuses(monkeypat
_extract_prompt=lambda raw: ("a prompt", {}),
_extract_text_conditioning=lambda raw: None,
_extract_prepared_reference_videos=lambda raw: None,
_prepare_request_inputs=lambda **kwargs: context,
_prepare_request_inputs=lambda *args, **kwargs: context,
_denoise_kwargs=lambda ctx: {},
diffuse=diffuse,
decode=decode,
@@ -318,7 +271,8 @@ def test_output_owner_rank_matches_executor_contract(monkeypatch, available, ini
assert MiniMaxH3Pipeline._is_output_owner_rank() is expected
def test_h3_model_cpu_offload_registers_direct_vae_stages(monkeypatch):
@pytest.mark.parametrize("load_text_encoder", [True, False])
def test_h3_model_cpu_offload_registers_direct_vae_stages(monkeypatch, load_text_encoder):
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as module
@@ -326,9 +280,11 @@ def test_h3_model_cpu_offload_registers_direct_vae_stages(monkeypatch):
torch.nn.Module.__init__(pipeline)
pipeline.transformer = torch.nn.Linear(2, 2)
pipeline.transformers_ref = torch.nn.Linear(2, 2)
pipeline.text_encoder = torch.nn.Linear(2, 2)
pipeline.video_vae = torch.nn.Linear(2, 2)
pipeline.audio_vae = torch.nn.Linear(2, 2)
pipeline._encoder_modules = ["text_encoder"] if load_text_encoder else []
if load_text_encoder:
pipeline.text_encoder = torch.nn.Linear(2, 2)
apply_offload = Mock()
remove_offload = Mock()
monkeypatch.setattr(module, "apply_sequential_offload", apply_offload)
@@ -341,7 +297,7 @@ def test_h3_model_cpu_offload_registers_direct_vae_stages(monkeypatch):
)
dits = [pipeline.transformer, pipeline.transformers_ref]
stages = [pipeline.text_encoder, pipeline.video_vae, pipeline.audio_vae]
stages = [*([pipeline.text_encoder] if load_text_encoder else []), pipeline.video_vae, pipeline.audio_vae]
apply_offload.assert_called_once_with(
dit_modules=dits,
encoder_modules=stages,
@@ -356,6 +312,46 @@ def test_h3_model_cpu_offload_registers_direct_vae_stages(monkeypatch):
remove_offload.assert_called_once_with([*dits, *stages])
@pytest.mark.parametrize("decode_fails", [False, True])
def test_h3_model_cpu_offload_scopes_direct_vae_call(monkeypatch, decode_fails):
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as module
pipeline = object.__new__(MiniMaxH3Pipeline)
torch.nn.Module.__init__(pipeline)
component = torch.nn.Linear(2, 2)
events: list[tuple[Any, ...]] = []
@contextmanager
def record_component(value):
events.append(("activate", value))
try:
yield
finally:
events.append(("offload", value))
monkeypatch.setattr(module, "sequential_offload_component", record_component)
pipeline._model_cpu_offload_modules = [component]
def decode():
with pipeline._component_on_device(component):
events.append(("decode", component))
if decode_fails:
raise RuntimeError("decode failed")
if decode_fails:
with pytest.raises(RuntimeError, match="decode failed"):
decode()
else:
decode()
assert events == [
("activate", component),
("decode", component),
("offload", component),
]
def test_h3_model_cpu_offload_keeps_unselected_vaes_resident(monkeypatch):
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as module
@@ -416,44 +412,20 @@ def test_h3_model_cpu_offload_rejects_unloaded_selected_encoder(monkeypatch):
apply_offload.assert_not_called()
@pytest.mark.parametrize("decode_fails", [False, True])
def test_h3_model_cpu_offload_scopes_direct_vae_call(monkeypatch, decode_fails):
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as module
def _encode_media(pipeline, **kwargs):
from vllm_omni.model_executor.models.minimax_h3.conditioning import MiniMaxH3EncoderMediaInput
from vllm_omni.model_executor.models.minimax_h3.encoder_processing import encode_media
pipeline = object.__new__(MiniMaxH3Pipeline)
torch.nn.Module.__init__(pipeline)
component = torch.nn.Linear(2, 2)
events: list[tuple[Any, ...]] = []
@contextmanager
def record_component(value):
events.append(("activate", value))
try:
yield
finally:
events.append(("offload", value))
monkeypatch.setattr(module, "sequential_offload_component", record_component)
pipeline._model_cpu_offload_modules = [component]
def decode():
with pipeline._component_on_device(component):
events.append(("decode", component))
if decode_fails:
raise RuntimeError("decode failed")
if decode_fails:
with pytest.raises(RuntimeError, match="decode failed"):
decode()
else:
decode()
assert events == [
("activate", component),
("decode", component),
("offload", component),
]
media = MiniMaxH3EncoderMediaInput(
task="ref2va", height=32, width=32, num_frames=124, latent_t=31, audio_t=208, **kwargs
)
return encode_media(
media,
video_vae=getattr(pipeline, "video_vae", None),
audio_vae=getattr(pipeline, "audio_vae", None),
emit_conditioning=True,
component_scope=pipeline._component_on_device,
)
@pytest.mark.parametrize("encode_fails", [False, True])
@@ -463,7 +435,6 @@ def test_h3_model_cpu_offload_batches_visual_reference_scope(monkeypatch, encode
pipeline = object.__new__(MiniMaxH3Pipeline)
torch.nn.Module.__init__(pipeline)
pipeline.device = torch.device("cpu")
pipeline.video_vae = Mock()
pipeline._model_cpu_offload_modules = [pipeline.video_vae]
events = []
@@ -484,17 +455,15 @@ def test_h3_model_cpu_offload_batches_visual_reference_scope(monkeypatch, encode
pipeline.video_vae.encode_image.side_effect = encode_image
monkeypatch.setattr(module, "sequential_offload_component", record_component)
monkeypatch.setattr(module, "_dit_rank_world", lambda: (None, 0, 1))
monkeypatch.setattr(module, "_broadcast_tensor", lambda value, **kwargs: value)
images = [Image.new("RGB", (16, 16)), Image.new("RGB", (32, 16))]
images = (torch.zeros(16, 16, 3, dtype=torch.uint8), torch.zeros(16, 32, 3, dtype=torch.uint8))
if encode_fails:
with pytest.raises(RuntimeError, match="encode failed"):
pipeline._encode_visual_conditions(images, None, video_count=0)
_encode_media(pipeline, images=images)
else:
rows, shapes = pipeline._encode_visual_conditions(images, None, video_count=0)
assert rows.shape == (3, 4)
assert shapes == [(1, 1, 1), (1, 1, 2)]
result = _encode_media(pipeline, images=images)
assert result.visual_condition.shape == (3, 4)
assert result.visual_condition_shapes == ((1, 1, 1), (1, 1, 2))
assert events == [
("activate", pipeline.video_vae),
@@ -510,9 +479,7 @@ def test_h3_model_cpu_offload_shares_image_and_video_scope(monkeypatch):
pipeline = object.__new__(MiniMaxH3Pipeline)
torch.nn.Module.__init__(pipeline)
pipeline.device = torch.device("cpu")
pipeline.video_vae = Mock()
pipeline.video_vae.is_distributed_enabled.return_value = False
pipeline._model_cpu_offload_modules = [pipeline.video_vae]
events = []
@@ -529,31 +496,26 @@ def test_h3_model_cpu_offload_shares_image_and_video_scope(monkeypatch):
return torch.ones(1, 4)
def encode_video(frames):
events.append(("video", frames))
events.append(("video", frames.shape))
return torch.ones(3, 4), (3, 2, 2)
pipeline.video_vae.encode_image.side_effect = encode_image
pipeline.video_vae.encode_video.side_effect = encode_video
monkeypatch.setattr(module, "sequential_offload_component", record_component)
monkeypatch.setattr(module, "_dit_rank_world", lambda: (None, 0, 1))
monkeypatch.setattr(module, "_broadcast_tensor", lambda value, **kwargs: value)
monkeypatch.setattr(module, "load_video_frames", lambda path: f"frames:{path}")
images = [Image.new("RGB", (16, 16)), Image.new("RGB", (32, 16))]
prepared_videos = [{"prepared_path": "reference.mp4"}]
rows, shapes = pipeline._encode_visual_conditions(
images,
prepared_videos,
video_count=1,
result = _encode_media(
pipeline,
images=(torch.zeros(16, 16, 3, dtype=torch.uint8), torch.zeros(16, 32, 3, dtype=torch.uint8)),
videos=(torch.zeros(12, 32, 32, 3, dtype=torch.uint8),),
video_audios=(None,),
)
assert rows.shape == (5, 4)
assert shapes == [(1, 1, 1), (1, 1, 2), (3, 2, 2)]
assert result.visual_condition.shape == (5, 4)
assert result.visual_condition_shapes == ((1, 1, 1), (1, 1, 2), (3, 2, 2))
assert events == [
("activate", pipeline.video_vae),
("image", (16, 16)),
("image", (32, 16)),
("video", "frames:reference.mp4"),
("video", (12, 32, 32, 3)),
("offload", pipeline.video_vae),
]
@@ -567,7 +529,7 @@ def test_h3_model_cpu_offload_shares_embedded_and_standalone_audio_scope(monkeyp
torch.nn.Module.__init__(pipeline)
pipeline.audio_vae = Mock()
pipeline._model_cpu_offload_modules = [pipeline.audio_vae]
events: list[tuple[Any, ...]] = []
events = []
@contextmanager
def record_component(value):
@@ -577,26 +539,20 @@ def test_h3_model_cpu_offload_shares_embedded_and_standalone_audio_scope(monkeyp
finally:
events.append(("offload", value))
def encode_embedded(*args, **kwargs):
events.append(("embedded",))
def encode_waveform(waveform, sample_rate):
events.append(("audio", int(waveform[0]), sample_rate))
if encode_fails:
raise RuntimeError("audio encode failed")
return torch.ones(1, 2), [1]
return torch.ones(80, 32), 80
def encode_standalone(*args, **kwargs):
events.append(("standalone",))
return torch.ones(1, 2), [1]
pipeline._encode_video_audio_conditions_resident = encode_embedded
pipeline._encode_audio_conditions_resident = encode_standalone
pipeline.audio_vae.encode_waveform.side_effect = encode_waveform
monkeypatch.setattr(module, "sequential_offload_component", record_component)
def call():
return pipeline._encode_reference_audio_conditions(
[{"input_has_audio": True}],
has_audio=[True],
standalone_audios=[(torch.ones(1), 16_000)],
max_duration_seconds=1.0,
return _encode_media(
pipeline,
video_audios=(),
audios=((torch.ones(32_000), 16_000),),
)
if encode_fails:
@@ -604,10 +560,61 @@ def test_h3_model_cpu_offload_shares_embedded_and_standalone_audio_scope(monkeyp
call()
else:
result = call()
assert result[0].shape == (1, 2)
assert result[2].shape == (1, 2)
assert result.audio_condition.shape == (80, 32)
assert result.audio_condition_lengths == (80,)
assert events[1:-1] == [("audio", 1, 16_000)]
assert events[0] == ("activate", pipeline.audio_vae)
assert events[-1] == ("offload", pipeline.audio_vae)
assert events.count(("activate", pipeline.audio_vae)) == 1
assert events.count(("offload", pipeline.audio_vae)) == 1
@pytest.mark.parametrize("distributed", [False, True])
def test_manual_component_offload_wins_over_requested_model_offload(distributed):
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
pipeline = object.__new__(MiniMaxH3Pipeline)
torch.nn.Module.__init__(pipeline)
pipeline.od_config = SimpleNamespace(
enable_cpu_offload=True,
enable_layerwise_offload=not distributed,
enable_distributed_layerwise_offload=distributed,
)
pipeline._model_cpu_offload_modules = []
pipeline.text_encoder = Mock()
expected = torch.ones(2, 3)
pipeline.text_encoder.encode_ids.return_value = expected
component = Mock()
actual = pipeline._encode_text_hidden(torch.tensor([1, 2]), {})
with pipeline._component_on_device(component):
component.load_to_device.assert_called_once_with()
component.offload_to_cpu.assert_not_called()
assert actual is expected
pipeline.text_encoder.load_to_device.assert_called_once_with()
pipeline.text_encoder.offload_to_cpu.assert_called_once_with()
component.offload_to_cpu.assert_called_once_with()
def test_requested_model_offload_without_backend_uses_resident_components():
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
pipeline = object.__new__(MiniMaxH3Pipeline)
torch.nn.Module.__init__(pipeline)
pipeline.od_config = SimpleNamespace(
enable_cpu_offload=True,
enable_layerwise_offload=False,
enable_distributed_layerwise_offload=False,
)
pipeline._model_cpu_offload_modules = []
pipeline.text_encoder = Mock()
expected = torch.ones(2, 3)
pipeline.text_encoder.encode_ids.return_value = expected
actual = pipeline._encode_text_hidden(torch.tensor([1, 2]), {})
assert actual is expected
pipeline.text_encoder.load_to_device.assert_called_once_with()
pipeline.text_encoder.encode_ids.assert_called_once()
@@ -186,9 +186,8 @@ def test_minimax_h3_quantization_quality(config: QualityTestConfig):
common_kwargs = {
"model": model_ref,
"enforce_eager": True,
"tensor_parallel_size": 2,
# The fused BF16 baseline only fits on H100-80GB with encoder TP.
"text_encoder_tp_size": 2,
"stage_0_tensor_parallel_size": 2,
"vae_use_tiling": True,
}
@@ -110,6 +110,8 @@ def _step_pipeline(model, *, packed_batch_supported: bool = True):
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import MiniMaxH3Pipeline
pipeline = object.__new__(MiniMaxH3Pipeline)
pipeline.load_text_encoder = False
pipeline.load_vae_encoder = False
pipeline.transformer = model
pipeline.device = torch.device("cpu")
pipeline._transformer_for_task = lambda task: model
@@ -304,8 +306,17 @@ def test_prepare_encode_seeds_runner_visible_state(monkeypatch, batch_frames):
}
pipeline = _step_pipeline(_SegmentMeanModel())
monkeypatch.setattr(mod.MiniMaxH3Pipeline, "_extract_prompt", staticmethod(lambda _: ("a prompt", {})))
monkeypatch.setattr(mod.MiniMaxH3Pipeline, "_prepare_request_inputs", lambda self, **_: context)
conditioning = object()
monkeypatch.setattr(
mod.MiniMaxH3Pipeline,
"_extract_encoder_conditioning",
staticmethod(lambda _: conditioning),
)
monkeypatch.setattr(
mod.MiniMaxH3Pipeline,
"_prepare_encoder_conditioning_inputs",
lambda self, value, sampling: context,
)
monkeypatch.setattr(
mod.MiniMaxH3Pipeline,
"_build_denoise_inputs",
@@ -439,50 +450,3 @@ def test_packed_batch_rejects_backends_that_cannot_isolate_requests(attention):
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import MiniMaxH3Pipeline
assert MiniMaxH3Pipeline._packed_batch_supported(_FakeTransformer([attention])) is False
def test_broadcast_rank0_exception_single_rank_reraises():
"""Single-rank execution has no group; the helper just reraises."""
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import (
_broadcast_rank0_exception,
)
from vllm_omni.errors import OmniClientError
_broadcast_rank0_exception(None)
with pytest.raises(OmniClientError, match="bad ref"):
_broadcast_rank0_exception(OmniClientError("bad ref"))
def test_broadcast_rank0_exception_propagates_to_non_zero_ranks(monkeypatch):
"""A rank-0 error becomes a matching client error on every other DiT rank."""
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as mod
from vllm_omni.errors import OmniClientError
def fake_rank_world(rank):
return lambda: (object(), rank, 4)
def make_broadcast(rank0_payload):
def fake_broadcast(payload_list, *, src, group):
payload_list[0] = rank0_payload
return fake_broadcast
monkeypatch.setattr(mod, "_dit_rank_world", fake_rank_world(0))
err = OmniClientError("invalid reference-video file", status_code=422, error_type="UnprocessableEntityError")
rank0_payload = {
"type": type(err).__name__,
"message": str(err),
"status_code": err.status_code,
"error_type": err.error_type,
}
monkeypatch.setattr(mod.dist, "broadcast_object_list", make_broadcast(rank0_payload))
with pytest.raises(OmniClientError) as rank0_info:
mod._broadcast_rank0_exception(err)
assert rank0_info.value is err
monkeypatch.setattr(mod, "_dit_rank_world", fake_rank_world(2))
with pytest.raises(OmniClientError) as rank2_info:
mod._broadcast_rank0_exception(None)
assert rank2_info.value.status_code == 422
assert rank2_info.value.error_type == "UnprocessableEntityError"
assert "invalid reference-video file" in str(rank2_info.value)
@@ -0,0 +1,406 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from contextlib import contextmanager
from types import SimpleNamespace
import pytest
import torch
from vllm_omni.data_entry_keys import flatten_payload
from vllm_omni.model_executor.models.minimax_h3.conditioning import (
MiniMaxH3EncoderConditioning,
MiniMaxH3EncoderMediaConditioning,
MiniMaxH3EncoderMediaInput,
)
from vllm_omni.model_executor.stage_input_processors.minimax_h3 import encoder2diffusion
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_pipeline_owns_three_encoders_then_dit_and_decoders() -> None:
from vllm_omni.model_executor.models.minimax_h3.pipeline import MINIMAX_H3_PIPELINE
encoder, diffusion = MINIMAX_H3_PIPELINE.stages
assert encoder.model_arch == "MiniMaxH3Encoder"
assert encoder.custom_process_next_stage_input_func is None
assert diffusion.model_arch == "MiniMaxH3Pipeline"
assert diffusion.requires_multimodal_data is False
assert diffusion.requires_full_payload_input is False
def _patch_encoder_constructors(monkeypatch, *, rank: int):
from vllm_omni.model_executor.models.minimax_h3 import encoder as encoder_module
group = SimpleNamespace(rank_in_group=rank, world_size=2, device_group=object())
class FakeVideoVAE(torch.nn.Module):
def __init__(self, path, *, device, encode_only, trust_remote_code):
super().__init__()
self.path = path
self.parallel_args = None
self.trust_remote_code = trust_remote_code
def set_parallel_size(self, size, *, process_group):
self.parallel_args = (size, process_group)
class FakeAudioVAE(torch.nn.Module):
def __init__(self, path, *, device, encode_only, trust_remote_code):
super().__init__()
self.path = path
self.trust_remote_code = trust_remote_code
def init_backbone(self, *, vllm_config, prefix):
torch.nn.Module.__init__(self)
monkeypatch.setattr(encoder_module.MiniMaxH3TextEncoderBackbone, "__init__", init_backbone)
monkeypatch.setattr(encoder_module, "get_tp_group", lambda: group)
monkeypatch.setattr(encoder_module, "get_local_device", lambda: torch.device("cpu"))
monkeypatch.setattr(encoder_module, "MiniMaxH3VideoVAE", FakeVideoVAE)
monkeypatch.setattr(encoder_module, "MiniMaxH3AudioVAE", FakeAudioVAE)
return encoder_module, group
def _component_root(tmp_path):
root = tmp_path / "FL2VA"
for name in ("text_encoder", "video_vae", "audio_vae"):
(root / name).mkdir(parents=True)
return root
def _encoder_config(root, *, video_mode="patch", roles=None, trust_remote_code=True):
components = (
{
"text_encoder": {"parallel_mode": "tp"},
"video_vae": {"parallel_mode": video_mode},
"audio_vae": {"parallel_mode": "leader"},
}
if roles is None
else roles
)
return SimpleNamespace(
model_config=SimpleNamespace(
model=str(root / "text_encoder"),
hf_config=SimpleNamespace(minimax_h3_encoder_components=components),
trust_remote_code=trust_remote_code,
)
)
def test_encoder_requires_exactly_three_components(monkeypatch, tmp_path) -> None:
encoder_module, _ = _patch_encoder_constructors(monkeypatch, rank=0)
root = tmp_path / "FL2VA"
(root / "text_encoder").mkdir(parents=True)
(root / "video_vae").mkdir()
config = _encoder_config(root)
with pytest.raises(RuntimeError, match=r"requires all three encoders.*audio_vae"):
encoder_module.MiniMaxH3Encoder(vllm_config=config)
@pytest.mark.parametrize(("rank", "has_audio_vae"), [(0, True), (1, False)])
@pytest.mark.parametrize("trust_remote_code", [False, True])
def test_encoder_uses_tp_patch_and_leader_on_the_same_rank_set(
monkeypatch,
tmp_path,
rank,
has_audio_vae,
trust_remote_code,
) -> None:
encoder_module, group = _patch_encoder_constructors(monkeypatch, rank=rank)
root = _component_root(tmp_path)
config = _encoder_config(root, trust_remote_code=trust_remote_code)
model = encoder_module.MiniMaxH3Encoder(vllm_config=config)
assert model.component_config.text_parallel_mode == "tp"
assert model.component_config.video_parallel_mode == "patch"
assert model.component_config.audio_parallel_mode == "leader"
assert model.video_vae.parallel_args == (2, group.device_group)
assert model.video_vae.trust_remote_code is trust_remote_code
assert (model.audio_vae is not None) is has_audio_vae
if has_audio_vae:
assert model.audio_vae.trust_remote_code is trust_remote_code
@pytest.mark.parametrize("component", ["video", "audio"])
@pytest.mark.parametrize("trusted", [False, True])
def test_encoder_only_vae_requires_remote_code_trust(monkeypatch, component, trusted) -> None:
from vllm_omni.diffusion.models.minimax_h3 import vae as vae_module
config = {"auto_map": {"AutoModel": "modeling.RemoteVAE"}, "sample_rate": 32000}
monkeypatch.setattr(vae_module, "_load_component_config", lambda _path: config)
remote = torch.nn.Module()
remote.model = torch.nn.Module()
calls = []
def load_encoder(path, config_dict):
calls.append((path, config_dict))
return remote
monkeypatch.setattr(vae_module, f"_load_{component}_vae_encoder", load_encoder)
vae_cls = vae_module.MiniMaxH3VideoVAE if component == "video" else vae_module.MiniMaxH3AudioVAE
if trusted:
model = vae_cls("unused", device=torch.device("cpu"), encode_only=True, trust_remote_code=True)
assert model.remote is remote
assert calls == [("unused", config)]
else:
with pytest.raises(ValueError, match="trust-remote-code"):
vae_cls("unused", device=torch.device("cpu"), encode_only=True)
assert calls == []
def test_encoder_requires_all_role_policies(monkeypatch, tmp_path) -> None:
encoder_module, _ = _patch_encoder_constructors(monkeypatch, rank=0)
root = _component_root(tmp_path)
config = _encoder_config(
root,
roles={
"text_encoder": {"parallel_mode": "tp"},
"video_vae": {"parallel_mode": "patch"},
},
)
with pytest.raises(ValueError, match=r"requires exactly.*audio_vae"):
encoder_module.MiniMaxH3Encoder(vllm_config=config)
def test_encoder_does_not_infer_role_policies(monkeypatch, tmp_path) -> None:
encoder_module, _ = _patch_encoder_constructors(monkeypatch, rank=0)
root = _component_root(tmp_path)
config = SimpleNamespace(
model_config=SimpleNamespace(
model=str(root / "text_encoder"),
hf_config=SimpleNamespace(),
)
)
with pytest.raises(ValueError, match="requires explicit minimax_h3_encoder_components"):
encoder_module.MiniMaxH3Encoder(vllm_config=config)
def test_video_leader_policy_skips_nonleader(monkeypatch, tmp_path) -> None:
encoder_module, _ = _patch_encoder_constructors(monkeypatch, rank=1)
root = _component_root(tmp_path)
model = encoder_module.MiniMaxH3Encoder(vllm_config=_encoder_config(root, video_mode="leader"))
assert model.video_vae is None
assert model.audio_vae is None
assert (
model._encode_media(
MiniMaxH3EncoderMediaInput(
task="ref2va",
height=32,
width=32,
num_frames=48,
latent_t=1,
audio_t=80,
images=(torch.zeros(32, 32, 3, dtype=torch.uint8),),
)
)
is None
)
def test_encoder_runs_video_and_audio_components_on_ar_model() -> None:
from vllm_omni.model_executor.models.minimax_h3.encoder import MiniMaxH3Encoder
class VideoVAE:
def encode_image(self, _image):
return torch.ones(1, 96)
def encode_video(self, _frames):
return torch.full((1, 96), 2.0), (1, 2, 2)
class AudioVAE:
def encode_waveform(self, waveform, _sample_rate):
return torch.ones(160, 32), waveform.shape[-1] // 800
model = MiniMaxH3Encoder.__new__(MiniMaxH3Encoder)
torch.nn.Module.__init__(model)
model._component_leader = True
model.video_vae = VideoVAE()
model.audio_vae = AudioVAE()
media = MiniMaxH3EncoderMediaInput(
task="ref2va",
height=32,
width=32,
num_frames=48,
latent_t=1,
audio_t=80,
images=(torch.zeros(32, 32, 3, dtype=torch.uint8),),
videos=(torch.zeros(1, 32, 32, 3, dtype=torch.uint8),),
video_audios=((torch.zeros(64_000), 32_000),),
)
conditioning = model._encode_media(media)
assert conditioning is not None
assert conditioning.visual_condition_shapes == ((1, 2, 2), (1, 2, 2))
assert conditioning.audio_condition_lengths == (80,)
assert conditioning.ref_blocks[1]["kind"] == "video_audio"
def test_prepare_encoder_inputs_keeps_reference_audio_budgets_separate(monkeypatch) -> None:
from vllm_omni.model_executor.models.minimax_h3 import encoder_processing as processing
frames = torch.zeros(1, 32, 32, 3, dtype=torch.uint8).numpy()
waveform = torch.zeros(320_000)
monkeypatch.setattr(processing, "resolve_minimax_h3_shape", lambda *_args: (32, 32, 240, 60, 400))
monkeypatch.setattr(
processing,
"prepare_reference_videos",
lambda *_args, **_kwargs: [
{"prepared_path": "prepared.mp4", "original_path": "reference.mp4", "input_has_audio": True}
],
)
monkeypatch.setattr(processing, "load_video_frames", lambda _path: frames)
monkeypatch.setattr(
processing,
"sample_reference_video_frames",
lambda *_args, **_kwargs: {"frames": [frames[0]], "block_timestamps": [[0.0]]},
)
monkeypatch.setattr(processing, "load_video_audio", lambda *_args, **_kwargs: (waveform, 32_000))
prepared = processing.prepare_encoder_inputs(
{
"prompt": "reference",
"multi_modal_data": {"video": "reference.mp4", "audio": (waveform, 32_000)},
},
SimpleNamespace(extra_args={"task": "ref2va"}),
)
assert prepared.media.video_audios[0][0].shape[-1] == 320_000
assert prepared.media.audios[0][0].shape[-1] == 320_000
assert prepared.condition_labels == [("audio", 1), ("video", 1), ("audio", 2)]
@pytest.mark.parametrize(
("embedded_lengths", "standalone_lengths", "valid"),
[((400,), (400,), True), ((320, 320), (400,), False), ((400,), (320, 320), False)],
)
def test_encode_media_keeps_audio_budgets_and_component_residency_separate(
embedded_lengths, standalone_lengths, valid
) -> None:
from vllm_omni.model_executor.models.minimax_h3.encoder_processing import encode_media
active = None
scopes = []
class VideoVAE:
def encode_image(self, _image):
assert active is self
return torch.ones(1, 96)
def encode_video(self, _frames):
assert active is self
return torch.ones(1, 96), (1, 2, 2)
class AudioVAE:
def encode_waveform(self, waveform, _sample_rate):
assert active is self
length = waveform.shape[-1] // 800
return torch.ones(2 * length, 32), length
@contextmanager
def component_scope(component):
nonlocal active
assert active is None
active = component
scopes.append(component)
try:
yield
finally:
active = None
video_vae = VideoVAE()
audio_vae = AudioVAE()
media = MiniMaxH3EncoderMediaInput(
task="ref2va",
height=32,
width=32,
num_frames=360,
latent_t=90,
audio_t=600,
images=(torch.zeros(32, 32, 3, dtype=torch.uint8),),
videos=tuple(torch.zeros(1, 32, 32, 3, dtype=torch.uint8) for _ in embedded_lengths),
video_audios=tuple((torch.zeros(length * 800), 32_000) for length in embedded_lengths),
audios=tuple((torch.zeros(length * 800), 32_000) for length in standalone_lengths),
)
kwargs = dict(video_vae=video_vae, audio_vae=audio_vae, emit_conditioning=True, component_scope=component_scope)
if valid:
conditioning = encode_media(media, **kwargs)
assert conditioning.audio_condition_lengths == embedded_lengths + standalone_lengths
assert [block["kind"] for block in conditioning.ref_blocks] == ["image", "video_audio", "audio"]
conditioning.to_omni_components()
else:
with pytest.raises(ValueError, match="at most 15 seconds"):
encode_media(media, **kwargs)
assert scopes == [video_vae, audio_vae]
def test_encoder_output_reuses_encoder_handoff_and_round_trips() -> None:
expected = MiniMaxH3EncoderConditioning(
hidden_states=torch.randn(3, 5120, dtype=torch.bfloat16),
token_tags=torch.tensor([1, 0, 1], dtype=torch.int64),
task="t2va",
height=256,
width=448,
num_frames=17,
latent_t=5,
audio_t=10,
)
source = SimpleNamespace(
finished=True,
request_id="request-1",
outputs=[SimpleNamespace(multimodal_output=flatten_payload(expected.to_omni_payload()))],
)
prompt = {
"prompt": "hello",
"additional_information": {"global_request_id": ["request-1"]},
}
result = encoder2diffusion([source], prompt)
assert result is not None
payload = result["additional_information"]["encoder_output"]
actual = MiniMaxH3EncoderConditioning.from_omni_payload(payload)
torch.testing.assert_close(actual.hidden_states, expected.hidden_states)
torch.testing.assert_close(actual.token_tags, expected.token_tags)
def test_encoder_releases_workspace_after_cpu_payload(monkeypatch) -> None:
from vllm_omni.model_executor.models.minimax_h3 import encoder as encoder_module
model = encoder_module.MiniMaxH3Encoder.__new__(encoder_module.MiniMaxH3Encoder)
torch.nn.Module.__init__(model)
model._component_leader = True
model._token_tags = torch.tensor([1])
info = {
"meta": {"minimax_h3_encoder_request": {}},
"_minimax_h3_encoder_media_conditioning": MiniMaxH3EncoderMediaConditioning(
task="t2va",
height=256,
width=448,
num_frames=17,
latent_t=5,
audio_t=10,
),
}
empty_cache_calls = 0
def empty_cache() -> None:
nonlocal empty_cache_calls
empty_cache_calls += 1
monkeypatch.setattr(torch.accelerator, "empty_cache", empty_cache)
output = model.make_omni_output(
torch.zeros(1, 5120),
model_intermediate_buffer=[info],
request_sample_eligible=[True],
)
assert empty_cache_calls == 1
assert output.multimodal_outputs is not None
assert output.multimodal_outputs["embed"]["embedding"][0].device.type == "cpu"
@@ -2,8 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Regression tests for MiniMax H3's disaggregated text-encoder contract."""
import os
import shutil
from types import SimpleNamespace
from unittest.mock import Mock
@@ -12,39 +10,37 @@ import pytest
import torch
from PIL import Image
from vllm_omni.data_entry_keys import OmniPayload, deserialize_payload, serialize_payload
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import (
_load_audio,
resolve_minimax_h3_diffusion_model_path,
from vllm_omni.data_entry_keys import flatten_payload
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import resolve_minimax_h3_diffusion_model_path
from vllm_omni.engine.serialization import (
deserialize_additional_information,
serialize_additional_information,
)
from vllm_omni.errors import OmniClientError
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.model_executor.models.minimax_h3.checkpoint import (
resolve_minimax_h3_model_root,
resolve_minimax_h3_encoder_model_root,
resolve_minimax_h3_partition,
)
from vllm_omni.model_executor.models.minimax_h3.conditioning import (
MINIMAX_H3_CONDITION_LABELS_KEY,
MINIMAX_H3_ENCODER_REQUEST_KEY,
MINIMAX_H3_PRESENTATION_TASK_KEY,
MINIMAX_H3_TEXT_CONDITIONING_SCHEMA,
MiniMaxH3TextConditioning,
MiniMaxH3EncoderConditioning,
)
from vllm_omni.model_executor.models.minimax_h3.encoder_processing import _audio_items, _load_audio
from vllm_omni.model_executor.models.minimax_h3.preprocessing import (
minimax_h3_ref2va_presentation,
minimax_h3_ref2va_video_presentation,
)
from vllm_omni.model_executor.models.minimax_h3.reference_video import (
MINIMAX_H3_PREPARED_REFERENCE_VIDEOS_KEY,
deserialize_prepared_reference_videos,
)
from vllm_omni.model_executor.models.minimax_h3.text_encoder import (
MiniMaxH3MultiModalProcessor,
_build_minimax_h3_presentation,
)
from vllm_omni.model_executor.stage_input_processors.minimax_h3 import (
_audio_items,
_diffusion_sampling_params,
prepare_text_encoder_prompt,
text_encoder2diffusion,
encoder2diffusion,
prepare_encoder_prompt,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -66,13 +62,14 @@ class _SegmentTokenizer:
return self._special_ids[token]
def test_h3_processor_reprocesses_media_instead_of_using_partial_sender_cache(mocker):
def test_h3_processor_reprocesses_media_instead_of_using_partial_sender_cache(monkeypatch):
processor = object.__new__(MiniMaxH3MultiModalProcessor)
sentinel = ([1, 2, 3], object(), True)
apply_processor = mocker.patch.object(
apply_processor = Mock(return_value=sentinel)
monkeypatch.setattr(
MiniMaxH3MultiModalProcessor,
"_apply_hf_processor",
return_value=sentinel,
apply_processor,
)
inputs = object()
timing_ctx = object()
@@ -119,9 +116,11 @@ def test_fused_audio_loader_accepts_list_waveform_pair():
def test_prepare_ref2va_keeps_original_text_and_exact_condition_order():
prompt = {
"prompt": "hello",
"additional_information": {"global_request_id": ["request-1"]},
"model_intermediate_buffer": {"private": "preserved"},
"multi_modal_data": {
"image": Image.new("RGB", (256, 256)),
"audio": [np.zeros(16), 16_000],
"audio": [np.zeros(32_000), 16_000],
},
}
sampling = OmniDiffusionSamplingParams(
@@ -130,7 +129,7 @@ def test_prepare_ref2va_keeps_original_text_and_exact_condition_order():
extra_args={"task": "ref2va"},
)
transformed = prepare_text_encoder_prompt(prompt, [sampling])
transformed = prepare_encoder_prompt(prompt, [sampling])
assert transformed["prompt"] == "hello"
assert len(transformed["multi_modal_data"]["image"]) == 1
@@ -140,223 +139,148 @@ def test_prepare_ref2va_keeps_original_text_and_exact_condition_order():
("image", 1),
("audio", 1),
]
assert transformed["model_intermediate_buffer"] == {"private": "preserved"}
runner_info = transformed["additional_information"]
for _ in range(2):
wire = serialize_additional_information(runner_info)
runner_info = deserialize_additional_information(wire)
assert runner_info["global_request_id"] == ["request-1"]
request_metadata = runner_info["meta"][MINIMAX_H3_ENCODER_REQUEST_KEY]
assert request_metadata["task"] == "ref2va"
assert isinstance(
runner_info["hidden_states"]["layers"][0],
torch.Tensor,
)
assert isinstance(
runner_info["hidden_states"]["layers"][1],
torch.Tensor,
)
from vllm_omni.model_executor.models.minimax_h3.encoder import MiniMaxH3Encoder
media = MiniMaxH3Encoder._media_input(runner_info)
assert media.task == "ref2va"
assert len(media.images) == 1
assert len(media.audios) == 1
def test_text_encoder_prompt_rejects_injected_prepared_video_descriptor():
prompt = {
"prompt": "hello",
"additional_information": {"meta": {MINIMAX_H3_PREPARED_REFERENCE_VIDEOS_KEY: '{"artifact_dir":"/tmp/user"}'}},
}
sampling = OmniDiffusionSamplingParams(height=256, width=448, extra_args={"task": "t2va"})
def _mock_ref2va_video_with_audio(monkeypatch, *, duration_seconds: float) -> None:
from vllm_omni.model_executor.models.minimax_h3 import encoder_processing
transformed = prepare_text_encoder_prompt(prompt, [sampling])
assert MINIMAX_H3_PREPARED_REFERENCE_VIDEOS_KEY not in transformed["additional_information"]["meta"]
def test_prepare_ref2va_video_uses_shared_frame_sampler_once(monkeypatch):
prepare_videos = Mock()
def prepare(_videos, *, target_frame_count, workdir, start_time_seconds):
prepare_videos(_videos, target_frame_count, workdir, start_time_seconds)
prepared_path = os.path.join(workdir, "prepared.mp4")
open(prepared_path, "wb").close()
return [
{
"original_path": "/tmp/input.mp4",
"prepared_path": prepared_path,
"input_has_audio": True,
"width": 448,
"height": 256,
"start_time_seconds": 0.0,
"duration_seconds": 5.2,
"audio_duration_seconds": 5.2,
}
]
sample_frames = Mock(return_value={"frames": [np.zeros((4, 4, 3), dtype=np.uint8)]})
monkeypatch.setattr("vllm_omni.model_executor.stage_input_processors.minimax_h3.prepare_reference_videos", prepare)
sample_rate = 16_000
frames = np.zeros((4, 32, 32, 3), dtype=np.uint8)
monkeypatch.setattr(
"vllm_omni.model_executor.stage_input_processors.minimax_h3.sample_reference_video_frames",
sample_frames,
encoder_processing,
"prepare_reference_videos",
lambda *args, **kwargs: [
{
"prepared_path": "prepared.mp4",
"original_path": "original.mp4",
"input_has_audio": True,
"duration_seconds": duration_seconds,
"audio_duration_seconds": duration_seconds,
}
],
)
monkeypatch.setattr(encoder_processing, "load_video_frames", lambda path: frames)
monkeypatch.setattr(
encoder_processing,
"sample_reference_video_frames",
lambda *args, **kwargs: {"block_timestamps": [0.0], "frames": frames[:1]},
)
monkeypatch.setattr(
encoder_processing,
"load_video_audio",
lambda *args, **kwargs: (torch.zeros(round(duration_seconds * sample_rate)), sample_rate),
)
def test_prepare_ref2va_rejects_short_embedded_video_audio(monkeypatch):
_mock_ref2va_video_with_audio(monkeypatch, duration_seconds=1.0)
sampling = OmniDiffusionSamplingParams(
height=256,
width=448,
num_frames=96,
extra_args={"task": "ref2va"},
)
prompt = {"prompt": "hello", "multi_modal_data": {"video": "original.mp4"}}
with pytest.raises(OmniClientError, match=r"duration must be in \[2, 15\] seconds, got 1\.000"):
prepare_encoder_prompt(prompt, [sampling])
def test_prepare_ref2va_keeps_embedded_and_standalone_audio_budgets_separate(monkeypatch):
duration_seconds = 8.0
sample_rate = 16_000
_mock_ref2va_video_with_audio(monkeypatch, duration_seconds=duration_seconds)
sampling = OmniDiffusionSamplingParams(
height=256,
width=448,
num_frames=240,
extra_args={"task": "ref2va"},
)
prompt = {
"prompt": "hello",
"multi_modal_data": {"video": "/tmp/input.mp4"},
"multi_modal_data": {
"video": "original.mp4",
"audio": (torch.zeros(int(duration_seconds * sample_rate)), sample_rate),
},
}
sampling = OmniDiffusionSamplingParams(height=256, width=448, num_frames=124, extra_args={"task": "ref2va"})
transformed = prepare_text_encoder_prompt(prompt, [sampling])
prepare_videos.assert_called_once()
_, target_frame_count, artifact_dir, start_time_seconds = prepare_videos.call_args.args
assert target_frame_count == 124
assert start_time_seconds is None
descriptor = transformed["additional_information"]["meta"][MINIMAX_H3_PREPARED_REFERENCE_VIDEOS_KEY]
described_dir, described_videos = deserialize_prepared_reference_videos(descriptor)
assert described_dir == artifact_dir
assert os.path.isfile(described_videos[0]["prepared_path"])
sample_frames.assert_called_once_with(described_videos[0]["prepared_path"])
assert prompt["multi_modal_data"] == {"video": "/tmp/input.mp4"}
transformed = prepare_encoder_prompt(prompt, [sampling])
assert transformed["mm_processor_kwargs"][MINIMAX_H3_CONDITION_LABELS_KEY] == [
("audio", 1),
("video", 1),
("audio", 2),
]
shutil.rmtree(artifact_dir)
def test_stage_wire_rejects_multiple_text_encoder_sources():
with pytest.raises(RuntimeError, match="exactly one text-encoder source"):
text_encoder2diffusion([SimpleNamespace(), SimpleNamespace()], prompt={"prompt": "hello"})
def _encoder_output() -> dict:
return MiniMaxH3EncoderConditioning(
hidden_states=torch.randn(3, 5120, dtype=torch.bfloat16),
token_tags=torch.tensor([1, 0, 1], dtype=torch.int64),
task="t2va",
height=256,
width=448,
num_frames=17,
latent_t=5,
audio_t=10,
).to_omni_payload()
def test_stage_wire_rejects_request_id_mismatch():
source = SimpleNamespace(request_id="other", outputs=[SimpleNamespace()])
def test_encoder2diffusion_reuses_encoder_handoff() -> None:
prompt = {
"prompt": "hello",
"additional_information": {"global_request_id": ["req-1"]},
}
with pytest.raises(RuntimeError, match="request ID does not match"):
text_encoder2diffusion([source], prompt=prompt)
def test_stage_wire_rejects_multiple_completions():
source = SimpleNamespace(request_id="req-1", outputs=[SimpleNamespace(), SimpleNamespace()])
with pytest.raises(RuntimeError, match="exactly one completion"):
text_encoder2diffusion([source], prompt={"prompt": "hello"})
def _source_output(payload: dict) -> SimpleNamespace:
completion = SimpleNamespace(multimodal_output=payload)
return SimpleNamespace(request_id="request-1", outputs=[completion])
def _diffusion_prompt() -> dict:
return {
"request_id": "request-1",
"prompt": "test prompt",
"additional_information": {},
}
def _valid_text_conditioning_payload() -> OmniPayload:
return {
"hidden_states": {
"output": torch.zeros(4, 5120, dtype=torch.bfloat16),
},
"meta": {
"token_role_ids": torch.tensor([[1], [1], [0], [0]], dtype=torch.int64),
"multi_modal_data": {"image": object()},
"additional_information": {
"private": "preserved",
"meta": {MINIMAX_H3_ENCODER_REQUEST_KEY: {"task": "t2va"}},
"hidden_states": {"layers": {0: torch.zeros(1)}},
},
"model_intermediate_buffer": {"private": "encoder-only"},
}
def test_text_encoder2diffusion_reads_hidden_states_output_and_token_role_ids() -> None:
payload = _valid_text_conditioning_payload()
hidden = payload["hidden_states"]["output"]
token_role_ids = payload["meta"]["token_role_ids"]
result = text_encoder2diffusion(
[_source_output(payload)],
_diffusion_prompt(),
source = SimpleNamespace(
finished=True,
request_id="request-1",
outputs=[SimpleNamespace(multimodal_output=flatten_payload(_encoder_output()))],
)
conditioning = result["additional_information"]["text_encoder_output"]
torch.testing.assert_close(conditioning["hidden_states"], hidden)
torch.testing.assert_close(conditioning["token_tags"], token_role_ids.squeeze(-1))
result = encoder2diffusion([source], prompt)
assert result["prompt"] == "test prompt"
assert result["multi_modal_data"] is None
assert "model_intermediate_buffer" not in result
additional_information = result["additional_information"]
assert additional_information["private"] == "preserved"
assert "hidden_states" not in additional_information
assert "meta" not in additional_information
parsed = MiniMaxH3EncoderConditioning.from_omni_payload(additional_information["encoder_output"])
assert parsed.task == "t2va"
def test_text_conditioning_artifact_round_trips_through_omni_payload_wire_format() -> None:
payload = _valid_text_conditioning_payload()
wire = serialize_payload(payload)
assert wire is not None
restored = deserialize_payload(wire)
conditioning = MiniMaxH3TextConditioning.from_omni_payload(restored)
assert MINIMAX_H3_TEXT_CONDITIONING_SCHEMA == "minimax_h3.text_conditioning/v1"
torch.testing.assert_close(
conditioning.hidden_states,
payload["hidden_states"]["output"],
)
torch.testing.assert_close(
conditioning.token_tags,
payload["meta"]["token_role_ids"].squeeze(-1),
)
assert conditioning.hidden_states.dtype == torch.bfloat16
assert conditioning.hidden_states.is_contiguous()
assert conditioning.token_tags.dtype == torch.int64
assert conditioning.token_tags.is_contiguous()
@pytest.mark.parametrize(
("payload", "message"),
[
({"meta": {"token_role_ids": torch.ones(4, 1)}}, "no hidden_states payload"),
({"hidden_states": {}, "meta": {"token_role_ids": torch.ones(4, 1)}}, "no hidden_states.output tensor"),
({"hidden_states": {"output": torch.randn(4, 5120)}}, "no conditioning metadata"),
({"hidden_states": {"output": torch.randn(4, 5120)}, "meta": {}}, "no token_role_ids tensor"),
(
{
"hidden_states": {"output": torch.randn(4, 5120)},
"meta": {"token_role_ids": torch.ones(4)},
},
r"must have shape \[tokens, 1\]",
),
],
)
def test_text_encoder2diffusion_rejects_invalid_conditioning_payload(payload: dict, message: str) -> None:
with pytest.raises(RuntimeError, match=message):
text_encoder2diffusion([_source_output(payload)], _diffusion_prompt())
@pytest.mark.parametrize(
("case", "message"),
[
("hidden_width", r"hidden_states must have shape \[tokens, 5120\]"),
("hidden_dtype", "hidden_states must have dtype torch.bfloat16"),
("hidden_noncontiguous", "hidden_states must use contiguous strided layout"),
("hidden_non_strided", "hidden_states must use contiguous strided layout"),
("token_dtype", "stage-wire token_role_ids must have dtype torch.int64"),
("token_layout", "stage-wire token_role_ids must use contiguous strided layout"),
("token_count", "token_tags must align with hidden_states"),
("token_value", "token_tags must contain only 0 and 1"),
],
)
def test_text_encoder2diffusion_rejects_h3_v1_contract_mismatch(case: str, message: str) -> None:
payload = _valid_text_conditioning_payload()
if case == "hidden_width":
payload["hidden_states"]["output"] = torch.zeros(4, 5119, dtype=torch.bfloat16)
elif case == "hidden_dtype":
payload["hidden_states"]["output"] = torch.zeros(4, 5120, dtype=torch.float32)
elif case == "hidden_noncontiguous":
payload["hidden_states"]["output"] = torch.empty(5120, 4, dtype=torch.bfloat16).t()
elif case == "hidden_non_strided":
payload["hidden_states"]["output"] = torch.empty(
(4, 5120),
dtype=torch.bfloat16,
layout=torch.sparse_coo,
)
elif case == "token_dtype":
payload["meta"]["token_role_ids"] = torch.tensor([[1], [1], [0], [0]], dtype=torch.int32)
elif case == "token_layout":
payload["meta"]["token_role_ids"] = torch.tensor(
[[1], [0], [1], [0], [1], [0], [1], [0]],
dtype=torch.int64,
)[::2]
elif case == "token_count":
payload["meta"]["token_role_ids"] = torch.tensor([[1], [0], [0]], dtype=torch.int64)
elif case == "token_value":
payload["meta"]["token_role_ids"] = torch.tensor([[1], [1], [2], [0]], dtype=torch.int64)
else: # pragma: no cover - parameterization is exhaustive
raise AssertionError(f"unknown test case: {case}")
with pytest.raises(RuntimeError, match=message) as exc_info:
text_encoder2diffusion([_source_output(payload)], _diffusion_prompt())
assert MINIMAX_H3_TEXT_CONDITIONING_SCHEMA in str(exc_info.value)
def test_encoder2diffusion_waits_for_one_finished_source() -> None:
assert encoder2diffusion([SimpleNamespace(finished=False)], {"prompt": "hello"}) is None
with pytest.raises(RuntimeError, match="exactly one encoder source"):
encoder2diffusion([SimpleNamespace(), SimpleNamespace()], {"prompt": "hello"})
def test_ref2va_one_image_tokens_and_tags_match_fused_presentation():
@@ -421,15 +345,20 @@ def test_checkpoint_resolver_selects_local_partition(tmp_path):
(root / "FL2VA" / "text_encoder").mkdir(parents=True)
(root / "Ref2VA" / "text_encoder").mkdir(parents=True)
assert resolve_minimax_h3_model_root(str(root), None, "fl2va") == str(root / "FL2VA" / "text_encoder")
assert resolve_minimax_h3_model_root(str(root), None, "ref2va") == str(root / "Ref2VA" / "text_encoder")
assert resolve_minimax_h3_model_root(str(root / "Ref2VA"), None, None) == str(root / "Ref2VA" / "text_encoder")
assert resolve_minimax_h3_model_root(str(root / "FL2VA"), None, "ref2va") == str(root / "Ref2VA" / "text_encoder")
assert resolve_minimax_h3_encoder_model_root(str(root), None, "fl2va") == str(root / "FL2VA" / "text_encoder")
assert resolve_minimax_h3_encoder_model_root(str(root), None, "ref2va") == str(root / "Ref2VA" / "text_encoder")
assert resolve_minimax_h3_encoder_model_root(str(root), None, "combined") == str(root / "FL2VA" / "text_encoder")
assert resolve_minimax_h3_encoder_model_root(str(root / "Ref2VA"), None, None) == str(
root / "Ref2VA" / "text_encoder"
)
assert resolve_minimax_h3_encoder_model_root(str(root / "FL2VA"), None, "ref2va") == str(
root / "Ref2VA" / "text_encoder"
)
def test_checkpoint_resolver_rejects_unknown_task(tmp_path):
with pytest.raises(ValueError, match="task_type must be one of"):
resolve_minimax_h3_model_root(str(tmp_path), None, "unknown")
resolve_minimax_h3_encoder_model_root(str(tmp_path), None, "unknown")
def test_partition_resolver_preserves_consumer_auto_default(tmp_path):