[feat] Extend disaggregated MiniMax-H3 stage with VAE encoders (#6939)
Signed-off-by: asukaqaq-s <1311722138@qq.com> Signed-off-by: Jader <yjader@foxmail.com> Co-authored-by: Jader <yjader@foxmail.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
27 files changed
+3663
-2399
No files matched your search
@@ -1562,7 +1562,10 @@ stages:
|
||||
monkeypatch.setattr(stage_init_utils, "resolve_worker_cls", lambda _engine_args: None)
|
||||
|
||||
deploy = load_deploy_config(get_deploy_config_path("minimax_h3_disaggregated.yaml"))
|
||||
assert deploy.stages[1].engine_extras["model_loaded"] == {"text_encoder": False}
|
||||
assert deploy.stages[1].engine_extras["model_loaded"] == {
|
||||
"text_encoder": False,
|
||||
"vae_encoder": False,
|
||||
}
|
||||
stages = merge_pipeline_deploy(OMNI_PIPELINES["minimax_h3_disaggregated"], deploy)
|
||||
resolved = [stage_init_utils.build_engine_args_dict(stage.to_omegaconf(), str(model_root)) for stage in stages]
|
||||
|
||||
@@ -1587,12 +1590,20 @@ stages:
|
||||
deploy = load_deploy_config(Path(get_deploy_config_path("minimax_h3_disaggregated.yaml")))
|
||||
stages = merge_pipeline_deploy(pipeline, deploy)
|
||||
|
||||
assert stages[0].yaml_engine_args["model_arch"] == "MiniMaxH3TextEncoder"
|
||||
assert stages[0].yaml_engine_args["model_arch"] == "MiniMaxH3Encoder"
|
||||
assert stages[1].yaml_engine_args["model_arch"] == "MiniMaxH3Pipeline"
|
||||
assert stages[0].yaml_runtime["num_replicas"] == 1
|
||||
assert stages[1].yaml_runtime["num_replicas"] == 1
|
||||
assert stages[1].yaml_engine_args["model_loaded"] == {"text_encoder": False}
|
||||
assert stages[1].yaml_engine_args["model_loaded"] == {
|
||||
"text_encoder": False,
|
||||
"vae_encoder": False,
|
||||
}
|
||||
assert stages[0].yaml_engine_args["max_num_seqs"] == 1
|
||||
assert stages[0].yaml_engine_args["hf_overrides"]["minimax_h3_encoder_components"] == {
|
||||
"text_encoder": {"parallel_mode": "tp"},
|
||||
"video_vae": {"parallel_mode": "patch"},
|
||||
"audio_vae": {"parallel_mode": "leader"},
|
||||
}
|
||||
assert stages[0].yaml_engine_args["model_path_resolver"].endswith(".resolve_minimax_h3_model_root")
|
||||
assert stages[1].yaml_engine_args["model_path_resolver"].endswith(".resolve_minimax_h3_diffusion_model_path")
|
||||
parallel = stages[1].yaml_engine_args["parallel_config"]
|
||||
@@ -1604,6 +1615,9 @@ stages:
|
||||
|
||||
turbo = load_deploy_config(Path(get_deploy_config_path("minimax_h3_disaggregated_turbo.yaml")))
|
||||
turbo_stages = merge_pipeline_deploy(pipeline, turbo)
|
||||
assert turbo_stages[0].yaml_engine_args["model_arch"] == stages[0].yaml_engine_args["model_arch"]
|
||||
assert turbo_stages[1].yaml_engine_args["model_loaded"] == stages[1].yaml_engine_args["model_loaded"]
|
||||
assert turbo_stages[0].yaml_engine_args["hf_overrides"] == stages[0].yaml_engine_args["hf_overrides"]
|
||||
turbo_sampling = turbo_stages[1].yaml_extras["default_sampling_params"]
|
||||
assert turbo_sampling["num_inference_steps"] == 5
|
||||
assert turbo_sampling["extra_args"] == {"flow_shift": 6.0, "audio_flow_shift": 3.0}
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
import os
|
||||
import time
|
||||
from datetime import timedelta
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import torch.multiprocessing as mp
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion]
|
||||
|
||||
|
||||
def _collective_worker(rank, rendezvous, backend):
|
||||
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as module
|
||||
from vllm_omni.errors import OmniClientError
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.model_executor.models.minimax_h3.conditioning import MiniMaxH3EncoderMediaInput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
torch.set_num_threads(1)
|
||||
device = torch.device("cuda", rank) if backend == "nccl" else torch.device("cpu")
|
||||
if backend == "nccl":
|
||||
current_omni_platform.set_device(device)
|
||||
dist.init_process_group(
|
||||
backend, init_method=f"file://{rendezvous}", rank=rank, world_size=2, timeout=timedelta(seconds=45)
|
||||
)
|
||||
try:
|
||||
module._dit_rank_world = lambda: (dist.group.WORLD, rank, 2)
|
||||
|
||||
class VisualCodec:
|
||||
fail = False
|
||||
|
||||
def is_distributed_enabled(self):
|
||||
return True
|
||||
|
||||
def collective(self):
|
||||
value = torch.ones(1, device=device)
|
||||
dist.all_reduce(value)
|
||||
assert value.item() == 2
|
||||
|
||||
def encode_image(self, image):
|
||||
self.collective()
|
||||
assert image.size == (32, 32)
|
||||
return torch.ones(1, 96)
|
||||
|
||||
def encode_video(self, frames):
|
||||
self.collective()
|
||||
assert frames.shape == (4, 32, 32, 3)
|
||||
if self.fail and rank == 1:
|
||||
raise ValueError("codec failed on rank 1")
|
||||
return torch.full((1, 96), 2.0), (1, 2, 2)
|
||||
|
||||
class AudioCodec:
|
||||
def encode_waveform(self, waveform, sample_rate):
|
||||
assert rank == 0
|
||||
assert sample_rate == 32000
|
||||
assert waveform.shape == (64000,)
|
||||
return torch.full((160, 32), 3.0), 80
|
||||
|
||||
model = object.__new__(module.MiniMaxH3Pipeline)
|
||||
torch.nn.Module.__init__(model)
|
||||
model.device = device
|
||||
model.od_config = SimpleNamespace(enable_layerwise_offload=False)
|
||||
model.video_vae = VisualCodec()
|
||||
model.audio_vae = AudioCodec()
|
||||
model.partition = "combined"
|
||||
model.supported_tasks = frozenset({"t2va", "fl2va", "ref2va"})
|
||||
model.text_encoder_tp_size = 2
|
||||
model.text_encoder_group = SimpleNamespace(
|
||||
world_size=2, ranks=[0, 1], rank_in_group=rank, device_group=dist.group.WORLD
|
||||
)
|
||||
model.processor = SimpleNamespace(image_processor=SimpleNamespace(merge_size=2))
|
||||
model.tokenizer = lambda text, **kwargs: {"input_ids": [1, 2, 3]}
|
||||
|
||||
def encode_text(ids, kwargs):
|
||||
assert ids.tolist() == [1, 2, 3]
|
||||
assert not kwargs
|
||||
hidden = torch.ones(3, 5120, device=device, dtype=torch.bfloat16)
|
||||
dist.all_reduce(hidden)
|
||||
return hidden
|
||||
|
||||
model._encode_text_hidden = encode_text
|
||||
sampling = OmniDiffusionSamplingParams(
|
||||
height=32, width=32, num_frames=96, extra_args={"task": "t2va", "aspect_ratio": "1:1"}
|
||||
)
|
||||
text_conditioning = model._prepare_local_conditioning({"prompt": "test"}, sampling)
|
||||
torch.testing.assert_close(text_conditioning.hidden_states, torch.full_like(text_conditioning.hidden_states, 2))
|
||||
assert text_conditioning.visual_condition is None
|
||||
|
||||
media = (
|
||||
MiniMaxH3EncoderMediaInput(
|
||||
task="ref2va",
|
||||
height=32,
|
||||
width=32,
|
||||
num_frames=107,
|
||||
latent_t=32,
|
||||
audio_t=178,
|
||||
images=(torch.zeros(32, 32, 3, dtype=torch.uint8),),
|
||||
videos=(torch.zeros(4, 32, 32, 3, dtype=torch.uint8),),
|
||||
video_audios=(None,),
|
||||
audios=((torch.zeros(64000), 32000),),
|
||||
)
|
||||
if rank == 0
|
||||
else None
|
||||
)
|
||||
conditioning = model._encode_local_media(media)
|
||||
assert [block["kind"] for block in conditioning.ref_blocks] == ["image", "video", "audio"]
|
||||
torch.testing.assert_close(
|
||||
conditioning.visual_condition.cpu(), torch.cat([torch.ones(1, 96), torch.full((1, 96), 2.0)])
|
||||
)
|
||||
torch.testing.assert_close(conditioning.audio_condition.cpu(), torch.full((160, 32), 3.0))
|
||||
assert conditioning.audio_condition_lengths == (80,)
|
||||
|
||||
model.video_vae.fail = True
|
||||
with pytest.raises(OmniClientError, match="codec failed on rank 1"):
|
||||
model._encode_local_media(media)
|
||||
with pytest.raises(OmniClientError, match="non-empty prompt"):
|
||||
model._prepare_local_conditioning({"prompt": ""}, sampling)
|
||||
model.video_vae.fail = False
|
||||
assert model._encode_local_media(media).visual_condition_shapes == ((1, 2, 2), (1, 2, 2))
|
||||
finally:
|
||||
dist.destroy_process_group()
|
||||
|
||||
|
||||
def _run_collectives(tmp_path, backend):
|
||||
context = mp.spawn(_collective_worker, args=(str(tmp_path / "rendezvous"), backend), nprocs=2, join=False)
|
||||
deadline = time.monotonic() + 150
|
||||
try:
|
||||
while not context.join(timeout=1):
|
||||
if time.monotonic() > deadline:
|
||||
pytest.fail("MiniMax H3 collectives did not finish within 150 seconds")
|
||||
finally:
|
||||
for process in context.processes:
|
||||
if process.is_alive():
|
||||
process.terminate()
|
||||
process.join(timeout=5)
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_two_rank_conditioning_and_errors_gloo(tmp_path):
|
||||
_run_collectives(tmp_path, "gloo")
|
||||
|
||||
|
||||
@pytest.mark.cuda
|
||||
@pytest.mark.parallel
|
||||
@pytest.mark.skipif(
|
||||
os.environ.get("VLLM_TEST_MINIMAX_H3_COLLECTIVES") != "1",
|
||||
reason="set VLLM_TEST_MINIMAX_H3_COLLECTIVES=1 with two reserved GPUs",
|
||||
)
|
||||
def test_two_rank_conditioning_and_errors_nccl(tmp_path):
|
||||
if torch.accelerator.device_count() < 2:
|
||||
pytest.skip("requires two GPUs")
|
||||
_run_collectives(tmp_path, "nccl")
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,133 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def pipeline():
|
||||
model = object.__new__(MiniMaxH3Pipeline)
|
||||
torch.nn.Module.__init__(model)
|
||||
model.load_text_encoder = True
|
||||
model.load_vae_encoder = True
|
||||
model.device = torch.device("cpu")
|
||||
model.partition = "combined"
|
||||
model.supported_tasks = frozenset({"t2va", "fl2va", "ref2va"})
|
||||
model.default_video_shift = 12.0
|
||||
model.default_audio_shift = 3.0
|
||||
model.od_config = SimpleNamespace(enable_layerwise_offload=False)
|
||||
model._quality_policy = SimpleNamespace(resolve=Mock(return_value=SimpleNamespace(cache_dit=None)))
|
||||
model._cache_dit_runtime = SimpleNamespace(prepare=Mock())
|
||||
model.encode_prompt = Mock(
|
||||
return_value=(torch.ones(2, 5120, dtype=torch.bfloat16), torch.ones(2, dtype=torch.long))
|
||||
)
|
||||
model.video_vae = Mock()
|
||||
model.video_vae.is_distributed_enabled.return_value = False
|
||||
model.video_vae.encode_image.side_effect = lambda image: torch.ones((image.height // 32) * (image.width // 32), 96)
|
||||
model.audio_vae = Mock()
|
||||
model.transformer = torch.nn.Identity()
|
||||
model.diffuse = Mock(return_value=(torch.zeros(1), torch.zeros(1)))
|
||||
model.decode = Mock(return_value=(torch.zeros(1, 3, 1, 32, 32), torch.zeros(1)))
|
||||
return model
|
||||
|
||||
|
||||
def _request(task):
|
||||
prompt: dict[str, Any] = {"prompt": "A person waves."}
|
||||
if task != "t2va":
|
||||
prompt["multi_modal_data"] = {"image": Image.new("RGB", (256, 256))}
|
||||
sampling = OmniDiffusionSamplingParams(
|
||||
height=32, width=32, num_frames=96, num_inference_steps=2, extra_args={"task": task, "aspect_ratio": "1:1"}
|
||||
)
|
||||
return SimpleNamespace(prompts=[prompt], sampling_params=sampling)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("task", ["t2va", "fl2va", "ref2va"])
|
||||
def test_single_stage_encodes_without_stage_zero(pipeline, task):
|
||||
output = pipeline.forward(_request(task))
|
||||
pipeline.encode_prompt.assert_called_once()
|
||||
assert pipeline.video_vae.encode_image.call_count == int(task != "t2va")
|
||||
pipeline.diffuse.assert_called_once()
|
||||
assert pipeline.diffuse.call_args.kwargs["task"] == task
|
||||
video, audio = output.output
|
||||
torch.testing.assert_close(video, torch.zeros(1, 1, 32, 32, 3, dtype=torch.uint8))
|
||||
torch.testing.assert_close(audio, pipeline.decode.return_value[1])
|
||||
|
||||
|
||||
def test_mixed_reference_conditioning_matches_stage_zero_handoff(pipeline, monkeypatch):
|
||||
import numpy as np
|
||||
|
||||
from vllm_omni.data_entry_keys import flatten_payload
|
||||
from vllm_omni.model_executor.models.minimax_h3 import encoder_processing as processing
|
||||
from vllm_omni.model_executor.models.minimax_h3.conditioning import (
|
||||
MiniMaxH3EncoderConditioning,
|
||||
MiniMaxH3TextConditioning,
|
||||
)
|
||||
from vllm_omni.model_executor.models.minimax_h3.encoder import MiniMaxH3Encoder
|
||||
from vllm_omni.model_executor.stage_input_processors.minimax_h3 import encoder2diffusion, prepare_encoder_prompt
|
||||
|
||||
frames = np.zeros((48, 32, 32, 3), dtype=np.uint8)
|
||||
decode_frames = Mock(return_value=frames)
|
||||
sampler = Mock(wraps=processing.sample_reference_video_frames)
|
||||
monkeypatch.setattr(processing, "load_video_frames", decode_frames)
|
||||
monkeypatch.setattr(processing, "sample_reference_video_frames", sampler)
|
||||
monkeypatch.setattr(
|
||||
processing,
|
||||
"prepare_reference_videos",
|
||||
Mock(
|
||||
return_value=[{"prepared_path": "prepared.mp4", "original_path": "original.mp4", "input_has_audio": True}]
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(processing, "load_video_audio", Mock(return_value=(torch.full((64000,), 3.0), 32000)))
|
||||
pipeline.video_vae.encode_video.return_value = (torch.full((1, 96), 2.0), (1, 2, 2))
|
||||
|
||||
def encode_audio(waveform, sample_rate):
|
||||
length = round(waveform.shape[-1] / sample_rate * 40)
|
||||
return torch.full((length * 2, 32), float(waveform[0])), length
|
||||
|
||||
pipeline.audio_vae.encode_waveform.side_effect = encode_audio
|
||||
request = _request("ref2va")
|
||||
request.prompts[0]["multi_modal_data"].update(video="original.mp4", audio=(torch.full((192000,), 4.0), 32000))
|
||||
pipeline.forward(request)
|
||||
local_kwargs = pipeline.diffuse.call_args.kwargs
|
||||
|
||||
transformed = prepare_encoder_prompt(request.prompts[0], [request.sampling_params])
|
||||
stage_zero = object.__new__(MiniMaxH3Encoder)
|
||||
torch.nn.Module.__init__(stage_zero)
|
||||
stage_zero._component_leader = True
|
||||
stage_zero.video_vae = pipeline.video_vae
|
||||
stage_zero.audio_vae = pipeline.audio_vae
|
||||
media = stage_zero._encode_media(stage_zero._media_input(transformed["additional_information"]))
|
||||
text = MiniMaxH3TextConditioning(*pipeline.encode_prompt.return_value)
|
||||
payload = MiniMaxH3EncoderConditioning.from_components(text, media).to_omni_payload()
|
||||
source = SimpleNamespace(finished=True, outputs=[SimpleNamespace(multimodal_output=flatten_payload(payload))])
|
||||
bridged = encoder2diffusion([source], request.prompts[0])
|
||||
pipeline.load_text_encoder = False
|
||||
pipeline.load_vae_encoder = False
|
||||
pipeline.encode_prompt.reset_mock()
|
||||
pipeline.forward(SimpleNamespace(prompts=[bridged], sampling_params=request.sampling_params))
|
||||
external_kwargs = pipeline.diffuse.call_args.kwargs
|
||||
for key, value in local_kwargs.items():
|
||||
if isinstance(value, torch.Tensor):
|
||||
torch.testing.assert_close(value, external_kwargs[key])
|
||||
else:
|
||||
assert value == external_kwargs[key]
|
||||
pipeline.encode_prompt.assert_not_called()
|
||||
assert decode_frames.call_count == sampler.call_count == 2
|
||||
assert all(call.kwargs["decoded_frames"] is frames for call in sampler.call_args_list)
|
||||
assert [block["kind"] for block in local_kwargs["ref_blocks"]] == ["image", "video_audio", "audio"]
|
||||
assert local_kwargs["audio_condition_lengths"] == [80, round(107 / 24 * 40)]
|
||||
assert all(
|
||||
call.args[0].shape[-1] == round(107 / 24 * 32000)
|
||||
for call in pipeline.audio_vae.encode_waveform.call_args_list[1::2]
|
||||
)
|
||||
@@ -9,72 +9,25 @@ from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||
|
||||
|
||||
def test_h3_profiler_targets_cover_reference_encoders_and_vae_decoders():
|
||||
def test_h3_full_pipeline_profiler_includes_local_encoding():
|
||||
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
|
||||
|
||||
targets = MiniMaxH3Pipeline._PROFILER_TARGETS
|
||||
assert "_encode_visual_conditions" in targets
|
||||
assert "_encode_reference_audio_conditions" in targets
|
||||
assert "video_vae.decode_latent" in targets
|
||||
assert "audio_vae.decode_latent" in targets
|
||||
assert "_encode_video_conditions" not in targets
|
||||
assert "_encode_video_audio_conditions" not in targets
|
||||
assert "_encode_audio_conditions" not in targets
|
||||
|
||||
|
||||
@pytest.mark.parametrize("distributed", [False, True])
|
||||
def test_manual_component_offload_wins_over_requested_model_offload(distributed):
|
||||
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
pipeline.od_config = SimpleNamespace(
|
||||
enable_cpu_offload=True,
|
||||
enable_layerwise_offload=not distributed,
|
||||
enable_distributed_layerwise_offload=distributed,
|
||||
)
|
||||
pipeline._model_cpu_offload_modules = []
|
||||
pipeline.text_encoder = Mock()
|
||||
expected = torch.ones(2, 3)
|
||||
pipeline.text_encoder.encode_ids.return_value = expected
|
||||
component = Mock()
|
||||
|
||||
actual = pipeline._encode_text_hidden(torch.tensor([1, 2]), {})
|
||||
with pipeline._component_on_device(component):
|
||||
component.load_to_device.assert_called_once_with()
|
||||
component.offload_to_cpu.assert_not_called()
|
||||
|
||||
assert actual is expected
|
||||
pipeline.text_encoder.load_to_device.assert_called_once_with()
|
||||
pipeline.text_encoder.offload_to_cpu.assert_called_once_with()
|
||||
component.offload_to_cpu.assert_called_once_with()
|
||||
|
||||
|
||||
def test_requested_model_offload_without_backend_uses_resident_components():
|
||||
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
pipeline.od_config = SimpleNamespace(
|
||||
enable_cpu_offload=True,
|
||||
enable_layerwise_offload=False,
|
||||
enable_distributed_layerwise_offload=False,
|
||||
)
|
||||
pipeline._model_cpu_offload_modules = []
|
||||
pipeline.text_encoder = Mock()
|
||||
expected = torch.ones(2, 3)
|
||||
pipeline.text_encoder.encode_ids.return_value = expected
|
||||
|
||||
actual = pipeline._encode_text_hidden(torch.tensor([1, 2]), {})
|
||||
|
||||
assert actual is expected
|
||||
pipeline.text_encoder.load_to_device.assert_called_once_with()
|
||||
pipeline.text_encoder.encode_ids.assert_called_once()
|
||||
assert targets == [
|
||||
"encode_prompt",
|
||||
"_encode_local_media",
|
||||
"diffuse",
|
||||
"decode",
|
||||
"video_vae.decode_latent",
|
||||
"audio_vae.decode_latent",
|
||||
"prepare_encode",
|
||||
"denoise_step",
|
||||
"post_decode",
|
||||
]
|
||||
|
||||
|
||||
def test_decoded_output_is_unchanged_without_active_model_offload():
|
||||
@@ -194,7 +147,7 @@ def test_forward_releases_video_after_quantization_and_before_next_seed(monkeypa
|
||||
_extract_prompt=lambda raw: ("a prompt", {}),
|
||||
_extract_text_conditioning=lambda raw: None,
|
||||
_extract_prepared_reference_videos=lambda raw: None,
|
||||
_prepare_request_inputs=lambda **kwargs: context,
|
||||
_prepare_request_inputs=lambda *args, **kwargs: context,
|
||||
_denoise_kwargs=lambda ctx: {},
|
||||
diffuse=diffuse,
|
||||
decode=decode,
|
||||
@@ -243,7 +196,7 @@ def test_forward_drops_the_decoded_video_before_the_next_seed_diffuses(monkeypat
|
||||
_extract_prompt=lambda raw: ("a prompt", {}),
|
||||
_extract_text_conditioning=lambda raw: None,
|
||||
_extract_prepared_reference_videos=lambda raw: None,
|
||||
_prepare_request_inputs=lambda **kwargs: context,
|
||||
_prepare_request_inputs=lambda *args, **kwargs: context,
|
||||
_denoise_kwargs=lambda ctx: {},
|
||||
diffuse=diffuse,
|
||||
decode=decode,
|
||||
@@ -318,7 +271,8 @@ def test_output_owner_rank_matches_executor_contract(monkeypatch, available, ini
|
||||
assert MiniMaxH3Pipeline._is_output_owner_rank() is expected
|
||||
|
||||
|
||||
def test_h3_model_cpu_offload_registers_direct_vae_stages(monkeypatch):
|
||||
@pytest.mark.parametrize("load_text_encoder", [True, False])
|
||||
def test_h3_model_cpu_offload_registers_direct_vae_stages(monkeypatch, load_text_encoder):
|
||||
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
|
||||
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as module
|
||||
|
||||
@@ -326,9 +280,11 @@ def test_h3_model_cpu_offload_registers_direct_vae_stages(monkeypatch):
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
pipeline.transformer = torch.nn.Linear(2, 2)
|
||||
pipeline.transformers_ref = torch.nn.Linear(2, 2)
|
||||
pipeline.text_encoder = torch.nn.Linear(2, 2)
|
||||
pipeline.video_vae = torch.nn.Linear(2, 2)
|
||||
pipeline.audio_vae = torch.nn.Linear(2, 2)
|
||||
pipeline._encoder_modules = ["text_encoder"] if load_text_encoder else []
|
||||
if load_text_encoder:
|
||||
pipeline.text_encoder = torch.nn.Linear(2, 2)
|
||||
apply_offload = Mock()
|
||||
remove_offload = Mock()
|
||||
monkeypatch.setattr(module, "apply_sequential_offload", apply_offload)
|
||||
@@ -341,7 +297,7 @@ def test_h3_model_cpu_offload_registers_direct_vae_stages(monkeypatch):
|
||||
)
|
||||
|
||||
dits = [pipeline.transformer, pipeline.transformers_ref]
|
||||
stages = [pipeline.text_encoder, pipeline.video_vae, pipeline.audio_vae]
|
||||
stages = [*([pipeline.text_encoder] if load_text_encoder else []), pipeline.video_vae, pipeline.audio_vae]
|
||||
apply_offload.assert_called_once_with(
|
||||
dit_modules=dits,
|
||||
encoder_modules=stages,
|
||||
@@ -356,6 +312,46 @@ def test_h3_model_cpu_offload_registers_direct_vae_stages(monkeypatch):
|
||||
remove_offload.assert_called_once_with([*dits, *stages])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("decode_fails", [False, True])
|
||||
def test_h3_model_cpu_offload_scopes_direct_vae_call(monkeypatch, decode_fails):
|
||||
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
|
||||
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as module
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
component = torch.nn.Linear(2, 2)
|
||||
events: list[tuple[Any, ...]] = []
|
||||
|
||||
@contextmanager
|
||||
def record_component(value):
|
||||
events.append(("activate", value))
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
events.append(("offload", value))
|
||||
|
||||
monkeypatch.setattr(module, "sequential_offload_component", record_component)
|
||||
pipeline._model_cpu_offload_modules = [component]
|
||||
|
||||
def decode():
|
||||
with pipeline._component_on_device(component):
|
||||
events.append(("decode", component))
|
||||
if decode_fails:
|
||||
raise RuntimeError("decode failed")
|
||||
|
||||
if decode_fails:
|
||||
with pytest.raises(RuntimeError, match="decode failed"):
|
||||
decode()
|
||||
else:
|
||||
decode()
|
||||
|
||||
assert events == [
|
||||
("activate", component),
|
||||
("decode", component),
|
||||
("offload", component),
|
||||
]
|
||||
|
||||
|
||||
def test_h3_model_cpu_offload_keeps_unselected_vaes_resident(monkeypatch):
|
||||
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
|
||||
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as module
|
||||
@@ -416,44 +412,20 @@ def test_h3_model_cpu_offload_rejects_unloaded_selected_encoder(monkeypatch):
|
||||
apply_offload.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("decode_fails", [False, True])
|
||||
def test_h3_model_cpu_offload_scopes_direct_vae_call(monkeypatch, decode_fails):
|
||||
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
|
||||
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as module
|
||||
def _encode_media(pipeline, **kwargs):
|
||||
from vllm_omni.model_executor.models.minimax_h3.conditioning import MiniMaxH3EncoderMediaInput
|
||||
from vllm_omni.model_executor.models.minimax_h3.encoder_processing import encode_media
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
component = torch.nn.Linear(2, 2)
|
||||
events: list[tuple[Any, ...]] = []
|
||||
|
||||
@contextmanager
|
||||
def record_component(value):
|
||||
events.append(("activate", value))
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
events.append(("offload", value))
|
||||
|
||||
monkeypatch.setattr(module, "sequential_offload_component", record_component)
|
||||
pipeline._model_cpu_offload_modules = [component]
|
||||
|
||||
def decode():
|
||||
with pipeline._component_on_device(component):
|
||||
events.append(("decode", component))
|
||||
if decode_fails:
|
||||
raise RuntimeError("decode failed")
|
||||
|
||||
if decode_fails:
|
||||
with pytest.raises(RuntimeError, match="decode failed"):
|
||||
decode()
|
||||
else:
|
||||
decode()
|
||||
|
||||
assert events == [
|
||||
("activate", component),
|
||||
("decode", component),
|
||||
("offload", component),
|
||||
]
|
||||
media = MiniMaxH3EncoderMediaInput(
|
||||
task="ref2va", height=32, width=32, num_frames=124, latent_t=31, audio_t=208, **kwargs
|
||||
)
|
||||
return encode_media(
|
||||
media,
|
||||
video_vae=getattr(pipeline, "video_vae", None),
|
||||
audio_vae=getattr(pipeline, "audio_vae", None),
|
||||
emit_conditioning=True,
|
||||
component_scope=pipeline._component_on_device,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("encode_fails", [False, True])
|
||||
@@ -463,7 +435,6 @@ def test_h3_model_cpu_offload_batches_visual_reference_scope(monkeypatch, encode
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
pipeline.device = torch.device("cpu")
|
||||
pipeline.video_vae = Mock()
|
||||
pipeline._model_cpu_offload_modules = [pipeline.video_vae]
|
||||
events = []
|
||||
@@ -484,17 +455,15 @@ def test_h3_model_cpu_offload_batches_visual_reference_scope(monkeypatch, encode
|
||||
|
||||
pipeline.video_vae.encode_image.side_effect = encode_image
|
||||
monkeypatch.setattr(module, "sequential_offload_component", record_component)
|
||||
monkeypatch.setattr(module, "_dit_rank_world", lambda: (None, 0, 1))
|
||||
monkeypatch.setattr(module, "_broadcast_tensor", lambda value, **kwargs: value)
|
||||
images = [Image.new("RGB", (16, 16)), Image.new("RGB", (32, 16))]
|
||||
images = (torch.zeros(16, 16, 3, dtype=torch.uint8), torch.zeros(16, 32, 3, dtype=torch.uint8))
|
||||
|
||||
if encode_fails:
|
||||
with pytest.raises(RuntimeError, match="encode failed"):
|
||||
pipeline._encode_visual_conditions(images, None, video_count=0)
|
||||
_encode_media(pipeline, images=images)
|
||||
else:
|
||||
rows, shapes = pipeline._encode_visual_conditions(images, None, video_count=0)
|
||||
assert rows.shape == (3, 4)
|
||||
assert shapes == [(1, 1, 1), (1, 1, 2)]
|
||||
result = _encode_media(pipeline, images=images)
|
||||
assert result.visual_condition.shape == (3, 4)
|
||||
assert result.visual_condition_shapes == ((1, 1, 1), (1, 1, 2))
|
||||
|
||||
assert events == [
|
||||
("activate", pipeline.video_vae),
|
||||
@@ -510,9 +479,7 @@ def test_h3_model_cpu_offload_shares_image_and_video_scope(monkeypatch):
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
pipeline.device = torch.device("cpu")
|
||||
pipeline.video_vae = Mock()
|
||||
pipeline.video_vae.is_distributed_enabled.return_value = False
|
||||
pipeline._model_cpu_offload_modules = [pipeline.video_vae]
|
||||
events = []
|
||||
|
||||
@@ -529,31 +496,26 @@ def test_h3_model_cpu_offload_shares_image_and_video_scope(monkeypatch):
|
||||
return torch.ones(1, 4)
|
||||
|
||||
def encode_video(frames):
|
||||
events.append(("video", frames))
|
||||
events.append(("video", frames.shape))
|
||||
return torch.ones(3, 4), (3, 2, 2)
|
||||
|
||||
pipeline.video_vae.encode_image.side_effect = encode_image
|
||||
pipeline.video_vae.encode_video.side_effect = encode_video
|
||||
monkeypatch.setattr(module, "sequential_offload_component", record_component)
|
||||
monkeypatch.setattr(module, "_dit_rank_world", lambda: (None, 0, 1))
|
||||
monkeypatch.setattr(module, "_broadcast_tensor", lambda value, **kwargs: value)
|
||||
monkeypatch.setattr(module, "load_video_frames", lambda path: f"frames:{path}")
|
||||
images = [Image.new("RGB", (16, 16)), Image.new("RGB", (32, 16))]
|
||||
prepared_videos = [{"prepared_path": "reference.mp4"}]
|
||||
|
||||
rows, shapes = pipeline._encode_visual_conditions(
|
||||
images,
|
||||
prepared_videos,
|
||||
video_count=1,
|
||||
result = _encode_media(
|
||||
pipeline,
|
||||
images=(torch.zeros(16, 16, 3, dtype=torch.uint8), torch.zeros(16, 32, 3, dtype=torch.uint8)),
|
||||
videos=(torch.zeros(12, 32, 32, 3, dtype=torch.uint8),),
|
||||
video_audios=(None,),
|
||||
)
|
||||
|
||||
assert rows.shape == (5, 4)
|
||||
assert shapes == [(1, 1, 1), (1, 1, 2), (3, 2, 2)]
|
||||
assert result.visual_condition.shape == (5, 4)
|
||||
assert result.visual_condition_shapes == ((1, 1, 1), (1, 1, 2), (3, 2, 2))
|
||||
assert events == [
|
||||
("activate", pipeline.video_vae),
|
||||
("image", (16, 16)),
|
||||
("image", (32, 16)),
|
||||
("video", "frames:reference.mp4"),
|
||||
("video", (12, 32, 32, 3)),
|
||||
("offload", pipeline.video_vae),
|
||||
]
|
||||
|
||||
@@ -567,7 +529,7 @@ def test_h3_model_cpu_offload_shares_embedded_and_standalone_audio_scope(monkeyp
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
pipeline.audio_vae = Mock()
|
||||
pipeline._model_cpu_offload_modules = [pipeline.audio_vae]
|
||||
events: list[tuple[Any, ...]] = []
|
||||
events = []
|
||||
|
||||
@contextmanager
|
||||
def record_component(value):
|
||||
@@ -577,26 +539,20 @@ def test_h3_model_cpu_offload_shares_embedded_and_standalone_audio_scope(monkeyp
|
||||
finally:
|
||||
events.append(("offload", value))
|
||||
|
||||
def encode_embedded(*args, **kwargs):
|
||||
events.append(("embedded",))
|
||||
def encode_waveform(waveform, sample_rate):
|
||||
events.append(("audio", int(waveform[0]), sample_rate))
|
||||
if encode_fails:
|
||||
raise RuntimeError("audio encode failed")
|
||||
return torch.ones(1, 2), [1]
|
||||
return torch.ones(80, 32), 80
|
||||
|
||||
def encode_standalone(*args, **kwargs):
|
||||
events.append(("standalone",))
|
||||
return torch.ones(1, 2), [1]
|
||||
|
||||
pipeline._encode_video_audio_conditions_resident = encode_embedded
|
||||
pipeline._encode_audio_conditions_resident = encode_standalone
|
||||
pipeline.audio_vae.encode_waveform.side_effect = encode_waveform
|
||||
monkeypatch.setattr(module, "sequential_offload_component", record_component)
|
||||
|
||||
def call():
|
||||
return pipeline._encode_reference_audio_conditions(
|
||||
[{"input_has_audio": True}],
|
||||
has_audio=[True],
|
||||
standalone_audios=[(torch.ones(1), 16_000)],
|
||||
max_duration_seconds=1.0,
|
||||
return _encode_media(
|
||||
pipeline,
|
||||
video_audios=(),
|
||||
audios=((torch.ones(32_000), 16_000),),
|
||||
)
|
||||
|
||||
if encode_fails:
|
||||
@@ -604,10 +560,61 @@ def test_h3_model_cpu_offload_shares_embedded_and_standalone_audio_scope(monkeyp
|
||||
call()
|
||||
else:
|
||||
result = call()
|
||||
assert result[0].shape == (1, 2)
|
||||
assert result[2].shape == (1, 2)
|
||||
assert result.audio_condition.shape == (80, 32)
|
||||
assert result.audio_condition_lengths == (80,)
|
||||
assert events[1:-1] == [("audio", 1, 16_000)]
|
||||
|
||||
assert events[0] == ("activate", pipeline.audio_vae)
|
||||
assert events[-1] == ("offload", pipeline.audio_vae)
|
||||
assert events.count(("activate", pipeline.audio_vae)) == 1
|
||||
assert events.count(("offload", pipeline.audio_vae)) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("distributed", [False, True])
|
||||
def test_manual_component_offload_wins_over_requested_model_offload(distributed):
|
||||
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
pipeline.od_config = SimpleNamespace(
|
||||
enable_cpu_offload=True,
|
||||
enable_layerwise_offload=not distributed,
|
||||
enable_distributed_layerwise_offload=distributed,
|
||||
)
|
||||
pipeline._model_cpu_offload_modules = []
|
||||
pipeline.text_encoder = Mock()
|
||||
expected = torch.ones(2, 3)
|
||||
pipeline.text_encoder.encode_ids.return_value = expected
|
||||
component = Mock()
|
||||
|
||||
actual = pipeline._encode_text_hidden(torch.tensor([1, 2]), {})
|
||||
with pipeline._component_on_device(component):
|
||||
component.load_to_device.assert_called_once_with()
|
||||
component.offload_to_cpu.assert_not_called()
|
||||
|
||||
assert actual is expected
|
||||
pipeline.text_encoder.load_to_device.assert_called_once_with()
|
||||
pipeline.text_encoder.offload_to_cpu.assert_called_once_with()
|
||||
component.offload_to_cpu.assert_called_once_with()
|
||||
|
||||
|
||||
def test_requested_model_offload_without_backend_uses_resident_components():
|
||||
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
torch.nn.Module.__init__(pipeline)
|
||||
pipeline.od_config = SimpleNamespace(
|
||||
enable_cpu_offload=True,
|
||||
enable_layerwise_offload=False,
|
||||
enable_distributed_layerwise_offload=False,
|
||||
)
|
||||
pipeline._model_cpu_offload_modules = []
|
||||
pipeline.text_encoder = Mock()
|
||||
expected = torch.ones(2, 3)
|
||||
pipeline.text_encoder.encode_ids.return_value = expected
|
||||
|
||||
actual = pipeline._encode_text_hidden(torch.tensor([1, 2]), {})
|
||||
|
||||
assert actual is expected
|
||||
pipeline.text_encoder.load_to_device.assert_called_once_with()
|
||||
pipeline.text_encoder.encode_ids.assert_called_once()
|
||||
@@ -186,9 +186,8 @@ def test_minimax_h3_quantization_quality(config: QualityTestConfig):
|
||||
common_kwargs = {
|
||||
"model": model_ref,
|
||||
"enforce_eager": True,
|
||||
"tensor_parallel_size": 2,
|
||||
# The fused BF16 baseline only fits on H100-80GB with encoder TP.
|
||||
"text_encoder_tp_size": 2,
|
||||
"stage_0_tensor_parallel_size": 2,
|
||||
"vae_use_tiling": True,
|
||||
}
|
||||
|
||||
|
||||
@@ -110,6 +110,8 @@ def _step_pipeline(model, *, packed_batch_supported: bool = True):
|
||||
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import MiniMaxH3Pipeline
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
pipeline.load_text_encoder = False
|
||||
pipeline.load_vae_encoder = False
|
||||
pipeline.transformer = model
|
||||
pipeline.device = torch.device("cpu")
|
||||
pipeline._transformer_for_task = lambda task: model
|
||||
@@ -304,8 +306,17 @@ def test_prepare_encode_seeds_runner_visible_state(monkeypatch, batch_frames):
|
||||
}
|
||||
|
||||
pipeline = _step_pipeline(_SegmentMeanModel())
|
||||
monkeypatch.setattr(mod.MiniMaxH3Pipeline, "_extract_prompt", staticmethod(lambda _: ("a prompt", {})))
|
||||
monkeypatch.setattr(mod.MiniMaxH3Pipeline, "_prepare_request_inputs", lambda self, **_: context)
|
||||
conditioning = object()
|
||||
monkeypatch.setattr(
|
||||
mod.MiniMaxH3Pipeline,
|
||||
"_extract_encoder_conditioning",
|
||||
staticmethod(lambda _: conditioning),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
mod.MiniMaxH3Pipeline,
|
||||
"_prepare_encoder_conditioning_inputs",
|
||||
lambda self, value, sampling: context,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
mod.MiniMaxH3Pipeline,
|
||||
"_build_denoise_inputs",
|
||||
@@ -439,50 +450,3 @@ def test_packed_batch_rejects_backends_that_cannot_isolate_requests(attention):
|
||||
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import MiniMaxH3Pipeline
|
||||
|
||||
assert MiniMaxH3Pipeline._packed_batch_supported(_FakeTransformer([attention])) is False
|
||||
|
||||
|
||||
def test_broadcast_rank0_exception_single_rank_reraises():
|
||||
"""Single-rank execution has no group; the helper just reraises."""
|
||||
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import (
|
||||
_broadcast_rank0_exception,
|
||||
)
|
||||
from vllm_omni.errors import OmniClientError
|
||||
|
||||
_broadcast_rank0_exception(None)
|
||||
with pytest.raises(OmniClientError, match="bad ref"):
|
||||
_broadcast_rank0_exception(OmniClientError("bad ref"))
|
||||
|
||||
|
||||
def test_broadcast_rank0_exception_propagates_to_non_zero_ranks(monkeypatch):
|
||||
"""A rank-0 error becomes a matching client error on every other DiT rank."""
|
||||
from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as mod
|
||||
from vllm_omni.errors import OmniClientError
|
||||
|
||||
def fake_rank_world(rank):
|
||||
return lambda: (object(), rank, 4)
|
||||
|
||||
def make_broadcast(rank0_payload):
|
||||
def fake_broadcast(payload_list, *, src, group):
|
||||
payload_list[0] = rank0_payload
|
||||
|
||||
return fake_broadcast
|
||||
|
||||
monkeypatch.setattr(mod, "_dit_rank_world", fake_rank_world(0))
|
||||
err = OmniClientError("invalid reference-video file", status_code=422, error_type="UnprocessableEntityError")
|
||||
rank0_payload = {
|
||||
"type": type(err).__name__,
|
||||
"message": str(err),
|
||||
"status_code": err.status_code,
|
||||
"error_type": err.error_type,
|
||||
}
|
||||
monkeypatch.setattr(mod.dist, "broadcast_object_list", make_broadcast(rank0_payload))
|
||||
with pytest.raises(OmniClientError) as rank0_info:
|
||||
mod._broadcast_rank0_exception(err)
|
||||
assert rank0_info.value is err
|
||||
|
||||
monkeypatch.setattr(mod, "_dit_rank_world", fake_rank_world(2))
|
||||
with pytest.raises(OmniClientError) as rank2_info:
|
||||
mod._broadcast_rank0_exception(None)
|
||||
assert rank2_info.value.status_code == 422
|
||||
assert rank2_info.value.error_type == "UnprocessableEntityError"
|
||||
assert "invalid reference-video file" in str(rank2_info.value)
|
||||
@@ -0,0 +1,406 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from contextlib import contextmanager
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.data_entry_keys import flatten_payload
|
||||
from vllm_omni.model_executor.models.minimax_h3.conditioning import (
|
||||
MiniMaxH3EncoderConditioning,
|
||||
MiniMaxH3EncoderMediaConditioning,
|
||||
MiniMaxH3EncoderMediaInput,
|
||||
)
|
||||
from vllm_omni.model_executor.stage_input_processors.minimax_h3 import encoder2diffusion
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_pipeline_owns_three_encoders_then_dit_and_decoders() -> None:
|
||||
from vllm_omni.model_executor.models.minimax_h3.pipeline import MINIMAX_H3_PIPELINE
|
||||
|
||||
encoder, diffusion = MINIMAX_H3_PIPELINE.stages
|
||||
assert encoder.model_arch == "MiniMaxH3Encoder"
|
||||
assert encoder.custom_process_next_stage_input_func is None
|
||||
assert diffusion.model_arch == "MiniMaxH3Pipeline"
|
||||
assert diffusion.requires_multimodal_data is False
|
||||
assert diffusion.requires_full_payload_input is False
|
||||
|
||||
|
||||
def _patch_encoder_constructors(monkeypatch, *, rank: int):
|
||||
from vllm_omni.model_executor.models.minimax_h3 import encoder as encoder_module
|
||||
|
||||
group = SimpleNamespace(rank_in_group=rank, world_size=2, device_group=object())
|
||||
|
||||
class FakeVideoVAE(torch.nn.Module):
|
||||
def __init__(self, path, *, device, encode_only, trust_remote_code):
|
||||
super().__init__()
|
||||
self.path = path
|
||||
self.parallel_args = None
|
||||
self.trust_remote_code = trust_remote_code
|
||||
|
||||
def set_parallel_size(self, size, *, process_group):
|
||||
self.parallel_args = (size, process_group)
|
||||
|
||||
class FakeAudioVAE(torch.nn.Module):
|
||||
def __init__(self, path, *, device, encode_only, trust_remote_code):
|
||||
super().__init__()
|
||||
self.path = path
|
||||
self.trust_remote_code = trust_remote_code
|
||||
|
||||
def init_backbone(self, *, vllm_config, prefix):
|
||||
torch.nn.Module.__init__(self)
|
||||
|
||||
monkeypatch.setattr(encoder_module.MiniMaxH3TextEncoderBackbone, "__init__", init_backbone)
|
||||
monkeypatch.setattr(encoder_module, "get_tp_group", lambda: group)
|
||||
monkeypatch.setattr(encoder_module, "get_local_device", lambda: torch.device("cpu"))
|
||||
monkeypatch.setattr(encoder_module, "MiniMaxH3VideoVAE", FakeVideoVAE)
|
||||
monkeypatch.setattr(encoder_module, "MiniMaxH3AudioVAE", FakeAudioVAE)
|
||||
return encoder_module, group
|
||||
|
||||
|
||||
def _component_root(tmp_path):
|
||||
root = tmp_path / "FL2VA"
|
||||
for name in ("text_encoder", "video_vae", "audio_vae"):
|
||||
(root / name).mkdir(parents=True)
|
||||
return root
|
||||
|
||||
|
||||
def _encoder_config(root, *, video_mode="patch", roles=None, trust_remote_code=True):
|
||||
components = (
|
||||
{
|
||||
"text_encoder": {"parallel_mode": "tp"},
|
||||
"video_vae": {"parallel_mode": video_mode},
|
||||
"audio_vae": {"parallel_mode": "leader"},
|
||||
}
|
||||
if roles is None
|
||||
else roles
|
||||
)
|
||||
return SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
model=str(root / "text_encoder"),
|
||||
hf_config=SimpleNamespace(minimax_h3_encoder_components=components),
|
||||
trust_remote_code=trust_remote_code,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_encoder_requires_exactly_three_components(monkeypatch, tmp_path) -> None:
|
||||
encoder_module, _ = _patch_encoder_constructors(monkeypatch, rank=0)
|
||||
root = tmp_path / "FL2VA"
|
||||
(root / "text_encoder").mkdir(parents=True)
|
||||
(root / "video_vae").mkdir()
|
||||
config = _encoder_config(root)
|
||||
|
||||
with pytest.raises(RuntimeError, match=r"requires all three encoders.*audio_vae"):
|
||||
encoder_module.MiniMaxH3Encoder(vllm_config=config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("rank", "has_audio_vae"), [(0, True), (1, False)])
|
||||
@pytest.mark.parametrize("trust_remote_code", [False, True])
|
||||
def test_encoder_uses_tp_patch_and_leader_on_the_same_rank_set(
|
||||
monkeypatch,
|
||||
tmp_path,
|
||||
rank,
|
||||
has_audio_vae,
|
||||
trust_remote_code,
|
||||
) -> None:
|
||||
encoder_module, group = _patch_encoder_constructors(monkeypatch, rank=rank)
|
||||
root = _component_root(tmp_path)
|
||||
config = _encoder_config(root, trust_remote_code=trust_remote_code)
|
||||
|
||||
model = encoder_module.MiniMaxH3Encoder(vllm_config=config)
|
||||
|
||||
assert model.component_config.text_parallel_mode == "tp"
|
||||
assert model.component_config.video_parallel_mode == "patch"
|
||||
assert model.component_config.audio_parallel_mode == "leader"
|
||||
assert model.video_vae.parallel_args == (2, group.device_group)
|
||||
assert model.video_vae.trust_remote_code is trust_remote_code
|
||||
assert (model.audio_vae is not None) is has_audio_vae
|
||||
if has_audio_vae:
|
||||
assert model.audio_vae.trust_remote_code is trust_remote_code
|
||||
|
||||
|
||||
@pytest.mark.parametrize("component", ["video", "audio"])
|
||||
@pytest.mark.parametrize("trusted", [False, True])
|
||||
def test_encoder_only_vae_requires_remote_code_trust(monkeypatch, component, trusted) -> None:
|
||||
from vllm_omni.diffusion.models.minimax_h3 import vae as vae_module
|
||||
|
||||
config = {"auto_map": {"AutoModel": "modeling.RemoteVAE"}, "sample_rate": 32000}
|
||||
monkeypatch.setattr(vae_module, "_load_component_config", lambda _path: config)
|
||||
remote = torch.nn.Module()
|
||||
remote.model = torch.nn.Module()
|
||||
calls = []
|
||||
|
||||
def load_encoder(path, config_dict):
|
||||
calls.append((path, config_dict))
|
||||
return remote
|
||||
|
||||
monkeypatch.setattr(vae_module, f"_load_{component}_vae_encoder", load_encoder)
|
||||
vae_cls = vae_module.MiniMaxH3VideoVAE if component == "video" else vae_module.MiniMaxH3AudioVAE
|
||||
if trusted:
|
||||
model = vae_cls("unused", device=torch.device("cpu"), encode_only=True, trust_remote_code=True)
|
||||
assert model.remote is remote
|
||||
assert calls == [("unused", config)]
|
||||
else:
|
||||
with pytest.raises(ValueError, match="trust-remote-code"):
|
||||
vae_cls("unused", device=torch.device("cpu"), encode_only=True)
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_encoder_requires_all_role_policies(monkeypatch, tmp_path) -> None:
|
||||
encoder_module, _ = _patch_encoder_constructors(monkeypatch, rank=0)
|
||||
root = _component_root(tmp_path)
|
||||
config = _encoder_config(
|
||||
root,
|
||||
roles={
|
||||
"text_encoder": {"parallel_mode": "tp"},
|
||||
"video_vae": {"parallel_mode": "patch"},
|
||||
},
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match=r"requires exactly.*audio_vae"):
|
||||
encoder_module.MiniMaxH3Encoder(vllm_config=config)
|
||||
|
||||
|
||||
def test_encoder_does_not_infer_role_policies(monkeypatch, tmp_path) -> None:
|
||||
encoder_module, _ = _patch_encoder_constructors(monkeypatch, rank=0)
|
||||
root = _component_root(tmp_path)
|
||||
config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
model=str(root / "text_encoder"),
|
||||
hf_config=SimpleNamespace(),
|
||||
)
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="requires explicit minimax_h3_encoder_components"):
|
||||
encoder_module.MiniMaxH3Encoder(vllm_config=config)
|
||||
|
||||
|
||||
def test_video_leader_policy_skips_nonleader(monkeypatch, tmp_path) -> None:
|
||||
encoder_module, _ = _patch_encoder_constructors(monkeypatch, rank=1)
|
||||
root = _component_root(tmp_path)
|
||||
model = encoder_module.MiniMaxH3Encoder(vllm_config=_encoder_config(root, video_mode="leader"))
|
||||
|
||||
assert model.video_vae is None
|
||||
assert model.audio_vae is None
|
||||
assert (
|
||||
model._encode_media(
|
||||
MiniMaxH3EncoderMediaInput(
|
||||
task="ref2va",
|
||||
height=32,
|
||||
width=32,
|
||||
num_frames=48,
|
||||
latent_t=1,
|
||||
audio_t=80,
|
||||
images=(torch.zeros(32, 32, 3, dtype=torch.uint8),),
|
||||
)
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_encoder_runs_video_and_audio_components_on_ar_model() -> None:
|
||||
from vllm_omni.model_executor.models.minimax_h3.encoder import MiniMaxH3Encoder
|
||||
|
||||
class VideoVAE:
|
||||
def encode_image(self, _image):
|
||||
return torch.ones(1, 96)
|
||||
|
||||
def encode_video(self, _frames):
|
||||
return torch.full((1, 96), 2.0), (1, 2, 2)
|
||||
|
||||
class AudioVAE:
|
||||
def encode_waveform(self, waveform, _sample_rate):
|
||||
return torch.ones(160, 32), waveform.shape[-1] // 800
|
||||
|
||||
model = MiniMaxH3Encoder.__new__(MiniMaxH3Encoder)
|
||||
torch.nn.Module.__init__(model)
|
||||
model._component_leader = True
|
||||
model.video_vae = VideoVAE()
|
||||
model.audio_vae = AudioVAE()
|
||||
media = MiniMaxH3EncoderMediaInput(
|
||||
task="ref2va",
|
||||
height=32,
|
||||
width=32,
|
||||
num_frames=48,
|
||||
latent_t=1,
|
||||
audio_t=80,
|
||||
images=(torch.zeros(32, 32, 3, dtype=torch.uint8),),
|
||||
videos=(torch.zeros(1, 32, 32, 3, dtype=torch.uint8),),
|
||||
video_audios=((torch.zeros(64_000), 32_000),),
|
||||
)
|
||||
|
||||
conditioning = model._encode_media(media)
|
||||
|
||||
assert conditioning is not None
|
||||
assert conditioning.visual_condition_shapes == ((1, 2, 2), (1, 2, 2))
|
||||
assert conditioning.audio_condition_lengths == (80,)
|
||||
assert conditioning.ref_blocks[1]["kind"] == "video_audio"
|
||||
|
||||
|
||||
def test_prepare_encoder_inputs_keeps_reference_audio_budgets_separate(monkeypatch) -> None:
|
||||
from vllm_omni.model_executor.models.minimax_h3 import encoder_processing as processing
|
||||
|
||||
frames = torch.zeros(1, 32, 32, 3, dtype=torch.uint8).numpy()
|
||||
waveform = torch.zeros(320_000)
|
||||
monkeypatch.setattr(processing, "resolve_minimax_h3_shape", lambda *_args: (32, 32, 240, 60, 400))
|
||||
monkeypatch.setattr(
|
||||
processing,
|
||||
"prepare_reference_videos",
|
||||
lambda *_args, **_kwargs: [
|
||||
{"prepared_path": "prepared.mp4", "original_path": "reference.mp4", "input_has_audio": True}
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(processing, "load_video_frames", lambda _path: frames)
|
||||
monkeypatch.setattr(
|
||||
processing,
|
||||
"sample_reference_video_frames",
|
||||
lambda *_args, **_kwargs: {"frames": [frames[0]], "block_timestamps": [[0.0]]},
|
||||
)
|
||||
monkeypatch.setattr(processing, "load_video_audio", lambda *_args, **_kwargs: (waveform, 32_000))
|
||||
|
||||
prepared = processing.prepare_encoder_inputs(
|
||||
{
|
||||
"prompt": "reference",
|
||||
"multi_modal_data": {"video": "reference.mp4", "audio": (waveform, 32_000)},
|
||||
},
|
||||
SimpleNamespace(extra_args={"task": "ref2va"}),
|
||||
)
|
||||
|
||||
assert prepared.media.video_audios[0][0].shape[-1] == 320_000
|
||||
assert prepared.media.audios[0][0].shape[-1] == 320_000
|
||||
assert prepared.condition_labels == [("audio", 1), ("video", 1), ("audio", 2)]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("embedded_lengths", "standalone_lengths", "valid"),
|
||||
[((400,), (400,), True), ((320, 320), (400,), False), ((400,), (320, 320), False)],
|
||||
)
|
||||
def test_encode_media_keeps_audio_budgets_and_component_residency_separate(
|
||||
embedded_lengths, standalone_lengths, valid
|
||||
) -> None:
|
||||
from vllm_omni.model_executor.models.minimax_h3.encoder_processing import encode_media
|
||||
|
||||
active = None
|
||||
scopes = []
|
||||
|
||||
class VideoVAE:
|
||||
def encode_image(self, _image):
|
||||
assert active is self
|
||||
return torch.ones(1, 96)
|
||||
|
||||
def encode_video(self, _frames):
|
||||
assert active is self
|
||||
return torch.ones(1, 96), (1, 2, 2)
|
||||
|
||||
class AudioVAE:
|
||||
def encode_waveform(self, waveform, _sample_rate):
|
||||
assert active is self
|
||||
length = waveform.shape[-1] // 800
|
||||
return torch.ones(2 * length, 32), length
|
||||
|
||||
@contextmanager
|
||||
def component_scope(component):
|
||||
nonlocal active
|
||||
assert active is None
|
||||
active = component
|
||||
scopes.append(component)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
active = None
|
||||
|
||||
video_vae = VideoVAE()
|
||||
audio_vae = AudioVAE()
|
||||
media = MiniMaxH3EncoderMediaInput(
|
||||
task="ref2va",
|
||||
height=32,
|
||||
width=32,
|
||||
num_frames=360,
|
||||
latent_t=90,
|
||||
audio_t=600,
|
||||
images=(torch.zeros(32, 32, 3, dtype=torch.uint8),),
|
||||
videos=tuple(torch.zeros(1, 32, 32, 3, dtype=torch.uint8) for _ in embedded_lengths),
|
||||
video_audios=tuple((torch.zeros(length * 800), 32_000) for length in embedded_lengths),
|
||||
audios=tuple((torch.zeros(length * 800), 32_000) for length in standalone_lengths),
|
||||
)
|
||||
kwargs = dict(video_vae=video_vae, audio_vae=audio_vae, emit_conditioning=True, component_scope=component_scope)
|
||||
if valid:
|
||||
conditioning = encode_media(media, **kwargs)
|
||||
assert conditioning.audio_condition_lengths == embedded_lengths + standalone_lengths
|
||||
assert [block["kind"] for block in conditioning.ref_blocks] == ["image", "video_audio", "audio"]
|
||||
conditioning.to_omni_components()
|
||||
else:
|
||||
with pytest.raises(ValueError, match="at most 15 seconds"):
|
||||
encode_media(media, **kwargs)
|
||||
assert scopes == [video_vae, audio_vae]
|
||||
|
||||
|
||||
def test_encoder_output_reuses_encoder_handoff_and_round_trips() -> None:
|
||||
expected = MiniMaxH3EncoderConditioning(
|
||||
hidden_states=torch.randn(3, 5120, dtype=torch.bfloat16),
|
||||
token_tags=torch.tensor([1, 0, 1], dtype=torch.int64),
|
||||
task="t2va",
|
||||
height=256,
|
||||
width=448,
|
||||
num_frames=17,
|
||||
latent_t=5,
|
||||
audio_t=10,
|
||||
)
|
||||
|
||||
source = SimpleNamespace(
|
||||
finished=True,
|
||||
request_id="request-1",
|
||||
outputs=[SimpleNamespace(multimodal_output=flatten_payload(expected.to_omni_payload()))],
|
||||
)
|
||||
prompt = {
|
||||
"prompt": "hello",
|
||||
"additional_information": {"global_request_id": ["request-1"]},
|
||||
}
|
||||
|
||||
result = encoder2diffusion([source], prompt)
|
||||
|
||||
assert result is not None
|
||||
payload = result["additional_information"]["encoder_output"]
|
||||
actual = MiniMaxH3EncoderConditioning.from_omni_payload(payload)
|
||||
torch.testing.assert_close(actual.hidden_states, expected.hidden_states)
|
||||
torch.testing.assert_close(actual.token_tags, expected.token_tags)
|
||||
|
||||
|
||||
def test_encoder_releases_workspace_after_cpu_payload(monkeypatch) -> None:
|
||||
from vllm_omni.model_executor.models.minimax_h3 import encoder as encoder_module
|
||||
|
||||
model = encoder_module.MiniMaxH3Encoder.__new__(encoder_module.MiniMaxH3Encoder)
|
||||
torch.nn.Module.__init__(model)
|
||||
model._component_leader = True
|
||||
model._token_tags = torch.tensor([1])
|
||||
info = {
|
||||
"meta": {"minimax_h3_encoder_request": {}},
|
||||
"_minimax_h3_encoder_media_conditioning": MiniMaxH3EncoderMediaConditioning(
|
||||
task="t2va",
|
||||
height=256,
|
||||
width=448,
|
||||
num_frames=17,
|
||||
latent_t=5,
|
||||
audio_t=10,
|
||||
),
|
||||
}
|
||||
empty_cache_calls = 0
|
||||
|
||||
def empty_cache() -> None:
|
||||
nonlocal empty_cache_calls
|
||||
empty_cache_calls += 1
|
||||
|
||||
monkeypatch.setattr(torch.accelerator, "empty_cache", empty_cache)
|
||||
output = model.make_omni_output(
|
||||
torch.zeros(1, 5120),
|
||||
model_intermediate_buffer=[info],
|
||||
request_sample_eligible=[True],
|
||||
)
|
||||
|
||||
assert empty_cache_calls == 1
|
||||
assert output.multimodal_outputs is not None
|
||||
assert output.multimodal_outputs["embed"]["embedding"][0].device.type == "cpu"
|
||||
@@ -2,8 +2,6 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Regression tests for MiniMax H3's disaggregated text-encoder contract."""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
@@ -12,39 +10,37 @@ import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
from vllm_omni.data_entry_keys import OmniPayload, deserialize_payload, serialize_payload
|
||||
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import (
|
||||
_load_audio,
|
||||
resolve_minimax_h3_diffusion_model_path,
|
||||
from vllm_omni.data_entry_keys import flatten_payload
|
||||
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import resolve_minimax_h3_diffusion_model_path
|
||||
from vllm_omni.engine.serialization import (
|
||||
deserialize_additional_information,
|
||||
serialize_additional_information,
|
||||
)
|
||||
from vllm_omni.errors import OmniClientError
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.model_executor.models.minimax_h3.checkpoint import (
|
||||
resolve_minimax_h3_model_root,
|
||||
resolve_minimax_h3_encoder_model_root,
|
||||
resolve_minimax_h3_partition,
|
||||
)
|
||||
from vllm_omni.model_executor.models.minimax_h3.conditioning import (
|
||||
MINIMAX_H3_CONDITION_LABELS_KEY,
|
||||
MINIMAX_H3_ENCODER_REQUEST_KEY,
|
||||
MINIMAX_H3_PRESENTATION_TASK_KEY,
|
||||
MINIMAX_H3_TEXT_CONDITIONING_SCHEMA,
|
||||
MiniMaxH3TextConditioning,
|
||||
MiniMaxH3EncoderConditioning,
|
||||
)
|
||||
from vllm_omni.model_executor.models.minimax_h3.encoder_processing import _audio_items, _load_audio
|
||||
from vllm_omni.model_executor.models.minimax_h3.preprocessing import (
|
||||
minimax_h3_ref2va_presentation,
|
||||
minimax_h3_ref2va_video_presentation,
|
||||
)
|
||||
from vllm_omni.model_executor.models.minimax_h3.reference_video import (
|
||||
MINIMAX_H3_PREPARED_REFERENCE_VIDEOS_KEY,
|
||||
deserialize_prepared_reference_videos,
|
||||
)
|
||||
from vllm_omni.model_executor.models.minimax_h3.text_encoder import (
|
||||
MiniMaxH3MultiModalProcessor,
|
||||
_build_minimax_h3_presentation,
|
||||
)
|
||||
from vllm_omni.model_executor.stage_input_processors.minimax_h3 import (
|
||||
_audio_items,
|
||||
_diffusion_sampling_params,
|
||||
prepare_text_encoder_prompt,
|
||||
text_encoder2diffusion,
|
||||
encoder2diffusion,
|
||||
prepare_encoder_prompt,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
@@ -66,13 +62,14 @@ class _SegmentTokenizer:
|
||||
return self._special_ids[token]
|
||||
|
||||
|
||||
def test_h3_processor_reprocesses_media_instead_of_using_partial_sender_cache(mocker):
|
||||
def test_h3_processor_reprocesses_media_instead_of_using_partial_sender_cache(monkeypatch):
|
||||
processor = object.__new__(MiniMaxH3MultiModalProcessor)
|
||||
sentinel = ([1, 2, 3], object(), True)
|
||||
apply_processor = mocker.patch.object(
|
||||
apply_processor = Mock(return_value=sentinel)
|
||||
monkeypatch.setattr(
|
||||
MiniMaxH3MultiModalProcessor,
|
||||
"_apply_hf_processor",
|
||||
return_value=sentinel,
|
||||
apply_processor,
|
||||
)
|
||||
inputs = object()
|
||||
timing_ctx = object()
|
||||
@@ -119,9 +116,11 @@ def test_fused_audio_loader_accepts_list_waveform_pair():
|
||||
def test_prepare_ref2va_keeps_original_text_and_exact_condition_order():
|
||||
prompt = {
|
||||
"prompt": "hello",
|
||||
"additional_information": {"global_request_id": ["request-1"]},
|
||||
"model_intermediate_buffer": {"private": "preserved"},
|
||||
"multi_modal_data": {
|
||||
"image": Image.new("RGB", (256, 256)),
|
||||
"audio": [np.zeros(16), 16_000],
|
||||
"audio": [np.zeros(32_000), 16_000],
|
||||
},
|
||||
}
|
||||
sampling = OmniDiffusionSamplingParams(
|
||||
@@ -130,7 +129,7 @@ def test_prepare_ref2va_keeps_original_text_and_exact_condition_order():
|
||||
extra_args={"task": "ref2va"},
|
||||
)
|
||||
|
||||
transformed = prepare_text_encoder_prompt(prompt, [sampling])
|
||||
transformed = prepare_encoder_prompt(prompt, [sampling])
|
||||
|
||||
assert transformed["prompt"] == "hello"
|
||||
assert len(transformed["multi_modal_data"]["image"]) == 1
|
||||
@@ -140,223 +139,148 @@ def test_prepare_ref2va_keeps_original_text_and_exact_condition_order():
|
||||
("image", 1),
|
||||
("audio", 1),
|
||||
]
|
||||
assert transformed["model_intermediate_buffer"] == {"private": "preserved"}
|
||||
runner_info = transformed["additional_information"]
|
||||
for _ in range(2):
|
||||
wire = serialize_additional_information(runner_info)
|
||||
runner_info = deserialize_additional_information(wire)
|
||||
assert runner_info["global_request_id"] == ["request-1"]
|
||||
request_metadata = runner_info["meta"][MINIMAX_H3_ENCODER_REQUEST_KEY]
|
||||
assert request_metadata["task"] == "ref2va"
|
||||
assert isinstance(
|
||||
runner_info["hidden_states"]["layers"][0],
|
||||
torch.Tensor,
|
||||
)
|
||||
assert isinstance(
|
||||
runner_info["hidden_states"]["layers"][1],
|
||||
torch.Tensor,
|
||||
)
|
||||
from vllm_omni.model_executor.models.minimax_h3.encoder import MiniMaxH3Encoder
|
||||
|
||||
media = MiniMaxH3Encoder._media_input(runner_info)
|
||||
assert media.task == "ref2va"
|
||||
assert len(media.images) == 1
|
||||
assert len(media.audios) == 1
|
||||
|
||||
|
||||
def test_text_encoder_prompt_rejects_injected_prepared_video_descriptor():
|
||||
prompt = {
|
||||
"prompt": "hello",
|
||||
"additional_information": {"meta": {MINIMAX_H3_PREPARED_REFERENCE_VIDEOS_KEY: '{"artifact_dir":"/tmp/user"}'}},
|
||||
}
|
||||
sampling = OmniDiffusionSamplingParams(height=256, width=448, extra_args={"task": "t2va"})
|
||||
def _mock_ref2va_video_with_audio(monkeypatch, *, duration_seconds: float) -> None:
|
||||
from vllm_omni.model_executor.models.minimax_h3 import encoder_processing
|
||||
|
||||
transformed = prepare_text_encoder_prompt(prompt, [sampling])
|
||||
|
||||
assert MINIMAX_H3_PREPARED_REFERENCE_VIDEOS_KEY not in transformed["additional_information"]["meta"]
|
||||
|
||||
|
||||
def test_prepare_ref2va_video_uses_shared_frame_sampler_once(monkeypatch):
|
||||
prepare_videos = Mock()
|
||||
|
||||
def prepare(_videos, *, target_frame_count, workdir, start_time_seconds):
|
||||
prepare_videos(_videos, target_frame_count, workdir, start_time_seconds)
|
||||
prepared_path = os.path.join(workdir, "prepared.mp4")
|
||||
open(prepared_path, "wb").close()
|
||||
return [
|
||||
{
|
||||
"original_path": "/tmp/input.mp4",
|
||||
"prepared_path": prepared_path,
|
||||
"input_has_audio": True,
|
||||
"width": 448,
|
||||
"height": 256,
|
||||
"start_time_seconds": 0.0,
|
||||
"duration_seconds": 5.2,
|
||||
"audio_duration_seconds": 5.2,
|
||||
}
|
||||
]
|
||||
|
||||
sample_frames = Mock(return_value={"frames": [np.zeros((4, 4, 3), dtype=np.uint8)]})
|
||||
monkeypatch.setattr("vllm_omni.model_executor.stage_input_processors.minimax_h3.prepare_reference_videos", prepare)
|
||||
sample_rate = 16_000
|
||||
frames = np.zeros((4, 32, 32, 3), dtype=np.uint8)
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.model_executor.stage_input_processors.minimax_h3.sample_reference_video_frames",
|
||||
sample_frames,
|
||||
encoder_processing,
|
||||
"prepare_reference_videos",
|
||||
lambda *args, **kwargs: [
|
||||
{
|
||||
"prepared_path": "prepared.mp4",
|
||||
"original_path": "original.mp4",
|
||||
"input_has_audio": True,
|
||||
"duration_seconds": duration_seconds,
|
||||
"audio_duration_seconds": duration_seconds,
|
||||
}
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(encoder_processing, "load_video_frames", lambda path: frames)
|
||||
monkeypatch.setattr(
|
||||
encoder_processing,
|
||||
"sample_reference_video_frames",
|
||||
lambda *args, **kwargs: {"block_timestamps": [0.0], "frames": frames[:1]},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
encoder_processing,
|
||||
"load_video_audio",
|
||||
lambda *args, **kwargs: (torch.zeros(round(duration_seconds * sample_rate)), sample_rate),
|
||||
)
|
||||
|
||||
|
||||
def test_prepare_ref2va_rejects_short_embedded_video_audio(monkeypatch):
|
||||
_mock_ref2va_video_with_audio(monkeypatch, duration_seconds=1.0)
|
||||
sampling = OmniDiffusionSamplingParams(
|
||||
height=256,
|
||||
width=448,
|
||||
num_frames=96,
|
||||
extra_args={"task": "ref2va"},
|
||||
)
|
||||
prompt = {"prompt": "hello", "multi_modal_data": {"video": "original.mp4"}}
|
||||
|
||||
with pytest.raises(OmniClientError, match=r"duration must be in \[2, 15\] seconds, got 1\.000"):
|
||||
prepare_encoder_prompt(prompt, [sampling])
|
||||
|
||||
|
||||
def test_prepare_ref2va_keeps_embedded_and_standalone_audio_budgets_separate(monkeypatch):
|
||||
duration_seconds = 8.0
|
||||
sample_rate = 16_000
|
||||
_mock_ref2va_video_with_audio(monkeypatch, duration_seconds=duration_seconds)
|
||||
sampling = OmniDiffusionSamplingParams(
|
||||
height=256,
|
||||
width=448,
|
||||
num_frames=240,
|
||||
extra_args={"task": "ref2va"},
|
||||
)
|
||||
prompt = {
|
||||
"prompt": "hello",
|
||||
"multi_modal_data": {"video": "/tmp/input.mp4"},
|
||||
"multi_modal_data": {
|
||||
"video": "original.mp4",
|
||||
"audio": (torch.zeros(int(duration_seconds * sample_rate)), sample_rate),
|
||||
},
|
||||
}
|
||||
sampling = OmniDiffusionSamplingParams(height=256, width=448, num_frames=124, extra_args={"task": "ref2va"})
|
||||
|
||||
transformed = prepare_text_encoder_prompt(prompt, [sampling])
|
||||
|
||||
prepare_videos.assert_called_once()
|
||||
_, target_frame_count, artifact_dir, start_time_seconds = prepare_videos.call_args.args
|
||||
assert target_frame_count == 124
|
||||
assert start_time_seconds is None
|
||||
descriptor = transformed["additional_information"]["meta"][MINIMAX_H3_PREPARED_REFERENCE_VIDEOS_KEY]
|
||||
described_dir, described_videos = deserialize_prepared_reference_videos(descriptor)
|
||||
assert described_dir == artifact_dir
|
||||
assert os.path.isfile(described_videos[0]["prepared_path"])
|
||||
sample_frames.assert_called_once_with(described_videos[0]["prepared_path"])
|
||||
assert prompt["multi_modal_data"] == {"video": "/tmp/input.mp4"}
|
||||
transformed = prepare_encoder_prompt(prompt, [sampling])
|
||||
assert transformed["mm_processor_kwargs"][MINIMAX_H3_CONDITION_LABELS_KEY] == [
|
||||
("audio", 1),
|
||||
("video", 1),
|
||||
("audio", 2),
|
||||
]
|
||||
shutil.rmtree(artifact_dir)
|
||||
|
||||
|
||||
def test_stage_wire_rejects_multiple_text_encoder_sources():
|
||||
with pytest.raises(RuntimeError, match="exactly one text-encoder source"):
|
||||
text_encoder2diffusion([SimpleNamespace(), SimpleNamespace()], prompt={"prompt": "hello"})
|
||||
def _encoder_output() -> dict:
|
||||
return MiniMaxH3EncoderConditioning(
|
||||
hidden_states=torch.randn(3, 5120, dtype=torch.bfloat16),
|
||||
token_tags=torch.tensor([1, 0, 1], dtype=torch.int64),
|
||||
task="t2va",
|
||||
height=256,
|
||||
width=448,
|
||||
num_frames=17,
|
||||
latent_t=5,
|
||||
audio_t=10,
|
||||
).to_omni_payload()
|
||||
|
||||
|
||||
def test_stage_wire_rejects_request_id_mismatch():
|
||||
source = SimpleNamespace(request_id="other", outputs=[SimpleNamespace()])
|
||||
def test_encoder2diffusion_reuses_encoder_handoff() -> None:
|
||||
prompt = {
|
||||
"prompt": "hello",
|
||||
"additional_information": {"global_request_id": ["req-1"]},
|
||||
}
|
||||
|
||||
with pytest.raises(RuntimeError, match="request ID does not match"):
|
||||
text_encoder2diffusion([source], prompt=prompt)
|
||||
|
||||
|
||||
def test_stage_wire_rejects_multiple_completions():
|
||||
source = SimpleNamespace(request_id="req-1", outputs=[SimpleNamespace(), SimpleNamespace()])
|
||||
|
||||
with pytest.raises(RuntimeError, match="exactly one completion"):
|
||||
text_encoder2diffusion([source], prompt={"prompt": "hello"})
|
||||
|
||||
|
||||
def _source_output(payload: dict) -> SimpleNamespace:
|
||||
completion = SimpleNamespace(multimodal_output=payload)
|
||||
return SimpleNamespace(request_id="request-1", outputs=[completion])
|
||||
|
||||
|
||||
def _diffusion_prompt() -> dict:
|
||||
return {
|
||||
"request_id": "request-1",
|
||||
"prompt": "test prompt",
|
||||
"additional_information": {},
|
||||
}
|
||||
|
||||
|
||||
def _valid_text_conditioning_payload() -> OmniPayload:
|
||||
return {
|
||||
"hidden_states": {
|
||||
"output": torch.zeros(4, 5120, dtype=torch.bfloat16),
|
||||
},
|
||||
"meta": {
|
||||
"token_role_ids": torch.tensor([[1], [1], [0], [0]], dtype=torch.int64),
|
||||
"multi_modal_data": {"image": object()},
|
||||
"additional_information": {
|
||||
"private": "preserved",
|
||||
"meta": {MINIMAX_H3_ENCODER_REQUEST_KEY: {"task": "t2va"}},
|
||||
"hidden_states": {"layers": {0: torch.zeros(1)}},
|
||||
},
|
||||
"model_intermediate_buffer": {"private": "encoder-only"},
|
||||
}
|
||||
|
||||
|
||||
def test_text_encoder2diffusion_reads_hidden_states_output_and_token_role_ids() -> None:
|
||||
payload = _valid_text_conditioning_payload()
|
||||
hidden = payload["hidden_states"]["output"]
|
||||
token_role_ids = payload["meta"]["token_role_ids"]
|
||||
|
||||
result = text_encoder2diffusion(
|
||||
[_source_output(payload)],
|
||||
_diffusion_prompt(),
|
||||
source = SimpleNamespace(
|
||||
finished=True,
|
||||
request_id="request-1",
|
||||
outputs=[SimpleNamespace(multimodal_output=flatten_payload(_encoder_output()))],
|
||||
)
|
||||
|
||||
conditioning = result["additional_information"]["text_encoder_output"]
|
||||
torch.testing.assert_close(conditioning["hidden_states"], hidden)
|
||||
torch.testing.assert_close(conditioning["token_tags"], token_role_ids.squeeze(-1))
|
||||
result = encoder2diffusion([source], prompt)
|
||||
|
||||
assert result["prompt"] == "test prompt"
|
||||
assert result["multi_modal_data"] is None
|
||||
assert "model_intermediate_buffer" not in result
|
||||
additional_information = result["additional_information"]
|
||||
assert additional_information["private"] == "preserved"
|
||||
assert "hidden_states" not in additional_information
|
||||
assert "meta" not in additional_information
|
||||
parsed = MiniMaxH3EncoderConditioning.from_omni_payload(additional_information["encoder_output"])
|
||||
assert parsed.task == "t2va"
|
||||
|
||||
|
||||
def test_text_conditioning_artifact_round_trips_through_omni_payload_wire_format() -> None:
|
||||
payload = _valid_text_conditioning_payload()
|
||||
|
||||
wire = serialize_payload(payload)
|
||||
|
||||
assert wire is not None
|
||||
restored = deserialize_payload(wire)
|
||||
conditioning = MiniMaxH3TextConditioning.from_omni_payload(restored)
|
||||
assert MINIMAX_H3_TEXT_CONDITIONING_SCHEMA == "minimax_h3.text_conditioning/v1"
|
||||
torch.testing.assert_close(
|
||||
conditioning.hidden_states,
|
||||
payload["hidden_states"]["output"],
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
conditioning.token_tags,
|
||||
payload["meta"]["token_role_ids"].squeeze(-1),
|
||||
)
|
||||
assert conditioning.hidden_states.dtype == torch.bfloat16
|
||||
assert conditioning.hidden_states.is_contiguous()
|
||||
assert conditioning.token_tags.dtype == torch.int64
|
||||
assert conditioning.token_tags.is_contiguous()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("payload", "message"),
|
||||
[
|
||||
({"meta": {"token_role_ids": torch.ones(4, 1)}}, "no hidden_states payload"),
|
||||
({"hidden_states": {}, "meta": {"token_role_ids": torch.ones(4, 1)}}, "no hidden_states.output tensor"),
|
||||
({"hidden_states": {"output": torch.randn(4, 5120)}}, "no conditioning metadata"),
|
||||
({"hidden_states": {"output": torch.randn(4, 5120)}, "meta": {}}, "no token_role_ids tensor"),
|
||||
(
|
||||
{
|
||||
"hidden_states": {"output": torch.randn(4, 5120)},
|
||||
"meta": {"token_role_ids": torch.ones(4)},
|
||||
},
|
||||
r"must have shape \[tokens, 1\]",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_text_encoder2diffusion_rejects_invalid_conditioning_payload(payload: dict, message: str) -> None:
|
||||
with pytest.raises(RuntimeError, match=message):
|
||||
text_encoder2diffusion([_source_output(payload)], _diffusion_prompt())
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("case", "message"),
|
||||
[
|
||||
("hidden_width", r"hidden_states must have shape \[tokens, 5120\]"),
|
||||
("hidden_dtype", "hidden_states must have dtype torch.bfloat16"),
|
||||
("hidden_noncontiguous", "hidden_states must use contiguous strided layout"),
|
||||
("hidden_non_strided", "hidden_states must use contiguous strided layout"),
|
||||
("token_dtype", "stage-wire token_role_ids must have dtype torch.int64"),
|
||||
("token_layout", "stage-wire token_role_ids must use contiguous strided layout"),
|
||||
("token_count", "token_tags must align with hidden_states"),
|
||||
("token_value", "token_tags must contain only 0 and 1"),
|
||||
],
|
||||
)
|
||||
def test_text_encoder2diffusion_rejects_h3_v1_contract_mismatch(case: str, message: str) -> None:
|
||||
payload = _valid_text_conditioning_payload()
|
||||
|
||||
if case == "hidden_width":
|
||||
payload["hidden_states"]["output"] = torch.zeros(4, 5119, dtype=torch.bfloat16)
|
||||
elif case == "hidden_dtype":
|
||||
payload["hidden_states"]["output"] = torch.zeros(4, 5120, dtype=torch.float32)
|
||||
elif case == "hidden_noncontiguous":
|
||||
payload["hidden_states"]["output"] = torch.empty(5120, 4, dtype=torch.bfloat16).t()
|
||||
elif case == "hidden_non_strided":
|
||||
payload["hidden_states"]["output"] = torch.empty(
|
||||
(4, 5120),
|
||||
dtype=torch.bfloat16,
|
||||
layout=torch.sparse_coo,
|
||||
)
|
||||
elif case == "token_dtype":
|
||||
payload["meta"]["token_role_ids"] = torch.tensor([[1], [1], [0], [0]], dtype=torch.int32)
|
||||
elif case == "token_layout":
|
||||
payload["meta"]["token_role_ids"] = torch.tensor(
|
||||
[[1], [0], [1], [0], [1], [0], [1], [0]],
|
||||
dtype=torch.int64,
|
||||
)[::2]
|
||||
elif case == "token_count":
|
||||
payload["meta"]["token_role_ids"] = torch.tensor([[1], [0], [0]], dtype=torch.int64)
|
||||
elif case == "token_value":
|
||||
payload["meta"]["token_role_ids"] = torch.tensor([[1], [1], [2], [0]], dtype=torch.int64)
|
||||
else: # pragma: no cover - parameterization is exhaustive
|
||||
raise AssertionError(f"unknown test case: {case}")
|
||||
|
||||
with pytest.raises(RuntimeError, match=message) as exc_info:
|
||||
text_encoder2diffusion([_source_output(payload)], _diffusion_prompt())
|
||||
|
||||
assert MINIMAX_H3_TEXT_CONDITIONING_SCHEMA in str(exc_info.value)
|
||||
def test_encoder2diffusion_waits_for_one_finished_source() -> None:
|
||||
assert encoder2diffusion([SimpleNamespace(finished=False)], {"prompt": "hello"}) is None
|
||||
with pytest.raises(RuntimeError, match="exactly one encoder source"):
|
||||
encoder2diffusion([SimpleNamespace(), SimpleNamespace()], {"prompt": "hello"})
|
||||
|
||||
|
||||
def test_ref2va_one_image_tokens_and_tags_match_fused_presentation():
|
||||
@@ -421,15 +345,20 @@ def test_checkpoint_resolver_selects_local_partition(tmp_path):
|
||||
(root / "FL2VA" / "text_encoder").mkdir(parents=True)
|
||||
(root / "Ref2VA" / "text_encoder").mkdir(parents=True)
|
||||
|
||||
assert resolve_minimax_h3_model_root(str(root), None, "fl2va") == str(root / "FL2VA" / "text_encoder")
|
||||
assert resolve_minimax_h3_model_root(str(root), None, "ref2va") == str(root / "Ref2VA" / "text_encoder")
|
||||
assert resolve_minimax_h3_model_root(str(root / "Ref2VA"), None, None) == str(root / "Ref2VA" / "text_encoder")
|
||||
assert resolve_minimax_h3_model_root(str(root / "FL2VA"), None, "ref2va") == str(root / "Ref2VA" / "text_encoder")
|
||||
assert resolve_minimax_h3_encoder_model_root(str(root), None, "fl2va") == str(root / "FL2VA" / "text_encoder")
|
||||
assert resolve_minimax_h3_encoder_model_root(str(root), None, "ref2va") == str(root / "Ref2VA" / "text_encoder")
|
||||
assert resolve_minimax_h3_encoder_model_root(str(root), None, "combined") == str(root / "FL2VA" / "text_encoder")
|
||||
assert resolve_minimax_h3_encoder_model_root(str(root / "Ref2VA"), None, None) == str(
|
||||
root / "Ref2VA" / "text_encoder"
|
||||
)
|
||||
assert resolve_minimax_h3_encoder_model_root(str(root / "FL2VA"), None, "ref2va") == str(
|
||||
root / "Ref2VA" / "text_encoder"
|
||||
)
|
||||
|
||||
|
||||
def test_checkpoint_resolver_rejects_unknown_task(tmp_path):
|
||||
with pytest.raises(ValueError, match="task_type must be one of"):
|
||||
resolve_minimax_h3_model_root(str(tmp_path), None, "unknown")
|
||||
resolve_minimax_h3_encoder_model_root(str(tmp_path), None, "unknown")
|
||||
|
||||
|
||||
def test_partition_resolver_preserves_consumer_auto_default(tmp_path):
|
||||
|
||||
Reference in new issue
Block a user