From 1f65814fab903abedea264369a7cc1b591452ba0 Mon Sep 17 00:00:00 2001 From: WeiQing Chen <40507679+david6666666@users.noreply.github.com> Date: Tue, 11 Aug 2026 13:36:37 +0800 Subject: [PATCH] [CI] Align MiniMax H3 Ref2VA inputs and add I2VA/Ref2VA accuracy coverage (#5978) Signed-off-by: david6666666 <530634352@qq.com> --- .buildkite/cuda/test-nightly.yml | 4 +- .../minimax_h3/test_minimax_h3_contract.py | 71 ++++ .../test_minimax_h3_i2va_ref2va_similarity.py | 386 ++++++++++++++++++ .../test_minimax_h3_t2va_similarity.py | 216 ---------- .../openai_api/test_video_server.py | 20 + .../models/minimax_h3/pipeline_minimax_h3.py | 30 +- .../models/minimax_h3/reference_video.py | 65 ++- vllm_omni/diffusion/utils/media_utils.py | 5 + 8 files changed, 535 insertions(+), 262 deletions(-) create mode 100644 tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py delete mode 100644 tests/e2e/accuracy/minimax_h3/test_minimax_h3_t2va_similarity.py diff --git a/.buildkite/cuda/test-nightly.yml b/.buildkite/cuda/test-nightly.yml index c482ae643..3640ddf21 100644 --- a/.buildkite/cuda/test-nightly.yml +++ b/.buildkite/cuda/test-nightly.yml @@ -427,10 +427,10 @@ steps: - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m full_model --run-level full_model mirror_hardwares: h100_2 - - label: ":full_moon: Diffusion X2V · MiniMax H3 T2VA Accuracy Test" + - label: ":full_moon: Diffusion X2V · MiniMax H3 I2VA/Ref2VA Accuracy Test" timeout_in_minutes: 180 commands: - - pytest -s -v tests/e2e/accuracy/minimax_h3/test_minimax_h3_t2va_similarity.py -m full_model --run-level full_model + - pytest -s -v tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py -m full_model --run-level full_model mirror_hardwares: h100_4 - label: ":full_moon: Diffusion X2V · Other Accuracy Test" diff --git a/tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py b/tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py index 66fb52a54..a195a6a8d 100644 --- a/tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py +++ b/tests/diffusion/models/minimax_h3/test_minimax_h3_contract.py @@ -1246,8 +1246,38 @@ def test_r7_r8_ref2va_video_segment_matrix(monkeypatch, tmp_path, case, start_ti ) assert prepared[0]["duration_seconds"] == pytest.approx(expected_duration) + assert prepared[0]["audio_duration_seconds"] == pytest.approx(min(expected_duration, 209 / 24.0)) assert prepared[0]["start_time_seconds"] == pytest.approx(start_time or 0.0) assert transcode_calls[0][1]["duration_seconds"] == pytest.approx(expected_duration) + assert transcode_calls[0][1]["target_frame_count"] == 209 + + +def test_ref2va_qwen_sampling_decodes_prepared_video_once(monkeypatch): + from vllm_omni.diffusion.models.minimax_h3 import reference_video as reference_video_module + + decoded = np.arange(25 * 2 * 2 * 3, dtype=np.uint8).reshape(25, 2, 2, 3) + load_calls = [] + monkeypatch.setattr( + reference_video_module, + "_probe_video", + lambda _path: {"frame_count": len(decoded)}, + ) + monkeypatch.setattr( + reference_video_module, + "load_video_frames", + lambda path: load_calls.append(path) or decoded, + ) + + sampled = reference_video_module.sample_reference_video_frames("prepared.mp4") + + assert load_calls == ["prepared.mp4"] + for frame, expected in zip( + sampled["frames"], + (decoded[0], decoded[12], decoded[24]), + strict=True, + ): + np.testing.assert_array_equal(frame, expected) + assert sampled["block_timestamps"] == [0.25, 1.0] def test_ref2va_two_video_recipe_tolerates_container_rounding(monkeypatch, tmp_path): @@ -1487,6 +1517,47 @@ def test_g4_standalone_audio_duration_and_total_duration_contract(): ) +def test_ref2va_audio_duration_validation_precedes_rank_branch(monkeypatch): + from vllm_omni.diffusion.models.minimax_h3 import pipeline_minimax_h3 as pipeline_module + + pipeline = object.__new__(pipeline_module.MiniMaxH3Pipeline) + pipeline.device = torch.device("cpu") + monkeypatch.setattr(pipeline_module, "_dit_rank_world", lambda: (None, 1, 2)) + + waveform = torch.zeros(1, 10) + with pytest.raises(ValueError, match="max_duration_seconds must be positive"): + pipeline._encode_audio_conditions( + [(waveform, 10)], + max_duration_seconds=0, + ) + + +def test_ref2va_standalone_audio_condition_is_bounded_to_output_duration(): + from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import MiniMaxH3Pipeline + + pipeline = object.__new__(MiniMaxH3Pipeline) + pipeline.device = torch.device("cpu") + observed = [] + + class FakeAudioVAE: + def encode_waveform(self, waveform, sample_rate): + observed.append((waveform.clone(), sample_rate)) + return waveform, waveform.shape[-1] + + pipeline.audio_vae = FakeAudioVAE() + waveform = torch.arange(40, dtype=torch.float32).reshape(1, 40) + + encoded, lengths = pipeline._encode_audio_conditions( + [(waveform, 10)], + max_duration_seconds=2.5, + ) + + assert observed[0][0].shape == (1, 25) + assert observed[0][1] == 10 + assert encoded.shape == (1, 25) + assert lengths == [25] + + @pytest.mark.parametrize( ("field", "value", "message"), [ diff --git a/tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py b/tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py new file mode 100644 index 000000000..b00e90d5e --- /dev/null +++ b/tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py @@ -0,0 +1,386 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from __future__ import annotations + +import hashlib +import json +import os +import subprocess +from pathlib import Path + +import pytest +import requests +import torch +from huggingface_hub import snapshot_download + +from tests.e2e.accuracy.helpers import ( + assert_video_metadata, + assert_video_similarity_metrics, + probe_binary, + probe_video, + reset_artifact_dir, +) +from tests.helpers.mark import hardware_test +from tests.helpers.runtime import OmniServer + +pytestmark = [pytest.mark.benchmark, pytest.mark.diffusion, pytest.mark.full_model] + +MODEL_REPO_ID = "MiniMaxAI/MiniMax-H3" +# The previously pinned revision was removed from the Hugging Face repository. +MODEL_REVISION = "main" +MODEL_ENV_VAR = "VLLM_TEST_MINIMAX_H3_FL2VA_MODEL" +REF2VA_MODEL_ENV_VAR = "VLLM_TEST_MINIMAX_H3_REF2VA_MODEL" +I2VA_REFERENCE_VIDEO_URL = "https://huggingface.co/MiniMaxAI/MiniMax-H3/blob/main/assets/i2va.mp4" +I2VA_REFERENCE_VIDEO_SHA256 = "5a5d6e0fcdda7e6a5d98cb92d9e5cb4b6b0b44d6e1dfd5a370d16dfaa79b9ae6" +I2VA_IMAGE_URL = ( + "https://cdn.hailuoai.com/prod/hailuo_demo/testsets/H3_AA_I2VA/gallery/" + "sr_v17_variants_seed42_43_20260724/inputs/4a3a90bf9100_KDmcbkhzYo5sjjxr9FqcVmWVnzb.png" +) +I2VA_IMAGE_SHA256 = "352e959e08b886406d3766cee25ebe7bbf679e6d53eeffbd1c607b8dc93e2068" +REF2VA_REFERENCE_VIDEO_URL = "https://huggingface.co/MiniMaxAI/MiniMax-H3/blob/main/assets/r2va.mp4" +REF2VA_REFERENCE_VIDEO_SHA256 = "5c9d33dfc517438c59e93da7b3d0b2a62e3cf69bc2adb7d1b2d33d7d8df7161f" +REF2VA_INPUT_VIDEO_URL = ( + "https://cdn.hailuoai.com/prod/hailuo_demo/testsets/h3_promo_eval_ref2va/" + "gallery/sr_v2p26_trio_seed42_20260724/inputs/" + "297573323635_00_%E8%A7%86%E9%A2%911_YnyRbxEwio_video_20260525_163755_1927e9d3.mp4" +) +REF2VA_INPUT_VIDEO_SHA256 = "d6edc20ac82aa980c9cbd0b05b371f60d12dc8d018492e571b42de0086819c41" +REF2VA_INPUT_AUDIO_URL = ( + "https://cdn.hailuoai.com/prod/hailuo_demo/testsets/h3_promo_eval_ref2va/" + "gallery/sr_v2p26_trio_seed42_20260724/inputs/" + "f463d523c5ce_01_%E9%9F%B3%E9%A2%911_RSLcbpzJPo_6%E6%9C%885%E6%97%A5(1).mp3" +) +REF2VA_INPUT_AUDIO_SHA256 = "2a48d447b343b38a2c504d5281a0c97c688db0ba87931ed9282c598bd6752454" + +I2VA_PROMPT = """For the target video, at 0.00 seconds into the target video, (from [Shot 1]) is fully referenced. + +integrated_multimodal_description: [Shot 1] This is a live-action, cinematic shot with a shallow depth of field. The camera holds a perfectly static shot throughout the entire eight-second duration, capturing a cozy family gathering in a traditional Japanese dining room. The scene opens with a large, intricately patterned blue and white ceramic bowl of ramen in the immediate foreground, rendered in crisp, sharp focus. The bowl sits on a smooth, polished long wooden table. Inside the bowl, a rich, oily golden-brown broth surrounds yellow wavy noodles, topped with two thick, round slices of chashu pork featuring visible fat marbling and a distinct spiral meat pattern. A generous mound of freshly chopped, bright green scallions rests in the center, and a crisp, dark green rectangular sheet of nori seaweed is tucked into the right edge. To the left of the bowl, a pair of light brown wooden chopsticks rests horizontally on a small, dark rectangular chopstick rest, near a small cylindrical ceramic teacup with blue painted patterns. On the right side of the table, a spherical paper lantern with a ribbed bamboo frame sits on a black wooden base. In the background, a large family of seven is gathered around the table, initially appearing as a soft, blurred presence. Behind them, traditional Japanese sliding shoji screens with wooden lattice frames are open, revealing a bright outdoor scene with lush green trees. Early in the clip, the thick, white steam rising from the hot ramen broth immediately intensifies, billowing upwards in thick, swirling clouds that dance continuously above the bowl. As the clip progresses into the middle seconds, the camera maintains its static position while the focus begins a deliberate, smooth shift deeper into the room. The foreground ramen bowl, its vibrant ingredients, and the rising steam gradually soften into a hazy, out-of-focus blur. Simultaneously, the family members in the background come into sharp, detailed clarity. The heavy steam continues to rise from the foreground, creating a dynamic, translucent veil between the camera and the family. With the focus now firmly locked on the background, the vibrant family dinner comes alive. The man in the dark navy blue long-sleeved shirt on the left leans forward, his mouth moving animatedly in a silent exchange. The young girl in the crisp white short-sleeved t-shirt beside him smiles brightly, looking toward the center of the table. The woman on the far left, wearing a soft light blue long-sleeved blouse, turns her head slightly, smiling gently. Across the table, the woman in the light grey button-down shirt smiles broadly, her eyes crinkling, as she rests her hands near her plate. The woman in the dark grey top further back uses her wooden chopsticks to pick up a small piece of food from a central ceramic dish filled with bright red pickled vegetables. The woman in the center back in the light grey sweater smiles gently, her hands clasped softly in front of her, observing the interaction. Throughout the remainder of the clip, the family continues their lively physical interaction, their mouths moving in continuous, silent cadences of conversation, while the thick, white steam from the blurred ramen bowl in the foreground never stops rising, adding a comforting atmosphere to the warm gathering. + +overall_soundscape: The soundscape begins with a quiet room tone mixed with the faint, airy rustle of the thick steam billowing from the hot ramen bowl in the foreground, accompanied by the subtle, continuous hissing and bubbling of the rich broth. As the visual focus shifts deeper into the room, the physical sounds of the bustling family dinner become dominant in the foreground. The clear, sharp clinking of ceramic bowls and wooden chopsticks touching plates is clearly heard as the family members reach for food. This is followed by the faint, muffled thud of a cup being set down on the smooth wooden table, and the subtle, rhythmic rustle of cotton and wool clothing as the family members lean forward and gesture, perfectly capturing the lively, physical atmosphere of the shared meal. + +non_diegetic_music: A gentle, heartwarming acoustic guitar melody plays softly in the background, accompanied by the subtle, resonant notes of a traditional Japanese koto. The music maintains a slow, comforting tempo that enhances the cozy, nostalgic, and joyful atmosphere of the family gathering.""" + +REF2VA_PROMPT = """subject_definitions: + is the young man with short wavy blonde hair, wearing a bright pink suit jacket, matching pink trousers, an unbuttoned white shirt, and silver rings, holding a small black lamb in his arms in