[Feature] Align MiniMax H3 official input matrix (#5752)
Signed-off-by: david6666666 <530634352@qq.com>
This commit is contained in:
1 parent
6a604be610
commit
ae7bee0c43
21 files changed
+1887
-331
No files matched your search
@@ -102,6 +102,52 @@ def test_reference_image_resize_contract():
|
||||
_reference_image_shape(Image.new("RGB", (100, 501)))
|
||||
|
||||
|
||||
def test_fl2va_supports_first_last_and_explicit_frame_index_contracts():
|
||||
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import (
|
||||
_resolve_fl2va_keyframe_indices,
|
||||
)
|
||||
|
||||
assert _resolve_fl2va_keyframe_indices({}, 1) == [0]
|
||||
assert _resolve_fl2va_keyframe_indices({}, 2) == [0, -1]
|
||||
assert _resolve_fl2va_keyframe_indices({"frame_index": -1}, 1) == [-1]
|
||||
assert _resolve_fl2va_keyframe_indices({"frame_indices": [0, -1]}, 2) == [0, -1]
|
||||
with pytest.raises(ValueError, match="frame_indices"):
|
||||
_resolve_fl2va_keyframe_indices({"frame_indices": [0, 1]}, 2)
|
||||
|
||||
|
||||
def test_minimax_h3_uses_the_official_output_canvas_policy():
|
||||
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import (
|
||||
_resolve_output_canvas,
|
||||
)
|
||||
|
||||
assert _resolve_output_canvas(21 / 9, 768) == (672, 1536)
|
||||
assert _resolve_output_canvas(16 / 9, 768) == (768, 1344)
|
||||
assert _resolve_output_canvas(9 / 16, 768) == (1344, 768)
|
||||
with pytest.raises(ValueError, match="short_edge"):
|
||||
_resolve_output_canvas(16 / 9, 720)
|
||||
|
||||
|
||||
def test_minimax_h3_accepts_sglang_auto_aspect_ratio_alias():
|
||||
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import MiniMaxH3Pipeline
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
sampling = SimpleNamespace(
|
||||
fps=24,
|
||||
num_frames=1,
|
||||
height=None,
|
||||
width=None,
|
||||
extra_args={"duration": 5.0, "aspect_ratio": "auto"},
|
||||
)
|
||||
height, width, *_ = pipeline._resolve_shape("ref2va", sampling, None)
|
||||
assert (height, width) == (768, 1344)
|
||||
|
||||
|
||||
def test_minimax_h3_advertises_the_official_ref2va_image_limit():
|
||||
from vllm_omni.diffusion.model_metadata import get_diffusion_model_metadata
|
||||
|
||||
assert get_diffusion_model_metadata("MiniMaxH3Pipeline").max_multimodal_image_inputs == 9
|
||||
|
||||
|
||||
def test_encoder_forward_uses_hook_compatible_encode_entrypoint():
|
||||
from vllm_omni.diffusion.models.minimax_h3.encoder import (
|
||||
MiniMaxH3Qwen3VLEncoder,
|
||||
@@ -373,3 +419,360 @@ def test_distributed_video_vae_encodes_references_sequentially(monkeypatch):
|
||||
torch.tensor([[1.0, 1.0], [2.0, 2.0]]),
|
||||
)
|
||||
assert shapes == [(1, 2, 3), (2, 2, 3)]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("case", "extra", "image_count", "expected"),
|
||||
[
|
||||
("F1", {"frame_index": -1}, 1, [-1]),
|
||||
("F2", {"frame_indices": [0, -1]}, 2, [0, -1]),
|
||||
],
|
||||
)
|
||||
def test_f1_f2_official_fl2va_keyframe_matrix(case, extra, image_count, expected):
|
||||
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import (
|
||||
_resolve_fl2va_keyframe_indices,
|
||||
)
|
||||
|
||||
assert case in {"F1", "F2"}
|
||||
assert _resolve_fl2va_keyframe_indices(extra, image_count) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("case", "counts"),
|
||||
[
|
||||
("R1", (3, 0, 0)),
|
||||
("R2", (1, 0, 0)),
|
||||
("R3", (1, 1, 0)),
|
||||
("R4", (0, 1, 1)),
|
||||
("R5", (1, 1, 1)),
|
||||
("R6", (1, 0, 2)),
|
||||
],
|
||||
)
|
||||
def test_r1_r6_ref2va_reference_count_matrix(case, counts):
|
||||
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import (
|
||||
_validate_ref2va_reference_counts,
|
||||
)
|
||||
|
||||
assert case.startswith("R")
|
||||
_validate_ref2va_reference_counts(*counts)
|
||||
|
||||
|
||||
def test_ref2va_reference_count_validation_preserves_client_error_metadata():
|
||||
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import (
|
||||
_validate_ref2va_reference_counts,
|
||||
)
|
||||
from vllm_omni.errors import OmniClientError
|
||||
|
||||
with pytest.raises(OmniClientError, match="at least one image or video"):
|
||||
_validate_ref2va_reference_counts(0, 0, 0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("case", "start_time", "expected_duration"),
|
||||
[
|
||||
("R7", None, 10.0),
|
||||
("R8", 4.0, 6.0),
|
||||
],
|
||||
)
|
||||
def test_r7_r8_ref2va_video_segment_matrix(monkeypatch, tmp_path, case, start_time, expected_duration):
|
||||
from vllm_omni.diffusion.models.minimax_h3 import reference_video as reference_video_module
|
||||
|
||||
source = tmp_path / f"{case}.mp4"
|
||||
source.touch()
|
||||
metadata = {
|
||||
"width": 1280,
|
||||
"height": 720,
|
||||
"fps": 24.0,
|
||||
"frame_count": 240,
|
||||
"duration": 10.0,
|
||||
"format_names": ("mp4",),
|
||||
"video_codec": "h264",
|
||||
"audio_codecs": (),
|
||||
"file_size": 1024,
|
||||
}
|
||||
transcode_calls = []
|
||||
monkeypatch.setattr(reference_video_module, "_probe_video", lambda _path: metadata)
|
||||
monkeypatch.setattr(
|
||||
reference_video_module,
|
||||
"_transcode_reference_video",
|
||||
lambda source, **kwargs: transcode_calls.append((source, kwargs)) or "prepared.mp4",
|
||||
)
|
||||
|
||||
prepared = reference_video_module.prepare_reference_videos(
|
||||
[str(source)],
|
||||
target_frame_count=209,
|
||||
workdir=str(tmp_path / "work"),
|
||||
start_time_seconds=start_time,
|
||||
)
|
||||
|
||||
assert prepared[0]["duration_seconds"] == pytest.approx(expected_duration)
|
||||
assert prepared[0]["start_time_seconds"] == pytest.approx(start_time or 0.0)
|
||||
assert transcode_calls[0][1]["duration_seconds"] == pytest.approx(expected_duration)
|
||||
|
||||
|
||||
def test_ref2va_two_video_recipe_tolerates_container_rounding(monkeypatch, tmp_path):
|
||||
from vllm_omni.diffusion.models.minimax_h3 import reference_video as reference_video_module
|
||||
|
||||
first = tmp_path / "first.mp4"
|
||||
second = tmp_path / "second.mp4"
|
||||
first.touch()
|
||||
second.touch()
|
||||
metadata = {
|
||||
str(first): {
|
||||
"width": 1280,
|
||||
"height": 720,
|
||||
"fps": 24.0,
|
||||
"frame_count": 241,
|
||||
"duration": 10.041667,
|
||||
"format_names": ("mp4",),
|
||||
"video_codec": "h264",
|
||||
"audio_codecs": (),
|
||||
"file_size": 1024,
|
||||
},
|
||||
str(second): {
|
||||
"width": 1280,
|
||||
"height": 720,
|
||||
"fps": 24.0,
|
||||
"frame_count": 120,
|
||||
"duration": 4.966667,
|
||||
"format_names": ("mp4",),
|
||||
"video_codec": "h264",
|
||||
"audio_codecs": (),
|
||||
"file_size": 1024,
|
||||
},
|
||||
}
|
||||
transcode_calls = []
|
||||
monkeypatch.setattr(reference_video_module, "_probe_video", lambda path: metadata[str(path)])
|
||||
monkeypatch.setattr(
|
||||
reference_video_module,
|
||||
"_transcode_reference_video",
|
||||
lambda source, **kwargs: transcode_calls.append((source, kwargs)) or f"{source}.prepared.mp4",
|
||||
)
|
||||
|
||||
prepared = reference_video_module.prepare_reference_videos(
|
||||
[str(first), str(second)],
|
||||
target_frame_count=124,
|
||||
workdir=str(tmp_path / "work"),
|
||||
)
|
||||
|
||||
assert [item["duration_seconds"] for item in prepared] == pytest.approx([10.041667, 4.958333])
|
||||
assert sum(item["duration_seconds"] for item in prepared) == pytest.approx(15.0)
|
||||
assert transcode_calls[1][1]["duration_seconds"] == pytest.approx(4.958333)
|
||||
|
||||
|
||||
def test_ref2va_two_video_recipe_rejects_real_duration_overflow(monkeypatch, tmp_path):
|
||||
from vllm_omni.diffusion.models.minimax_h3 import reference_video as reference_video_module
|
||||
from vllm_omni.errors import OmniClientError
|
||||
|
||||
first = tmp_path / "first.mp4"
|
||||
second = tmp_path / "second.mp4"
|
||||
first.touch()
|
||||
second.touch()
|
||||
metadata = {
|
||||
str(first): {
|
||||
"width": 1280,
|
||||
"height": 720,
|
||||
"fps": 24.0,
|
||||
"frame_count": 240,
|
||||
"duration": 10.0,
|
||||
"format_names": ("mp4",),
|
||||
"video_codec": "h264",
|
||||
"audio_codecs": (),
|
||||
"file_size": 1024,
|
||||
},
|
||||
str(second): {
|
||||
"width": 1280,
|
||||
"height": 720,
|
||||
"fps": 24.0,
|
||||
"frame_count": 120,
|
||||
"duration": 5.02,
|
||||
"format_names": ("mp4",),
|
||||
"video_codec": "h264",
|
||||
"audio_codecs": (),
|
||||
"file_size": 1024,
|
||||
},
|
||||
}
|
||||
monkeypatch.setattr(reference_video_module, "_probe_video", lambda path: metadata[str(path)])
|
||||
monkeypatch.setattr(reference_video_module, "_transcode_reference_video", lambda source, **kwargs: "prepared.mp4")
|
||||
|
||||
with pytest.raises(OmniClientError, match="15 seconds"):
|
||||
reference_video_module.prepare_reference_videos(
|
||||
[str(first), str(second)],
|
||||
target_frame_count=124,
|
||||
workdir=str(tmp_path / "work"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("duration", [4.0, 15.0])
|
||||
def test_g2_output_duration_accepts_official_boundaries(duration):
|
||||
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
sampling = SimpleNamespace(
|
||||
fps=24,
|
||||
num_frames=1,
|
||||
height=None,
|
||||
width=None,
|
||||
extra_args={"duration": duration},
|
||||
)
|
||||
|
||||
height, width, *_ = pipeline._resolve_shape("ref2va", sampling, None)
|
||||
assert height > 0 and width > 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("duration", [3.99, 15.01, float("nan"), "not-a-duration"])
|
||||
def test_g2_output_duration_rejects_out_of_contract_values(duration):
|
||||
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
sampling = SimpleNamespace(
|
||||
fps=24,
|
||||
num_frames=1,
|
||||
height=None,
|
||||
width=None,
|
||||
extra_args={"duration": duration},
|
||||
)
|
||||
with pytest.raises(ValueError, match="duration"):
|
||||
pipeline._resolve_shape("ref2va", sampling, None)
|
||||
|
||||
|
||||
def test_g1_fanout_uses_incrementing_output_seeds():
|
||||
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import (
|
||||
_minimax_h3_output_seeds,
|
||||
_resolve_minimax_h3_num_outputs,
|
||||
)
|
||||
|
||||
assert _resolve_minimax_h3_num_outputs(3) == 3
|
||||
assert _minimax_h3_output_seeds(100, 3) == [100, 101, 102]
|
||||
with pytest.raises(ValueError, match="num_outputs_per_prompt"):
|
||||
_resolve_minimax_h3_num_outputs(11)
|
||||
|
||||
|
||||
def test_g3_task_specific_aspect_ratio_policy():
|
||||
from PIL import Image
|
||||
|
||||
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import (
|
||||
_resolve_minimax_h3_aspect_ratio,
|
||||
)
|
||||
|
||||
image = Image.new("RGB", (1280, 720))
|
||||
assert _resolve_minimax_h3_aspect_ratio("fl2va", "9:16", image) == pytest.approx(16 / 9)
|
||||
assert _resolve_minimax_h3_aspect_ratio("ref2va", None, None) == pytest.approx(16 / 9)
|
||||
assert _resolve_minimax_h3_aspect_ratio("ref2va", "auto", None) == pytest.approx(16 / 9)
|
||||
with pytest.raises(ValueError, match="requires an explicit"):
|
||||
_resolve_minimax_h3_aspect_ratio("t2va", None, None)
|
||||
with pytest.raises(ValueError, match="one of"):
|
||||
_resolve_minimax_h3_aspect_ratio("t2va", "2:1", None)
|
||||
|
||||
|
||||
def test_g3_t2va_shape_requires_a_named_ratio_and_fl2va_ignores_override():
|
||||
from PIL import Image
|
||||
|
||||
from vllm_omni.diffusion.models.minimax_h3 import MiniMaxH3Pipeline
|
||||
|
||||
pipeline = object.__new__(MiniMaxH3Pipeline)
|
||||
t2va_sampling = SimpleNamespace(
|
||||
fps=24,
|
||||
num_frames=1,
|
||||
height=None,
|
||||
width=None,
|
||||
extra_args={"duration": 5.0},
|
||||
)
|
||||
with pytest.raises(ValueError, match="requires an explicit aspect_ratio"):
|
||||
pipeline._resolve_shape("t2va", t2va_sampling, None)
|
||||
|
||||
fl2va_sampling = SimpleNamespace(
|
||||
fps=24,
|
||||
num_frames=1,
|
||||
height=None,
|
||||
width=None,
|
||||
extra_args={"duration": 5.0, "aspect_ratio": "9:16"},
|
||||
)
|
||||
height, width, *_ = pipeline._resolve_shape("fl2va", fl2va_sampling, Image.new("RGB", (1280, 720)))
|
||||
assert (height, width) == (768, 1344)
|
||||
|
||||
|
||||
def test_g4_reference_image_boundaries_and_aspect_ratio():
|
||||
from PIL import Image
|
||||
|
||||
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import (
|
||||
_validate_reference_image,
|
||||
)
|
||||
|
||||
_validate_reference_image(Image.new("RGB", (256, 256)))
|
||||
_validate_reference_image(Image.new("RGB", (5760, 2304)))
|
||||
with pytest.raises(ValueError, match="dimensions"):
|
||||
_validate_reference_image(Image.new("RGB", (255, 255)))
|
||||
with pytest.raises(ValueError, match="aspect ratio"):
|
||||
_validate_reference_image(Image.new("RGB", (256, 641)))
|
||||
|
||||
|
||||
def test_g4_reference_image_file_format_and_size_contract(tmp_path):
|
||||
from PIL import Image
|
||||
|
||||
from vllm_omni.diffusion.models.minimax_h3.pipeline_minimax_h3 import (
|
||||
_load_image,
|
||||
)
|
||||
|
||||
valid_path = tmp_path / "reference.png"
|
||||
Image.new("RGB", (256, 256)).save(valid_path)
|
||||
assert _load_image(valid_path).size == (256, 256)
|
||||
|
||||
invalid_path = tmp_path / "reference.bmp"
|
||||
Image.new("RGB", (256, 256)).save(invalid_path)
|
||||
with pytest.raises(ValueError, match="must use"):
|
||||
_load_image(invalid_path)
|
||||
|
||||
|
||||
def test_g4_standalone_audio_duration_and_total_duration_contract():
|
||||
from vllm_omni.diffusion.models.minimax_h3.reference_video import (
|
||||
validate_reference_audio_waveforms,
|
||||
)
|
||||
|
||||
sample_rate = 16000
|
||||
validate_reference_audio_waveforms(
|
||||
[
|
||||
(torch.zeros(1, 2 * sample_rate), sample_rate),
|
||||
(torch.zeros(1, 13 * sample_rate), sample_rate),
|
||||
]
|
||||
)
|
||||
with pytest.raises(ValueError, match="duration"):
|
||||
validate_reference_audio_waveforms([(torch.zeros(1, int(1.9 * sample_rate)), sample_rate)])
|
||||
with pytest.raises(ValueError, match="at most 15 seconds in total"):
|
||||
validate_reference_audio_waveforms(
|
||||
[
|
||||
(torch.zeros(1, 8 * sample_rate), sample_rate),
|
||||
(torch.zeros(1, 8 * sample_rate), sample_rate),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "value", "message"),
|
||||
[
|
||||
("fps", 23.0, "FPS"),
|
||||
("duration", 1.0, "duration"),
|
||||
("format_names", ("avi",), "container"),
|
||||
("video_codec", "vp9", "H.264"),
|
||||
("audio_codecs", ("opus",), "AAC"),
|
||||
("file_size", 50 * 1024 * 1024 + 1, "size"),
|
||||
],
|
||||
)
|
||||
def test_g4_reference_video_metadata_validation(field, value, message, tmp_path):
|
||||
from vllm_omni.diffusion.models.minimax_h3.reference_video import (
|
||||
_validate_reference_video_metadata,
|
||||
)
|
||||
|
||||
metadata = {
|
||||
"width": 1280,
|
||||
"height": 720,
|
||||
"fps": 24.0,
|
||||
"duration": 10.0,
|
||||
"format_names": ("mp4",),
|
||||
"video_codec": "h264",
|
||||
"audio_codecs": ("aac",),
|
||||
"file_size": 1024,
|
||||
}
|
||||
metadata[field] = value
|
||||
with pytest.raises(ValueError, match=message):
|
||||
_validate_reference_video_metadata(metadata, index=0, source=str(tmp_path / "reference.mp4"))
|
||||
@@ -61,6 +61,8 @@ def test_minimax_h3_t2va_ulysses8_smoke():
|
||||
output_type="np",
|
||||
extra_args={
|
||||
"task": "t2va",
|
||||
"duration": 4.0,
|
||||
"aspect_ratio": "16:9",
|
||||
"flow_shift": 12.0,
|
||||
"audio_flow_shift": 3.0,
|
||||
},
|
||||
@@ -70,7 +72,14 @@ def test_minimax_h3_t2va_ulysses8_smoke():
|
||||
finally:
|
||||
engine.close()
|
||||
|
||||
_assert_joint_output(outputs)
|
||||
assert len(outputs) == 1
|
||||
frames = np.asarray(outputs[0].images[0])
|
||||
assert frames.shape == (107, 256, 448, 3)
|
||||
multimodal = outputs[0].multimodal_output
|
||||
assert multimodal is not None
|
||||
assert np.asarray(multimodal["audio"]).shape[1] == 2
|
||||
assert multimodal["audio_sample_rate"] == 32000
|
||||
assert multimodal["fps"] == 24
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
|
||||
@@ -144,3 +144,20 @@ def test_condition_noise_is_seeded_and_keeps_clean_anchor_at_timestep_one():
|
||||
noise_aug=0.25,
|
||||
)
|
||||
torch.testing.assert_close(first, second)
|
||||
|
||||
|
||||
def test_condition_noise_accepts_a_reference_video_longer_than_the_target():
|
||||
from vllm_omni.diffusion.models.minimax_h3.condition_noise import (
|
||||
minimax_h3_imgvid_cond_noise_aug_rows,
|
||||
)
|
||||
|
||||
rows = torch.zeros(4 * 2 * 2, 96)
|
||||
result = minimax_h3_imgvid_cond_noise_aug_rows(
|
||||
rows,
|
||||
condition_shapes=[(4, 4, 4)],
|
||||
target_latent_t=2,
|
||||
imgvid_cond_num_frames=1,
|
||||
seed=7,
|
||||
noise_aug=0.5,
|
||||
)
|
||||
assert result.shape == rows.shape
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
@@ -26,9 +27,13 @@ from tests.helpers.runtime import OmniServer
|
||||
pytestmark = [pytest.mark.benchmark, pytest.mark.diffusion, pytest.mark.full_model]
|
||||
|
||||
MODEL_REPO_ID = "MiniMaxAI/MiniMax-H3"
|
||||
MODEL_REVISION = "73372e6cf53e414edd3ab03e357717fb0602e758"
|
||||
# The previously pinned revision was removed from the Hugging Face repository.
|
||||
MODEL_REVISION = "main"
|
||||
MODEL_ENV_VAR = "VLLM_TEST_MINIMAX_H3_FL2VA_MODEL"
|
||||
REFERENCE_VIDEO_URL = f"https://huggingface.co/{MODEL_REPO_ID}/resolve/main/assets/t2va.mp4"
|
||||
# Keep the official asset link visible in the test report. Hugging Face's
|
||||
# ``blob`` URL is an HTML page, so resolve it to the raw asset for download.
|
||||
REFERENCE_VIDEO_URL = "https://huggingface.co/MiniMaxAI/MiniMax-H3/blob/main/assets/t2va.mp4"
|
||||
REFERENCE_VIDEO_SHA256 = "d66903241362e224085cc93f7a5e70fba6ab378d0ac6ff6af83acf2559849a42"
|
||||
|
||||
# This is the official reproducible 768p T2VA prompt associated with
|
||||
# ``assets/t2va.mp4`` in the MiniMax H3 repository.
|
||||
@@ -93,9 +98,15 @@ def _server_args() -> list[str]:
|
||||
|
||||
def _download_reference_video(output_dir: Path) -> Path:
|
||||
reference_path = output_dir / "reference.mp4"
|
||||
response = requests.get(REFERENCE_VIDEO_URL, timeout=120)
|
||||
download_url = REFERENCE_VIDEO_URL.replace("/blob/", "/resolve/", 1)
|
||||
response = requests.get(download_url, timeout=120)
|
||||
response.raise_for_status()
|
||||
reference_path.write_bytes(response.content)
|
||||
content = response.content
|
||||
digest = hashlib.sha256(content).hexdigest()
|
||||
assert digest == REFERENCE_VIDEO_SHA256, (
|
||||
f"Official MiniMax H3 T2VA reference changed: got {digest}, expected {REFERENCE_VIDEO_SHA256}"
|
||||
)
|
||||
reference_path.write_bytes(content)
|
||||
return reference_path
|
||||
|
||||
|
||||
@@ -159,6 +170,7 @@ def test_minimax_h3_t2va_matches_official_reference(
|
||||
{
|
||||
"task": "t2va",
|
||||
"duration": DURATION_SECONDS,
|
||||
"aspect_ratio": "16:9",
|
||||
"audio_flow_shift": AUDIO_FLOW_SHIFT,
|
||||
}
|
||||
),
|
||||
|
||||
@@ -83,7 +83,11 @@ class FakeDeadLLMStageClient(FakeStageClient):
|
||||
|
||||
async def get_output_async(self):
|
||||
if not self.add_request_calls:
|
||||
return SimpleNamespace(outputs=[])
|
||||
return SimpleNamespace(
|
||||
outputs=[],
|
||||
scheduler_stats=None,
|
||||
finished_requests=None,
|
||||
)
|
||||
raise EngineDeadError("Stage-0 engine core is dead")
|
||||
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from fastapi.testclient import TestClient
|
||||
from PIL import Image
|
||||
from pytest_mock import MockerFixture
|
||||
@@ -62,10 +62,14 @@ class FakeAsyncOmni:
|
||||
def __init__(self):
|
||||
self.stage_configs = [SimpleNamespace(stage_type="diffusion")]
|
||||
self.default_sampling_params_list = [OmniDiffusionSamplingParams()]
|
||||
self.model_class_name = "WanPipeline"
|
||||
self.captured_prompt = None
|
||||
self.captured_reference_video_bytes = None
|
||||
self.captured_sampling_params_list = None
|
||||
|
||||
def get_diffusion_od_config(self):
|
||||
return SimpleNamespace(model_class_name=self.model_class_name)
|
||||
|
||||
async def generate(self, prompt, request_id, sampling_params_list):
|
||||
self.captured_prompt = prompt
|
||||
self.captured_sampling_params_list = sampling_params_list
|
||||
@@ -487,7 +491,7 @@ def test_v2v_video_generation_form(test_client, mocker: MockerFixture):
|
||||
assert input_video[0].size == (32, 24)
|
||||
|
||||
|
||||
def test_v2v_video_generation_with_video_reference_form(test_client, mocker: MockerFixture):
|
||||
def test_r9_typed_video_reference_form(test_client, mocker: MockerFixture):
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
||||
return_value=b"fake-video",
|
||||
@@ -506,8 +510,53 @@ def test_v2v_video_generation_with_video_reference_form(test_client, mocker: Moc
|
||||
|
||||
engine = test_client.app.state.openai_serving_video._engine_client
|
||||
input_video = engine.captured_prompt["multi_modal_data"]["video"]
|
||||
assert len(input_video) == 2
|
||||
assert input_video[0].size == (32, 24)
|
||||
assert len(input_video) == 1
|
||||
assert all(isinstance(item, str) for item in input_video)
|
||||
assert engine.captured_reference_video_bytes is not None
|
||||
assert len(engine.captured_reference_video_bytes) == 1
|
||||
assert b"ftyp" in engine.captured_reference_video_bytes[0][:32]
|
||||
|
||||
|
||||
def test_r5_mixed_image_video_audio_references_and_fanout(test_client, mocker: MockerFixture):
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
||||
return_value=b"fake-video",
|
||||
)
|
||||
test_client.app.state.openai_serving_video._engine_client.model_class_name = "MiniMaxH3Pipeline"
|
||||
response = test_client.post(
|
||||
"/v1/videos",
|
||||
data={
|
||||
"prompt": "Use both references.",
|
||||
"image_reference": json.dumps(
|
||||
[
|
||||
{"image_url": _make_test_image_data_url((40, 24))},
|
||||
{"image_url": _make_test_image_data_url((40, 24))},
|
||||
]
|
||||
),
|
||||
"video_reference": json.dumps(
|
||||
{"video_url": _make_test_video_data_url((32, 24), 2)},
|
||||
),
|
||||
"audio_reference": json.dumps(
|
||||
[{"audio_url": "data:audio/mp3;base64,ZmFrZQ=="}],
|
||||
),
|
||||
"extra_params": json.dumps({"task": "ref2va", "frame_indices": [0, -1], "start_time_seconds": 1.5}),
|
||||
"num_outputs_per_prompt": "2",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
video_id = response.json()["id"]
|
||||
_wait_for_status(test_client, video_id, VideoGenerationStatus.COMPLETED.value)
|
||||
engine = test_client.app.state.openai_serving_video._engine_client
|
||||
multi_modal_data = engine.captured_prompt["multi_modal_data"]
|
||||
assert len(multi_modal_data["image"]) == 2
|
||||
assert len(multi_modal_data["video"]) == 1
|
||||
assert all(isinstance(item, str) for item in multi_modal_data["video"])
|
||||
assert engine.captured_reference_video_bytes is not None
|
||||
assert len(engine.captured_reference_video_bytes) == 1
|
||||
assert b"ftyp" in engine.captured_reference_video_bytes[0][:32]
|
||||
assert isinstance(multi_modal_data["audio"], str)
|
||||
assert engine.captured_sampling_params_list[0].num_outputs_per_prompt == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("endpoint", ["/v1/videos", "/v1/videos/sync"])
|
||||
@@ -520,6 +569,7 @@ def test_multi_video_generation_preserves_uploaded_files_until_generation(
|
||||
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
||||
return_value=b"fake-video",
|
||||
)
|
||||
test_client.app.state.openai_serving_video._engine_client.model_class_name = "MiniMaxH3Pipeline"
|
||||
response = test_client.post(
|
||||
endpoint,
|
||||
data={
|
||||
@@ -527,8 +577,10 @@ def test_multi_video_generation_preserves_uploaded_files_until_generation(
|
||||
"extra_params": json.dumps({"task": "ref2va", "duration": 15.0}),
|
||||
},
|
||||
files=[
|
||||
("input_references", ("subject.png", _make_test_image_bytes((40, 24)), "image/png")),
|
||||
("input_references", ("subject.mp4", b"subject-video", "video/mp4")),
|
||||
("input_references", ("background.mov", b"background-video", "video/quicktime")),
|
||||
("input_references", ("voice.wav", b"reference-audio", "audio/wav")),
|
||||
],
|
||||
)
|
||||
|
||||
@@ -544,6 +596,8 @@ def test_multi_video_generation_preserves_uploaded_files_until_generation(
|
||||
assert engine.captured_reference_video_bytes == [b"subject-video", b"background-video"]
|
||||
assert len(input_videos) == 2
|
||||
assert all(not Path(path).exists() for path in input_videos)
|
||||
assert isinstance(engine.captured_prompt["multi_modal_data"]["image"], Image.Image)
|
||||
assert isinstance(engine.captured_prompt["multi_modal_data"]["audio"], str)
|
||||
assert engine.captured_sampling_params_list[0].extra_args["task"] == "ref2va"
|
||||
assert engine.captured_sampling_params_list[0].extra_args["duration"] == 15.0
|
||||
|
||||
@@ -1200,17 +1254,67 @@ def test_rejects_input_reference_and_image_reference_together(test_client):
|
||||
assert "only one of input_reference, image_reference, or video_reference" in response.json()["detail"].lower()
|
||||
|
||||
|
||||
def test_rejects_image_reference_and_video_reference_together(test_client):
|
||||
def test_r10_typed_image_and_video_reference_form(test_client, mocker: MockerFixture):
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.openai.serving_video._encode_video_bytes",
|
||||
return_value=b"fake-video",
|
||||
)
|
||||
test_client.app.state.openai_serving_video._engine_client.model_class_name = "MiniMaxH3Pipeline"
|
||||
response = test_client.post(
|
||||
"/v1/videos",
|
||||
data={
|
||||
"prompt": "bad refs",
|
||||
"image_reference": '{"image_url": "https://example.com/cat.png"}',
|
||||
"video_reference": '{"video_url": "https://example.com/cat.mp4"}',
|
||||
"prompt": "use both references",
|
||||
"image_reference": json.dumps({"image_url": _make_test_image_data_url((40, 24))}),
|
||||
"video_reference": json.dumps({"video_url": _make_test_video_data_url((32, 24), 2)}),
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
_wait_for_status(test_client, response.json()["id"], VideoGenerationStatus.COMPLETED.value)
|
||||
|
||||
|
||||
def test_generic_video_model_rejects_mixed_image_and_video_references(test_client):
|
||||
response = test_client.post(
|
||||
"/v1/videos/sync",
|
||||
data={
|
||||
"prompt": "use both references",
|
||||
"image_reference": json.dumps({"image_url": _make_test_image_data_url((40, 24))}),
|
||||
"video_reference": json.dumps({"video_url": _make_test_video_data_url((32, 24), 2)}),
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 400
|
||||
assert "only one of input_reference, image_reference, or video_reference" in response.json()["detail"].lower()
|
||||
assert "does not support mixed image and video" in response.json()["detail"].lower()
|
||||
|
||||
|
||||
def test_h3_multipart_rejects_bmp_image_reference(test_client):
|
||||
image = Image.new("RGB", (64, 64), color="blue")
|
||||
image_buffer = io.BytesIO()
|
||||
image.save(image_buffer, format="BMP")
|
||||
|
||||
test_client.app.state.openai_serving_video._engine_client.model_class_name = "MiniMaxH3Pipeline"
|
||||
response = test_client.post(
|
||||
"/v1/videos/sync",
|
||||
data={"prompt": "reject unsupported image container", "extra_params": '{"task":"ref2va"}'},
|
||||
files=[("input_references", ("reference.bmp", image_buffer.getvalue(), "image/bmp"))],
|
||||
)
|
||||
|
||||
assert response.status_code == 400
|
||||
assert "must use jpg" in response.json()["detail"].lower()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_h3_upload_limit_checks_declared_size_before_read():
|
||||
class OversizedUpload:
|
||||
size = api_server.MINIMAX_H3_MAX_REFERENCE_IMAGE_BYTES + 1
|
||||
|
||||
async def read(self, _size):
|
||||
raise AssertionError("the oversized upload must be rejected before reading")
|
||||
|
||||
with pytest.raises(HTTPException, match="size limit"):
|
||||
await api_server._read_upload_limited(
|
||||
OversizedUpload(),
|
||||
max_bytes=api_server.MINIMAX_H3_MAX_REFERENCE_IMAGE_BYTES,
|
||||
)
|
||||
|
||||
|
||||
def test_invalid_seconds_returns_422(test_client):
|
||||
|
||||
Reference in new issue
Block a user