diff --git a/.buildkite/cuda/test-nightly.yml b/.buildkite/cuda/test-nightly.yml index 18bd82a9d..9be72d3d5 100644 --- a/.buildkite/cuda/test-nightly.yml +++ b/.buildkite/cuda/test-nightly.yml @@ -460,6 +460,21 @@ steps: exit $$EXIT2 mirror_hardwares: h100_2 + - label: ":full_moon: Diffusion X2V · Perf Test · MiniMax-H3" + key: nightly-diffusion-x2v-performance-minimax-h3 + timeout_in_minutes: 180 + commands: + - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results + - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json + EXIT=$$? + buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" + buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" + exit $$EXIT + mirror_hardwares: h100_4 + - group: ":card_index_dividers: Diffusion Test" key: nightly-diffusion-test-group depends_on: upload-nightly-pipeline diff --git a/benchmarks/diffusion/backends.py b/benchmarks/diffusion/backends.py index f8ad1a71e..aeff3ebbb 100644 --- a/benchmarks/diffusion/backends.py +++ b/benchmarks/diffusion/backends.py @@ -1,5 +1,6 @@ import asyncio import base64 +import json import mimetypes import os import time @@ -13,6 +14,17 @@ from tqdm import tqdm DEFAULT_EDITS_BOT_TASK = "think" +def _is_minimax_h3_mixed_reference(model: str) -> bool: + """MiniMax-H3 Ref2VA consumes reference videos as file paths. + + Its reference preparer ffprobes / ffmpegs the reference file, so single + ``video_reference`` uploads are sent as a base64 data URL: the server + persists them to disk and keeps ``source_path``, handing the pipeline a + real file path instead of a decoded frame list. + """ + return "MiniMax-H3" in (model or "") + + @dataclass class RequestFuncInput: prompt: str @@ -355,13 +367,21 @@ async def async_request_v1_videos( ) elif input.video_paths and len(input.video_paths) > 0: video_path = input.video_paths[0] - image_file = open(video_path, "rb") - form.add_field( - "input_reference", - image_file, - filename=os.path.basename(video_path), - content_type=_guess_mime_type(video_path), - ) + if _is_minimax_h3_mixed_reference(input.model): + with open(video_path, "rb") as reference_file: + video_b64 = base64.b64encode(reference_file.read()).decode("ascii") + form.add_field( + "video_reference", + json.dumps([{"video_url": f"data:video/mp4;base64,{video_b64}"}]), + ) + else: + image_file = open(video_path, "rb") + form.add_field( + "input_reference", + image_file, + filename=os.path.basename(video_path), + content_type=_guess_mime_type(video_path), + ) job_id = None job_status = None diff --git a/benchmarks/diffusion/diffusion_benchmark_serving.py b/benchmarks/diffusion/diffusion_benchmark_serving.py index 8f7a02858..6daac9df4 100644 --- a/benchmarks/diffusion/diffusion_benchmark_serving.py +++ b/benchmarks/diffusion/diffusion_benchmark_serving.py @@ -858,7 +858,12 @@ class RandomDataset(BaseDataset): height = int(self.args.height or profile.get("height") or 720) num_frames = int(self.args.num_frames or profile.get("num_frames") or 16) fps = float(self.args.fps or profile.get("fps") or 8) - writer = cv2.VideoWriter(video_path, cv2.VideoWriter_fourcc(*"mp4v"), fps, (width, height)) + # cv2 typically writes MPEG-4 Part 2 (``mp4v``), which pipelines such + # as MiniMax-H3 Ref2VA reject because they require H.264/H.265 + # reference videos. Write the synthetic clip first, then re-encode + # it to H.264 when ffmpeg is available; otherwise keep ``mp4v``. + raw_path = video_path + ".raw.mp4" + writer = cv2.VideoWriter(raw_path, cv2.VideoWriter_fourcc(*"mp4v"), fps, (width, height)) if not writer.isOpened(): raise RuntimeError("cv2.VideoWriter failed to open") for frame_idx in range(num_frames): @@ -871,6 +876,10 @@ class RandomDataset(BaseDataset): frame[max(y - 36, 0) : min(y + 36, height), x : min(x + 96, width), :] = (64, 128, 220) writer.write(frame) writer.release() + if _reencode_video_to_h264(raw_path, video_path): + os.unlink(raw_path) + else: + os.replace(raw_path, video_path) return [video_path] except Exception as e: raise RuntimeError( @@ -878,6 +887,40 @@ class RandomDataset(BaseDataset): ) from e +def _reencode_video_to_h264(source: str, destination: str) -> bool: + """Re-encode a video to H.264/MP4 using ffmpeg when available. + + Returns False when ffmpeg is missing or the re-encode fails so callers can + fall back to the original container/codec. + """ + import shutil + import subprocess + + if shutil.which("ffmpeg") is None: + return False + try: + result = subprocess.run( + [ + "ffmpeg", + "-y", + "-loglevel", + "error", + "-i", + source, + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + destination, + ], + capture_output=True, + text=True, + ) + return result.returncode == 0 and os.path.exists(destination) + except Exception: + return False + + def _compute_expected_latency_ms_from_base(req: RequestFuncInput, args, base_time_ms: float | None) -> float | None: """Compute expected execution time (ms) based on a base per-step-per-frame unit time. diff --git a/tests/dfx/perf/scripts/run_diffusion_benchmark.py b/tests/dfx/perf/scripts/run_diffusion_benchmark.py index 92f85587c..75d1bdc21 100644 --- a/tests/dfx/perf/scripts/run_diffusion_benchmark.py +++ b/tests/dfx/perf/scripts/run_diffusion_benchmark.py @@ -350,10 +350,49 @@ def _kill_process_tree(pid: int) -> None: def _resolve_offline_model(model: str) -> str: - """Under HF_HUB_OFFLINE, resolve a HF repo id to its local snapshot dir.""" + """Resolve a model string to a local directory. + + Local paths pass through unchanged. HF repo ids are resolved to their local + snapshot dir when ``HF_HUB_OFFLINE`` is set. ``repo_id/subfolder`` (e.g. + ``MiniMaxAI/MiniMax-H3/FL2VA``) is resolved to the matching partition + subfolder of the snapshot, downloading only that subfolder when online. + """ import huggingface_hub - if not model or os.path.isdir(model) or not huggingface_hub.constants.HF_HUB_OFFLINE: + if not model or os.path.isdir(model): + return model + + # MiniMax-H3 custom code uses relative imports across files, which breaks + # when the HF cache snapshot is symlinked (``get_class_from_dynamic_module`` + # resolves ``realpath`` into the blobs dir). Mirror the accuracy test's env + # overrides so CI / local runs can point at a materialized partition. The + # repo-root key (used with ``--task-type fl2va|ref2va``) expects a + # materialized root containing the FL2VA/Ref2VA subfolders. + model_env_overrides = { + "MiniMaxAI/MiniMax-H3": "VLLM_TEST_MINIMAX_H3_MODEL", + "MiniMaxAI/MiniMax-H3/FL2VA": "VLLM_TEST_MINIMAX_H3_FL2VA_MODEL", + "MiniMaxAI/MiniMax-H3/Ref2VA": "VLLM_TEST_MINIMAX_H3_REF2VA_MODEL", + } + env_name = model_env_overrides.get(model) + if env_name: + env_model = os.environ.get(env_name) + if env_model: + return env_model + + parts = model.split("/") + if len(parts) >= 3: + repo_id = "/".join(parts[:2]) + subfolder = "/".join(parts[2:]) + from huggingface_hub import snapshot_download + + snapshot_root = snapshot_download( + repo_id, + allow_patterns=[f"{subfolder}/**"], + local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE, + ) + return str(Path(snapshot_root) / subfolder) + + if not huggingface_hub.constants.HF_HUB_OFFLINE: return model from huggingface_hub import snapshot_download @@ -387,6 +426,11 @@ class DiffusionServer: def _start_server(self) -> None: env = os.environ.copy() env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + # Give each run its own writable video-storage dir. The server default + # (/tmp/storage) may be missing or not writable on shared hosts / CI. + storage_path = Path(os.environ.get("VLLM_OMNI_STORAGE_PATH", str(BENCHMARK_RESULT_DIR / "storage"))) + storage_path.mkdir(parents=True, exist_ok=True) + env["VLLM_OMNI_STORAGE_PATH"] = str(storage_path) cmd = [ sys.executable, diff --git a/tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json b/tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json new file mode 100644 index 000000000..a28aab96c --- /dev/null +++ b/tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json @@ -0,0 +1,295 @@ +[ + { + "test_name": "test_minimax_h3_t2v_4gpu", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 4 + } + }, + "full_model", + "diffusion" + ], + "description": "MiniMax-H3 T2VA text-to-video+audio on 4 GPUs (USP4 + HSDP4, encoder-TP4, VAE-PP4 tile)", + "server_type": "vllm-omni", + "benchmark_endpoint": "/v1/videos", + "server_params": { + "model": "MiniMaxAI/MiniMax-H3", + "serve_args": { + "trust-remote-code": true, + "task-type": "fl2va", + "num-gpus": 4, + "usp": 4, + "ring": 1, + "use-hsdp": true, + "hsdp-shard-size": 4, + "text-encoder-tp-size": 4, + "vae-patch-parallel-size": 4, + "vae-parallel-mode": "tile", + "vae-use-tiling": true, + "diffusion-attention-backend": "FLASH_ATTN", + "stage-init-timeout": 1800, + "init-timeout": 1800, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "1344x768_frames209_steps8", + "dataset": "random", + "task": "t2v", + "num-prompts": 3, + "max-concurrency": 1, + "warmup-requests": 1, + "warmup-concurrency": 1, + "seed": 42, + "random-request-config": [ + { + "width": 1344, + "height": 768, + "prompt": "Cinematic drone shot pushing through misty mountain ridges at sunrise, golden light catching the clouds, slow and smooth.", + "num_inference_steps": 8, + "num_frames": 209, + "fps": 24, + "weight": 1 + } + ], + "extra-body": { + "flow_shift": 12.0, + "extra_params": "{\"task\":\"t2va\",\"duration\":8.7,\"aspect_ratio\":\"16:9\",\"audio_flow_shift\":3.0}" + }, + "baseline": { + "H100": { + "throughput_qps": 0.0211, + "latency_mean": 38.3252, + "peak_memory_mb_mean": 54219.6 + } + } + } + ] + }, + { + "test_name": "test_minimax_h3_ti2v_4gpu", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 4 + } + }, + "full_model", + "diffusion" + ], + "description": "MiniMax-H3 FL2VA first-frame image-to-video+audio on 4 GPUs (USP4 + HSDP4, encoder-TP4, VAE-PP4 tile)", + "server_type": "vllm-omni", + "benchmark_endpoint": "/v1/videos", + "server_params": { + "model": "MiniMaxAI/MiniMax-H3", + "serve_args": { + "trust-remote-code": true, + "task-type": "fl2va", + "num-gpus": 4, + "usp": 4, + "ring": 1, + "use-hsdp": true, + "hsdp-shard-size": 4, + "text-encoder-tp-size": 4, + "vae-patch-parallel-size": 4, + "vae-parallel-mode": "tile", + "vae-use-tiling": true, + "diffusion-attention-backend": "FLASH_ATTN", + "stage-init-timeout": 1800, + "init-timeout": 1800, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "1344x768_frames209_steps8", + "dataset": "random", + "task": "ti2v", + "num-prompts": 3, + "max-concurrency": 1, + "warmup-requests": 1, + "warmup-concurrency": 1, + "seed": 42, + "random-request-config": [ + { + "width": 1344, + "height": 768, + "prompt": "Continue the scene: the camera slowly pushes in on the subject as the background city lights drift and blur, with the actor singing softly and emotionally.", + "num_inference_steps": 8, + "num_frames": 209, + "fps": 24, + "weight": 1 + } + ], + "extra-body": { + "flow_shift": 12.0, + "extra_params": "{\"task\":\"fl2va\",\"duration\":8.7,\"audio_flow_shift\":3.0}" + }, + "num-input-images": 1, + "baseline": { + "H100": { + "throughput_qps": 0.0212, + "latency_mean": 38.1385, + "peak_memory_mb_mean": 54622.8 + } + } + } + ] + }, + { + "test_name": "test_minimax_h3_v2v_4gpu", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 4 + } + }, + "full_model", + "diffusion" + ], + "description": "MiniMax-H3 Ref2VA video-reference-to-video+audio on 4 GPUs (USP4 + HSDP4, encoder-TP4, VAE-PP4 tile)", + "server_type": "vllm-omni", + "benchmark_endpoint": "/v1/videos", + "server_params": { + "model": "MiniMaxAI/MiniMax-H3", + "serve_args": { + "trust-remote-code": true, + "task-type": "ref2va", + "num-gpus": 4, + "usp": 4, + "ring": 1, + "use-hsdp": true, + "hsdp-shard-size": 4, + "text-encoder-tp-size": 4, + "vae-patch-parallel-size": 4, + "vae-parallel-mode": "tile", + "vae-use-tiling": true, + "diffusion-attention-backend": "FLASH_ATTN", + "stage-init-timeout": 1800, + "init-timeout": 1800, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "1344x768_frames209_steps8", + "dataset": "random", + "task": "v2v", + "num-prompts": 3, + "max-concurrency": 1, + "warmup-requests": 1, + "warmup-concurrency": 1, + "seed": 42, + "random-request-config": [ + { + "width": 1344, + "height": 768, + "prompt": "Restyle the reference footage into a cinematic night scene with neon reflections, keeping the same camera motion and subject.", + "num_inference_steps": 8, + "num_frames": 209, + "fps": 24, + "weight": 1 + } + ], + "extra-body": { + "flow_shift": 12.0, + "extra_params": "{\"task\":\"ref2va\",\"duration\":8.7,\"aspect_ratio\":\"16:9\",\"audio_flow_shift\":3.0}" + }, + "baseline": { + "H100": { + "throughput_qps": 0.0077, + "latency_mean": 105.7508, + "peak_memory_mb_mean": 55180.8 + } + } + } + ] + }, + { + "test_name": "test_minimax_h3_t2v_dlo_4gpu", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 4 + } + }, + "full_model", + "diffusion" + ], + "description": "MiniMax-H3 T2VA with 4-card distributed layerwise offload (DLO), 8 denoise steps (USP4 + HSDP4, encoder-TP4, VAE-PP4 tile)", + "server_type": "vllm-omni", + "benchmark_endpoint": "/v1/videos", + "server_params": { + "model": "MiniMaxAI/MiniMax-H3", + "serve_args": { + "trust-remote-code": true, + "task-type": "fl2va", + "num-gpus": 4, + "usp": 4, + "ring": 1, + "use-hsdp": true, + "hsdp-shard-size": 4, + "text-encoder-tp-size": 4, + "vae-patch-parallel-size": 4, + "vae-parallel-mode": "tile", + "vae-use-tiling": true, + "diffusion-attention-backend": "FLASH_ATTN", + "stage-init-timeout": 1800, + "init-timeout": 1800, + "enable-diffusion-pipeline-profiler": true, + "enable-distributed-layerwise-offload": true, + "dlo-no-use-allgather": true, + "dlo-resident-layers": 8, + "enforce-eager": true + } + }, + "benchmark_params": [ + { + "name": "1344x768_frames209_steps8", + "dataset": "random", + "task": "t2v", + "num-prompts": 3, + "max-concurrency": 1, + "warmup-requests": 1, + "warmup-concurrency": 1, + "seed": 42, + "random-request-config": [ + { + "width": 1344, + "height": 768, + "prompt": "Cinematic drone shot pushing through misty mountain ridges at sunrise, golden light catching the clouds, slow and smooth.", + "num_inference_steps": 8, + "num_frames": 209, + "fps": 24, + "weight": 1 + } + ], + "extra-body": { + "flow_shift": 12.0, + "extra_params": "{\"task\":\"t2va\",\"duration\":8.7,\"aspect_ratio\":\"16:9\",\"audio_flow_shift\":3.0}" + }, + "baseline": { + "H100": { + "throughput_qps": 0.0212, + "latency_mean": 38.2089, + "peak_memory_mb_mean": 36218.4 + } + } + } + ] + } +]