[Perf][CI] Add MiniMax-H3 4xH100 diffusion perf config (#5836)

Signed-off-by: David Chen <530634352@qq.com>
2026-08-07 00:12:07 +08:00
1 parent 62bf8d630d
commit a874b8e09b
@@ -460,6 +460,21 @@ steps:
            exit $$EXIT2
        mirror_hardwares: h100_2

+      - label: ":full_moon: Diffusion X2V · Perf Test · MiniMax-H3"
+        key: nightly-diffusion-x2v-performance-minimax-h3
+        timeout_in_minutes: 180
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+        mirror_hardwares: h100_4
+
  - group: ":card_index_dividers: Diffusion Test"
    key: nightly-diffusion-test-group
    depends_on: upload-nightly-pipeline
@@ -1,5 +1,6 @@
 import asyncio
 import base64
+import json
 import mimetypes
 import os
 import time
@@ -13,6 +14,17 @@ from tqdm import tqdm
 DEFAULT_EDITS_BOT_TASK = "think"


+def _is_minimax_h3_mixed_reference(model: str) -> bool:
+    """MiniMax-H3 Ref2VA consumes reference videos as file paths.
+
+    Its reference preparer ffprobes / ffmpegs the reference file, so single
+    ``video_reference`` uploads are sent as a base64 data URL: the server
+    persists them to disk and keeps ``source_path``, handing the pipeline a
+    real file path instead of a decoded frame list.
+    """
+    return "MiniMax-H3" in (model or "")
+
+
@dataclass
 class RequestFuncInput:
    prompt: str
@@ -355,13 +367,21 @@ async def async_request_v1_videos(
        )
    elif input.video_paths and len(input.video_paths) > 0:
        video_path = input.video_paths[0]
-        image_file = open(video_path, "rb")
-        form.add_field(
-            "input_reference",
-            image_file,
-            filename=os.path.basename(video_path),
-            content_type=_guess_mime_type(video_path),
-        )
+        if _is_minimax_h3_mixed_reference(input.model):
+            with open(video_path, "rb") as reference_file:
+                video_b64 = base64.b64encode(reference_file.read()).decode("ascii")
+            form.add_field(
+                "video_reference",
+                json.dumps([{"video_url": f"data:video/mp4;base64,{video_b64}"}]),
+            )
+        else:
+            image_file = open(video_path, "rb")
+            form.add_field(
+                "input_reference",
+                image_file,
+                filename=os.path.basename(video_path),
+                content_type=_guess_mime_type(video_path),
+            )

    job_id = None
    job_status = None
@@ -858,7 +858,12 @@ class RandomDataset(BaseDataset):
            height = int(self.args.height or profile.get("height") or 720)
            num_frames = int(self.args.num_frames or profile.get("num_frames") or 16)
            fps = float(self.args.fps or profile.get("fps") or 8)
-            writer = cv2.VideoWriter(video_path, cv2.VideoWriter_fourcc(*"mp4v"), fps, (width, height))
+            # cv2 typically writes MPEG-4 Part 2 (``mp4v``), which pipelines such
+            # as MiniMax-H3 Ref2VA reject because they require H.264/H.265
+            # reference videos. Write the synthetic clip first, then re-encode
+            # it to H.264 when ffmpeg is available; otherwise keep ``mp4v``.
+            raw_path = video_path + ".raw.mp4"
+            writer = cv2.VideoWriter(raw_path, cv2.VideoWriter_fourcc(*"mp4v"), fps, (width, height))
            if not writer.isOpened():
                raise RuntimeError("cv2.VideoWriter failed to open")
            for frame_idx in range(num_frames):
@@ -871,6 +876,10 @@ class RandomDataset(BaseDataset):
                frame[max(y - 36, 0) : min(y + 36, height), x : min(x + 96, width), :] = (64, 128, 220)
                writer.write(frame)
            writer.release()
+            if _reencode_video_to_h264(raw_path, video_path):
+                os.unlink(raw_path)
+            else:
+                os.replace(raw_path, video_path)
            return [video_path]
        except Exception as e:
            raise RuntimeError(
@@ -878,6 +887,40 @@ class RandomDataset(BaseDataset):
            ) from e


+def _reencode_video_to_h264(source: str, destination: str) -> bool:
+    """Re-encode a video to H.264/MP4 using ffmpeg when available.
+
+    Returns False when ffmpeg is missing or the re-encode fails so callers can
+    fall back to the original container/codec.
+    """
+    import shutil
+    import subprocess
+
+    if shutil.which("ffmpeg") is None:
+        return False
+    try:
+        result = subprocess.run(
+            [
+                "ffmpeg",
+                "-y",
+                "-loglevel",
+                "error",
+                "-i",
+                source,
+                "-c:v",
+                "libx264",
+                "-pix_fmt",
+                "yuv420p",
+                destination,
+            ],
+            capture_output=True,
+            text=True,
+        )
+        return result.returncode == 0 and os.path.exists(destination)
+    except Exception:
+        return False
+
+
 def _compute_expected_latency_ms_from_base(req: RequestFuncInput, args, base_time_ms: float | None) -> float | None:
    """Compute expected execution time (ms) based on a base per-step-per-frame unit time.

@@ -350,10 +350,49 @@ def _kill_process_tree(pid: int) -> None:


 def _resolve_offline_model(model: str) -> str:
-    """Under HF_HUB_OFFLINE, resolve a HF repo id to its local snapshot dir."""
+    """Resolve a model string to a local directory.
+
+    Local paths pass through unchanged. HF repo ids are resolved to their local
+    snapshot dir when ``HF_HUB_OFFLINE`` is set. ``repo_id/subfolder`` (e.g.
+    ``MiniMaxAI/MiniMax-H3/FL2VA``) is resolved to the matching partition
+    subfolder of the snapshot, downloading only that subfolder when online.
+    """
    import huggingface_hub

-    if not model or os.path.isdir(model) or not huggingface_hub.constants.HF_HUB_OFFLINE:
+    if not model or os.path.isdir(model):
+        return model
+
+    # MiniMax-H3 custom code uses relative imports across files, which breaks
+    # when the HF cache snapshot is symlinked (``get_class_from_dynamic_module``
+    # resolves ``realpath`` into the blobs dir). Mirror the accuracy test's env
+    # overrides so CI / local runs can point at a materialized partition. The
+    # repo-root key (used with ``--task-type fl2va|ref2va``) expects a
+    # materialized root containing the FL2VA/Ref2VA subfolders.
+    model_env_overrides = {
+        "MiniMaxAI/MiniMax-H3": "VLLM_TEST_MINIMAX_H3_MODEL",
+        "MiniMaxAI/MiniMax-H3/FL2VA": "VLLM_TEST_MINIMAX_H3_FL2VA_MODEL",
+        "MiniMaxAI/MiniMax-H3/Ref2VA": "VLLM_TEST_MINIMAX_H3_REF2VA_MODEL",
+    }
+    env_name = model_env_overrides.get(model)
+    if env_name:
+        env_model = os.environ.get(env_name)
+        if env_model:
+            return env_model
+
+    parts = model.split("/")
+    if len(parts) >= 3:
+        repo_id = "/".join(parts[:2])
+        subfolder = "/".join(parts[2:])
+        from huggingface_hub import snapshot_download
+
+        snapshot_root = snapshot_download(
+            repo_id,
+            allow_patterns=[f"{subfolder}/**"],
+            local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
+        )
+        return str(Path(snapshot_root) / subfolder)
+
+    if not huggingface_hub.constants.HF_HUB_OFFLINE:
        return model
    from huggingface_hub import snapshot_download

@@ -387,6 +426,11 @@ class DiffusionServer:
    def _start_server(self) -> None:
        env = os.environ.copy()
        env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
+        # Give each run its own writable video-storage dir. The server default
+        # (/tmp/storage) may be missing or not writable on shared hosts / CI.
+        storage_path = Path(os.environ.get("VLLM_OMNI_STORAGE_PATH", str(BENCHMARK_RESULT_DIR / "storage")))
+        storage_path.mkdir(parents=True, exist_ok=True)
+        env["VLLM_OMNI_STORAGE_PATH"] = str(storage_path)

        cmd = [
            sys.executable,
@@ -0,0 +1,295 @@
+[
+  {
+    "test_name": "test_minimax_h3_t2v_4gpu",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 4
+        }
+      },
+      "full_model",
+      "diffusion"
+    ],
+    "description": "MiniMax-H3 T2VA text-to-video+audio on 4 GPUs (USP4 + HSDP4, encoder-TP4, VAE-PP4 tile)",
+    "server_type": "vllm-omni",
+    "benchmark_endpoint": "/v1/videos",
+    "server_params": {
+      "model": "MiniMaxAI/MiniMax-H3",
+      "serve_args": {
+        "trust-remote-code": true,
+        "task-type": "fl2va",
+        "num-gpus": 4,
+        "usp": 4,
+        "ring": 1,
+        "use-hsdp": true,
+        "hsdp-shard-size": 4,
+        "text-encoder-tp-size": 4,
+        "vae-patch-parallel-size": 4,
+        "vae-parallel-mode": "tile",
+        "vae-use-tiling": true,
+        "diffusion-attention-backend": "FLASH_ATTN",
+        "stage-init-timeout": 1800,
+        "init-timeout": 1800,
+        "enable-diffusion-pipeline-profiler": true
+      }
+    },
+    "benchmark_params": [
+      {
+        "name": "1344x768_frames209_steps8",
+        "dataset": "random",
+        "task": "t2v",
+        "num-prompts": 3,
+        "max-concurrency": 1,
+        "warmup-requests": 1,
+        "warmup-concurrency": 1,
+        "seed": 42,
+        "random-request-config": [
+          {
+            "width": 1344,
+            "height": 768,
+            "prompt": "Cinematic drone shot pushing through misty mountain ridges at sunrise, golden light catching the clouds, slow and smooth.",
+            "num_inference_steps": 8,
+            "num_frames": 209,
+            "fps": 24,
+            "weight": 1
+          }
+        ],
+        "extra-body": {
+          "flow_shift": 12.0,
+          "extra_params": "{\"task\":\"t2va\",\"duration\":8.7,\"aspect_ratio\":\"16:9\",\"audio_flow_shift\":3.0}"
+        },
+        "baseline": {
+          "H100": {
+            "throughput_qps": 0.0211,
+            "latency_mean": 38.3252,
+            "peak_memory_mb_mean": 54219.6
+          }
+        }
+      }
+    ]
+  },
+  {
+    "test_name": "test_minimax_h3_ti2v_4gpu",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 4
+        }
+      },
+      "full_model",
+      "diffusion"
+    ],
+    "description": "MiniMax-H3 FL2VA first-frame image-to-video+audio on 4 GPUs (USP4 + HSDP4, encoder-TP4, VAE-PP4 tile)",
+    "server_type": "vllm-omni",
+    "benchmark_endpoint": "/v1/videos",
+    "server_params": {
+      "model": "MiniMaxAI/MiniMax-H3",
+      "serve_args": {
+        "trust-remote-code": true,
+        "task-type": "fl2va",
+        "num-gpus": 4,
+        "usp": 4,
+        "ring": 1,
+        "use-hsdp": true,
+        "hsdp-shard-size": 4,
+        "text-encoder-tp-size": 4,
+        "vae-patch-parallel-size": 4,
+        "vae-parallel-mode": "tile",
+        "vae-use-tiling": true,
+        "diffusion-attention-backend": "FLASH_ATTN",
+        "stage-init-timeout": 1800,
+        "init-timeout": 1800,
+        "enable-diffusion-pipeline-profiler": true
+      }
+    },
+    "benchmark_params": [
+      {
+        "name": "1344x768_frames209_steps8",
+        "dataset": "random",
+        "task": "ti2v",
+        "num-prompts": 3,
+        "max-concurrency": 1,
+        "warmup-requests": 1,
+        "warmup-concurrency": 1,
+        "seed": 42,
+        "random-request-config": [
+          {
+            "width": 1344,
+            "height": 768,
+            "prompt": "Continue the scene: the camera slowly pushes in on the subject as the background city lights drift and blur, with the actor singing softly and emotionally.",
+            "num_inference_steps": 8,
+            "num_frames": 209,
+            "fps": 24,
+            "weight": 1
+          }
+        ],
+        "extra-body": {
+          "flow_shift": 12.0,
+          "extra_params": "{\"task\":\"fl2va\",\"duration\":8.7,\"audio_flow_shift\":3.0}"
+        },
+        "num-input-images": 1,
+        "baseline": {
+          "H100": {
+            "throughput_qps": 0.0212,
+            "latency_mean": 38.1385,
+            "peak_memory_mb_mean": 54622.8
+          }
+        }
+      }
+    ]
+  },
+  {
+    "test_name": "test_minimax_h3_v2v_4gpu",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 4
+        }
+      },
+      "full_model",
+      "diffusion"
+    ],
+    "description": "MiniMax-H3 Ref2VA video-reference-to-video+audio on 4 GPUs (USP4 + HSDP4, encoder-TP4, VAE-PP4 tile)",
+    "server_type": "vllm-omni",
+    "benchmark_endpoint": "/v1/videos",
+    "server_params": {
+      "model": "MiniMaxAI/MiniMax-H3",
+      "serve_args": {
+        "trust-remote-code": true,
+        "task-type": "ref2va",
+        "num-gpus": 4,
+        "usp": 4,
+        "ring": 1,
+        "use-hsdp": true,
+        "hsdp-shard-size": 4,
+        "text-encoder-tp-size": 4,
+        "vae-patch-parallel-size": 4,
+        "vae-parallel-mode": "tile",
+        "vae-use-tiling": true,
+        "diffusion-attention-backend": "FLASH_ATTN",
+        "stage-init-timeout": 1800,
+        "init-timeout": 1800,
+        "enable-diffusion-pipeline-profiler": true
+      }
+    },
+    "benchmark_params": [
+      {
+        "name": "1344x768_frames209_steps8",
+        "dataset": "random",
+        "task": "v2v",
+        "num-prompts": 3,
+        "max-concurrency": 1,
+        "warmup-requests": 1,
+        "warmup-concurrency": 1,
+        "seed": 42,
+        "random-request-config": [
+          {
+            "width": 1344,
+            "height": 768,
+            "prompt": "Restyle the reference footage into a cinematic night scene with neon reflections, keeping the same camera motion and subject.",
+            "num_inference_steps": 8,
+            "num_frames": 209,
+            "fps": 24,
+            "weight": 1
+          }
+        ],
+        "extra-body": {
+          "flow_shift": 12.0,
+          "extra_params": "{\"task\":\"ref2va\",\"duration\":8.7,\"aspect_ratio\":\"16:9\",\"audio_flow_shift\":3.0}"
+        },
+        "baseline": {
+          "H100": {
+            "throughput_qps": 0.0077,
+            "latency_mean": 105.7508,
+            "peak_memory_mb_mean": 55180.8
+          }
+        }
+      }
+    ]
+  },
+  {
+    "test_name": "test_minimax_h3_t2v_dlo_4gpu",
+    "mark": [
+      {
+        "hardware_marks": {
+          "res": {
+            "cuda": "H100"
+          },
+          "num_cards": 4
+        }
+      },
+      "full_model",
+      "diffusion"
+    ],
+    "description": "MiniMax-H3 T2VA with 4-card distributed layerwise offload (DLO), 8 denoise steps (USP4 + HSDP4, encoder-TP4, VAE-PP4 tile)",
+    "server_type": "vllm-omni",
+    "benchmark_endpoint": "/v1/videos",
+    "server_params": {
+      "model": "MiniMaxAI/MiniMax-H3",
+      "serve_args": {
+        "trust-remote-code": true,
+        "task-type": "fl2va",
+        "num-gpus": 4,
+        "usp": 4,
+        "ring": 1,
+        "use-hsdp": true,
+        "hsdp-shard-size": 4,
+        "text-encoder-tp-size": 4,
+        "vae-patch-parallel-size": 4,
+        "vae-parallel-mode": "tile",
+        "vae-use-tiling": true,
+        "diffusion-attention-backend": "FLASH_ATTN",
+        "stage-init-timeout": 1800,
+        "init-timeout": 1800,
+        "enable-diffusion-pipeline-profiler": true,
+        "enable-distributed-layerwise-offload": true,
+        "dlo-no-use-allgather": true,
+        "dlo-resident-layers": 8,
+        "enforce-eager": true
+      }
+    },
+    "benchmark_params": [
+      {
+        "name": "1344x768_frames209_steps8",
+        "dataset": "random",
+        "task": "t2v",
+        "num-prompts": 3,
+        "max-concurrency": 1,
+        "warmup-requests": 1,
+        "warmup-concurrency": 1,
+        "seed": 42,
+        "random-request-config": [
+          {
+            "width": 1344,
+            "height": 768,
+            "prompt": "Cinematic drone shot pushing through misty mountain ridges at sunrise, golden light catching the clouds, slow and smooth.",
+            "num_inference_steps": 8,
+            "num_frames": 209,
+            "fps": 24,
+            "weight": 1
+          }
+        ],
+        "extra-body": {
+          "flow_shift": 12.0,
+          "extra_params": "{\"task\":\"t2va\",\"duration\":8.7,\"aspect_ratio\":\"16:9\",\"audio_flow_shift\":3.0}"
+        },
+        "baseline": {
+          "H100": {
+            "throughput_qps": 0.0212,
+            "latency_mean": 38.2089,
+            "peak_memory_mb_mean": 36218.4
+          }
+        }
+      }
+    ]
+  }
+]