[CI] Add HunyuanVideo 1.5 X2V accuracy tests (#3852)

Signed-off-by: david6666666 <530634352@qq.com>

.buildkite/test-nightly.yml

@@ -1091,6 +1091,8 @@ steps:
         timeout_in_minutes: 180
         commands:
           - pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py -m full_model --run-level full_model
+          - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py -m full_model --run-level full_model
+          - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m full_model --run-level full_model
           - pytest -s -v tests/e2e/accuracy/test_ltx2_3_video_similarity.py -m full_model --run-level full_model
           - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m full_model --run-level full_model
         agents:

tests/e2e/accuracy/conftest.py

@@ -62,6 +62,19 @@ def pytest_addoption(parser):
         default=1200,
         help="Online serving timeout in seconds for Wan2.2 I2V accuracy tests.",
     )
+    group.addoption(
+        "--hunyuanvideo15-i2v-image-source",
+        action="store",
+        default=None,
+        help="Image source for HunyuanVideo-1.5 I2V accuracy tests. Can be local path or remote URL",
+    )
+    group.addoption(
+        "--hunyuanvideo15-online-timeout-seconds",
+        action="store",
+        type=int,
+        default=3600,
+        help="Online serving timeout in seconds for HunyuanVideo-1.5 accuracy tests.",
+    )
 
 
 def _hf_cache_root() -> Path:
@@ -168,6 +181,17 @@ def wan22_i2v_online_timeout_seconds(request: pytest.FixtureRequest) -> int:
     return int(request.config.getoption("wan22_i2v_online_timeout_seconds"))
 
 
+@pytest.fixture(scope="session")
+def hunyuanvideo15_i2v_image_source(request: pytest.FixtureRequest) -> str | None:
+    value = request.config.getoption("hunyuanvideo15_i2v_image_source")
+    return str(value) if value else None
+
+
+@pytest.fixture(scope="session")
+def hunyuanvideo15_online_timeout_seconds(request: pytest.FixtureRequest) -> int:
+    return int(request.config.getoption("hunyuanvideo15_online_timeout_seconds"))
+
+
 @pytest.fixture(scope="session")
 def gebench_samples_per_type(request: pytest.FixtureRequest) -> int:
     return int(request.config.getoption("gebench_samples_per_type"))

tests/e2e/accuracy/helpers.py

@@ -1,7 +1,17 @@
+import json
 import os
+import re
+import shutil
+import subprocess
+import time
+from base64 import b64decode, b64encode
+from fractions import Fraction
+from hashlib import sha1
 from io import BytesIO
+from mimetypes import guess_type
 from pathlib import Path
 from typing import Any
+from urllib.parse import urlparse
 
 import numpy as np
 import pytest
@@ -63,6 +73,274 @@ def model_output_dir(parent_dir: Path, model: str) -> Path:
     return path
 
 
+_SSIM_RE = re.compile(r"All:(?P<score>[0-9.]+)")
+_PSNR_RE = re.compile(r"average:(?P<score>[0-9.]+)")
+
+
+def parse_video_metadata(payload: dict[str, Any]) -> dict[str, int | float]:
+    streams = payload.get("streams")
+    if not isinstance(streams, list) or not streams:
+        raise ValueError(f"ffprobe payload did not include video streams: {payload}")
+
+    stream = streams[0]
+    width = int(stream["width"])
+    height = int(stream["height"])
+    fps = float(Fraction(stream["avg_frame_rate"]))
+    frame_count_value = stream.get("nb_read_frames") or stream.get("nb_frames")
+    if frame_count_value is None:
+        raise ValueError(f"ffprobe payload did not include frame count: {payload}")
+    frame_count = int(frame_count_value)
+    return {
+        "width": width,
+        "height": height,
+        "fps": fps,
+        "frame_count": frame_count,
+    }
+
+
+def parse_ssim_score(output: str) -> float:
+    match = _SSIM_RE.search(output)
+    if match is None:
+        raise ValueError(f"Could not parse SSIM score from ffmpeg output:\n{output}")
+    return float(match.group("score"))
+
+
+def parse_psnr_score(output: str) -> float:
+    match = _PSNR_RE.search(output)
+    if match is None:
+        raise ValueError(f"Could not parse PSNR score from ffmpeg output:\n{output}")
+    return float(match.group("score"))
+
+
+def probe_binary(binary: str, *, test_name: str = "video similarity e2e test") -> str:
+    resolved = shutil.which(binary)
+    if resolved is None:
+        pytest.skip(f"{binary} is required for {test_name}.")
+    return resolved
+
+
+def probe_video(path: Path) -> dict[str, int | float]:
+    probe_binary("ffprobe")
+    result = subprocess.run(
+        [
+            "ffprobe",
+            "-v",
+            "error",
+            "-select_streams",
+            "v:0",
+            "-count_frames",
+            "-show_entries",
+            "stream=width,height,avg_frame_rate,nb_read_frames",
+            "-of",
+            "json",
+            str(path),
+        ],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    return parse_video_metadata(json.loads(result.stdout))
+
+
+def run_ffmpeg_similarity(filter_name: str, first: Path, second: Path) -> str:
+    probe_binary("ffmpeg")
+    result = subprocess.run(
+        [
+            "ffmpeg",
+            "-hide_banner",
+            "-i",
+            str(first),
+            "-i",
+            str(second),
+            "-lavfi",
+            f"[0:v][1:v]{filter_name}",
+            "-f",
+            "null",
+            "-",
+        ],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    return result.stderr
+
+
+def online_timeout_seconds(configured: int | None, *, default: int = 1200) -> int:
+    return int(configured) if configured is not None else default
+
+
+def is_remote_image_source(source: str) -> bool:
+    return source.startswith("http://") or source.startswith("https://")
+
+
+def validate_image_source(source: str) -> None:
+    if is_remote_image_source(source):
+        response = requests.get(source, timeout=60)
+        response.raise_for_status()
+        return
+
+    image_path = Path(source)
+    if not image_path.exists():
+        raise FileNotFoundError(f"Local image source does not exist: {image_path}")
+
+
+def build_online_image_reference(source: str) -> str:
+    if source.startswith("data:image"):
+        return source
+
+    if is_remote_image_source(source):
+        response = requests.get(source, timeout=60)
+        response.raise_for_status()
+        mime_type = response.headers.get("Content-Type", "").split(";", 1)[0].strip()
+        if not mime_type.startswith("image/"):
+            mime_type = guess_type(urlparse(source).path)[0] or "image/png"
+        encoded = b64encode(response.content).decode("ascii")
+        return f"data:{mime_type};base64,{encoded}"
+
+    image_path = Path(source)
+    mime_type = guess_type(image_path.name)[0] or "image/png"
+    encoded = b64encode(image_path.read_bytes()).decode("ascii")
+    return f"data:{mime_type};base64,{encoded}"
+
+
+def materialize_image_source(source: str, output_dir: Path) -> str:
+    if not is_remote_image_source(source):
+        return source
+
+    from diffusers.utils import load_image
+
+    output_dir.mkdir(parents=True, exist_ok=True)
+    source_name = Path(urlparse(source).path).name or "input.png"
+    safe_name = re.sub(r"[^A-Za-z0-9_.-]+", "_", source_name)
+    if "." not in safe_name:
+        safe_name = f"{safe_name}.png"
+    image_path = output_dir / safe_name
+    if not image_path.exists():
+        image = load_image(source).convert("RGB")
+        image.save(image_path)
+    return str(image_path)
+
+
+def video_artifact_dir(result_root: Path, source: str) -> Path:
+    if is_remote_image_source(source):
+        source_name = Path(urlparse(source).path).stem or "remote"
+    else:
+        source_name = Path(source).stem or "local"
+
+    safe_name = re.sub(r"[^A-Za-z0-9_.-]+", "_", source_name)
+    digest = sha1(source.encode("utf-8")).hexdigest()[:8]
+    return result_root / f"{safe_name}-{digest}"
+
+
+def send_video_request_with_timeout(
+    openai_client,
+    request_config: dict[str, Any],
+    *,
+    timeout_seconds: int,
+) -> bytes:
+    form_data = request_config.get("form_data")
+    if not isinstance(form_data, dict):
+        raise ValueError("Video request_config must contain 'form_data'")
+
+    if not form_data.get("prompt"):
+        raise ValueError("Video request_config['form_data'] must contain 'prompt'")
+
+    normalized_form_data = {key: str(value) for key, value in form_data.items() if value is not None}
+
+    files: dict[str, tuple[str, BytesIO, str]] = {}
+    image_reference = request_config.get("image_reference")
+    if image_reference:
+        if image_reference.startswith("data:image"):
+            header, encoded = image_reference.split(",", 1)
+            content_type = header.split(";")[0].removeprefix("data:")
+            extension = content_type.split("/")[-1]
+            file_data = b64decode(encoded)
+            files["input_reference"] = (
+                f"reference.{extension}",
+                BytesIO(file_data),
+                content_type,
+            )
+        else:
+            normalized_form_data["image_reference"] = json.dumps({"image_url": image_reference})
+
+    start_time = time.perf_counter()
+    create_url = openai_client._build_url("/v1/videos")
+    response = requests.post(
+        create_url,
+        data=normalized_form_data,
+        files=files,
+        headers={"Accept": "application/json"},
+        timeout=60,
+    )
+    try:
+        response.raise_for_status()
+    except requests.HTTPError as exc:
+        raise requests.HTTPError(f"{exc}; response body: {response.text}", response=response) from exc
+    job_data = response.json()
+    video_id = job_data["id"]
+    openai_client._wait_until_video_completed(video_id, timeout_seconds=timeout_seconds)
+    video_content = openai_client._download_video_content(video_id)
+    print(f"online_video_e2e_latency_s={time.perf_counter() - start_time:.3f}")
+    return video_content
+
+
+def assert_video_metadata(
+    metadata: dict[str, int | float],
+    *,
+    width: int,
+    height: int,
+    fps: int | float,
+    frame_count: int,
+) -> None:
+    assert metadata["width"] == width
+    assert metadata["height"] == height
+    assert metadata["fps"] == float(fps)
+    assert metadata["frame_count"] == frame_count
+
+
+def assert_video_similarity_metrics(
+    *,
+    label: str,
+    online_path: Path,
+    offline_path: Path,
+    ssim_threshold: float,
+    psnr_threshold: float,
+) -> None:
+    ssim_output = run_ffmpeg_similarity("ssim", online_path, offline_path)
+    psnr_output = run_ffmpeg_similarity("psnr", online_path, offline_path)
+    ssim_score = parse_ssim_score(ssim_output)
+    psnr_score = parse_psnr_score(psnr_output)
+
+    print(f"{label} similarity metrics:")
+    print(
+        "  SSIM:"
+        f" value={ssim_score:.6f},"
+        f" threshold>={ssim_threshold:.6f},"
+        " range=[0, 1],"
+        " higher_is_better=True,"
+        " interpretation=structural_similarity"
+    )
+    print(
+        "  PSNR:"
+        f" value={psnr_score:.6f} dB,"
+        f" threshold>={psnr_threshold:.6f} dB,"
+        " range=[0, +inf),"
+        " higher_is_better=True,"
+        " interpretation=pixel_error_in_decibels"
+    )
+    print(f"online_video={online_path}")
+    print(f"offline_video={offline_path}")
+
+    assert ssim_score >= ssim_threshold, (
+        f"SSIM below threshold: got {ssim_score:.6f}, expected >= {ssim_threshold:.6f}. "
+        f"online={online_path} offline={offline_path}"
+    )
+    assert psnr_score >= psnr_threshold, (
+        f"PSNR below threshold: got {psnr_score:.6f}, expected >= {psnr_threshold:.6f}. "
+        f"online={online_path} offline={offline_path}"
+    )
+
+
 def assert_similarity(
     *,
     model_name: str,

tests/e2e/accuracy/hunyuanvideo15_i2v/__init__.py

@@ -0,0 +1 @@
+"""HunyuanVideo-1.5 I2V accuracy test cases."""

tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py

@@ -0,0 +1,262 @@
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+import torch
+
+from tests.e2e.accuracy.helpers import (
+    assert_video_metadata,
+    assert_video_similarity_metrics,
+    build_online_image_reference,
+    materialize_image_source,
+    probe_binary,
+    probe_video,
+    send_video_request_with_timeout,
+    validate_image_source,
+    video_artifact_dir,
+)
+from tests.helpers.mark import hardware_test
+from tests.helpers.runtime import OmniServerParams
+
+pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
+
+MODEL_NAME = "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v"
+IMAGE_URL = "https://huggingface.co/datasets/YiYiXu/testing-images/resolve/main/wan_i2v_input.JPG"
+PROMPT = (
+    "Summer beach vacation style, a white cat wearing sunglasses sits on a surfboard. "
+    "The fluffy-furred feline gazes directly at the camera with a relaxed expression. "
+    "Blurred beach scenery forms the background featuring crystal-clear waters, distant green hills, "
+    "and a blue sky dotted with white clouds. The cat assumes a naturally relaxed posture, "
+    "as if savoring the sea breeze and warm sunlight. A close-up shot highlights the feline's "
+    "intricate details and the refreshing atmosphere of the seaside."
+)
+WIDTH = 832
+HEIGHT = 480
+FPS = 24
+NUM_FRAMES = 33
+NUM_INFERENCE_STEPS = 50
+GUIDANCE_SCALE = 6.0
+FLOW_SHIFT = 5.0
+SEED = 42
+SSIM_THRESHOLD = 0.94
+PSNR_THRESHOLD = 28.0
+
+REPO_ROOT = Path(__file__).resolve().parents[4]
+WORKSPACE_ROOT = REPO_ROOT.parent
+RUNNER_PATH = REPO_ROOT / "examples" / "offline_inference" / "image_to_video" / "image_to_video.py"
+RESULT_ROOT = Path(__file__).parent / "result"
+VIDEO_TIMEOUT_SECONDS = 60 * 60
+SERVER_CASES = [
+    pytest.param(
+        OmniServerParams(
+            model=MODEL_NAME,
+            server_args=[
+                "--flow-shift",
+                str(FLOW_SHIFT),
+                "--enforce-eager",
+            ],
+            env_dict={"VLLM_OMNI_STORAGE_PATH": str(RESULT_ROOT / "storage")},
+            use_omni=True,
+        ),
+        id="hunyuanvideo15_i2v_default",
+    )
+]
+
+
+def _runner_env() -> dict[str, str]:
+    env = os.environ.copy()
+    pythonpath_parts = [
+        str(REPO_ROOT),
+        str(WORKSPACE_ROOT / "diffusers" / "src"),
+    ]
+    existing_pythonpath = env.get("PYTHONPATH")
+    if existing_pythonpath:
+        pythonpath_parts.append(existing_pythonpath)
+    env["PYTHONPATH"] = os.pathsep.join(pythonpath_parts)
+    env.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
+    return env
+
+
+def _resolve_image_source(configured: str | None) -> str:
+    if configured:
+        candidate = Path(configured)
+        if candidate.exists():
+            return str(candidate.resolve())
+        return configured
+    return IMAGE_URL
+
+
+def _artifact_paths(image_source: str) -> tuple[Path, Path]:
+    artifact_dir = video_artifact_dir(RESULT_ROOT, image_source)
+    artifact_dir.mkdir(parents=True, exist_ok=True)
+    return artifact_dir / "online.mp4", artifact_dir / "offline.mp4"
+
+
+def _offline_image_source(image_source: str) -> str:
+    artifact_dir = video_artifact_dir(RESULT_ROOT, image_source)
+    return materialize_image_source(image_source, artifact_dir)
+
+
+def _build_offline_command(*, image_source: str, output_path: Path) -> list[str]:
+    return [
+        sys.executable,
+        str(RUNNER_PATH),
+        "--model",
+        MODEL_NAME,
+        "--image",
+        image_source,
+        "--prompt",
+        PROMPT,
+        "--height",
+        str(HEIGHT),
+        "--width",
+        str(WIDTH),
+        "--num-frames",
+        str(NUM_FRAMES),
+        "--num-inference-steps",
+        str(NUM_INFERENCE_STEPS),
+        "--guidance-scale",
+        str(GUIDANCE_SCALE),
+        "--flow-shift",
+        str(FLOW_SHIFT),
+        "--fps",
+        str(FPS),
+        "--seed",
+        str(SEED),
+        "--enforce-eager",
+        "--output",
+        str(output_path),
+    ]
+
+
+def _generate_offline_video(*, image_source: str) -> Path:
+    _, offline_path = _artifact_paths(image_source)
+    offline_image_source = _offline_image_source(image_source)
+    subprocess.run(
+        _build_offline_command(image_source=offline_image_source, output_path=offline_path),
+        cwd=REPO_ROOT,
+        env=_runner_env(),
+        check=True,
+        timeout=VIDEO_TIMEOUT_SECONDS,
+    )
+    return offline_path
+
+
+def _generate_online_video(
+    *,
+    omni_server,
+    openai_client,
+    image_source: str,
+    timeout_seconds: int,
+) -> Path:
+    online_path, _ = _artifact_paths(image_source)
+    online_image_source = _offline_image_source(image_source)
+    request_config = {
+        "model": omni_server.model,
+        "form_data": {
+            "prompt": PROMPT,
+            "height": HEIGHT,
+            "width": WIDTH,
+            "fps": FPS,
+            "num_frames": NUM_FRAMES,
+            "guidance_scale": GUIDANCE_SCALE,
+            "flow_shift": FLOW_SHIFT,
+            "num_inference_steps": NUM_INFERENCE_STEPS,
+            "seed": SEED,
+        },
+        "image_reference": build_online_image_reference(online_image_source),
+    }
+    online_video_bytes = send_video_request_with_timeout(
+        openai_client,
+        request_config,
+        timeout_seconds=timeout_seconds,
+    )
+    online_path.write_bytes(online_video_bytes)
+    return online_path
+
+
+@pytest.mark.benchmark
+@hardware_test(res={"cuda": "H100"}, num_cards=1)
+def test_hunyuanvideo15_i2v_diffusers_offline_generates_video(
+    hunyuanvideo15_i2v_image_source: str | None,
+) -> None:
+    if not torch.cuda.is_available():
+        pytest.skip("HunyuanVideo-1.5 I2V offline accuracy test requires CUDA.")
+
+    probe_binary("ffprobe")
+    if not RUNNER_PATH.exists():
+        raise AssertionError(f"Offline runner does not exist: {RUNNER_PATH}")
+
+    image_source = _resolve_image_source(hunyuanvideo15_i2v_image_source)
+    validate_image_source(image_source)
+    offline_path = _generate_offline_video(image_source=image_source)
+    assert offline_path.exists(), f"Expected offline video artifact at {offline_path}"
+    offline_metadata = probe_video(offline_path)
+    assert_video_metadata(offline_metadata, width=WIDTH, height=HEIGHT, fps=FPS, frame_count=NUM_FRAMES)
+
+
+@pytest.mark.benchmark
+@hardware_test(res={"cuda": "H100"}, num_cards=1)
+@pytest.mark.parametrize("omni_server", SERVER_CASES, indirect=True)
+def test_hunyuanvideo15_i2v_online_serving_generates_video(
+    omni_server,
+    openai_client,
+    hunyuanvideo15_i2v_image_source: str | None,
+    hunyuanvideo15_online_timeout_seconds: int,
+) -> None:
+    if not torch.cuda.is_available():
+        pytest.skip("HunyuanVideo-1.5 I2V online accuracy test requires CUDA.")
+
+    probe_binary("ffprobe")
+    image_source = _resolve_image_source(hunyuanvideo15_i2v_image_source)
+    validate_image_source(image_source)
+    online_path = _generate_online_video(
+        omni_server=omni_server,
+        openai_client=openai_client,
+        image_source=image_source,
+        timeout_seconds=hunyuanvideo15_online_timeout_seconds,
+    )
+    assert online_path.exists(), f"Expected online video artifact at {online_path}"
+    online_metadata = probe_video(online_path)
+    assert_video_metadata(online_metadata, width=WIDTH, height=HEIGHT, fps=FPS, frame_count=NUM_FRAMES)
+
+
+@pytest.mark.benchmark
+@hardware_test(res={"cuda": "H100"}, num_cards=1)
+def test_hunyuanvideo15_i2v_serving_matches_offline_video_similarity(
+    hunyuanvideo15_i2v_image_source: str | None,
+) -> None:
+    if not torch.cuda.is_available():
+        pytest.skip("HunyuanVideo-1.5 I2V video similarity test requires CUDA.")
+
+    probe_binary("ffmpeg")
+    probe_binary("ffprobe")
+    image_source = _resolve_image_source(hunyuanvideo15_i2v_image_source)
+    validate_image_source(image_source)
+    online_path, offline_path = _artifact_paths(image_source)
+    if not online_path.exists():
+        pytest.skip(f"Missing online artifact from prerequisite test: {online_path}")
+    if not offline_path.exists():
+        pytest.skip(f"Missing offline artifact from prerequisite test: {offline_path}")
+
+    online_metadata = probe_video(online_path)
+    offline_metadata = probe_video(offline_path)
+    assert online_metadata == offline_metadata, (
+        f"Video metadata mismatch:\n"
+        f"online={online_metadata}\n"
+        f"offline={offline_metadata}\n"
+        f"online_path={online_path}\n"
+        f"offline_path={offline_path}"
+    )
+    assert_video_metadata(online_metadata, width=WIDTH, height=HEIGHT, fps=FPS, frame_count=NUM_FRAMES)
+    assert_video_similarity_metrics(
+        label="hunyuanvideo15_i2v",
+        online_path=online_path,
+        offline_path=offline_path,
+        ssim_threshold=SSIM_THRESHOLD,
+        psnr_threshold=PSNR_THRESHOLD,
+    )

tests/e2e/accuracy/hunyuanvideo15_t2v/__init__.py

@@ -0,0 +1 @@
+"""HunyuanVideo-1.5 T2V accuracy test cases."""

tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py

@@ -0,0 +1,211 @@
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+import torch
+
+from tests.e2e.accuracy.helpers import (
+    assert_video_metadata,
+    assert_video_similarity_metrics,
+    probe_binary,
+    probe_video,
+    send_video_request_with_timeout,
+)
+from tests.helpers.mark import hardware_test
+from tests.helpers.runtime import OmniServerParams
+
+pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
+
+MODEL_NAME = "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v"
+PROMPT = "A cat walks on the grass, realistic style."
+WIDTH = 832
+HEIGHT = 480
+FPS = 24
+NUM_FRAMES = 33
+NUM_INFERENCE_STEPS = 50
+GUIDANCE_SCALE = 6.0
+FLOW_SHIFT = 5.0
+SEED = 42
+SSIM_THRESHOLD = 0.94
+PSNR_THRESHOLD = 28.0
+
+REPO_ROOT = Path(__file__).resolve().parents[4]
+WORKSPACE_ROOT = REPO_ROOT.parent
+RUNNER_PATH = REPO_ROOT / "examples" / "offline_inference" / "text_to_video" / "text_to_video.py"
+RESULT_ROOT = Path(__file__).parent / "result"
+ARTIFACT_DIR = RESULT_ROOT / "cat_grass"
+VIDEO_TIMEOUT_SECONDS = 60 * 60
+SERVER_CASES = [
+    pytest.param(
+        OmniServerParams(
+            model=MODEL_NAME,
+            server_args=[
+                "--flow-shift",
+                str(FLOW_SHIFT),
+            ],
+            env_dict={"VLLM_OMNI_STORAGE_PATH": str(RESULT_ROOT / "storage")},
+            use_omni=True,
+        ),
+        id="hunyuanvideo15_t2v_default",
+    )
+]
+
+
+def _runner_env() -> dict[str, str]:
+    env = os.environ.copy()
+    pythonpath_parts = [
+        str(REPO_ROOT),
+        str(WORKSPACE_ROOT / "diffusers" / "src"),
+    ]
+    existing_pythonpath = env.get("PYTHONPATH")
+    if existing_pythonpath:
+        pythonpath_parts.append(existing_pythonpath)
+    env["PYTHONPATH"] = os.pathsep.join(pythonpath_parts)
+    env.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
+    return env
+
+
+def _artifact_paths() -> tuple[Path, Path]:
+    ARTIFACT_DIR.mkdir(parents=True, exist_ok=True)
+    return ARTIFACT_DIR / "online.mp4", ARTIFACT_DIR / "offline.mp4"
+
+
+def _build_offline_command(*, output_path: Path) -> list[str]:
+    return [
+        sys.executable,
+        str(RUNNER_PATH),
+        "--model",
+        MODEL_NAME,
+        "--prompt",
+        PROMPT,
+        "--height",
+        str(HEIGHT),
+        "--width",
+        str(WIDTH),
+        "--num-frames",
+        str(NUM_FRAMES),
+        "--num-inference-steps",
+        str(NUM_INFERENCE_STEPS),
+        "--guidance-scale",
+        str(GUIDANCE_SCALE),
+        "--flow-shift",
+        str(FLOW_SHIFT),
+        "--fps",
+        str(FPS),
+        "--seed",
+        str(SEED),
+        "--output",
+        str(output_path),
+    ]
+
+
+def _generate_offline_video() -> Path:
+    _, offline_path = _artifact_paths()
+    subprocess.run(
+        _build_offline_command(output_path=offline_path),
+        cwd=REPO_ROOT,
+        env=_runner_env(),
+        check=True,
+        timeout=VIDEO_TIMEOUT_SECONDS,
+    )
+    return offline_path
+
+
+def _generate_online_video(*, omni_server, openai_client, timeout_seconds: int) -> Path:
+    online_path, _ = _artifact_paths()
+    request_config = {
+        "model": omni_server.model,
+        "form_data": {
+            "prompt": PROMPT,
+            "height": HEIGHT,
+            "width": WIDTH,
+            "fps": FPS,
+            "num_frames": NUM_FRAMES,
+            "guidance_scale": GUIDANCE_SCALE,
+            "flow_shift": FLOW_SHIFT,
+            "num_inference_steps": NUM_INFERENCE_STEPS,
+            "seed": SEED,
+        },
+    }
+    online_video_bytes = send_video_request_with_timeout(
+        openai_client,
+        request_config,
+        timeout_seconds=timeout_seconds,
+    )
+    online_path.write_bytes(online_video_bytes)
+    return online_path
+
+
+@pytest.mark.benchmark
+@hardware_test(res={"cuda": "H100"}, num_cards=1)
+def test_hunyuanvideo15_t2v_diffusers_offline_generates_video() -> None:
+    if not torch.cuda.is_available():
+        pytest.skip("HunyuanVideo-1.5 T2V offline accuracy test requires CUDA.")
+
+    probe_binary("ffprobe")
+    if not RUNNER_PATH.exists():
+        raise AssertionError(f"Offline runner does not exist: {RUNNER_PATH}")
+
+    offline_path = _generate_offline_video()
+    assert offline_path.exists(), f"Expected offline video artifact at {offline_path}"
+    offline_metadata = probe_video(offline_path)
+    assert_video_metadata(offline_metadata, width=WIDTH, height=HEIGHT, fps=FPS, frame_count=NUM_FRAMES)
+
+
+@pytest.mark.benchmark
+@hardware_test(res={"cuda": "H100"}, num_cards=1)
+@pytest.mark.parametrize("omni_server", SERVER_CASES, indirect=True)
+def test_hunyuanvideo15_t2v_online_serving_generates_video(
+    omni_server,
+    openai_client,
+    hunyuanvideo15_online_timeout_seconds: int,
+) -> None:
+    if not torch.cuda.is_available():
+        pytest.skip("HunyuanVideo-1.5 T2V online accuracy test requires CUDA.")
+
+    probe_binary("ffprobe")
+    online_path = _generate_online_video(
+        omni_server=omni_server,
+        openai_client=openai_client,
+        timeout_seconds=hunyuanvideo15_online_timeout_seconds,
+    )
+    assert online_path.exists(), f"Expected online video artifact at {online_path}"
+    online_metadata = probe_video(online_path)
+    assert_video_metadata(online_metadata, width=WIDTH, height=HEIGHT, fps=FPS, frame_count=NUM_FRAMES)
+
+
+@pytest.mark.benchmark
+@hardware_test(res={"cuda": "H100"}, num_cards=1)
+def test_hunyuanvideo15_t2v_serving_matches_offline_video_similarity() -> None:
+    if not torch.cuda.is_available():
+        pytest.skip("HunyuanVideo-1.5 T2V video similarity test requires CUDA.")
+
+    probe_binary("ffmpeg")
+    probe_binary("ffprobe")
+    online_path, offline_path = _artifact_paths()
+    if not online_path.exists():
+        pytest.skip(f"Missing online artifact from prerequisite test: {online_path}")
+    if not offline_path.exists():
+        pytest.skip(f"Missing offline artifact from prerequisite test: {offline_path}")
+
+    online_metadata = probe_video(online_path)
+    offline_metadata = probe_video(offline_path)
+    assert online_metadata == offline_metadata, (
+        f"Video metadata mismatch:\n"
+        f"online={online_metadata}\n"
+        f"offline={offline_metadata}\n"
+        f"online_path={online_path}\n"
+        f"offline_path={offline_path}"
+    )
+    assert_video_metadata(online_metadata, width=WIDTH, height=HEIGHT, fps=FPS, frame_count=NUM_FRAMES)
+    assert_video_similarity_metrics(
+        label="hunyuanvideo15_t2v",
+        online_path=online_path,
+        offline_path=offline_path,
+        ssim_threshold=SSIM_THRESHOLD,
+        psnr_threshold=PSNR_THRESHOLD,
+    )

tests/e2e/accuracy/test_diffusers_backend_similarity.py

@@ -30,10 +30,14 @@ from tests.e2e.accuracy.helpers import (
     env_to_apply_ftfy_mock_in_subproc,
     model_output_dir,
 )
-from tests.e2e.accuracy.wan22_i2v.test_wan22_i2v_video_similarity import (
-    _parse_psnr_score,
-    _parse_ssim_score,
-    _run_ffmpeg_similarity,
+from tests.e2e.accuracy.helpers import (
+    parse_psnr_score as _parse_psnr_score,
+)
+from tests.e2e.accuracy.helpers import (
+    parse_ssim_score as _parse_ssim_score,
+)
+from tests.e2e.accuracy.helpers import (
+    run_ffmpeg_similarity as _run_ffmpeg_similarity,
 )
 from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup
 from tests.helpers.mark import hardware_test

tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py

@@ -1,20 +1,10 @@
 from __future__ import annotations
 
-import json
 import os
-import re
-import shutil
 import subprocess
 import sys
-import time
-from base64 import b64decode, b64encode
-from fractions import Fraction
-from hashlib import sha1
-from io import BytesIO
-from mimetypes import guess_type
+from base64 import b64decode
 from pathlib import Path
-from typing import Any
-from urllib.parse import urlparse
 
 import pytest
 import requests
@@ -22,6 +12,38 @@ import torch
 from diffusers import UniPCMultistepScheduler
 from PIL import Image
 
+from tests.e2e.accuracy.helpers import (
+    assert_video_metadata,
+    assert_video_similarity_metrics,
+    video_artifact_dir,
+)
+from tests.e2e.accuracy.helpers import (
+    build_online_image_reference as _build_online_image_reference,
+)
+from tests.e2e.accuracy.helpers import (
+    online_timeout_seconds as _online_timeout_seconds,
+)
+from tests.e2e.accuracy.helpers import (
+    parse_psnr_score as _parse_psnr_score,
+)
+from tests.e2e.accuracy.helpers import (
+    parse_ssim_score as _parse_ssim_score,
+)
+from tests.e2e.accuracy.helpers import (
+    parse_video_metadata as _parse_video_metadata,
+)
+from tests.e2e.accuracy.helpers import (
+    probe_binary as _probe_binary,
+)
+from tests.e2e.accuracy.helpers import (
+    probe_video as _probe_video,
+)
+from tests.e2e.accuracy.helpers import (
+    send_video_request_with_timeout as _send_video_request_with_timeout,
+)
+from tests.e2e.accuracy.helpers import (
+    validate_image_source as _validate_image_source,
+)
 from tests.e2e.accuracy.wan22_i2v.run_wan22_i2v_diffusers_cp import (
     _configure_scheduler,
     _ensure_wan_ftfy_fallback,
@@ -218,8 +240,6 @@ WORKSPACE_ROOT = REPO_ROOT.parent
 RUNNER_PATH = Path(__file__).with_name("run_wan22_i2v_diffusers_cp.py")
 RESULT_ROOT = Path(__file__).parent / "result"
 VIDEO_TIMEOUT_SECONDS = 60 * 60
-_SSIM_RE = re.compile(r"All:(?P<score>[0-9.]+)")
-_PSNR_RE = re.compile(r"average:(?P<score>[0-9.]+)")
 SERVER_CASES = [
     pytest.param(
         OmniServerParams(
@@ -238,48 +258,6 @@ SERVER_CASES = [
 ]
 
 
-def _parse_video_metadata(payload: dict[str, Any]) -> dict[str, int | float]:
-    streams = payload.get("streams")
-    if not isinstance(streams, list) or not streams:
-        raise ValueError(f"ffprobe payload did not include video streams: {payload}")
-
-    stream = streams[0]
-    width = int(stream["width"])
-    height = int(stream["height"])
-    fps = float(Fraction(stream["avg_frame_rate"]))
-    frame_count_value = stream.get("nb_read_frames") or stream.get("nb_frames")
-    if frame_count_value is None:
-        raise ValueError(f"ffprobe payload did not include frame count: {payload}")
-    frame_count = int(frame_count_value)
-    return {
-        "width": width,
-        "height": height,
-        "fps": fps,
-        "frame_count": frame_count,
-    }
-
-
-def _parse_ssim_score(output: str) -> float:
-    match = _SSIM_RE.search(output)
-    if match is None:
-        raise ValueError(f"Could not parse SSIM score from ffmpeg output:\n{output}")
-    return float(match.group("score"))
-
-
-def _parse_psnr_score(output: str) -> float:
-    match = _PSNR_RE.search(output)
-    if match is None:
-        raise ValueError(f"Could not parse PSNR score from ffmpeg output:\n{output}")
-    return float(match.group("score"))
-
-
-def _probe_binary(binary: str) -> str:
-    resolved = shutil.which(binary)
-    if resolved is None:
-        pytest.skip(f"{binary} is required for Wan2.2 video similarity e2e test.")
-    return resolved
-
-
 def _build_diffusers_command(
     *,
     runner_path: Path,
@@ -343,99 +321,8 @@ def _resolve_image_source(configured: str | None) -> str:
     return configured
 
 
-def _is_remote_image_source(source: str) -> bool:
-    return source.startswith("http://") or source.startswith("https://")
-
-
-def _validate_image_source(source: str) -> None:
-    if _is_remote_image_source(source):
-        response = requests.get(source, timeout=60)
-        response.raise_for_status()
-        return
-
-    image_path = Path(source)
-    if not image_path.exists():
-        raise FileNotFoundError(f"Local image source does not exist: {image_path}")
-
-
-def _build_online_image_reference(source: str) -> str:
-    if source.startswith("data:image"):
-        return source
-
-    if _is_remote_image_source(source):
-        response = requests.get(source, timeout=60)
-        response.raise_for_status()
-        mime_type = response.headers.get("Content-Type", "").split(";", 1)[0].strip()
-        if not mime_type.startswith("image/"):
-            mime_type = guess_type(urlparse(source).path)[0] or "image/png"
-        encoded = b64encode(response.content).decode("ascii")
-        return f"data:{mime_type};base64,{encoded}"
-
-    image_path = Path(source)
-    mime_type = guess_type(image_path.name)[0] or "image/png"
-    encoded = b64encode(image_path.read_bytes()).decode("ascii")
-    return f"data:{mime_type};base64,{encoded}"
-
-
-def _probe_video(path: Path) -> dict[str, int | float]:
-    _probe_binary("ffprobe")
-    result = subprocess.run(
-        [
-            "ffprobe",
-            "-v",
-            "error",
-            "-select_streams",
-            "v:0",
-            "-count_frames",
-            "-show_entries",
-            "stream=width,height,avg_frame_rate,nb_read_frames",
-            "-of",
-            "json",
-            str(path),
-        ],
-        capture_output=True,
-        text=True,
-        check=True,
-    )
-    return _parse_video_metadata(json.loads(result.stdout))
-
-
-def _run_ffmpeg_similarity(filter_name: str, first: Path, second: Path) -> str:
-    _probe_binary("ffmpeg")
-    result = subprocess.run(
-        [
-            "ffmpeg",
-            "-hide_banner",
-            "-i",
-            str(first),
-            "-i",
-            str(second),
-            "-lavfi",
-            f"[0:v][1:v]{filter_name}",
-            "-f",
-            "null",
-            "-",
-        ],
-        capture_output=True,
-        text=True,
-        check=True,
-    )
-    return result.stderr
-
-
-def _online_timeout_seconds(configured: int | None) -> int:
-    return int(configured) if configured is not None else 1200
-
-
 def _artifact_dir(image_source: str) -> Path:
-    if _is_remote_image_source(image_source):
-        source_name = Path(urlparse(image_source).path).stem or "remote"
-    else:
-        source_name = Path(image_source).stem or "local"
-
-    safe_name = re.sub(r"[^A-Za-z0-9_.-]+", "_", source_name)
-    digest = sha1(image_source.encode("utf-8")).hexdigest()[:8]
-    return RESULT_ROOT / f"{safe_name}-{digest}"
+    return video_artifact_dir(RESULT_ROOT, image_source)
 
 
 def _artifact_paths(image_source: str) -> tuple[Path, Path, Path]:
@@ -448,58 +335,6 @@ def _artifact_paths(image_source: str) -> tuple[Path, Path, Path]:
     )
 
 
-def _send_video_request_with_timeout(
-    openai_client,
-    request_config: dict[str, Any],
-    *,
-    timeout_seconds: int,
-) -> bytes:
-    form_data = request_config.get("form_data")
-    if not isinstance(form_data, dict):
-        raise ValueError("Video request_config must contain 'form_data'")
-
-    if not form_data.get("prompt"):
-        raise ValueError("Video request_config['form_data'] must contain 'prompt'")
-
-    normalized_form_data = {key: str(value) for key, value in form_data.items() if value is not None}
-
-    files: dict[str, tuple[str, BytesIO, str]] = {}
-    image_reference = request_config.get("image_reference")
-    if image_reference:
-        if image_reference.startswith("data:image"):
-            header, encoded = image_reference.split(",", 1)
-            content_type = header.split(";")[0].removeprefix("data:")
-            extension = content_type.split("/")[-1]
-            file_data = b64decode(encoded)
-            files["input_reference"] = (
-                f"reference.{extension}",
-                BytesIO(file_data),
-                content_type,
-            )
-        else:
-            normalized_form_data["image_reference"] = json.dumps({"image_url": image_reference})
-
-    start_time = time.perf_counter()
-    create_url = openai_client._build_url("/v1/videos")
-    response = requests.post(
-        create_url,
-        data=normalized_form_data,
-        files=files,
-        headers={"Accept": "application/json"},
-        timeout=60,
-    )
-    try:
-        response.raise_for_status()
-    except requests.HTTPError as exc:
-        raise requests.HTTPError(f"{exc}; response body: {response.text}", response=response) from exc
-    job_data = response.json()
-    video_id = job_data["id"]
-    openai_client._wait_until_video_completed(video_id, timeout_seconds=timeout_seconds)
-    video_content = openai_client._download_video_content(video_id)
-    print(f"online_video_e2e_latency_s={time.perf_counter() - start_time:.3f}")
-    return video_content
-
-
 def _generate_online_video(
     *,
     omni_server,
@@ -569,10 +404,7 @@ def test_wan22_i2v_diffusers_offline_generates_video(
     assert offline_path.exists(), f"Expected offline video artifact at {offline_path}"
     assert offline_metadata_path.exists(), f"Expected offline metadata artifact at {offline_metadata_path}"
     offline_metadata = _probe_video(offline_path)
-    assert offline_metadata["width"] == WIDTH
-    assert offline_metadata["height"] == HEIGHT
-    assert offline_metadata["fps"] == float(FPS)
-    assert offline_metadata["frame_count"] == NUM_FRAMES
+    assert_video_metadata(offline_metadata, width=WIDTH, height=HEIGHT, fps=FPS, frame_count=NUM_FRAMES)
 
 
 @pytest.mark.benchmark
@@ -598,10 +430,7 @@ def test_wan22_i2v_online_serving_generates_video(
     )
     assert online_path.exists(), f"Expected online video artifact at {online_path}"
     online_metadata = _probe_video(online_path)
-    assert online_metadata["width"] == WIDTH
-    assert online_metadata["height"] == HEIGHT
-    assert online_metadata["fps"] == float(FPS)
-    assert online_metadata["frame_count"] == NUM_FRAMES
+    assert_video_metadata(online_metadata, width=WIDTH, height=HEIGHT, fps=FPS, frame_count=NUM_FRAMES)
 
 
 @pytest.mark.benchmark
@@ -639,42 +468,12 @@ def test_wan22_i2v_serving_matches_diffusers_video_similarity(
         f"online_path={online_path}\n"
         f"offline_path={offline_path}"
     )
-    assert online_metadata["width"] == WIDTH
-    assert online_metadata["height"] == HEIGHT
-    assert online_metadata["fps"] == float(FPS)
-    assert online_metadata["frame_count"] == NUM_FRAMES
-
-    ssim_output = _run_ffmpeg_similarity("ssim", online_path, offline_path)
-    psnr_output = _run_ffmpeg_similarity("psnr", online_path, offline_path)
-    ssim_score = _parse_ssim_score(ssim_output)
-    psnr_score = _parse_psnr_score(psnr_output)
-
-    print("wan22_i2v_similarity metrics:")
-    print(
-        "  SSIM:"
-        f" value={ssim_score:.6f},"
-        f" threshold>={SSIM_THRESHOLD:.6f},"
-        " range=[0, 1],"
-        " higher_is_better=True,"
-        " interpretation=structural_similarity"
+    assert_video_metadata(online_metadata, width=WIDTH, height=HEIGHT, fps=FPS, frame_count=NUM_FRAMES)
+    assert_video_similarity_metrics(
+        label="wan22_i2v",
+        online_path=online_path,
+        offline_path=offline_path,
+        ssim_threshold=SSIM_THRESHOLD,
+        psnr_threshold=PSNR_THRESHOLD,
     )
-    print(
-        "  PSNR:"
-        f" value={psnr_score:.6f} dB,"
-        f" threshold>={PSNR_THRESHOLD:.6f} dB,"
-        " range=[0, +inf),"
-        " higher_is_better=True,"
-        " interpretation=pixel_error_in_decibels"
-    )
-    print(f"online_video={online_path}")
-    print(f"offline_video={offline_path}")
     print(f"offline_metadata={offline_metadata_path}")
-
-    assert ssim_score >= SSIM_THRESHOLD, (
-        f"SSIM below threshold: got {ssim_score:.6f}, expected >= {SSIM_THRESHOLD:.6f}. "
-        f"online={online_path} offline={offline_path}"
-    )
-    assert psnr_score >= PSNR_THRESHOLD, (
-        f"PSNR below threshold: got {psnr_score:.6f}, expected >= {PSNR_THRESHOLD:.6f}. "
-        f"online={online_path} offline={offline_path}"
-    )

vllm_omni/diffusion/models/hunyuan_video/hunyuan_video_15_transformer.py

@@ -837,9 +837,10 @@ class HunyuanVideo15Transformer3DModel(nn.Module):
             for param_name, weight_name, shard_id in stacked_params_mapping:
                 if weight_name not in original_name:
                     continue
-                lookup_name = original_name.replace(weight_name, param_name)
-                if lookup_name not in params_dict:
-                    break
+                maybe_lookup_name = original_name.replace(weight_name, param_name)
+                if maybe_lookup_name not in params_dict:
+                    continue
+                lookup_name = maybe_lookup_name
                 param = params_dict[lookup_name]
                 weight_loader = param.weight_loader
                 weight_loader(param, loaded_weight, shard_id)