From 6031c7d393db87a17fdad96982eb5ff1d2af4ab5 Mon Sep 17 00:00:00 2001 From: WeiQing Chen <40507679+david6666666@users.noreply.github.com> Date: Thu, 28 May 2026 11:09:45 +0800 Subject: [PATCH] [CI] Add HunyuanVideo 1.5 X2V accuracy tests (#3852) Signed-off-by: david6666666 <530634352@qq.com> --- .buildkite/test-nightly.yml | 2 + tests/e2e/accuracy/conftest.py | 24 ++ tests/e2e/accuracy/helpers.py | 278 +++++++++++++++++ .../accuracy/hunyuanvideo15_i2v/__init__.py | 1 + ...est_hunyuanvideo15_i2v_video_similarity.py | 262 ++++++++++++++++ .../accuracy/hunyuanvideo15_t2v/__init__.py | 1 + ...est_hunyuanvideo15_t2v_video_similarity.py | 211 +++++++++++++ .../test_diffusers_backend_similarity.py | 12 +- .../test_wan22_i2v_video_similarity.py | 287 +++--------------- .../hunyuan_video_15_transformer.py | 7 +- 10 files changed, 834 insertions(+), 251 deletions(-) create mode 100644 tests/e2e/accuracy/hunyuanvideo15_i2v/__init__.py create mode 100644 tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py create mode 100644 tests/e2e/accuracy/hunyuanvideo15_t2v/__init__.py create mode 100644 tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py diff --git a/.buildkite/test-nightly.yml b/.buildkite/test-nightly.yml index 3b3d986aa..45df0afa4 100644 --- a/.buildkite/test-nightly.yml +++ b/.buildkite/test-nightly.yml @@ -1091,6 +1091,8 @@ steps: timeout_in_minutes: 180 commands: - pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py -m full_model --run-level full_model + - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py -m full_model --run-level full_model + - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m full_model --run-level full_model - pytest -s -v tests/e2e/accuracy/test_ltx2_3_video_similarity.py -m full_model --run-level full_model - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m full_model --run-level full_model agents: diff --git a/tests/e2e/accuracy/conftest.py b/tests/e2e/accuracy/conftest.py index 431a8cc9b..6c0258aa2 100644 --- a/tests/e2e/accuracy/conftest.py +++ b/tests/e2e/accuracy/conftest.py @@ -62,6 +62,19 @@ def pytest_addoption(parser): default=1200, help="Online serving timeout in seconds for Wan2.2 I2V accuracy tests.", ) + group.addoption( + "--hunyuanvideo15-i2v-image-source", + action="store", + default=None, + help="Image source for HunyuanVideo-1.5 I2V accuracy tests. Can be local path or remote URL", + ) + group.addoption( + "--hunyuanvideo15-online-timeout-seconds", + action="store", + type=int, + default=3600, + help="Online serving timeout in seconds for HunyuanVideo-1.5 accuracy tests.", + ) def _hf_cache_root() -> Path: @@ -168,6 +181,17 @@ def wan22_i2v_online_timeout_seconds(request: pytest.FixtureRequest) -> int: return int(request.config.getoption("wan22_i2v_online_timeout_seconds")) +@pytest.fixture(scope="session") +def hunyuanvideo15_i2v_image_source(request: pytest.FixtureRequest) -> str | None: + value = request.config.getoption("hunyuanvideo15_i2v_image_source") + return str(value) if value else None + + +@pytest.fixture(scope="session") +def hunyuanvideo15_online_timeout_seconds(request: pytest.FixtureRequest) -> int: + return int(request.config.getoption("hunyuanvideo15_online_timeout_seconds")) + + @pytest.fixture(scope="session") def gebench_samples_per_type(request: pytest.FixtureRequest) -> int: return int(request.config.getoption("gebench_samples_per_type")) diff --git a/tests/e2e/accuracy/helpers.py b/tests/e2e/accuracy/helpers.py index 38b045dce..593a9960a 100644 --- a/tests/e2e/accuracy/helpers.py +++ b/tests/e2e/accuracy/helpers.py @@ -1,7 +1,17 @@ +import json import os +import re +import shutil +import subprocess +import time +from base64 import b64decode, b64encode +from fractions import Fraction +from hashlib import sha1 from io import BytesIO +from mimetypes import guess_type from pathlib import Path from typing import Any +from urllib.parse import urlparse import numpy as np import pytest @@ -63,6 +73,274 @@ def model_output_dir(parent_dir: Path, model: str) -> Path: return path +_SSIM_RE = re.compile(r"All:(?P[0-9.]+)") +_PSNR_RE = re.compile(r"average:(?P[0-9.]+)") + + +def parse_video_metadata(payload: dict[str, Any]) -> dict[str, int | float]: + streams = payload.get("streams") + if not isinstance(streams, list) or not streams: + raise ValueError(f"ffprobe payload did not include video streams: {payload}") + + stream = streams[0] + width = int(stream["width"]) + height = int(stream["height"]) + fps = float(Fraction(stream["avg_frame_rate"])) + frame_count_value = stream.get("nb_read_frames") or stream.get("nb_frames") + if frame_count_value is None: + raise ValueError(f"ffprobe payload did not include frame count: {payload}") + frame_count = int(frame_count_value) + return { + "width": width, + "height": height, + "fps": fps, + "frame_count": frame_count, + } + + +def parse_ssim_score(output: str) -> float: + match = _SSIM_RE.search(output) + if match is None: + raise ValueError(f"Could not parse SSIM score from ffmpeg output:\n{output}") + return float(match.group("score")) + + +def parse_psnr_score(output: str) -> float: + match = _PSNR_RE.search(output) + if match is None: + raise ValueError(f"Could not parse PSNR score from ffmpeg output:\n{output}") + return float(match.group("score")) + + +def probe_binary(binary: str, *, test_name: str = "video similarity e2e test") -> str: + resolved = shutil.which(binary) + if resolved is None: + pytest.skip(f"{binary} is required for {test_name}.") + return resolved + + +def probe_video(path: Path) -> dict[str, int | float]: + probe_binary("ffprobe") + result = subprocess.run( + [ + "ffprobe", + "-v", + "error", + "-select_streams", + "v:0", + "-count_frames", + "-show_entries", + "stream=width,height,avg_frame_rate,nb_read_frames", + "-of", + "json", + str(path), + ], + capture_output=True, + text=True, + check=True, + ) + return parse_video_metadata(json.loads(result.stdout)) + + +def run_ffmpeg_similarity(filter_name: str, first: Path, second: Path) -> str: + probe_binary("ffmpeg") + result = subprocess.run( + [ + "ffmpeg", + "-hide_banner", + "-i", + str(first), + "-i", + str(second), + "-lavfi", + f"[0:v][1:v]{filter_name}", + "-f", + "null", + "-", + ], + capture_output=True, + text=True, + check=True, + ) + return result.stderr + + +def online_timeout_seconds(configured: int | None, *, default: int = 1200) -> int: + return int(configured) if configured is not None else default + + +def is_remote_image_source(source: str) -> bool: + return source.startswith("http://") or source.startswith("https://") + + +def validate_image_source(source: str) -> None: + if is_remote_image_source(source): + response = requests.get(source, timeout=60) + response.raise_for_status() + return + + image_path = Path(source) + if not image_path.exists(): + raise FileNotFoundError(f"Local image source does not exist: {image_path}") + + +def build_online_image_reference(source: str) -> str: + if source.startswith("data:image"): + return source + + if is_remote_image_source(source): + response = requests.get(source, timeout=60) + response.raise_for_status() + mime_type = response.headers.get("Content-Type", "").split(";", 1)[0].strip() + if not mime_type.startswith("image/"): + mime_type = guess_type(urlparse(source).path)[0] or "image/png" + encoded = b64encode(response.content).decode("ascii") + return f"data:{mime_type};base64,{encoded}" + + image_path = Path(source) + mime_type = guess_type(image_path.name)[0] or "image/png" + encoded = b64encode(image_path.read_bytes()).decode("ascii") + return f"data:{mime_type};base64,{encoded}" + + +def materialize_image_source(source: str, output_dir: Path) -> str: + if not is_remote_image_source(source): + return source + + from diffusers.utils import load_image + + output_dir.mkdir(parents=True, exist_ok=True) + source_name = Path(urlparse(source).path).name or "input.png" + safe_name = re.sub(r"[^A-Za-z0-9_.-]+", "_", source_name) + if "." not in safe_name: + safe_name = f"{safe_name}.png" + image_path = output_dir / safe_name + if not image_path.exists(): + image = load_image(source).convert("RGB") + image.save(image_path) + return str(image_path) + + +def video_artifact_dir(result_root: Path, source: str) -> Path: + if is_remote_image_source(source): + source_name = Path(urlparse(source).path).stem or "remote" + else: + source_name = Path(source).stem or "local" + + safe_name = re.sub(r"[^A-Za-z0-9_.-]+", "_", source_name) + digest = sha1(source.encode("utf-8")).hexdigest()[:8] + return result_root / f"{safe_name}-{digest}" + + +def send_video_request_with_timeout( + openai_client, + request_config: dict[str, Any], + *, + timeout_seconds: int, +) -> bytes: + form_data = request_config.get("form_data") + if not isinstance(form_data, dict): + raise ValueError("Video request_config must contain 'form_data'") + + if not form_data.get("prompt"): + raise ValueError("Video request_config['form_data'] must contain 'prompt'") + + normalized_form_data = {key: str(value) for key, value in form_data.items() if value is not None} + + files: dict[str, tuple[str, BytesIO, str]] = {} + image_reference = request_config.get("image_reference") + if image_reference: + if image_reference.startswith("data:image"): + header, encoded = image_reference.split(",", 1) + content_type = header.split(";")[0].removeprefix("data:") + extension = content_type.split("/")[-1] + file_data = b64decode(encoded) + files["input_reference"] = ( + f"reference.{extension}", + BytesIO(file_data), + content_type, + ) + else: + normalized_form_data["image_reference"] = json.dumps({"image_url": image_reference}) + + start_time = time.perf_counter() + create_url = openai_client._build_url("/v1/videos") + response = requests.post( + create_url, + data=normalized_form_data, + files=files, + headers={"Accept": "application/json"}, + timeout=60, + ) + try: + response.raise_for_status() + except requests.HTTPError as exc: + raise requests.HTTPError(f"{exc}; response body: {response.text}", response=response) from exc + job_data = response.json() + video_id = job_data["id"] + openai_client._wait_until_video_completed(video_id, timeout_seconds=timeout_seconds) + video_content = openai_client._download_video_content(video_id) + print(f"online_video_e2e_latency_s={time.perf_counter() - start_time:.3f}") + return video_content + + +def assert_video_metadata( + metadata: dict[str, int | float], + *, + width: int, + height: int, + fps: int | float, + frame_count: int, +) -> None: + assert metadata["width"] == width + assert metadata["height"] == height + assert metadata["fps"] == float(fps) + assert metadata["frame_count"] == frame_count + + +def assert_video_similarity_metrics( + *, + label: str, + online_path: Path, + offline_path: Path, + ssim_threshold: float, + psnr_threshold: float, +) -> None: + ssim_output = run_ffmpeg_similarity("ssim", online_path, offline_path) + psnr_output = run_ffmpeg_similarity("psnr", online_path, offline_path) + ssim_score = parse_ssim_score(ssim_output) + psnr_score = parse_psnr_score(psnr_output) + + print(f"{label} similarity metrics:") + print( + " SSIM:" + f" value={ssim_score:.6f}," + f" threshold>={ssim_threshold:.6f}," + " range=[0, 1]," + " higher_is_better=True," + " interpretation=structural_similarity" + ) + print( + " PSNR:" + f" value={psnr_score:.6f} dB," + f" threshold>={psnr_threshold:.6f} dB," + " range=[0, +inf)," + " higher_is_better=True," + " interpretation=pixel_error_in_decibels" + ) + print(f"online_video={online_path}") + print(f"offline_video={offline_path}") + + assert ssim_score >= ssim_threshold, ( + f"SSIM below threshold: got {ssim_score:.6f}, expected >= {ssim_threshold:.6f}. " + f"online={online_path} offline={offline_path}" + ) + assert psnr_score >= psnr_threshold, ( + f"PSNR below threshold: got {psnr_score:.6f}, expected >= {psnr_threshold:.6f}. " + f"online={online_path} offline={offline_path}" + ) + + def assert_similarity( *, model_name: str, diff --git a/tests/e2e/accuracy/hunyuanvideo15_i2v/__init__.py b/tests/e2e/accuracy/hunyuanvideo15_i2v/__init__.py new file mode 100644 index 000000000..08dfa08ca --- /dev/null +++ b/tests/e2e/accuracy/hunyuanvideo15_i2v/__init__.py @@ -0,0 +1 @@ +"""HunyuanVideo-1.5 I2V accuracy test cases.""" diff --git a/tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py b/tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py new file mode 100644 index 000000000..84e032770 --- /dev/null +++ b/tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py @@ -0,0 +1,262 @@ +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +import pytest +import torch + +from tests.e2e.accuracy.helpers import ( + assert_video_metadata, + assert_video_similarity_metrics, + build_online_image_reference, + materialize_image_source, + probe_binary, + probe_video, + send_video_request_with_timeout, + validate_image_source, + video_artifact_dir, +) +from tests.helpers.mark import hardware_test +from tests.helpers.runtime import OmniServerParams + +pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] + +MODEL_NAME = "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v" +IMAGE_URL = "https://huggingface.co/datasets/YiYiXu/testing-images/resolve/main/wan_i2v_input.JPG" +PROMPT = ( + "Summer beach vacation style, a white cat wearing sunglasses sits on a surfboard. " + "The fluffy-furred feline gazes directly at the camera with a relaxed expression. " + "Blurred beach scenery forms the background featuring crystal-clear waters, distant green hills, " + "and a blue sky dotted with white clouds. The cat assumes a naturally relaxed posture, " + "as if savoring the sea breeze and warm sunlight. A close-up shot highlights the feline's " + "intricate details and the refreshing atmosphere of the seaside." +) +WIDTH = 832 +HEIGHT = 480 +FPS = 24 +NUM_FRAMES = 33 +NUM_INFERENCE_STEPS = 50 +GUIDANCE_SCALE = 6.0 +FLOW_SHIFT = 5.0 +SEED = 42 +SSIM_THRESHOLD = 0.94 +PSNR_THRESHOLD = 28.0 + +REPO_ROOT = Path(__file__).resolve().parents[4] +WORKSPACE_ROOT = REPO_ROOT.parent +RUNNER_PATH = REPO_ROOT / "examples" / "offline_inference" / "image_to_video" / "image_to_video.py" +RESULT_ROOT = Path(__file__).parent / "result" +VIDEO_TIMEOUT_SECONDS = 60 * 60 +SERVER_CASES = [ + pytest.param( + OmniServerParams( + model=MODEL_NAME, + server_args=[ + "--flow-shift", + str(FLOW_SHIFT), + "--enforce-eager", + ], + env_dict={"VLLM_OMNI_STORAGE_PATH": str(RESULT_ROOT / "storage")}, + use_omni=True, + ), + id="hunyuanvideo15_i2v_default", + ) +] + + +def _runner_env() -> dict[str, str]: + env = os.environ.copy() + pythonpath_parts = [ + str(REPO_ROOT), + str(WORKSPACE_ROOT / "diffusers" / "src"), + ] + existing_pythonpath = env.get("PYTHONPATH") + if existing_pythonpath: + pythonpath_parts.append(existing_pythonpath) + env["PYTHONPATH"] = os.pathsep.join(pythonpath_parts) + env.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn") + return env + + +def _resolve_image_source(configured: str | None) -> str: + if configured: + candidate = Path(configured) + if candidate.exists(): + return str(candidate.resolve()) + return configured + return IMAGE_URL + + +def _artifact_paths(image_source: str) -> tuple[Path, Path]: + artifact_dir = video_artifact_dir(RESULT_ROOT, image_source) + artifact_dir.mkdir(parents=True, exist_ok=True) + return artifact_dir / "online.mp4", artifact_dir / "offline.mp4" + + +def _offline_image_source(image_source: str) -> str: + artifact_dir = video_artifact_dir(RESULT_ROOT, image_source) + return materialize_image_source(image_source, artifact_dir) + + +def _build_offline_command(*, image_source: str, output_path: Path) -> list[str]: + return [ + sys.executable, + str(RUNNER_PATH), + "--model", + MODEL_NAME, + "--image", + image_source, + "--prompt", + PROMPT, + "--height", + str(HEIGHT), + "--width", + str(WIDTH), + "--num-frames", + str(NUM_FRAMES), + "--num-inference-steps", + str(NUM_INFERENCE_STEPS), + "--guidance-scale", + str(GUIDANCE_SCALE), + "--flow-shift", + str(FLOW_SHIFT), + "--fps", + str(FPS), + "--seed", + str(SEED), + "--enforce-eager", + "--output", + str(output_path), + ] + + +def _generate_offline_video(*, image_source: str) -> Path: + _, offline_path = _artifact_paths(image_source) + offline_image_source = _offline_image_source(image_source) + subprocess.run( + _build_offline_command(image_source=offline_image_source, output_path=offline_path), + cwd=REPO_ROOT, + env=_runner_env(), + check=True, + timeout=VIDEO_TIMEOUT_SECONDS, + ) + return offline_path + + +def _generate_online_video( + *, + omni_server, + openai_client, + image_source: str, + timeout_seconds: int, +) -> Path: + online_path, _ = _artifact_paths(image_source) + online_image_source = _offline_image_source(image_source) + request_config = { + "model": omni_server.model, + "form_data": { + "prompt": PROMPT, + "height": HEIGHT, + "width": WIDTH, + "fps": FPS, + "num_frames": NUM_FRAMES, + "guidance_scale": GUIDANCE_SCALE, + "flow_shift": FLOW_SHIFT, + "num_inference_steps": NUM_INFERENCE_STEPS, + "seed": SEED, + }, + "image_reference": build_online_image_reference(online_image_source), + } + online_video_bytes = send_video_request_with_timeout( + openai_client, + request_config, + timeout_seconds=timeout_seconds, + ) + online_path.write_bytes(online_video_bytes) + return online_path + + +@pytest.mark.benchmark +@hardware_test(res={"cuda": "H100"}, num_cards=1) +def test_hunyuanvideo15_i2v_diffusers_offline_generates_video( + hunyuanvideo15_i2v_image_source: str | None, +) -> None: + if not torch.cuda.is_available(): + pytest.skip("HunyuanVideo-1.5 I2V offline accuracy test requires CUDA.") + + probe_binary("ffprobe") + if not RUNNER_PATH.exists(): + raise AssertionError(f"Offline runner does not exist: {RUNNER_PATH}") + + image_source = _resolve_image_source(hunyuanvideo15_i2v_image_source) + validate_image_source(image_source) + offline_path = _generate_offline_video(image_source=image_source) + assert offline_path.exists(), f"Expected offline video artifact at {offline_path}" + offline_metadata = probe_video(offline_path) + assert_video_metadata(offline_metadata, width=WIDTH, height=HEIGHT, fps=FPS, frame_count=NUM_FRAMES) + + +@pytest.mark.benchmark +@hardware_test(res={"cuda": "H100"}, num_cards=1) +@pytest.mark.parametrize("omni_server", SERVER_CASES, indirect=True) +def test_hunyuanvideo15_i2v_online_serving_generates_video( + omni_server, + openai_client, + hunyuanvideo15_i2v_image_source: str | None, + hunyuanvideo15_online_timeout_seconds: int, +) -> None: + if not torch.cuda.is_available(): + pytest.skip("HunyuanVideo-1.5 I2V online accuracy test requires CUDA.") + + probe_binary("ffprobe") + image_source = _resolve_image_source(hunyuanvideo15_i2v_image_source) + validate_image_source(image_source) + online_path = _generate_online_video( + omni_server=omni_server, + openai_client=openai_client, + image_source=image_source, + timeout_seconds=hunyuanvideo15_online_timeout_seconds, + ) + assert online_path.exists(), f"Expected online video artifact at {online_path}" + online_metadata = probe_video(online_path) + assert_video_metadata(online_metadata, width=WIDTH, height=HEIGHT, fps=FPS, frame_count=NUM_FRAMES) + + +@pytest.mark.benchmark +@hardware_test(res={"cuda": "H100"}, num_cards=1) +def test_hunyuanvideo15_i2v_serving_matches_offline_video_similarity( + hunyuanvideo15_i2v_image_source: str | None, +) -> None: + if not torch.cuda.is_available(): + pytest.skip("HunyuanVideo-1.5 I2V video similarity test requires CUDA.") + + probe_binary("ffmpeg") + probe_binary("ffprobe") + image_source = _resolve_image_source(hunyuanvideo15_i2v_image_source) + validate_image_source(image_source) + online_path, offline_path = _artifact_paths(image_source) + if not online_path.exists(): + pytest.skip(f"Missing online artifact from prerequisite test: {online_path}") + if not offline_path.exists(): + pytest.skip(f"Missing offline artifact from prerequisite test: {offline_path}") + + online_metadata = probe_video(online_path) + offline_metadata = probe_video(offline_path) + assert online_metadata == offline_metadata, ( + f"Video metadata mismatch:\n" + f"online={online_metadata}\n" + f"offline={offline_metadata}\n" + f"online_path={online_path}\n" + f"offline_path={offline_path}" + ) + assert_video_metadata(online_metadata, width=WIDTH, height=HEIGHT, fps=FPS, frame_count=NUM_FRAMES) + assert_video_similarity_metrics( + label="hunyuanvideo15_i2v", + online_path=online_path, + offline_path=offline_path, + ssim_threshold=SSIM_THRESHOLD, + psnr_threshold=PSNR_THRESHOLD, + ) diff --git a/tests/e2e/accuracy/hunyuanvideo15_t2v/__init__.py b/tests/e2e/accuracy/hunyuanvideo15_t2v/__init__.py new file mode 100644 index 000000000..bb679db4c --- /dev/null +++ b/tests/e2e/accuracy/hunyuanvideo15_t2v/__init__.py @@ -0,0 +1 @@ +"""HunyuanVideo-1.5 T2V accuracy test cases.""" diff --git a/tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py b/tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py new file mode 100644 index 000000000..f2c14e2b0 --- /dev/null +++ b/tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py @@ -0,0 +1,211 @@ +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +import pytest +import torch + +from tests.e2e.accuracy.helpers import ( + assert_video_metadata, + assert_video_similarity_metrics, + probe_binary, + probe_video, + send_video_request_with_timeout, +) +from tests.helpers.mark import hardware_test +from tests.helpers.runtime import OmniServerParams + +pytestmark = [pytest.mark.diffusion, pytest.mark.full_model] + +MODEL_NAME = "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v" +PROMPT = "A cat walks on the grass, realistic style." +WIDTH = 832 +HEIGHT = 480 +FPS = 24 +NUM_FRAMES = 33 +NUM_INFERENCE_STEPS = 50 +GUIDANCE_SCALE = 6.0 +FLOW_SHIFT = 5.0 +SEED = 42 +SSIM_THRESHOLD = 0.94 +PSNR_THRESHOLD = 28.0 + +REPO_ROOT = Path(__file__).resolve().parents[4] +WORKSPACE_ROOT = REPO_ROOT.parent +RUNNER_PATH = REPO_ROOT / "examples" / "offline_inference" / "text_to_video" / "text_to_video.py" +RESULT_ROOT = Path(__file__).parent / "result" +ARTIFACT_DIR = RESULT_ROOT / "cat_grass" +VIDEO_TIMEOUT_SECONDS = 60 * 60 +SERVER_CASES = [ + pytest.param( + OmniServerParams( + model=MODEL_NAME, + server_args=[ + "--flow-shift", + str(FLOW_SHIFT), + ], + env_dict={"VLLM_OMNI_STORAGE_PATH": str(RESULT_ROOT / "storage")}, + use_omni=True, + ), + id="hunyuanvideo15_t2v_default", + ) +] + + +def _runner_env() -> dict[str, str]: + env = os.environ.copy() + pythonpath_parts = [ + str(REPO_ROOT), + str(WORKSPACE_ROOT / "diffusers" / "src"), + ] + existing_pythonpath = env.get("PYTHONPATH") + if existing_pythonpath: + pythonpath_parts.append(existing_pythonpath) + env["PYTHONPATH"] = os.pathsep.join(pythonpath_parts) + env.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn") + return env + + +def _artifact_paths() -> tuple[Path, Path]: + ARTIFACT_DIR.mkdir(parents=True, exist_ok=True) + return ARTIFACT_DIR / "online.mp4", ARTIFACT_DIR / "offline.mp4" + + +def _build_offline_command(*, output_path: Path) -> list[str]: + return [ + sys.executable, + str(RUNNER_PATH), + "--model", + MODEL_NAME, + "--prompt", + PROMPT, + "--height", + str(HEIGHT), + "--width", + str(WIDTH), + "--num-frames", + str(NUM_FRAMES), + "--num-inference-steps", + str(NUM_INFERENCE_STEPS), + "--guidance-scale", + str(GUIDANCE_SCALE), + "--flow-shift", + str(FLOW_SHIFT), + "--fps", + str(FPS), + "--seed", + str(SEED), + "--output", + str(output_path), + ] + + +def _generate_offline_video() -> Path: + _, offline_path = _artifact_paths() + subprocess.run( + _build_offline_command(output_path=offline_path), + cwd=REPO_ROOT, + env=_runner_env(), + check=True, + timeout=VIDEO_TIMEOUT_SECONDS, + ) + return offline_path + + +def _generate_online_video(*, omni_server, openai_client, timeout_seconds: int) -> Path: + online_path, _ = _artifact_paths() + request_config = { + "model": omni_server.model, + "form_data": { + "prompt": PROMPT, + "height": HEIGHT, + "width": WIDTH, + "fps": FPS, + "num_frames": NUM_FRAMES, + "guidance_scale": GUIDANCE_SCALE, + "flow_shift": FLOW_SHIFT, + "num_inference_steps": NUM_INFERENCE_STEPS, + "seed": SEED, + }, + } + online_video_bytes = send_video_request_with_timeout( + openai_client, + request_config, + timeout_seconds=timeout_seconds, + ) + online_path.write_bytes(online_video_bytes) + return online_path + + +@pytest.mark.benchmark +@hardware_test(res={"cuda": "H100"}, num_cards=1) +def test_hunyuanvideo15_t2v_diffusers_offline_generates_video() -> None: + if not torch.cuda.is_available(): + pytest.skip("HunyuanVideo-1.5 T2V offline accuracy test requires CUDA.") + + probe_binary("ffprobe") + if not RUNNER_PATH.exists(): + raise AssertionError(f"Offline runner does not exist: {RUNNER_PATH}") + + offline_path = _generate_offline_video() + assert offline_path.exists(), f"Expected offline video artifact at {offline_path}" + offline_metadata = probe_video(offline_path) + assert_video_metadata(offline_metadata, width=WIDTH, height=HEIGHT, fps=FPS, frame_count=NUM_FRAMES) + + +@pytest.mark.benchmark +@hardware_test(res={"cuda": "H100"}, num_cards=1) +@pytest.mark.parametrize("omni_server", SERVER_CASES, indirect=True) +def test_hunyuanvideo15_t2v_online_serving_generates_video( + omni_server, + openai_client, + hunyuanvideo15_online_timeout_seconds: int, +) -> None: + if not torch.cuda.is_available(): + pytest.skip("HunyuanVideo-1.5 T2V online accuracy test requires CUDA.") + + probe_binary("ffprobe") + online_path = _generate_online_video( + omni_server=omni_server, + openai_client=openai_client, + timeout_seconds=hunyuanvideo15_online_timeout_seconds, + ) + assert online_path.exists(), f"Expected online video artifact at {online_path}" + online_metadata = probe_video(online_path) + assert_video_metadata(online_metadata, width=WIDTH, height=HEIGHT, fps=FPS, frame_count=NUM_FRAMES) + + +@pytest.mark.benchmark +@hardware_test(res={"cuda": "H100"}, num_cards=1) +def test_hunyuanvideo15_t2v_serving_matches_offline_video_similarity() -> None: + if not torch.cuda.is_available(): + pytest.skip("HunyuanVideo-1.5 T2V video similarity test requires CUDA.") + + probe_binary("ffmpeg") + probe_binary("ffprobe") + online_path, offline_path = _artifact_paths() + if not online_path.exists(): + pytest.skip(f"Missing online artifact from prerequisite test: {online_path}") + if not offline_path.exists(): + pytest.skip(f"Missing offline artifact from prerequisite test: {offline_path}") + + online_metadata = probe_video(online_path) + offline_metadata = probe_video(offline_path) + assert online_metadata == offline_metadata, ( + f"Video metadata mismatch:\n" + f"online={online_metadata}\n" + f"offline={offline_metadata}\n" + f"online_path={online_path}\n" + f"offline_path={offline_path}" + ) + assert_video_metadata(online_metadata, width=WIDTH, height=HEIGHT, fps=FPS, frame_count=NUM_FRAMES) + assert_video_similarity_metrics( + label="hunyuanvideo15_t2v", + online_path=online_path, + offline_path=offline_path, + ssim_threshold=SSIM_THRESHOLD, + psnr_threshold=PSNR_THRESHOLD, + ) diff --git a/tests/e2e/accuracy/test_diffusers_backend_similarity.py b/tests/e2e/accuracy/test_diffusers_backend_similarity.py index da05d77ec..26b39ea78 100644 --- a/tests/e2e/accuracy/test_diffusers_backend_similarity.py +++ b/tests/e2e/accuracy/test_diffusers_backend_similarity.py @@ -30,10 +30,14 @@ from tests.e2e.accuracy.helpers import ( env_to_apply_ftfy_mock_in_subproc, model_output_dir, ) -from tests.e2e.accuracy.wan22_i2v.test_wan22_i2v_video_similarity import ( - _parse_psnr_score, - _parse_ssim_score, - _run_ffmpeg_similarity, +from tests.e2e.accuracy.helpers import ( + parse_psnr_score as _parse_psnr_score, +) +from tests.e2e.accuracy.helpers import ( + parse_ssim_score as _parse_ssim_score, +) +from tests.e2e.accuracy.helpers import ( + run_ffmpeg_similarity as _run_ffmpeg_similarity, ) from tests.helpers.env import run_post_test_cleanup, run_pre_test_cleanup from tests.helpers.mark import hardware_test diff --git a/tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py b/tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py index 5300f1c6e..a09d2330a 100644 --- a/tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py +++ b/tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py @@ -1,20 +1,10 @@ from __future__ import annotations -import json import os -import re -import shutil import subprocess import sys -import time -from base64 import b64decode, b64encode -from fractions import Fraction -from hashlib import sha1 -from io import BytesIO -from mimetypes import guess_type +from base64 import b64decode from pathlib import Path -from typing import Any -from urllib.parse import urlparse import pytest import requests @@ -22,6 +12,38 @@ import torch from diffusers import UniPCMultistepScheduler from PIL import Image +from tests.e2e.accuracy.helpers import ( + assert_video_metadata, + assert_video_similarity_metrics, + video_artifact_dir, +) +from tests.e2e.accuracy.helpers import ( + build_online_image_reference as _build_online_image_reference, +) +from tests.e2e.accuracy.helpers import ( + online_timeout_seconds as _online_timeout_seconds, +) +from tests.e2e.accuracy.helpers import ( + parse_psnr_score as _parse_psnr_score, +) +from tests.e2e.accuracy.helpers import ( + parse_ssim_score as _parse_ssim_score, +) +from tests.e2e.accuracy.helpers import ( + parse_video_metadata as _parse_video_metadata, +) +from tests.e2e.accuracy.helpers import ( + probe_binary as _probe_binary, +) +from tests.e2e.accuracy.helpers import ( + probe_video as _probe_video, +) +from tests.e2e.accuracy.helpers import ( + send_video_request_with_timeout as _send_video_request_with_timeout, +) +from tests.e2e.accuracy.helpers import ( + validate_image_source as _validate_image_source, +) from tests.e2e.accuracy.wan22_i2v.run_wan22_i2v_diffusers_cp import ( _configure_scheduler, _ensure_wan_ftfy_fallback, @@ -218,8 +240,6 @@ WORKSPACE_ROOT = REPO_ROOT.parent RUNNER_PATH = Path(__file__).with_name("run_wan22_i2v_diffusers_cp.py") RESULT_ROOT = Path(__file__).parent / "result" VIDEO_TIMEOUT_SECONDS = 60 * 60 -_SSIM_RE = re.compile(r"All:(?P[0-9.]+)") -_PSNR_RE = re.compile(r"average:(?P[0-9.]+)") SERVER_CASES = [ pytest.param( OmniServerParams( @@ -238,48 +258,6 @@ SERVER_CASES = [ ] -def _parse_video_metadata(payload: dict[str, Any]) -> dict[str, int | float]: - streams = payload.get("streams") - if not isinstance(streams, list) or not streams: - raise ValueError(f"ffprobe payload did not include video streams: {payload}") - - stream = streams[0] - width = int(stream["width"]) - height = int(stream["height"]) - fps = float(Fraction(stream["avg_frame_rate"])) - frame_count_value = stream.get("nb_read_frames") or stream.get("nb_frames") - if frame_count_value is None: - raise ValueError(f"ffprobe payload did not include frame count: {payload}") - frame_count = int(frame_count_value) - return { - "width": width, - "height": height, - "fps": fps, - "frame_count": frame_count, - } - - -def _parse_ssim_score(output: str) -> float: - match = _SSIM_RE.search(output) - if match is None: - raise ValueError(f"Could not parse SSIM score from ffmpeg output:\n{output}") - return float(match.group("score")) - - -def _parse_psnr_score(output: str) -> float: - match = _PSNR_RE.search(output) - if match is None: - raise ValueError(f"Could not parse PSNR score from ffmpeg output:\n{output}") - return float(match.group("score")) - - -def _probe_binary(binary: str) -> str: - resolved = shutil.which(binary) - if resolved is None: - pytest.skip(f"{binary} is required for Wan2.2 video similarity e2e test.") - return resolved - - def _build_diffusers_command( *, runner_path: Path, @@ -343,99 +321,8 @@ def _resolve_image_source(configured: str | None) -> str: return configured -def _is_remote_image_source(source: str) -> bool: - return source.startswith("http://") or source.startswith("https://") - - -def _validate_image_source(source: str) -> None: - if _is_remote_image_source(source): - response = requests.get(source, timeout=60) - response.raise_for_status() - return - - image_path = Path(source) - if not image_path.exists(): - raise FileNotFoundError(f"Local image source does not exist: {image_path}") - - -def _build_online_image_reference(source: str) -> str: - if source.startswith("data:image"): - return source - - if _is_remote_image_source(source): - response = requests.get(source, timeout=60) - response.raise_for_status() - mime_type = response.headers.get("Content-Type", "").split(";", 1)[0].strip() - if not mime_type.startswith("image/"): - mime_type = guess_type(urlparse(source).path)[0] or "image/png" - encoded = b64encode(response.content).decode("ascii") - return f"data:{mime_type};base64,{encoded}" - - image_path = Path(source) - mime_type = guess_type(image_path.name)[0] or "image/png" - encoded = b64encode(image_path.read_bytes()).decode("ascii") - return f"data:{mime_type};base64,{encoded}" - - -def _probe_video(path: Path) -> dict[str, int | float]: - _probe_binary("ffprobe") - result = subprocess.run( - [ - "ffprobe", - "-v", - "error", - "-select_streams", - "v:0", - "-count_frames", - "-show_entries", - "stream=width,height,avg_frame_rate,nb_read_frames", - "-of", - "json", - str(path), - ], - capture_output=True, - text=True, - check=True, - ) - return _parse_video_metadata(json.loads(result.stdout)) - - -def _run_ffmpeg_similarity(filter_name: str, first: Path, second: Path) -> str: - _probe_binary("ffmpeg") - result = subprocess.run( - [ - "ffmpeg", - "-hide_banner", - "-i", - str(first), - "-i", - str(second), - "-lavfi", - f"[0:v][1:v]{filter_name}", - "-f", - "null", - "-", - ], - capture_output=True, - text=True, - check=True, - ) - return result.stderr - - -def _online_timeout_seconds(configured: int | None) -> int: - return int(configured) if configured is not None else 1200 - - def _artifact_dir(image_source: str) -> Path: - if _is_remote_image_source(image_source): - source_name = Path(urlparse(image_source).path).stem or "remote" - else: - source_name = Path(image_source).stem or "local" - - safe_name = re.sub(r"[^A-Za-z0-9_.-]+", "_", source_name) - digest = sha1(image_source.encode("utf-8")).hexdigest()[:8] - return RESULT_ROOT / f"{safe_name}-{digest}" + return video_artifact_dir(RESULT_ROOT, image_source) def _artifact_paths(image_source: str) -> tuple[Path, Path, Path]: @@ -448,58 +335,6 @@ def _artifact_paths(image_source: str) -> tuple[Path, Path, Path]: ) -def _send_video_request_with_timeout( - openai_client, - request_config: dict[str, Any], - *, - timeout_seconds: int, -) -> bytes: - form_data = request_config.get("form_data") - if not isinstance(form_data, dict): - raise ValueError("Video request_config must contain 'form_data'") - - if not form_data.get("prompt"): - raise ValueError("Video request_config['form_data'] must contain 'prompt'") - - normalized_form_data = {key: str(value) for key, value in form_data.items() if value is not None} - - files: dict[str, tuple[str, BytesIO, str]] = {} - image_reference = request_config.get("image_reference") - if image_reference: - if image_reference.startswith("data:image"): - header, encoded = image_reference.split(",", 1) - content_type = header.split(";")[0].removeprefix("data:") - extension = content_type.split("/")[-1] - file_data = b64decode(encoded) - files["input_reference"] = ( - f"reference.{extension}", - BytesIO(file_data), - content_type, - ) - else: - normalized_form_data["image_reference"] = json.dumps({"image_url": image_reference}) - - start_time = time.perf_counter() - create_url = openai_client._build_url("/v1/videos") - response = requests.post( - create_url, - data=normalized_form_data, - files=files, - headers={"Accept": "application/json"}, - timeout=60, - ) - try: - response.raise_for_status() - except requests.HTTPError as exc: - raise requests.HTTPError(f"{exc}; response body: {response.text}", response=response) from exc - job_data = response.json() - video_id = job_data["id"] - openai_client._wait_until_video_completed(video_id, timeout_seconds=timeout_seconds) - video_content = openai_client._download_video_content(video_id) - print(f"online_video_e2e_latency_s={time.perf_counter() - start_time:.3f}") - return video_content - - def _generate_online_video( *, omni_server, @@ -569,10 +404,7 @@ def test_wan22_i2v_diffusers_offline_generates_video( assert offline_path.exists(), f"Expected offline video artifact at {offline_path}" assert offline_metadata_path.exists(), f"Expected offline metadata artifact at {offline_metadata_path}" offline_metadata = _probe_video(offline_path) - assert offline_metadata["width"] == WIDTH - assert offline_metadata["height"] == HEIGHT - assert offline_metadata["fps"] == float(FPS) - assert offline_metadata["frame_count"] == NUM_FRAMES + assert_video_metadata(offline_metadata, width=WIDTH, height=HEIGHT, fps=FPS, frame_count=NUM_FRAMES) @pytest.mark.benchmark @@ -598,10 +430,7 @@ def test_wan22_i2v_online_serving_generates_video( ) assert online_path.exists(), f"Expected online video artifact at {online_path}" online_metadata = _probe_video(online_path) - assert online_metadata["width"] == WIDTH - assert online_metadata["height"] == HEIGHT - assert online_metadata["fps"] == float(FPS) - assert online_metadata["frame_count"] == NUM_FRAMES + assert_video_metadata(online_metadata, width=WIDTH, height=HEIGHT, fps=FPS, frame_count=NUM_FRAMES) @pytest.mark.benchmark @@ -639,42 +468,12 @@ def test_wan22_i2v_serving_matches_diffusers_video_similarity( f"online_path={online_path}\n" f"offline_path={offline_path}" ) - assert online_metadata["width"] == WIDTH - assert online_metadata["height"] == HEIGHT - assert online_metadata["fps"] == float(FPS) - assert online_metadata["frame_count"] == NUM_FRAMES - - ssim_output = _run_ffmpeg_similarity("ssim", online_path, offline_path) - psnr_output = _run_ffmpeg_similarity("psnr", online_path, offline_path) - ssim_score = _parse_ssim_score(ssim_output) - psnr_score = _parse_psnr_score(psnr_output) - - print("wan22_i2v_similarity metrics:") - print( - " SSIM:" - f" value={ssim_score:.6f}," - f" threshold>={SSIM_THRESHOLD:.6f}," - " range=[0, 1]," - " higher_is_better=True," - " interpretation=structural_similarity" + assert_video_metadata(online_metadata, width=WIDTH, height=HEIGHT, fps=FPS, frame_count=NUM_FRAMES) + assert_video_similarity_metrics( + label="wan22_i2v", + online_path=online_path, + offline_path=offline_path, + ssim_threshold=SSIM_THRESHOLD, + psnr_threshold=PSNR_THRESHOLD, ) - print( - " PSNR:" - f" value={psnr_score:.6f} dB," - f" threshold>={PSNR_THRESHOLD:.6f} dB," - " range=[0, +inf)," - " higher_is_better=True," - " interpretation=pixel_error_in_decibels" - ) - print(f"online_video={online_path}") - print(f"offline_video={offline_path}") print(f"offline_metadata={offline_metadata_path}") - - assert ssim_score >= SSIM_THRESHOLD, ( - f"SSIM below threshold: got {ssim_score:.6f}, expected >= {SSIM_THRESHOLD:.6f}. " - f"online={online_path} offline={offline_path}" - ) - assert psnr_score >= PSNR_THRESHOLD, ( - f"PSNR below threshold: got {psnr_score:.6f}, expected >= {PSNR_THRESHOLD:.6f}. " - f"online={online_path} offline={offline_path}" - ) diff --git a/vllm_omni/diffusion/models/hunyuan_video/hunyuan_video_15_transformer.py b/vllm_omni/diffusion/models/hunyuan_video/hunyuan_video_15_transformer.py index 1ef618003..123652edd 100644 --- a/vllm_omni/diffusion/models/hunyuan_video/hunyuan_video_15_transformer.py +++ b/vllm_omni/diffusion/models/hunyuan_video/hunyuan_video_15_transformer.py @@ -837,9 +837,10 @@ class HunyuanVideo15Transformer3DModel(nn.Module): for param_name, weight_name, shard_id in stacked_params_mapping: if weight_name not in original_name: continue - lookup_name = original_name.replace(weight_name, param_name) - if lookup_name not in params_dict: - break + maybe_lookup_name = original_name.replace(weight_name, param_name) + if maybe_lookup_name not in params_dict: + continue + lookup_name = maybe_lookup_name param = params_dict[lookup_name] weight_loader = param.weight_loader weight_loader(param, loaded_weight, shard_id)