From c201fd77c63a9041f13e757d9c796f08abf15bb3 Mon Sep 17 00:00:00 2001 From: zzh <67720593+BLANKETusers@users.noreply.github.com> Date: Fri, 28 Aug 2026 14:31:03 +0800 Subject: [PATCH] Add HYvideo1.5 benchmark (#6349) Signed-off-by: zzh <943967662@qq.com> --- .buildkite/cuda/test-merge.yml | 12 ++ .buildkite/cuda/test-nightly.yml | 23 ++++ .buildkite/cuda/test-ready.yml | 12 ++ .../test_hunyuanvideo15_i2v_vllm_omni.json | 106 ++++++++++++++++++ .../test_hunyuanvideo15_t2v_vllm_omni.json | 104 +++++++++++++++++ .../test_hunyuan_video_15_expansion.py | 33 ++++-- 6 files changed, 282 insertions(+), 8 deletions(-) create mode 100644 tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json create mode 100644 tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json diff --git a/.buildkite/cuda/test-merge.yml b/.buildkite/cuda/test-merge.yml index 0aed5f492..aba70c286 100644 --- a/.buildkite/cuda/test-merge.yml +++ b/.buildkite/cuda/test-merge.yml @@ -216,3 +216,15 @@ steps: pytest -s -v tests/e2e/offline_inference/test_wan22_t2v.py tests/e2e/online_serving/test_wan22_t2v.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model' " mirror_hardwares: h100_1 + + - label: "Diffusion · HunyuanVideo-1.5 Test" + source_file_dependencies: + - tests/e2e/online_serving/test_hunyuan_video_15_expansion.py + - vllm_omni/diffusion/models/hunyuan_video/ + commands: + - | + timeout 40m bash -c " + export VLLM_IMAGE_FETCH_TIMEOUT=60 + pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model' + " + mirror_hardwares: h100_1 diff --git a/.buildkite/cuda/test-nightly.yml b/.buildkite/cuda/test-nightly.yml index 65f1dfa12..a3dad553b 100644 --- a/.buildkite/cuda/test-nightly.yml +++ b/.buildkite/cuda/test-nightly.yml @@ -272,6 +272,12 @@ steps: - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda" --run-level "full_model" mirror_hardwares: h100_2 + - label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Function Test" + timeout_in_minutes: 90 + commands: + - pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m "full_model and cuda" --run-level "full_model" + mirror_hardwares: h100_2 + - label: ":full_moon: Diffusion X2V · Doc Test" timeout_in_minutes: 60 commands: @@ -322,6 +328,23 @@ steps: exit $$EXIT2 mirror_hardwares: h100_2 + - label: ":full_moon: Diffusion X2V · Perf Test · HunyuanVideo-1.5" + key: nightly-diffusion-x2v-performance-hunyuanvideo15 + timeout_in_minutes: 180 + commands: + - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json + EXIT1=$$? + pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json + EXIT2=$$? + buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" + buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" + if [ $$EXIT1 -ne 0 ]; then exit $$EXIT1; fi + exit $$EXIT2 + mirror_hardwares: h100_2 + - label: ":full_moon: Diffusion X2V · Perf Test · MiniMax-H3" key: nightly-diffusion-x2v-performance-minimax-h3 timeout_in_minutes: 180 diff --git a/.buildkite/cuda/test-ready.yml b/.buildkite/cuda/test-ready.yml index 32c4bbec1..418d8f996 100644 --- a/.buildkite/cuda/test-ready.yml +++ b/.buildkite/cuda/test-ready.yml @@ -337,3 +337,15 @@ steps: pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model' --run-level 'core_model' " mirror_hardwares: h100_1 + + - label: "Diffusion · HunyuanVideo-1.5 Test" + source_file_dependencies: + - tests/e2e/online_serving/test_hunyuan_video_15_expansion.py + - vllm_omni/diffusion/models/hunyuan_video/ + commands: + - | + timeout 40m bash -c " + export VLLM_IMAGE_FETCH_TIMEOUT=60 + pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m 'core_model and diffusion' --run-level 'core_model' + " + mirror_hardwares: h100_1 diff --git a/tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json b/tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json new file mode 100644 index 000000000..8186edf2c --- /dev/null +++ b/tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json @@ -0,0 +1,106 @@ +[ + { + "test_name": "test_hunyuanvideo15_i2v_single_device", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], + "description": "Single-device baseline", + "server_type": "vllm-omni", + "server_params": { + "model": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v", + "serve_args": { + "flow-shift": 5.0, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "832x480_frames33_steps4", + "dataset": "random", + "task": "i2v", + "num-prompts": 10, + "max-concurrency": 1, + "num-input-images": 1, + "seed": 42, + "enable-negative-prompt": true, + "random-request-config": [ + { + "width": 832, + "height": 480, + "num_inference_steps": 4, + "num_frames": 33, + "fps": 24, + "weight": 1 + } + ], + "extra-body": { + "guidance_scale": 6.0, + "flow_shift": 5.0 + } + } + ] + }, + { + "test_name": "test_hunyuanvideo15_i2v_cachedit_tp2_vae_patch2", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 2 + } + }, + "full_model", + "diffusion" + ], + "description": "CacheDiT + TP=2 + VAE patch parallel=2 + VAE tiling", + "server_type": "vllm-omni", + "server_params": { + "model": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v", + "serve_args": { + "flow-shift": 5.0, + "cache-backend": "cache_dit", + "tensor-parallel-size": 2, + "vae-patch-parallel-size": 2, + "vae-use-tiling": true, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "832x480_frames33_steps4", + "dataset": "random", + "task": "i2v", + "num-prompts": 10, + "max-concurrency": 1, + "num-input-images": 1, + "seed": 42, + "enable-negative-prompt": true, + "random-request-config": [ + { + "width": 832, + "height": 480, + "num_inference_steps": 4, + "num_frames": 33, + "fps": 24, + "weight": 1 + } + ], + "extra-body": { + "guidance_scale": 6.0, + "flow_shift": 5.0 + } + } + ] + } +] diff --git a/tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json b/tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json new file mode 100644 index 000000000..3771c1c81 --- /dev/null +++ b/tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json @@ -0,0 +1,104 @@ +[ + { + "test_name": "test_hunyuanvideo15_t2v_single_device", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 1 + } + }, + "full_model", + "diffusion" + ], + "description": "Single-device baseline", + "server_type": "vllm-omni", + "server_params": { + "model": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", + "serve_args": { + "flow-shift": 5.0, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "832x480_frames33_steps4", + "dataset": "random", + "task": "t2v", + "num-prompts": 10, + "max-concurrency": 1, + "seed": 42, + "enable-negative-prompt": true, + "random-request-config": [ + { + "width": 832, + "height": 480, + "num_inference_steps": 4, + "num_frames": 33, + "fps": 24, + "weight": 1 + } + ], + "extra-body": { + "guidance_scale": 6.0, + "flow_shift": 5.0 + } + } + ] + }, + { + "test_name": "test_hunyuanvideo15_t2v_cachedit_tp2_vae_patch2", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H100" + }, + "num_cards": 2 + } + }, + "full_model", + "diffusion" + ], + "description": "CacheDiT + TP=2 + VAE patch parallel=2 + VAE tiling", + "server_type": "vllm-omni", + "server_params": { + "model": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", + "serve_args": { + "flow-shift": 5.0, + "cache-backend": "cache_dit", + "tensor-parallel-size": 2, + "vae-patch-parallel-size": 2, + "vae-use-tiling": true, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "832x480_frames33_steps4", + "dataset": "random", + "task": "t2v", + "num-prompts": 10, + "max-concurrency": 1, + "seed": 42, + "enable-negative-prompt": true, + "random-request-config": [ + { + "width": 832, + "height": 480, + "num_inference_steps": 4, + "num_frames": 33, + "fps": 24, + "weight": 1 + } + ], + "extra-body": { + "guidance_scale": 6.0, + "flow_shift": 5.0 + } + } + ] + } +] diff --git a/tests/e2e/online_serving/test_hunyuan_video_15_expansion.py b/tests/e2e/online_serving/test_hunyuan_video_15_expansion.py index dfe545327..0bc3f449f 100644 --- a/tests/e2e/online_serving/test_hunyuan_video_15_expansion.py +++ b/tests/e2e/online_serving/test_hunyuan_video_15_expansion.py @@ -5,8 +5,20 @@ Tests of common diffusion feature combinations in online serving mode for HunyuanVideo-1.5-T2V (480p). Coverage (H100, since model cannot fit L4): -- CacheDiT + Layerwise CPU offloading (1 GPU) -- CacheDiT + TP=2 + VAE patch parallel=2 (2 GPUs) +- CPU offloading (1 GPU) — ``core_model`` + ``advanced_model`` +- CacheDiT + Layerwise CPU offloading (1 GPU) — ``full_model`` +- CacheDiT + TP=2 + VAE patch parallel=2 (2 GPUs) — ``full_model`` + +HunyuanVideo-1.5 is a high-priority model, so only the most basic single-card deployment +row runs on every PR (L2) and on merge (L3). The heavyweight feature combinations stay +nightly-only (L4), together with the video similarity suites in +``tests/e2e/accuracy/hunyuanvideo15_{t2v,i2v}/``. + +From ``tests/``:: + + pytest -s -v e2e/online_serving/test_hunyuan_video_15_expansion.py -m "core_model and diffusion" --run-level=core_model + pytest -s -v e2e/online_serving/test_hunyuan_video_15_expansion.py -m "advanced_model and diffusion" --run-level=advanced_model + pytest -s -v e2e/online_serving/test_hunyuan_video_15_expansion.py -m "full_model and diffusion" --run-level=full_model """ import pytest @@ -14,7 +26,7 @@ import pytest from tests.helpers.mark import hardware_marks from tests.helpers.runtime import OmniServer, OmniServerParams, OnlineOmniClient -pytestmark = [pytest.mark.diffusion, pytest.mark.slow] +pytestmark = [pytest.mark.diffusion] PROMPT = "A cat walking across a sunlit garden, cinematic lighting, slow motion." NEGATIVE_PROMPT = "low quality, blurry, distorted" @@ -24,9 +36,14 @@ MODEL = "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v" SINGLE_CARD_MARKS = hardware_marks(res={"cuda": "H100"}) PARALLEL_MARKS = hardware_marks(res={"cuda": "H100"}, num_cards=2) +# Only the most basic single-card deployment row is cheap enough for PR (L2) / merge (L3) +# CI; the CacheDiT / parallel feature combinations run nightly (L4). +_CORE_MARKS = [pytest.mark.core_model, pytest.mark.advanced_model] +_FULL_MARKS = [pytest.mark.full_model] + def _get_diffusion_feature_cases(model: str): - """Return L4 diffusion feature cases for HunyuanVideo-1.5. + """Return diffusion feature cases for HunyuanVideo-1.5. Designed for 2x H100 environment per issue #1832. """ @@ -40,7 +57,7 @@ def _get_diffusion_feature_cases(model: str): ], ), id="single_card_cpu_offload", - marks=SINGLE_CARD_MARKS, + marks=SINGLE_CARD_MARKS + _CORE_MARKS, ), # (1 GPU) CacheDiT + Layerwise CPU offloading pytest.param( @@ -53,7 +70,7 @@ def _get_diffusion_feature_cases(model: str): ], ), id="single_card_cachedit_layerwise", - marks=SINGLE_CARD_MARKS, + marks=SINGLE_CARD_MARKS + _FULL_MARKS, ), # (2 GPUs) CacheDiT + TP=2 + VAE patch parallel=2 pytest.param( @@ -70,7 +87,7 @@ def _get_diffusion_feature_cases(model: str): ], ), id="parallel_cachedit_tp2_vae2", - marks=PARALLEL_MARKS, + marks=PARALLEL_MARKS + _FULL_MARKS, ), ] @@ -84,7 +101,7 @@ def test_hunyuan_video_15_t2v( omni_server: OmniServer, online_client: OnlineOmniClient, ): - """L4 diffusion feature coverage for HunyuanVideo-1.5-T2V on H100.""" + """Diffusion feature coverage for HunyuanVideo-1.5-T2V on H100.""" form_data = { "prompt": PROMPT, "negative_prompt": NEGATIVE_PROMPT,