anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Add HYvideo1.5 benchmark (#6349)

Browse Source 
Signed-off-by: zzh <943967662@qq.com>
This commit is contained in:
zzh authored and GitHub committed 2026-08-28 14:31:03 +08:00
1 parent b6a4baa659
commit c201fd77c6
6 files changed
+282 -8
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.buildkite/cuda/test-merge.yml
+12
View File
@@ -216,3 +216,15 @@ steps:
pytest -s -v tests/e2e/offline_inference/test_wan22_t2v.py tests/e2e/online_serving/test_wan22_t2v.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
"
mirror_hardwares: h100_1
- label: "Diffusion · HunyuanVideo-1.5 Test"
source_file_dependencies:
- tests/e2e/online_serving/test_hunyuan_video_15_expansion.py
- vllm_omni/diffusion/models/hunyuan_video/
commands:
- |
timeout 40m bash -c "
export VLLM_IMAGE_FETCH_TIMEOUT=60
pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
"
mirror_hardwares: h100_1
.buildkite/cuda/test-nightly.yml
+23
View File
@@ -272,6 +272,12 @@ steps:
- pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda" --run-level "full_model"
mirror_hardwares: h100_2
- label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Function Test"
timeout_in_minutes: 90
commands:
- pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m "full_model and cuda" --run-level "full_model"
mirror_hardwares: h100_2
- label: ":full_moon: Diffusion X2V · Doc Test"
timeout_in_minutes: 60
commands:
@@ -322,6 +328,23 @@ steps:
exit $$EXIT2
mirror_hardwares: h100_2
- label: ":full_moon: Diffusion X2V · Perf Test · HunyuanVideo-1.5"
key: nightly-diffusion-x2v-performance-hunyuanvideo15
timeout_in_minutes: 180
commands:
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
- |
set +e
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json
EXIT1=$$?
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json
EXIT2=$$?
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
if [ $$EXIT1 -ne 0 ]; then exit $$EXIT1; fi
exit $$EXIT2
mirror_hardwares: h100_2
- label: ":full_moon: Diffusion X2V · Perf Test · MiniMax-H3"
key: nightly-diffusion-x2v-performance-minimax-h3
timeout_in_minutes: 180
.buildkite/cuda/test-ready.yml
+12
View File
@@ -337,3 +337,15 @@ steps:
pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model' --run-level 'core_model'
"
mirror_hardwares: h100_1
- label: "Diffusion · HunyuanVideo-1.5 Test"
source_file_dependencies:
- tests/e2e/online_serving/test_hunyuan_video_15_expansion.py
- vllm_omni/diffusion/models/hunyuan_video/
commands:
- |
timeout 40m bash -c "
export VLLM_IMAGE_FETCH_TIMEOUT=60
pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m 'core_model and diffusion' --run-level 'core_model'
"
mirror_hardwares: h100_1
tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json
+106
View File
@@ -0,0 +1,106 @@
[
{
"test_name": "test_hunyuanvideo15_i2v_single_device",
"mark": [
{
"hardware_marks": {
"res": {
"cuda": "H100"
},
"num_cards": 1
}
},
"full_model",
"diffusion"
],
"description": "Single-device baseline",
"server_type": "vllm-omni",
"server_params": {
"model": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v",
"serve_args": {
"flow-shift": 5.0,
"enable-diffusion-pipeline-profiler": true
}
},
"benchmark_params": [
{
"name": "832x480_frames33_steps4",
"dataset": "random",
"task": "i2v",
"num-prompts": 10,
"max-concurrency": 1,
"num-input-images": 1,
"seed": 42,
"enable-negative-prompt": true,
"random-request-config": [
{
"width": 832,
"height": 480,
"num_inference_steps": 4,
"num_frames": 33,
"fps": 24,
"weight": 1
}
],
"extra-body": {
"guidance_scale": 6.0,
"flow_shift": 5.0
}
}
]
},
{
"test_name": "test_hunyuanvideo15_i2v_cachedit_tp2_vae_patch2",
"mark": [
{
"hardware_marks": {
"res": {
"cuda": "H100"
},
"num_cards": 2
}
},
"full_model",
"diffusion"
],
"description": "CacheDiT + TP=2 + VAE patch parallel=2 + VAE tiling",
"server_type": "vllm-omni",
"server_params": {
"model": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v",
"serve_args": {
"flow-shift": 5.0,
"cache-backend": "cache_dit",
"tensor-parallel-size": 2,
"vae-patch-parallel-size": 2,
"vae-use-tiling": true,
"enable-diffusion-pipeline-profiler": true
}
},
"benchmark_params": [
{
"name": "832x480_frames33_steps4",
"dataset": "random",
"task": "i2v",
"num-prompts": 10,
"max-concurrency": 1,
"num-input-images": 1,
"seed": 42,
"enable-negative-prompt": true,
"random-request-config": [
{
"width": 832,
"height": 480,
"num_inference_steps": 4,
"num_frames": 33,
"fps": 24,
"weight": 1
}
],
"extra-body": {
"guidance_scale": 6.0,
"flow_shift": 5.0
}
}
]
}
]
tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json
+104
View File
@@ -0,0 +1,104 @@
[
{
"test_name": "test_hunyuanvideo15_t2v_single_device",
"mark": [
{
"hardware_marks": {
"res": {
"cuda": "H100"
},
"num_cards": 1
}
},
"full_model",
"diffusion"
],
"description": "Single-device baseline",
"server_type": "vllm-omni",
"server_params": {
"model": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v",
"serve_args": {
"flow-shift": 5.0,
"enable-diffusion-pipeline-profiler": true
}
},
"benchmark_params": [
{
"name": "832x480_frames33_steps4",
"dataset": "random",
"task": "t2v",
"num-prompts": 10,
"max-concurrency": 1,
"seed": 42,
"enable-negative-prompt": true,
"random-request-config": [
{
"width": 832,
"height": 480,
"num_inference_steps": 4,
"num_frames": 33,
"fps": 24,
"weight": 1
}
],
"extra-body": {
"guidance_scale": 6.0,
"flow_shift": 5.0
}
}
]
},
{
"test_name": "test_hunyuanvideo15_t2v_cachedit_tp2_vae_patch2",
"mark": [
{
"hardware_marks": {
"res": {
"cuda": "H100"
},
"num_cards": 2
}
},
"full_model",
"diffusion"
],
"description": "CacheDiT + TP=2 + VAE patch parallel=2 + VAE tiling",
"server_type": "vllm-omni",
"server_params": {
"model": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v",
"serve_args": {
"flow-shift": 5.0,
"cache-backend": "cache_dit",
"tensor-parallel-size": 2,
"vae-patch-parallel-size": 2,
"vae-use-tiling": true,
"enable-diffusion-pipeline-profiler": true
}
},
"benchmark_params": [
{
"name": "832x480_frames33_steps4",
"dataset": "random",
"task": "t2v",
"num-prompts": 10,
"max-concurrency": 1,
"seed": 42,
"enable-negative-prompt": true,
"random-request-config": [
{
"width": 832,
"height": 480,
"num_inference_steps": 4,
"num_frames": 33,
"fps": 24,
"weight": 1
}
],
"extra-body": {
"guidance_scale": 6.0,
"flow_shift": 5.0
}
}
]
}
]
tests/e2e/online_serving/test_hunyuan_video_15_expansion.py
+25 -8
View File
@@ -5,8 +5,20 @@ Tests of common diffusion feature combinations in online serving mode
for HunyuanVideo-1.5-T2V (480p).
Coverage (H100, since model cannot fit L4):
- CacheDiT + Layerwise CPU offloading (1 GPU)
- CacheDiT + TP=2 + VAE patch parallel=2 (2 GPUs)
- CPU offloading (1 GPU) — ``core_model`` + ``advanced_model``
- CacheDiT + Layerwise CPU offloading (1 GPU) — ``full_model``
- CacheDiT + TP=2 + VAE patch parallel=2 (2 GPUs) — ``full_model``
HunyuanVideo-1.5 is a high-priority model, so only the most basic single-card deployment
row runs on every PR (L2) and on merge (L3). The heavyweight feature combinations stay
nightly-only (L4), together with the video similarity suites in
``tests/e2e/accuracy/hunyuanvideo15_{t2v,i2v}/``.
From ``tests/``::
pytest -s -v e2e/online_serving/test_hunyuan_video_15_expansion.py -m "core_model and diffusion" --run-level=core_model
pytest -s -v e2e/online_serving/test_hunyuan_video_15_expansion.py -m "advanced_model and diffusion" --run-level=advanced_model
pytest -s -v e2e/online_serving/test_hunyuan_video_15_expansion.py -m "full_model and diffusion" --run-level=full_model
"""
import pytest
@@ -14,7 +26,7 @@ import pytest
from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import OmniServer, OmniServerParams, OnlineOmniClient
pytestmark = [pytest.mark.diffusion, pytest.mark.slow]
pytestmark = [pytest.mark.diffusion]
PROMPT = "A cat walking across a sunlit garden, cinematic lighting, slow motion."
NEGATIVE_PROMPT = "low quality, blurry, distorted"
@@ -24,9 +36,14 @@ MODEL = "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v"
SINGLE_CARD_MARKS = hardware_marks(res={"cuda": "H100"})
PARALLEL_MARKS = hardware_marks(res={"cuda": "H100"}, num_cards=2)
# Only the most basic single-card deployment row is cheap enough for PR (L2) / merge (L3)
# CI; the CacheDiT / parallel feature combinations run nightly (L4).
_CORE_MARKS = [pytest.mark.core_model, pytest.mark.advanced_model]
_FULL_MARKS = [pytest.mark.full_model]
def _get_diffusion_feature_cases(model: str):
"""Return L4 diffusion feature cases for HunyuanVideo-1.5.
"""Return diffusion feature cases for HunyuanVideo-1.5.
Designed for 2x H100 environment per issue #1832.
"""
@@ -40,7 +57,7 @@ def _get_diffusion_feature_cases(model: str):
],
),
id="single_card_cpu_offload",
marks=SINGLE_CARD_MARKS,
marks=SINGLE_CARD_MARKS + _CORE_MARKS,
),
# (1 GPU) CacheDiT + Layerwise CPU offloading
pytest.param(
@@ -53,7 +70,7 @@ def _get_diffusion_feature_cases(model: str):
],
),
id="single_card_cachedit_layerwise",
marks=SINGLE_CARD_MARKS,
marks=SINGLE_CARD_MARKS + _FULL_MARKS,
),
# (2 GPUs) CacheDiT + TP=2 + VAE patch parallel=2
pytest.param(
@@ -70,7 +87,7 @@ def _get_diffusion_feature_cases(model: str):
],
),
id="parallel_cachedit_tp2_vae2",
marks=PARALLEL_MARKS,
marks=PARALLEL_MARKS + _FULL_MARKS,
),
]
@@ -84,7 +101,7 @@ def test_hunyuan_video_15_t2v(
omni_server: OmniServer,
online_client: OnlineOmniClient,
):
"""L4 diffusion feature coverage for HunyuanVideo-1.5-T2V on H100."""
"""Diffusion feature coverage for HunyuanVideo-1.5-T2V on H100."""
form_data = {
"prompt": PROMPT,
"negative_prompt": NEGATIVE_PROMPT,