diff --git a/.buildkite/npu/test-npu-nightly.yml b/.buildkite/npu/test-npu-nightly.yml index f64fa711f..0c39a3f93 100644 --- a/.buildkite/npu/test-npu-nightly.yml +++ b/.buildkite/npu/test-npu-nightly.yml @@ -235,6 +235,37 @@ steps: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" steps: + - label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Function Test" + source_file_dependencies: diffusion_hunyuan_video_function + timeout_in_minutes: 180 + mirror_hardwares: a2b3_npu_4 + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + HF_TOKEN: "${HF_TOKEN}" + HF_HUB_OFFLINE: "1" + commands: + - pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m "npu and A2" --run-level "full_model" + - label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Perf Test" + source_file_dependencies: diffusion_hunyuan_video_perf + key: nightly-diffusion-x2v-performance-hunyuanvideo15 + timeout_in_minutes: 180 + mirror_hardwares: a3_npu_2 + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + HF_TOKEN: "${HF_TOKEN}" + HF_HUB_OFFLINE: "1" + DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA + commands: + - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m "npu and A3" --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json + EXIT=$$? + buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" + buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" + exit $$EXIT - label: ":full_moon: Diffusion X2V · Wan 2.2 Function Test" source_file_dependencies: diffusion_wan22_function mirror_hardwares: a2b3_npu_4 diff --git a/tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json b/tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json index ffaaba436..6a3433dbe 100644 --- a/tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json +++ b/tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json @@ -54,7 +54,8 @@ { "hardware_marks": { "res": { - "cuda": ["H100", "B200"] + "cuda": ["H100", "B200"], + "npu": "A3" }, "num_cards": 2 } diff --git a/tests/e2e/online_serving/test_hunyuan_video_15_expansion.py b/tests/e2e/online_serving/test_hunyuan_video_15_expansion.py index 1d6df3317..3d54303f7 100644 --- a/tests/e2e/online_serving/test_hunyuan_video_15_expansion.py +++ b/tests/e2e/online_serving/test_hunyuan_video_15_expansion.py @@ -4,14 +4,15 @@ Tests of common diffusion feature combinations in online serving mode for HunyuanVideo-1.5-T2V (480p). -Coverage (H100, since model cannot fit L4): -- CPU offloading (1 GPU) — ``core_model`` + ``advanced_model`` -- CacheDiT + Layerwise CPU offloading (1 GPU) — ``full_model`` -- CacheDiT + TP=2 + VAE patch parallel=2 (2 GPUs) — ``full_model`` +Coverage: +- Default single-card deployment (H100, B200, or Ascend A2) — ``full_model`` +- CPU offloading (1 H100 or Ascend A2) — ``core_model`` + ``advanced_model`` +- CacheDiT + Layerwise CPU offloading (1 H100, B200, or Ascend A2) — ``full_model`` +- CacheDiT + TP=2 + VAE patch parallel=2 (2 H100s, B200s, or Ascend A2s) — ``full_model`` -HunyuanVideo-1.5 is a high-priority model, so only the most basic single-card deployment -row runs on every PR (L2) and on merge (L3). The heavyweight feature combinations stay -nightly-only (L4), together with the video similarity suites in +HunyuanVideo-1.5 is a high-priority model. The CPU-offload row runs in the existing +CUDA PR (L2) and merge (L3) lanes. The shared default and CacheDiT / parallel rows +carry ``full_model`` (L4) marks, together with the video similarity suites in ``tests/e2e/accuracy/hunyuanvideo15_{t2v,i2v}/``. From ``tests/``:: @@ -23,8 +24,9 @@ From ``tests/``:: import pytest -from tests.helpers.mark import hardware_marks +from tests.helpers.mark import hardware_marks, hardware_test from tests.helpers.runtime import OmniServer, OmniServerParams, OnlineOmniClient +from tests.helpers.stage_config import get_deploy_config_path pytestmark = [pytest.mark.diffusion] @@ -32,17 +34,25 @@ PROMPT = "A cat walking across a sunlit garden, cinematic lighting, slow motion. NEGATIVE_PROMPT = "low quality, blurry, distorted" MODEL = "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v" +DEPLOY_CONFIG = get_deploy_config_path("hunyuan_video_15.yaml") + +DEFAULT_TEST_PARAMS = [ + pytest.param( + OmniServerParams(model=MODEL, stage_config_path=DEPLOY_CONFIG), + id="default", + ) +] def _get_diffusion_feature_cases(model: str): """Return diffusion feature cases for HunyuanVideo-1.5. - Designed for 2x H100 environment per issue #1832. + Designed for up to two accelerators per issue #1832. Only the CPU-offload row is cheap enough for PR (L2) / merge (L3); CacheDiT / parallel combinations run nightly (L4). """ return [ - # (1 GPU) CPU offload + # (1 accelerator) CPU offload pytest.param( OmniServerParams( model=model, @@ -52,12 +62,12 @@ def _get_diffusion_feature_cases(model: str): ), id="single_card_cpu_offload", marks=[ - *hardware_marks(res={"cuda": "H100"}), + *hardware_marks(res={"cuda": "H100", "npu": "A2"}), pytest.mark.core_model, pytest.mark.advanced_model, ], ), - # (1 GPU) CacheDiT + Layerwise CPU offloading + # (1 accelerator) CacheDiT + Layerwise CPU offloading pytest.param( OmniServerParams( model=model, @@ -69,11 +79,11 @@ def _get_diffusion_feature_cases(model: str): ), id="single_card_cachedit_layerwise", marks=[ - *hardware_marks(res={"cuda": ["H100", "B200"]}), + *hardware_marks(res={"cuda": ["H100", "B200"], "npu": "A2"}), pytest.mark.full_model, ], ), - # (2 GPUs) CacheDiT + TP=2 + VAE patch parallel=2 + # (2 accelerators) CacheDiT + TP=2 + VAE patch parallel=2 pytest.param( OmniServerParams( model=model, @@ -89,23 +99,17 @@ def _get_diffusion_feature_cases(model: str): ), id="parallel_cachedit_tp2_vae2", marks=[ - *hardware_marks(res={"cuda": ["H100", "B200"]}, num_cards=2), + *hardware_marks(res={"cuda": ["H100", "B200"], "npu": "A2"}, num_cards=2), pytest.mark.full_model, ], ), ] -@pytest.mark.parametrize( - "omni_server", - _get_diffusion_feature_cases(MODEL), - indirect=True, -) -def test_hunyuan_video_15_t2v( +def _run_hunyuan_video_15_t2v( omni_server: OmniServer, online_client: OnlineOmniClient, -): - """Diffusion feature coverage for HunyuanVideo-1.5-T2V on H100.""" +) -> None: form_data = { "prompt": PROMPT, "negative_prompt": NEGATIVE_PROMPT, @@ -123,3 +127,27 @@ def test_hunyuan_video_15_t2v( } online_client.send_video_diffusion_request(request_config) + + +@hardware_test(res={"cuda": ["H100", "B200"], "npu": "A2"}, num_cards=1) +@pytest.mark.full_model +@pytest.mark.parametrize("omni_server", DEFAULT_TEST_PARAMS, indirect=True) +def test_hunyuan_video_15_t2v_default( + omni_server: OmniServer, + online_client: OnlineOmniClient, +) -> None: + """Generate a short video with the shared default GPU/NPU configuration.""" + _run_hunyuan_video_15_t2v(omni_server, online_client) + + +@pytest.mark.parametrize( + "omni_server", + _get_diffusion_feature_cases(MODEL), + indirect=True, +) +def test_hunyuan_video_15_t2v_features( + omni_server: OmniServer, + online_client: OnlineOmniClient, +) -> None: + """Exercise shared diffusion features for HunyuanVideo-1.5-T2V.""" + _run_hunyuan_video_15_t2v(omni_server, online_client)