anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

[CI]Split GPU jobs by cards_* and reject hand-written SKU marks (#6650)

Browse Source 
Signed-off-by: wangyu <410167048@qq.com>
Signed-off-by: [Your Name] <your.email@example.com>
This commit is contained in:
wangyu authored and GitHub committed 2026-08-29 18:42:33 +08:00
1 parent c50b7de167
commit e338f0ece3
31 files changed
+896 -402
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.buildkite/cuda/test-nightly.yml
+108 -34
View File
@@ -12,16 +12,22 @@ steps:
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "omni-test"
steps:
- label: ":full_moon: Omni · Function Test with H100"
timeout_in_minutes: 120
- label: ":full_moon: Omni · Function Test with H100 · Single-GPU"
timeout_in_minutes: 90
commands:
- pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
- pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni and cards_1" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
mirror_hardwares: h100_1
- label: ":full_moon: Omni · Function Test with H100 · 2-GPU"
timeout_in_minutes: 90
commands:
- pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni and cards_2" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py
mirror_hardwares: h100_2
- label: ":full_moon: Omni · MiniCPM-o 4.5 Duplex Test"
timeout_in_minutes: 50
commands:
- pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and omni" --run-level "full_model"
- pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and omni and cards_1" --run-level "full_model"
mirror_hardwares: h100_1
- label: ":full_moon: Omni · Doc Test with L4"
@@ -31,10 +37,10 @@ steps:
- pytest -s -v tests/examples/ -m "full_model and omni and L4" --run-level "full_model"
mirror_hardwares: l4_4
- label: ":full_moon: Omni · Doc Test with H100"
- label: ":full_moon: Omni · Doc Test with H100 · 2-GPU"
timeout_in_minutes: 90
commands:
- pytest -s -v tests/examples/ -m "full_model and omni and H100" --run-level "full_model"
- pytest -s -v tests/examples/ -m "full_model and omni and H100 and cards_2" --run-level "full_model"
mirror_hardwares: h100_2
- label: ":full_moon: Omni · Accuracy Test"
@@ -136,15 +142,29 @@ steps:
- pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy
mirror_hardwares: l4_1
- label: ":full_moon: TTS · Perf Test"
key: nightly-tts-performance
- label: ":full_moon: TTS · Perf Test · Single-GPU"
key: nightly-tts-performance-single
timeout_in_minutes: 180
commands:
- export BENCHMARK_DIR=tests/dfx/perf/results
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
- |
set +e
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and cards_1"
EXIT=$$?
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
exit $$EXIT
mirror_hardwares: h100_1
- label: ":full_moon: TTS · Perf Test · 2-GPU"
key: nightly-tts-performance-2gpu
timeout_in_minutes: 180
commands:
- export BENCHMARK_DIR=tests/dfx/perf/results
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
- |
set +e
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and cards_2"
EXIT=$$?
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
exit $$EXIT
@@ -179,7 +199,6 @@ steps:
build.pull_request.labels includes "nightly-test" ||
build.pull_request.labels includes "diffusion-x2iat-test"
steps:
# Use explicit file shards so single-card cases do not reserve 2+ GPUs.
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Single-GPU"
timeout_in_minutes: 120
@@ -187,24 +206,24 @@ steps:
- >-
pytest -sv
tests/e2e/online_serving/test_qwen_image_expansion.py
-m "full_model and diffusion and H100 and not distributed_cuda"
-m "full_model and diffusion and H100 and cards_1"
--run-level "full_model"
mirror_hardwares: h100_1
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Qwen-Image"
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · 2-GPU"
timeout_in_minutes: 120
commands:
- >-
pytest -sv
tests/e2e/online_serving/test_qwen_image_expansion.py
-m "full_model and diffusion and H100 and distributed_cuda"
-m "full_model and diffusion and H100 and cards_2"
--run-level "full_model"
mirror_hardwares: h100_2
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4"
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4 · Multi-GPU"
timeout_in_minutes: 120
commands:
- pytest -sv tests/e2e/ -k "not test_wan and not hunyuan" -m "full_model and diffusion and L4" --run-level "full_model" --ignore=tests/e2e/accuracy
- pytest -sv tests/e2e/ -k "not test_wan and not hunyuan" -m "full_model and diffusion and L4 and not cards_1" --run-level "full_model" --ignore=tests/e2e/accuracy
mirror_hardwares: l4_4
- label: ":full_moon: Diffusion X2I(&A&T) · Doc Test"
@@ -235,8 +254,8 @@ steps:
- pytest -s -v tests/e2e/accuracy/test_hidream_o1_image.py -m full_model --run-level full_model
mirror_hardwares: h100_2
- label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image"
key: nightly-diffusion-x2iat-performance-qwen-image
- label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image · Single-GPU"
key: nightly-diffusion-x2iat-performance-qwen-image-single
timeout_in_minutes: 180
commands:
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
@@ -244,7 +263,23 @@ steps:
- export CACHE_DIT_VERSION=1.5.0
- |
set +e
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and cards_1"
EXIT=$$?
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
exit $$EXIT
mirror_hardwares: h100_1
- label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image · 4-GPU"
key: nightly-diffusion-x2iat-performance-qwen-image-4gpu
timeout_in_minutes: 180
commands:
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
- export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
- export CACHE_DIT_VERSION=1.5.0
- |
set +e
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and cards_4"
EXIT=$$?
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
@@ -260,16 +295,28 @@ steps:
build.pull_request.labels includes "diffusion-x2v-test"
steps:
- label: ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test"
- label: ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test · Single-GPU"
timeout_in_minutes: 90
commands:
- pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda" --run-level "full_model"
- pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda and H100 and cards_1" --run-level "full_model"
mirror_hardwares: h100_1
- label: ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test · 2-GPU"
timeout_in_minutes: 90
commands:
- pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda and H100 and cards_2" --run-level "full_model"
mirror_hardwares: h100_2
- label: ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test"
- label: ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test · Single-GPU"
timeout_in_minutes: 90
commands:
- pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda" --run-level "full_model"
- pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda and H100 and cards_1" --run-level "full_model"
mirror_hardwares: h100_1
- label: ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test · 2-GPU"
timeout_in_minutes: 90
commands:
- pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda and H100 and cards_2" --run-level "full_model"
mirror_hardwares: h100_2
- label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Function Test"
@@ -310,17 +357,32 @@ steps:
- pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m full_model --run-level full_model
mirror_hardwares: h100_2
- label: ":full_moon: Diffusion X2V · Perf Test"
key: nightly-diffusion-x2v-performance
- label: ":full_moon: Diffusion X2V · Perf Test · Single-GPU"
key: nightly-diffusion-x2v-performance-single
timeout_in_minutes: 180
commands:
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
- export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
- |
set +e
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and cards_1"
EXIT=$$?
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
exit $$EXIT
mirror_hardwares: h100_1
- label: ":full_moon: Diffusion X2V · Perf Test · 2-GPU"
key: nightly-diffusion-x2v-performance-2gpu
timeout_in_minutes: 180
commands:
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
- export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
- |
set +e
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and cards_2"
EXIT1=$$?
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_cosmos3_vllm_omni.json
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_cosmos3_vllm_omni.json -m "H100 and cards_2"
EXIT2=$$?
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
@@ -371,10 +433,16 @@ steps:
- pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4' --run-level "full_model"
mirror_hardwares: l4_4
- label: ":full_moon: Diffusion · Distributed Test with H100"
- label: ":full_moon: Diffusion · Distributed Test with H100 · 2-GPU"
timeout_in_minutes: 60
commands:
- pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100' --run-level "full_model"
- pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100 and cards_2' --run-level "full_model"
mirror_hardwares: h100_2
- label: ":full_moon: Diffusion · Distributed Test with H100 · 4-GPU"
timeout_in_minutes: 60
commands:
- pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100 and cards_4' --run-level "full_model"
mirror_hardwares: h100_4
- label: ":full_moon: Diffusion · Offloader Test"
@@ -409,19 +477,25 @@ steps:
depends_on:
- nightly-omni-performance-no-async-chunk
- nightly-omni-performance-async-chunk
- nightly-tts-performance
- nightly-diffusion-x2iat-performance-qwen-image
- nightly-diffusion-x2v-performance
- nightly-tts-performance-single
- nightly-tts-performance-2gpu
- nightly-diffusion-x2iat-performance-qwen-image-single
- nightly-diffusion-x2iat-performance-qwen-image-4gpu
- nightly-diffusion-x2v-performance-single
- nightly-diffusion-x2v-performance-2gpu
if: build.env("NIGHTLY") == "1" && build.env("EMAIL_DISTRIBUTION") == "1"
commands:
- pip install openpyxl
- export DEFAULT_INPUT_DIR=tests/dfx/perf/results
- export DEFAULT_OUTPUT_DIR=tests/dfx/perf/results
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance-single
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance-2gpu
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-no-async-chunk
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-async-chunk
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-single
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-4gpu
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance-single
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance-2gpu
- python tools/nightly/generate_nightly_perf_excel.py
- python tools/nightly/generate_nightly_perf_html.py
- python tools/nightly/send_nightly_email.py --report-file "tests/dfx/perf/results/*.xlsx, tests/dfx/perf/results/*.html"
.buildkite/cuda/test-weekly.yml
+48 -14
View File
@@ -44,10 +44,16 @@ steps:
- pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m "slow"
mirror_hardwares: h100_4
- label: "Reliability Test · Invalid parameters · H100"
timeout_in_minutes: 120
- label: "Reliability Test · Invalid parameters · H100 · Single-GPU"
timeout_in_minutes: 90
commands:
- pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100"
- pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100 and cards_1"
mirror_hardwares: h100_1
- label: "Reliability Test · Invalid parameters · H100 · 2-GPU"
timeout_in_minutes: 90
commands:
- pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100 and cards_2"
mirror_hardwares: h100_2
- label: "Reliability Test · Invalid parameters · L4"
@@ -105,27 +111,37 @@ steps:
if: build.env("NON_CRITICAL") == "1"
steps:
- label: "Omni · L4"
# Qwen2.5 expansion (CUDA 4-card / online 2-card) stays on this l4_4 job.
- label: "Omni · L4 · Multi-GPU"
timeout_in_minutes: 90
commands:
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
- pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model"
- pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py -m "slow and L4 and omni and not cards_1" --run-level "core_model"
- >-
pytest -s -v tests/e2e/
--ignore=tests/e2e/online_serving/test_qwen2_5_omni_expansion.py
--ignore=tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py
-m "slow and L4 and omni"
-m "slow and L4 and omni and not cards_1"
--run-level "full_model"
mirror_hardwares: l4_4
- label: "Omni · H100"
- label: "Omni · H100 · Single-GPU"
timeout_in_minutes: 90
commands:
- >-
pytest -sv
tests/e2e/ -m "slow and H100 and omni"
tests/e2e/ -m "slow and H100 and omni and cards_1"
--run-level "full_model"
mirror_hardwares: h100_2
mirror_hardwares: h100_1
- label: "Omni · H100 · 4-GPU"
timeout_in_minutes: 90
commands:
- >-
pytest -sv
tests/e2e/ -m "slow and H100 and omni and cards_4"
--run-level "full_model"
mirror_hardwares: h100_4
- label: "TTS · L4"
timeout_in_minutes: 120
@@ -169,12 +185,21 @@ steps:
--run-level "full_model"
mirror_hardwares: l4_1
- label: "Diffusion · L4"
- label: "Diffusion · L4 · Single-GPU"
timeout_in_minutes: 60
commands:
- >-
pytest -sv
tests/e2e/ -m "slow and diffusion and L4"
tests/e2e/ -m "slow and diffusion and L4 and cards_1"
--run-level "full_model"
mirror_hardwares: l4_1
- label: "Diffusion · L4 · Multi-GPU"
timeout_in_minutes: 60
commands:
- >-
pytest -sv
tests/e2e/ -m "slow and diffusion and L4 and not cards_1"
--run-level "full_model"
mirror_hardwares: l4_4
@@ -183,15 +208,24 @@ steps:
commands:
- >-
pytest -sv
tests/e2e/ -m "slow and diffusion and H100 and not distributed_cuda"
tests/e2e/ -m "slow and diffusion and H100 and cards_1"
--run-level "full_model"
mirror_hardwares: h100_1
- label: "Diffusion · H100 · Multi-GPU"
- label: "Diffusion · H100 · 2-GPU"
timeout_in_minutes: 90
commands:
- >-
pytest -sv
tests/e2e/ -m "slow and diffusion and H100 and distributed_cuda"
tests/e2e/ -m "slow and diffusion and H100 and cards_2"
--run-level "full_model"
mirror_hardwares: h100_2
- label: "Diffusion · H100 · 4-GPU"
timeout_in_minutes: 90
commands:
- >-
pytest -sv
tests/e2e/ -m "slow and diffusion and H100 and cards_4"
--run-level "full_model"
mirror_hardwares: h100_4
.claude/skills/add-diffusion-model/SKILL.md
+1 -1
View File
@@ -338,7 +338,7 @@ Classify the model's **CI priority** first:
**L2 & L3 online — same file, dual marks on the baseline smoke:** The **first / simplest** case in `test_{slug}.py` (default deploy, minimal steps, single prompt) should carry **both** `@pytest.mark.core_model` and `@pytest.mark.advanced_model` on the **same** function so L2 (`test-ready.yml`) and L3 (`test-merge.yml`) share one smoke test. Heavier deploy variants or API paths in the same file use **`advanced_model` only**. When L3 moves to nightly, migrate those heavier cases into `test_{slug}_expansion.py` with `full_model` and remove the dedicated `test-merge.yml` job (see `test_longcat_image_expansion.py`, `test_qwen_image_expansion.py`).
**L4 design (high priority):** Combine multiple supported features (Cache-DiT, TP, USP, CFG, HSDP, CPU offload, quantization) into **few parametrized** `OmniServerParams` rows so each feature appears in at least one case without exploding GPU jobs. Shard single-GPU vs multi-GPU cases across the nightly X2I/X2V function steps (1-GPU vs `distributed_cuda`).
**L4 design (high priority):** Combine multiple supported features (Cache-DiT, TP, USP, CFG, HSDP, CPU offload, quantization) into **few parametrized** `OmniServerParams` rows so each feature appears in at least one case without exploding GPU jobs. Shard single-GPU vs multi-GPU cases across the nightly X2I/X2V function steps (`cards_1` vs `not cards_1`).
**L4 design (medium / low):** One or two parametrized rows covering the best quality/perf trade-off; skip perf JSON unless the model is high priority.
.claude/skills/add-tts-model/references/precommit-dco.md
+1 -1
View File
@@ -34,7 +34,7 @@ but the changes are correct — stage the modified files and re-commit.
| `check-forbidden-imports` | Stdlib `re`/`base64`, pickle, Hugging Face Hub API, or direct Triton/TileLang | `import regex as re` and `pybase64`; use `vllm.transformers_utils.repo_utils`. Do **not** add the file to `allowed_files` without review |
| `check-torch-cuda-call` | New `torch.cuda.*` call site | Use `current_omni_platform` / `OmniPlatform`; do not grow `ALLOWED_FILES` without review |
| `check-tts-adapter-migration` | New `self._tts_model_type` branch in `serving_speech.py` | Put per-model logic in `tts_adapters/`. Lower `MAX_MODEL_TYPE_BRANCHES` when removing branches; do not raise it without review |
| `check-test-ci-coverage` | New `tests/**/test_*.py` missing level or hardware mark | Add `core_model`/`advanced_model`/… plus `cpu`/`cuda`/`hardware_test(` |
| `check-mark` | New `tests/**/test_*.py` missing level/hardware mark, or direct `pytest.mark.H100` | Add `core_model`/`advanced_model`/… plus `cpu`/`cuda`/`hardware_test(`; SKU marks only via helpers |
| `mypy-3.10` | Type error on changed `vllm_omni/` files | Fix the types; model trees are excluded. Extra versions: `pre-commit run --hook-stage manual mypy-3.12` |
| `markdownlint-cli2` | Docs/README markdown lint | Accept auto-fix under `docs/` / `recipes/` / root README |
| `check-buildkite` | Invalid `.buildkite/*.yml` | Fix the YAML; do not grow `SKIP_FILES` without review |
.claude/skills/vllm-omni-test/SKILL.md
+3 -3
View File
@@ -52,8 +52,8 @@ Always attach markers deliberately:
- `tts` — speech synthesis / TTS-only models (`/v1/audio/speech`, voice clone, etc.)
- `diffusion` — generative diffusion models (image / audio / text / video from noise)
- **Cross-cutting area** (when relevant): `parallel`, `cache`, `example`, `benchmark`
- **Hardware**: `cpu`, `gpu`, `cuda`, `rocm`, `npu`, `L4`, `H100`, `distributed_cuda`, …
- Optional: `slow`, distributed markers when multi-card is required
- **Hardware**: `cpu`, `gpu`, `cuda`, `rocm`, `npu`, …; SKU (`H100`, `L4`, …) and `cards_{n}` via `@hardware_test(...)` / `hardware_marks(...)` only — never `pytest.mark.H100` (`check-mark` rejects it)
- Optional: `slow`. Multi-card: `num_cards=N` on the helper (filter with `cards_N` / `not cards_1`)
**Baseline smoke (L2 + L3):** The simplest e2e case per model — default deploy, minimal request — should usually carry **both** `@pytest.mark.core_model` and `@pytest.mark.advanced_model` on the **same** test function so `test-ready.yml` and `test-merge.yml` share one test. `send_*_request` picks validation depth from `--run-level`. References: `test_voxcpm2_tts.py::test_text_to_audio_001`, `test_qwen3_tts_customvoice.py::test_text_to_audio_001`. Heavier scenarios use **`advanced_model` only**; L4 expansion uses **`full_model`**.
@@ -274,7 +274,7 @@ def test_text_to_video_001(omni_server, online_client) -> None:
**3. Cross-cutting rules**
- New `.py` files need the Omni SPDX header (`Copyright contributors to the vLLM-Omni project`, not `vLLM project`). `tests/` may keep stdlib `re`/`base64`; pickle is still banned unless the file is already on the pickle allowlist. Every collected `tests/**/test_*.py` needs a CI **level** mark and a **hardware** mark/helper (`check-test-ci-coverage` runs locally; GHA skips it). Do not grow `allowed_files` to land a test. Policy: [docs/contributing/README.md](../../../docs/contributing/README.md#linting).
- New `.py` files need the Omni SPDX header (`Copyright contributors to the vLLM-Omni project`, not `vLLM project`). `tests/` may keep stdlib `re`/`base64`; pickle is still banned unless the file is already on the pickle allowlist. Every collected `tests/**/test_*.py` needs a CI **level** mark and a **hardware** platform mark/helper (`check-mark` runs locally; GHA skips it). Do not write `pytest.mark.H100` / `pytest.mark.L4`; use the helpers so `cards_{n}` is attached. Do not grow `allowed_files` to land a test. Policy: [docs/contributing/README.md](../../../docs/contributing/README.md#linting).
- Reuse existing fixtures for the chosen scenario; do not mix “online client” assumptions into offline `OmniRunner` tests without a clear reason.
- Avoid external network dependency in assertions unless the scenario is explicitly “online serving” or doc examples that require a model hub (then align with CI secrets/cache).
- Keep **one test function = one intent** (one modality combo, one endpoint contract, or one acceleration combo).
.claude/skills/vllm-omni-test/references/test-routing.md
+2 -2
View File
@@ -79,7 +79,7 @@ E2E (L2+) should not rely on mocks unless documenting a rare exception; prefer r
- `full_model` — L4 nightly (`test-nightly.yml`); some expansion tests still carry both `advanced_model` and `full_model` during migration
2. **Model type** (pick one): `omni`, `tts`, or `diffusion`
3. **Cross-cutting** when relevant: `parallel`, `cache`, `example`, `benchmark`
4. **Hardware**: `cpu`, `cuda`, `rocm`, `npu`, `L4`, `H100`, `distributed_cuda`, …
4. **Hardware**: `cpu`, `cuda`, `rocm`, `npu`, `L4`, `H100`, `cards_1`, …
5. Multi-card: `@hardware_test(...)` in `tests/helpers/mark.py`
## Command Templates
@@ -244,7 +244,7 @@ pytest -s -v dfx/reliability/invalid_param_test/test_invalid_image_generation.py
```bash
cd tests
pytest -s -v -m "core_model and distributed_cuda and L4" --run-level=core_model
pytest -s -v -m "core_model and L4 and not cards_1" --run-level=core_model
```
### Concrete e2e paths (common in-tree)
.github/workflows/pre-commit.yml
+1 -1
View File
@@ -27,4 +27,4 @@ jobs:
# Don't run historical-debt hooks against the whole tree in CI yet.
# They still run locally on changed files.
env:
SKIP: check-test-ci-coverage,markdownlint-cli2,shellcheck,check-spdx-header,mypy-3.10
SKIP: check-mark,markdownlint-cli2,shellcheck,check-spdx-header,mypy-3.10
.pre-commit-config.yaml
+2 -2
View File
@@ -105,8 +105,8 @@ repos:
<<: *mypy_common
stages: [manual]
- id: check-test-ci-coverage
name: Ensure test files have pytestmarks
- id: check-mark
name: Ensure test files have CI marks (no direct SKU)
entry: python tools/pre_commit/check_test_marks.py
language: python
types: [python]
docs/contributing/README.md
+8 -6
View File
@@ -49,7 +49,7 @@ vLLM-Omni's `pre-commit` hooks will now run automatically every time you commit.
`--all-files` run stays green while historical debt is cleaned up. The
current `SKIP` list in
[`.github/workflows/pre-commit.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.github/workflows/pre-commit.yml)
is `check-test-ci-coverage`, `markdownlint-cli2`, `shellcheck`,
is `check-mark`, `markdownlint-cli2`, `shellcheck`,
`check-spdx-header`, and `mypy-3.10`. Those hooks still run on **your
commit** for changed files. A passing GitHub pre-commit check does not mean
they passed locally. Hooks **not** on that list (forbidden imports,
@@ -66,7 +66,7 @@ already there). `check-pickle-imports` is gone: pickle is now one rule inside
| `markdownlint-cli2` | Markdown in `docs/`, `recipes/`, `README.md`, `CONTRIBUTING.md` (not `.claude/` / `.cursor/` / `CLAUDE.md`) | skipped |
| `mypy-3.10` | Type-check changed `vllm_omni/` files (model trees excluded). `tests/` uses `--follow-imports skip` | skipped |
| `mypy-3.11` / `3.12` / `3.13` | Same checker, extra Python versions | not installed; `pre-commit run --hook-stage manual mypy-3.12` |
| `check-test-ci-coverage` | Every `tests/**/test_*.py` has a CI level mark and a hardware mark/helper | skipped |
| `check-mark` | Every `tests/**/test_*.py` has a CI level mark and a hardware platform mark or helper; no direct `pytest.mark.H100` / SKU | skipped |
| `check-tts-adapter-migration` | `self._tts_model_type` branches in `serving_speech.py` must not increase | runs |
| `shellcheck` | `*.sh` quoting / undefined vars | skipped |
| `check-spdx-header` | Omni SPDX header on `.py` / `.pyi` / `.sh` / `.rs` / `.proto` | skipped |
@@ -144,11 +144,13 @@ pre-commit run --hook-stage manual mypy-3.12
#### Test CI marks
`check-test-ci-coverage` requires each collected test module under `tests/` to
`check-mark` requires each collected test module under `tests/` to
have at least one CI **level** mark (`core_model`, `advanced_model`,
`full_model`, `local_model`, or `slow`) and a **hardware** mark (`cpu`, `cuda`,
`H100`, …) or helper (`hardware_test(` / `hardware_marks(`). GitHub Actions
skips this hook; local commit does not. See the
`full_model`, `local_model`, or `slow`) and a **hardware** platform mark
(`cpu`, `cuda`, …) or helper (`hardware_test(` / `hardware_marks(`). SKU marks
(`H100`, `L4`, …) must come from those helpers so `cards_{n}` is attached;
direct `pytest.mark.H100` is rejected. GitHub Actions skips this hook; local
commit does not. See the
[test writing guide](./ci/test_writing_guide.md).
#### TTS adapter ratchet
docs/contributing/ci/ci_settings.md
+1 -1
View File
@@ -402,7 +402,7 @@ loads the model once per mode.
Before splitting, confirm each half actually collects at least one test under the job's forwarded filters — pytest exits with code 5 ("no tests collected") if a half is empty, which the script reports as a failure, red-ing a job that used to pass as one combined invocation.
Also check the tests' `num_cards` against the job's `mirror_hardwares` preset.
`cuda_marks` turns `num_cards > 1` into `skipif(device_count() < num_cards)`, so a
`_cuda_marks` turns `num_cards > 1` into `skipif(device_count() < num_cards)`, so a
test asking for more GPUs than the preset provides is silently skipped and its
coverage artifact comes back empty. Splitting such a job produces a file that
looks fine and measures nothing — "Diffusion · Z Image Test" is the current
docs/contributing/ci/test_examples/l4_performance_tests.inc.md
+3 -3
View File
@@ -81,8 +81,8 @@ Supported form:
| Form | Example | Effect |
| ---- | ------- | ------ |
| Array (single platform) | `"mark": [{"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 2}}, "full_model", "diffusion"]` | Hardware marks from `hardware_marks(...)`; `num_cards` > 1 adds `distributed_*` (+ CUDA `skipif_cuda` when applicable). String entries become extra pytest marks. |
| Array (multi-platform) | `"mark": [{"hardware_marks": {"res": {"cuda": "H100", "rocm": "MI325", "npu": "A2"}, "num_cards": {"cuda": 2, "rocm": 2, "npu": 1}}}, "full_model", "omni"]` | Same as above, but declares **multiple platforms** in one case. Filter examples: `-m "full_model and H100 and cuda"`, `-m "full_model and MI325 and rocm"`. |
| Array (single platform) | `"mark": [{"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 2}}, "full_model", "diffusion"]` | Hardware marks from `hardware_marks(...)`; `num_cards` adds `cards_{n}` (including `cards_1` when omitted). String entries become extra pytest marks. |
| Array (multi-platform) | `"mark": [{"hardware_marks": {"res": {"cuda": "H100", "rocm": "MI325"}, "num_cards": 2}}, "full_model", "omni"]` | Same as above, but declares **multiple platforms** on one item. All platforms must share the same `num_cards` (int or identical dict values). Different counts need `@hardware_test` or one `hardware_marks()` per `pytest.param`. Filter examples: `-m "full_model and H100 and cards_2 and cuda"`, `-m "full_model and MI325 and cards_2 and rocm"`. |
Recommended for L4 perf cases:
@@ -157,7 +157,7 @@ pytest -s -v tests/dfx/perf/scripts/run_benchmark.py \
```
<!-- markdownlint-disable-next-line MD051 -->
See also [Markers for Tests](#markers-for-tests) for registered hardware markers (`H100`, `L4`, `cuda`, `distributed_cuda`, …).
See also [Markers for Tests](#markers-for-tests) for registered hardware markers (`H100`, `L4`, `cuda`, `cards_1`, …).
#### `server_params` Configuration
docs/contributing/ci/test_execution_guide.md
+4 -4
View File
@@ -88,7 +88,7 @@ Failed jobs: 1/2
```bash
cd tests
pytest -s -v test_xxxx.py --run-level=core_model
pytest -s -v -m "core_model and distributed_cuda and L4" --run-level=core_model
pytest -s -v -m "core_model and L4 and not cards_1" --run-level=core_model
```
=== "L3 level"
@@ -119,7 +119,7 @@ Failed jobs: 1/2
```bash
pytest -s -v test_xxxx.py --run-level=advanced_model
pytest -s -v -m "advanced_model and distributed_cuda and L4" --run-level=advanced_model
pytest -s -v -m "advanced_model and L4 and not cards_1" --run-level=advanced_model
```
=== "L4 level"
@@ -153,12 +153,12 @@ Failed jobs: 1/2
```bash
cd tests
pytest -s -v test_xxxx.py --run-level=full_model
pytest -s -v -m "full_model and distributed_cuda and L4" --run-level=full_model
pytest -s -v -m "full_model and L4 and not cards_1" --run-level=full_model
pytest -s -v -m "full_model and (omni or tts) and H100" --run-level=full_model
```
If you only want to run specific test cases on a particular platform, you can use:
```bash
pytest -s -v -m "full_model and distributed_cuda and L4" --run-level=full_model
pytest -s -v -m "full_model and L4 and not cards_1" --run-level=full_model
```
Note: ``run_benchmark.py`` and ``run_diffusion_benchmark.py`` accept an optional ``--test-config-file``. If omitted, each loads every ``*.json`` under ``tests/dfx/perf/tests/`` (omni/tts vs diffusion split by ``is_diffusion_perf_config``) and pytest ``-m`` filters by each case's JSON ``mark``:
```bash
docs/contributing/ci/test_writing_guide.md
+34 -47
View File
@@ -48,17 +48,12 @@ Defined in `pyproject.toml`:
| `MI325` | Tests that require MI325 GPU (AMD/ROCm) * |
| `A2` | Tests that require A2 NPU * |
| `A3` | Tests that require A3 NPU * |
| `distributed_cuda` | Tests that require multi cards on CUDA platform * |
| `distributed_rocm` | Tests that require multi cards on ROCm platform * |
| `distributed_npu` | Tests that require multi cards on NPU platform * |
| `skipif_cuda` | Skip if the num of CUDA cards is less than the required * |
| `skipif_rocm` | Skip if the num of ROCm cards is less than the required * |
| `skipif_npu` | Skip if the num of NPU cards is less than the required * |
| `cards_{n}` | Tests that require *n* accelerator cards (`cards_1` … `cards_8`); auto-added from `num_cards` * |
| `slow` | Slow tests (may skip in quick CI) |
| `benchmark` | Benchmark tests (decorator on runner test functions; perf JSON uses `full_model` + type marker instead) |
| `local_model` | Tests requiring local / non-HF-hub model weights |
\* Means those markers are auto-added by `@hardware_test` (parametrization decorator) or `hardware_marks` (only returning the list of marks for flexibility).
\* Means those markers are auto-added by `@hardware_test` (parametrization decorator) or `hardware_marks` (only returning the list of marks for flexibility). Do **not** write `pytest.mark.H100` / `pytest.mark.L4` (or other SKU names) by hand: `check-mark` rejects that so `cards_{n}` cannot be skipped.
#### Example usage for markers
@@ -81,25 +76,23 @@ def test_video_to_audio()
This decorator is intended to make hardware-aware, cross-platform test authoring easier and more robust for CI/CD environments. The `hardware_test` decorator in `vllm-omni/tests/helpers/mark.py` performs the following actions:
1. **Applies platform and resource markers**
Adds the appropriate pytest markers for each specified hardware platform (e.g., `cuda`, `rocm`, `xpu`, `npu`) and resource type (e.g., `L4`, `H100`, `MI325`, `B60`, `A2`, `A3`).
Adds the appropriate pytest markers for each specified hardware platform (e.g., `cuda`, `rocm`, `xpu`, `npu`) and resource type (e.g., `L4`, `H100`, `H200`, `B200`, `MI325`, `B60`, `A2`, `A3`).
```python
@pytest.mark.cuda
@pytest.mark.L4
```
2. **Handles multi-card (distributed) scenarios**
For tests requiring multiple cards, it automatically adds distributed markers such as `distributed_cuda`, `distributed_rocm`, or `distributed_npu`.
```python
@pytest.mark.distributed_cuda(num_cards=num_cards)
```
2. **Handles card-count filtering**
Adds a filterable marker `cards_{n}` (including `cards_1` when `num_cards` is omitted / 1). Use this with the SKU and platform, e.g. `-m "H100 and cards_4 and cuda"` vs `-m "H100 and cards_2 and cuda"`. Any multi-card CUDA test is `-m "cuda and not cards_1"`.
When `res` lists more than one platform **and** `num_cards` differs by platform, `@hardware_test` collects **one variant per platform** so those `cards_{n}` marks do not stack. `-m "H100 and cards_4 and cuda"` then selects only the CUDA/H100 4-card variant, not a ROCm 2-card sibling. The same count on every platform stays a single item.
3. **Supports flexible card requirements**
Accepts `num_cards` as either a single integer for all platforms or as a dictionary with per-platform values. If not specified, defaults to 1 card per platform.
4. **Integrates resource validation**
On CUDA, adds a skip marker (`skipif_cuda`) if the system does not have the required number of devices.
Support for `skipif_rocm` and `skipif_npu` will be implemented later.
On CUDA and XPU, when `num_cards > 1`, attaches pytest's built-in `skipif` if the machine has fewer devices than requested. This is not a named `skipif_*` marker and is not used for `-m` filtering.
5. **Works with pytest filtering**
Allows tests to be filtered and selected at runtime using standard pytest marker expressions (e.g., `-m "distributed_cuda and L4"`).
Allows tests to be filtered and selected at runtime using standard pytest marker expressions (e.g., `-m "H100 and cards_2"`, `-m "L4 and not cards_1"`).
##### Example usage for decorator
@@ -117,13 +110,14 @@ This decorator is intended to make hardware-aware, cross-platform test authoring
num_cards=2,
)
```
- `res` must be a dict; supported resources: CUDA (L4/H100), ROCm (MI325), XPU (B60), MUSA (S5000), NPU (A2/A3)
- `res` must be a dict. SKU values come from `pyproject.toml` markers tagged `[hardware-resource]` plus the platform key (`[cuda]`, `[rocm]`, `[xpu]`, `[npu]`, `[musa]`).
- `num_cards` can be int (all platforms) or dict (per platform); defaults to 1 when missing
- Distributed markers (`distributed_cuda`, `distributed_rocm`, `distributed_npu`) are auto-added for multi-card cases
- Filtering examples:
- CUDA only: `pytest -m "distributed_cuda and L4"`
- ROCm only: `pytest -m "distributed_rocm and MI325"`
- NPU only: `pytest -m "distributed_npu"`
- CUDA H100 4-card: `pytest -m "H100 and cards_4 and cuda"`
- CUDA H100 2-card: `pytest -m "H100 and cards_2 and cuda"`
- CUDA multi-card (any count): `pytest -m "L4 and not cards_1"`
- ROCm multi-card: `pytest -m "MI325 and not cards_1"`
- NPU multi-card: `pytest -m "npu and not cards_1"`
#### Function: `hardware_marks`
@@ -179,37 +173,30 @@ If you want to add support for a new platform (e.g., "tpu" for a new accelerator
[tool.pytest.ini_options]
markers = [
# ... existing markers ...
"tpu: Tests that require TPU device",
"TPU_V3: Tests that require TPU v3 hardware",
"distributed_tpu: Tests that require multiple TPU devices",
"tpu: [hardware-platform] Tests that require TPU device",
# Add [gpu] next to [hardware-platform] if this platform should also get pytest.mark.gpu.
"TPU_V3: [hardware-resource] [tpu] Tests that require TPU v3 hardware",
"cards_1: [hardware-cards] Tests that require 1 accelerator card",
"cards_2: [hardware-cards] Tests that require 2 accelerator cards",
]
```
`check-mark` treats `[hardware-platform]` names (including `cpu`/`gpu`) as allowed `pytest.mark.<name>` values. `hardware_test(res=...)` still rejects `cpu`/`gpu` as `res` keys.
2. **Implement a marker construction function for your platform** in `vllm-omni/tests/helpers/mark.py`:
```python
# In vllm-omni/tests/helpers/mark.py
def tpu_marks(*, res: str, num_cards: int):
test_platform = pytest.mark.tpu
if res == "TPU_V3":
test_resource = pytest.mark.TPU_V3
else:
raise ValueError(
f"Invalid TPU resource type: {res}. Supported: TPU_V3")
if num_cards == 1:
return [test_platform, test_resource]
else:
test_distributed = pytest.mark.distributed_tpu(num_cards=num_cards)
# Optionally: add skipif_tpu when implemented
return [test_platform, test_resource, test_distributed]
def _tpu_marks(*, res: str, num_cards: int):
_require_sku("tpu", res)
return [pytest.mark.tpu, getattr(pytest.mark, res), _cards_mark(num_cards)]
```
3. **Update `hardware_marks` to recognize your new platform**:
In the relevant place (see the `hardware_marks` implementation), add:
3. **Register the builder** in `_marks_for_platform`:
```python
if platform == "tpu":
marks = tpu_marks(res=resource, num_cards=cards)
builders = {
# ... existing platforms ...
"tpu": _tpu_marks,
}
```
(`hardware_test` calls `hardware_marks` internally, so both will pick up the change.)
(`hardware_test` / `hardware_marks` both go through `_marks_for_platform`.)
4. **(Recommended) Add a test using your new markers**:
```python
@hardware_test(
@@ -223,11 +210,11 @@ If you want to add support for a new platform (e.g., "tpu" for a new accelerator
**Summary**:
- Add pytest markers for your new platform/resources
- Implement a marker function (`xxx_marks`)
- Plug into `hardware_marks`
- You're done: tests using `@hardware_test` or `hardware_marks` with your platform now automatically get the correct markers, distribution, and isolation!
- Implement a private marker function (`_xxx_marks`)
- Register it in `_marks_for_platform`
- You're done: tests using `@hardware_test` or `hardware_marks` with your platform now automatically get the correct markers and `cards_{n}` filtering.
See code in `vllm-omni/tests/helpers/mark.py` for existing examples (`cuda_marks`, `rocm_marks`, `npu_marks`).
See code in `vllm-omni/tests/helpers/mark.py` for existing examples (`_cuda_marks`, `_rocm_marks`, `_npu_marks`).
## Test case style
docs/contributing/model/adding_tts_model.md
+2 -2
View File
@@ -438,7 +438,7 @@ selects the scheduler and worker family:
Key topology fields belong to `StagePipelineConfig`:
| Field | Description |
|-------|-------------|
| ------- | ------------- |
| `model_stage` | Logical stage name (`ar_stage`, `decoder`, etc.). |
| `execution_type` | Runtime family for this stage. |
| `input_sources` | Upstream stage IDs that provide input. |
@@ -960,7 +960,7 @@ Common failures and fixes:
| `check-forbidden-imports` | Stdlib `re`/`base64`, pickle, Hugging Face Hub API, or direct Triton/TileLang | `import regex as re` and `pybase64`; do not grow the allowlist without review |
| `check-torch-cuda-call` | New `torch.cuda.*` call site | Use `current_omni_platform`; do not grow `ALLOWED_FILES` without review |
| `check-tts-adapter-migration` | New `_tts_model_type` branch in `serving_speech.py` | Put logic in `tts_adapters/`; lower `MAX_MODEL_TYPE_BRANCHES` when removing branches |
| `check-test-ci-coverage` | Test file missing level or hardware mark | Add `core_model`/… plus `cpu`/`cuda`/`hardware_test(` |
| `check-mark` | Test file missing level/hardware mark, or direct `pytest.mark.H100` | Add `core_model`/… plus `cpu`/`cuda`/`hardware_test(`; SKU marks only via helpers |
| `shellcheck` | Native `shellcheck` missing, or script warning | Install via apt/dnf/brew (or `shellcheck.exe` on Windows). See [Linting](../README.md#linting) |
| `mypy-3.10` / `markdownlint-cli2` / `check-buildkite` | Types, docs markdown, or Buildkite YAML | See [Linting](../README.md#linting) |
docs/design/feature/vae_parallel.md
+69 -49
View File
@@ -22,59 +22,60 @@ We use **Qwen-Image** as the reference implementation for decode parallel, and *
**VAE Patch Parallelism** is an acceleration technique for both **encoding** and **decoding**. Instead of processing the entire tensor at once, the tensor is:
+ Split into multiple spatial tiles
- Split into multiple spatial tiles
+ Distributed across multiple ranks
- Distributed across multiple ranks
+ Encoded/Decoded in parallel
- Encoded/Decoded in parallel
+ Merged to reconstruct the final output
- Merged to reconstruct the final output
This approach:
+ Distributes computation across multiple devices
- Distributes computation across multiple devices
+ Reduces peak memory usage per device
- Reduces peak memory usage per device
+ Accelerates encoding/decoding latency
- Accelerates encoding/decoding latency
### When to Use Encode vs Decode Parallel
| Operation | Use Case | Example |
|-----------|----------|---------|
| **Decode Parallel** | Text-to-Image, Text-to-Video | Latent → Image/Video |
| **Encode Parallel** | Image-to-Video (I2V) | Image → Latent (for conditioning) |
| Operation | Use Case | Example |
| ------------------- | ---------------------------- | --------------------------------- |
| **Decode Parallel** | Text-to-Image, Text-to-Video | Latent → Image/Video |
| **Encode Parallel** | Image-to-Video (I2V) | Image → Latent (for conditioning) |
### Architecture
We introduce **DistributedVaeExecutor** as the core component responsible for distributed VAE encoding/decoding.
The executor is model-agnostic and accepts three function parameters:
+ split – Partition the latent into tiles
- split – Partition the latent into tiles
+ exec – Decode a single tile
- exec – Decode a single tile
+ merge – Combine decoded tiles into the final output
- merge – Combine decoded tiles into the final output
#### Execution Flow
+ Call split(z) to generate a list of TileTask and a GridSpec
- Call split(z) to generate a list of TileTask and a GridSpec
+ Dispatch tasks across ranks using workload-based balancing
- Dispatch tasks across ranks using workload-based balancing
+ Each rank executes exec(task) on its assigned tiles
- Each rank executes exec(task) on its assigned tiles
+ Gather decoded tile results to rank 0
- Gather decoded tile results to rank 0
+ Rank 0 performs merge(...)
- Rank 0 performs merge(...)
+ (Optional) Broadcast final result to all ranks
- (Optional) Broadcast final result to all ranks
This design separates:
+ Distributed execution logic
- Distributed execution logic
+ Model-specific tiling and merging logic
- Model-specific tiling and merging logic
#### Why split / exec / merge is necessary?
@@ -82,23 +83,23 @@ The latent tensor cannot be arbitrarily partitioned.
During decoding:
+ Each output pixel may depend on neighboring pixels
- Each output pixel may depend on neighboring pixels
+ The receptive field is model-dependent
- The receptive field is model-dependent
Therefore:
+ Tiles must include overlap
- Tiles must include overlap
+ Merge must perform blending to avoid seams
- Merge must perform blending to avoid seams
## Step-by-Step Implementation (Decode)
### Step 1: Implement DistributedAutoencoderKLQwenImage
`QwenImagePipeline` use `AutoencoderKLQwenImage` for vae, so implement a distributed version:
```
```python
class DistributedAutoencoderKLQwenImage(AutoencoderKLQwenImage, DistributedVaeMixin):
@classmethod
def from_pretrained(cls, *args: Any, **kwargs: Any):
@@ -106,28 +107,35 @@ class DistributedAutoencoderKLQwenImage(AutoencoderKLQwenImage, DistributedVaeMi
model.init_distributed()
return model
```
**Key points**:
+ Inherit both AutoencoderKLQwenImage and DistributedVaeMixin
+ Call init_distributed() after loading weights
- Inherit both AutoencoderKLQwenImage and DistributedVaeMixin
- Call init_distributed() after loading weights
### Step 2: Implement split/exec/merge
Reuse `AutoencoderKLQwenImage.tiled_decode` logic and divide it into three stages. And we need return tiles with `GridSpec` and `TileTask`:
```
```python
class GridSpec:
split_dims: tuple[int, ...] # Tensor dimensions being split (e.g., (2, 3) for (B, C, H, W))
grid_shape: tuple[int, ...] # Tile grid layout (num_rows, num_cols)
tile_spec: dict = field(default_factory=dict) # Metadata required for merging
output_dtype: torch.dtype | None = None # Final output dtype
```
```
```python
class TileTask:
tile_id: int # task id
grid_coord: tuple[int, ...] # Tile position in grid
tensor: torch.Tensor | list[torch.Tensor] # The tile tensor
workload: int | float = 1 # Used for load balancing (e.g., tile area)
```
And tiled base split/exec/merge as follow:
```
```python
def tile_split(self, z: torch.Tensor) -> tuple[list[TileTask], GridSpec]:
# mostly copy from AutoencoderKL
_, _, num_frames, height, width = z.shape
@@ -210,11 +218,14 @@ def tile_merge(self, coord_tensor_map: dict[tuple[int, ...], torch.Tensor], grid
```
### Step 3: Override tiled_decode
We need to override tiled_decode, the main logic is:
+ check distributed is enabled
+ select split/exec/merge
+ Invoke self.distributed_executor.execute to decode
```
- check distributed is enabled
- select split/exec/merge
- Invoke self.distributed_executor.execute to decode
```python
def tiled_decode(self, z: torch.Tensor, return_dict: bool = True):
if not self.is_distributed_enabled():
return super().tiled_decode(z, return_dict=return_dict)
@@ -230,11 +241,14 @@ def tiled_decode(self, z: torch.Tensor, return_dict: bool = True):
return DecoderOutput(sample=result)
```
`broadcast_result` is set to True or False depending on the model; when enabled, the result will be used even on ranks other than 0.
### Step 4: Modify Pipeline
Change vae model from AutoencoderKLQwenImage to DistributedAutoencoderKLQwenImage
```
```diff
class YourModelPipeline(nn.Module):
def __init__(
self,
@@ -259,8 +273,8 @@ For models that require VAE encoding (e.g., Image-to-Video), you can also parall
Similar to decode, split the input tensor into tiles. Key considerations:
+ **Patchify handling**: If the model uses `patch_size`, scale tile parameters accordingly
+ **Temporal chunking**: Video VAEs may have temporal compression (e.g., 4x)
- **Patchify handling**: If the model uses `patch_size`, scale tile parameters accordingly
- **Temporal chunking**: Video VAEs may have temporal compression (e.g., 4x)
```python
def encode_tile_split(self, x: torch.Tensor) -> tuple[list[TileTask], GridSpec]:
@@ -406,23 +420,28 @@ def tiled_encode(self, x: torch.Tensor) -> torch.Tensor:
**Key differences from decode parallel:**
| Aspect | Decode Parallel | Encode Parallel |
|--------|-----------------|-----------------|
| -------- | ----------------- | ----------------- |
| `broadcast_result` | Often `False` (only rank 0 needs output) | `True` (all ranks need latents for diffusion) |
| Patchify | Applied in merge (unpatchify) | Handled by parent `_encode()` before `tiled_encode()` |
| Temporal chunking | Frame-by-frame | Chunk-based (e.g., 1 + 4n frames) |
## Testing
Verify numerical consistency between:
+ vae_patch_parallel_size = 1
+ vae_patch_parallel_size = N
Verify numerical consistency between:
- vae_patch_parallel_size = 1
- vae_patch_parallel_size = N
Example:
```python
torch.allclose(output_1, output_n, atol=1e-5)
```
Testing requirements:
+ Fix random seed
+ Use identical tiling strategy
- Fix random seed
- Use identical tiling strategy
```python
m = Omni(
@@ -434,6 +453,7 @@ m = Omni(
),
)
```
When vae_patch_parallel_size is larger than the DiT world size, it will automatically fall back to using the DiT world size instead.
## Reference Implementations
@@ -441,7 +461,7 @@ When vae_patch_parallel_size is larger than the DiT world size, it will automati
Complete examples in the codebase:
| Model | Path | Decode Parallel | Encode Parallel |
|-------|------|-----------------|-----------------|
| ------- | ------ | ----------------- | ----------------- |
| **Z-Image** | `vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl.py` | ✅ | ❌ |
| **Wan2.2** | `vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_wan.py` | ✅ | ✅ |
| **Qwen-Image** | `vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_qwenimage.py` | ✅ | ❌ |
@@ -456,7 +476,7 @@ The tile-parallel executor above assigns independent spatial **tiles** to ranks.
### How it differs from tile parallel
| Aspect | Tile parallel (`"tile"`) | Spatially-sharded (`"spatial_shard_height"`/`"spatial_shard_width"`) |
|--------|--------------------------|-----------------------------------------------|
| -------- | -------------------------- | ----------------------------------------------- |
| Unit of work | Independent overlapping tiles | A single global feature map sharded along H or W |
| Cross-rank communication | Gather tiles to rank 0, stitch + blend | Per-conv **halo exchange** of boundary rows/cols (P2P) |
| Output assembly | Blend overlapping tiles | All-gather shards on rank 0, trim padding (matches `broadcast_result=False`) |
@@ -481,7 +501,7 @@ serve.py (--vae-parallel-mode) / OmniEngineArgs
### Notes
- The decoder is patched **in place** the first time spatial-shard decode runs and is bound to a single split dimension for the lifetime of the VAE instance.
- Numerical correctness vs. single-GPU decode is covered by `tests/diffusion/distributed/test_wan_spatial_shard.py::test_spatial_shard_decode_matches_reference` (multi-GPU, nightly `full_model` + `distributed_cuda`).
- Numerical correctness vs. single-GPU decode is covered by `tests/diffusion/distributed/test_wan_spatial_shard.py::test_spatial_shard_decode_matches_reference` (multi-GPU, nightly `full_model` + `cards_2`).
---
pyproject.toml
+37 -34
View File
@@ -248,11 +248,11 @@ addopts = [
"--dist=loadgroup"
]
markers = [
# ci/cd required
"core_model: L1&L2 tests (run in each PR)",
"advanced_model: L3 level tests (run on each merge)",
"full_model: L4 level tests (run nightly)",
"local_model: tests (run locally)",
# ci/cd required — tag with [ci-level] for tests.helpers.mark.get_level_markers().
"core_model: [ci-level] L1&L2 tests (run in each PR)",
"advanced_model: [ci-level] L3 level tests (run on each merge)",
"full_model: [ci-level] L4 level tests (run nightly)",
"local_model: [ci-level] tests (run locally)",
# function module markers
"diffusion: Diffusion model tests",
"omni: Omni model tests",
@@ -262,38 +262,41 @@ markers = [
"sp: Sequence parallelism tests",
"example: Doc example code tests",
# platform markers
"cpu: Tests that run on CPU",
"gpu: Tests that run on GPU (auto-added)",
"cuda: Tests that run on CUDA (auto-added)",
"rocm: Tests that run on AMD/ROCm (auto-added)",
"xpu: Tests that run on XPU (auto-added)",
"npu: Tests that run on NPU/Ascend (auto-added)",
"musa: Tests that run on MUSA/Moore Threads (auto-added)",
# Tag with [hardware-platform] for tests.helpers.mark.get_supported_platforms().
# Add [gpu] on accelerator platforms that should also get pytest.mark.gpu.
# cpu/gpu are pytest marks; hardware_test(res=...) rejects them as keys.
"cpu: [hardware-platform] Tests that run on CPU",
"gpu: [hardware-platform] Tests that run on GPU (auto-added)",
"cuda: [hardware-platform] [gpu] Tests that run on CUDA (auto-added)",
"rocm: [hardware-platform] [gpu] Tests that run on AMD/ROCm (auto-added)",
"xpu: [hardware-platform] [gpu] Tests that run on XPU (auto-added)",
"npu: [hardware-platform] Tests that run on NPU/Ascend (auto-added)",
"musa: [hardware-platform] [gpu] Tests that run on MUSA/Moore Threads (auto-added)",
# specified computation resources marks (auto-added)
# Tag with [hardware-resource] for tests.helpers.mark.get_hardware_mark_list().
"H100: [hardware-resource] Tests that require H100 GPU",
"H800: [hardware-resource] Tests that require H800 GPU",
"H200: [hardware-resource] Tests that require H200 GPU",
"L4: [hardware-resource] Tests that require L4 GPU",
"B200: [hardware-resource] Tests that require B200 GPU",
"MI325: [hardware-resource] Tests that require MI325 GPU (AMD/ROCm)",
"B60: [hardware-resource] Tests that require Intel Arc Pro B60 XPU",
"S5000: [hardware-resource] Tests that require S5000 GPU (Moore Threads/MUSA)",
"A2: [hardware-resource] Tests that require A2 NPU",
"A3: [hardware-resource] Tests that require A3 NPU",
"distributed_cuda: Tests that require multi cards on CUDA platform",
"distributed_rocm: Tests that require multi cards on ROCm platform",
"distributed_xpu: Tests that require multi cards on XPU platform",
"distributed_npu: Tests that require multi cards on NPU platform",
"distributed_musa: Tests that require multi cards on MUSA platform",
"skipif_cuda: Skip if the num of CUDA cards is less than the required",
"skipif_rocm: Skip if the num of ROCm cards is less than the required",
"skipif_xpu: Skip if the num of XPU cards is less than the required",
"skipif_npu: Skip if the num of NPU cards is less than the required",
"skipif_musa: Skip if the num of MUSA cards is less than the required",
# [hardware-resource] + [cuda]/[rocm]/[xpu]/[npu]/[musa] for hardware_test(res=...).
# Do not put [gpu] on SKU lines: that would make gpu a res key.
"H100: [hardware-resource] [cuda] Tests that require H100 GPU",
"H800: [hardware-resource] [cuda] Tests that require H800 GPU",
"H200: [hardware-resource] [cuda] Tests that require H200 GPU",
"L4: [hardware-resource] [cuda] Tests that require L4 GPU",
"B200: [hardware-resource] [cuda] Tests that require B200 GPU",
"MI325: [hardware-resource] [rocm] Tests that require MI325 GPU (AMD/ROCm)",
"B60: [hardware-resource] [xpu] Tests that require Intel Arc Pro B60 XPU",
"S5000: [hardware-resource] [musa] Tests that require S5000 GPU (Moore Threads/MUSA)",
"A2: [hardware-resource] [npu] Tests that require A2 NPU",
"A3: [hardware-resource] [npu] Tests that require A3 NPU",
# Tag with [hardware-cards] for tests.helpers.mark.get_supported_card_counts().
"cards_1: [hardware-cards] Tests that require 1 accelerator card",
"cards_2: [hardware-cards] Tests that require 2 accelerator cards",
"cards_3: [hardware-cards] Tests that require 3 accelerator cards",
"cards_4: [hardware-cards] Tests that require 4 accelerator cards",
"cards_5: [hardware-cards] Tests that require 5 accelerator cards",
"cards_6: [hardware-cards] Tests that require 6 accelerator cards",
"cards_7: [hardware-cards] Tests that require 7 accelerator cards",
"cards_8: [hardware-cards] Tests that require 8 accelerator cards",
# more detailed markers
"sp: Sequence parallelism tests (multi-GPU)",
"slow: Slow tests (may skip in quick CI)",
"slow: [ci-level] Slow tests (may skip in quick CI)",
"benchmark: Benchmark tests",
"xdist: xdist-parallelizable tests",
]
tests/conftest.py
+2 -2
View File
@@ -25,9 +25,9 @@ pytest_plugins = (
"tests.helpers.fixtures.clean",
"tests.helpers.fixtures.log",
"tests.helpers.fixtures.media",
"tests.helpers.fixtures.run_args",
"tests.helpers.fixtures.pytest_collection",
"tests.helpers.fixtures.pytest_run_args",
"tests.helpers.fixtures.runtime",
"tests.helpers.fixtures.xdist",
)
tests/dfx/perf/tests/test_runner_metadata.py
+4 -1
View File
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Tests for DFX runner metadata field exclusion."""
import json
@@ -105,7 +108,7 @@ def test_resolve_pytest_marks_hardware_dict_with_extra():
assert "H100" in names
assert "cuda" in names
assert "gpu" in names
assert "distributed_cuda" in names
assert "cards_2" in names
assert "full_model" in names
assert "diffusion" in names
assert "local_model" in names
tests/diffusion/distributed/test_sp_plan_hooks.py
+4 -2
View File
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Tests for the Sequence Parallelism (SP) framework.
These tests verify the SP plan mechanism and hooks work correctly without
@@ -17,6 +17,7 @@ import pytest
import torch
import torch.nn as nn
from tests.helpers.mark import hardware_marks
from vllm_omni.diffusion.distributed.sp_plan import (
SequenceParallelInput,
SequenceParallelOutput,
@@ -419,7 +420,6 @@ class TestHookRegistration:
# =============================================================================
@pytest.mark.L4
class TestModelSpPlans:
"""Test that model _sp_plan definitions are valid.
@@ -427,6 +427,8 @@ class TestModelSpPlans:
May require GPU for model imports.
"""
pytestmark = hardware_marks(res={"cuda": "L4"})
def test_zimage_transformer_sp_plan(self):
"""Test ZImageTransformer2DModel _sp_plan structure.
tests/e2e/features/fullduplex/test_personaplex_elastic.py
+10 -5
View File
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""GPU e2e for PersonaPlex elastic batching (bit-exact isolation + recycle).
Greedy decoding makes every cross-slot leak observable as a bit difference:
@@ -24,12 +24,17 @@ from __future__ import annotations
import numpy as np
import pytest
from tests.helpers.mark import hardware_marks
torch = pytest.importorskip("torch")
# slow + H100 + omni routes this module into the weekly Buildkite E2E group
# ("Omni · H100" in .buildkite/cuda/test-weekly.yml: -m "slow and H100 and
# omni" --run-level full_model); the ready/merge CPU sweeps exclude it by mark.
pytestmark = [pytest.mark.full_model, pytest.mark.omni, pytest.mark.slow, pytest.mark.H100]
# slow + H100 + omni + cards_1 routes this module into weekly
# "Omni · H100 · Single-GPU" (test-weekly.yml).
pytestmark = [
pytest.mark.omni,
pytest.mark.slow,
*hardware_marks(res={"cuda": "H100"}),
]
if not torch.cuda.is_available(): # noqa: E402
pytest.skip("needs CUDA", allow_module_level=True)
tests/e2e/offline_inference/test_hunyuanimage3_i2t_expansion.py
+4 -4
View File
@@ -1,13 +1,13 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Smoke test for HunyuanImage-3.0 Image-to-Text (I2T) pipeline."""
from collections.abc import Generator
import pytest
import torch
from PIL import Image
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from tests.helpers.stage_config import get_deploy_config_path
from vllm_omni import Omni
@@ -44,7 +44,7 @@ EXPECTED_PREFIX_TOKEN_IDS: list[int] = [
# Decoded form, kept only for human-readable assertion messages.
EXPECTED_PREFIX_TEXT = "The image is a solid, uniform green color with no variations, objects, or details present. It"
pytestmark = [pytest.mark.full_model, pytest.mark.diffusion, pytest.mark.cuda]
pytestmark = [pytest.mark.full_model, pytest.mark.diffusion]
@pytest.fixture(scope="module")
@@ -57,7 +57,7 @@ def omni() -> Generator[Omni, None, None]:
yield runner.omni
@pytest.mark.skipif(torch.accelerator.device_count() < 4, reason="Need at least 4 CUDA GPUs.")
@hardware_test(res={"cuda": "H100"}, num_cards=4)
def test_i2t_generates_text(omni: Omni) -> None:
"""Verify I2T output's first 20 token IDs match the HF greedy baseline."""
# Solid-color image keeps the input self-contained and reproducible.
tests/e2e/offline_inference/test_krea2_expansion.py
+3 -3
View File
@@ -91,8 +91,8 @@ def test_krea2_cache_dit(offline_client: OfflineOmniClient) -> None:
def test_krea2_hsdp(offline_client: OfflineOmniClient) -> None:
"""Exercise Hybrid Sharded Data Parallel (HSDP) weight sharding on the DiT across 2 GPUs.
Validates the docs' HSDP-support claim for Krea 2. Scheduled on the multi-GPU (distributed_cuda)
shard by ``num_cards=2``; skipped automatically on boxes with fewer than 2 CUDA devices.
Validates the docs' HSDP-support claim for Krea 2. Scheduled on the multi-GPU
shard by ``num_cards=2`` (``not cards_1``); skipped automatically on boxes with fewer than 2 CUDA devices.
"""
offline_client.send_diffusion_request({"model": MODEL, "prompt": PROMPT, "sampling_params": _sampling()})
@@ -111,7 +111,7 @@ def test_krea2_vae_patch_parallel(offline_client: OfflineOmniClient) -> None:
"""Exercise VAE patch-parallel (decode) across 2 GPUs.
Validates the docs' VAE-patch-parallel (decode) support claim for Krea 2. Scheduled on the
multi-GPU (distributed_cuda) shard by ``num_cards=2``; skipped automatically on boxes with
multi-GPU shard by ``num_cards=2`` (``not cards_1``); skipped automatically on boxes with
fewer than 2 CUDA devices.
"""
offline_client.send_diffusion_request({"model": MODEL, "prompt": PROMPT, "sampling_params": _sampling()})
tests/e2e/online_serving/test_mimo_audio_expansion.py
+1 -1
View File
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""
E2E expansion tests for MiMo-Audio online serving (nightly Omni · Function Test with H100).
E2E expansion tests for MiMo-Audio online serving (nightly Omni · Function Test with H100 · Single-GPU).
"""
import os
tests/helpers/fixtures/pytest_collection.py
+30
View File
@@ -0,0 +1,30 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Pytest collection-phase hooks.
All ``pytest_collection_modifyitems`` policies live here so their order is
explicit. Add a named helper and call it from the hook; do not register a
second ``pytest_collection_modifyitems`` in another plugin.
"""
from __future__ import annotations
import pytest
def pytest_collection_modifyitems(config, items):
_pin_xdist_serial(config, items)
def _pin_xdist_serial(config, items):
"""Pin unmarked tests to one xdist worker.
Only tests explicitly marked ``pytest.mark.xdist`` run concurrently.
Others are pinned to a single xdist worker.
"""
if not config.pluginmanager.hasplugin("xdist"):
return
for item in items:
if not item.get_closest_marker("xdist"):
item.add_marker(pytest.mark.xdist_group(name="serial"))
tests/helpers/fixtures/run_args.py → tests/helpers/fixtures/pytest_run_args.py
+7
View File
@@ -1,3 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Pytest CLI options and the fixtures that expose them."""
from __future__ import annotations
import pytest
tests/helpers/fixtures/runtime.py
+12
View File
@@ -24,6 +24,18 @@ if TYPE_CHECKING:
omni_fixture_lock = threading.Lock()
@pytest.fixture
def _normalized_hardware_marks(request: pytest.FixtureRequest):
"""Dummy carrier for mixed-``num_cards`` ``@hardware_test`` mark normalization.
Pytest has no API to clone an item and only change marks. ``@hardware_test``
therefore parametrizes this fixture (``indirect=True``) so each collected
item gets one platform's SKU and ``cards_{n}`` without changing test
signatures. The returned platform id is unused by test bodies.
"""
return request.param
@pytest.fixture(scope="function")
def omni_server_function(
request: pytest.FixtureRequest,
tests/helpers/fixtures/xdist.py
-14
View File
@@ -1,14 +0,0 @@
"""
Only tests explicitly marked ``pytest.mark.xdist`` run concurrently.
Other test are pinned to a single xdist worker.
"""
import pytest
def pytest_collection_modifyitems(config, items):
if not config.pluginmanager.hasplugin("xdist"):
return
for item in items:
if not item.get_closest_marker("xdist"):
item.add_marker(pytest.mark.xdist_group(name="serial"))
tests/helpers/mark.py
+251 -126
View File
@@ -1,19 +1,33 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Pytest marks and decorators for hardware / resource selection (CUDA, ROCm, …)."""
"""Pytest marks and decorators for hardware / resource selection (CUDA, ROCm, …).
Module import is stdlib-only so ``tools/pre_commit/check_test_marks.py`` can load
this file without pytest or vllm. Decorator helpers import pytest lazily.
"""
from __future__ import annotations
import re
from functools import lru_cache
from pathlib import Path
from typing import TYPE_CHECKING
import pytest
from vllm.platforms import current_platform
if TYPE_CHECKING:
import pytest
# Marker description tag in ``pyproject.toml`` ``tool.pytest.ini_options.markers``.
# Example: ``"H100: [hardware-resource] Tests that require H100 GPU"``.
# Marker description tags in ``pyproject.toml`` ``tool.pytest.ini_options.markers``.
# Example: ``"H100: [hardware-resource] [cuda] Tests that require H100 GPU"``.
_HARDWARE_RESOURCE_MARKER_TAG = "[hardware-resource]"
_HARDWARE_CARDS_MARKER_TAG = "[hardware-cards]"
_HARDWARE_PLATFORM_MARKER_TAG = "[hardware-platform]"
_CI_LEVEL_MARKER_TAG = "[ci-level]"
# On ``[hardware-platform]`` lines (not SKUs): also attach ``pytest.mark.gpu``.
_GPU_DEVICE_CLASS_TAG = "[gpu]"
_CARDS_MARK_NAME_RE = re.compile(r"^cards_(\d+)$")
# Indirect fixture in ``tests.helpers.fixtures.runtime``: mixed-count ``@hardware_test``
# parametrizes it so each collected item carries one platform's SKU and ``cards_{n}``.
HARDWARE_MARK_NORMALIZATION_FIXTURE = "_normalized_hardware_marks"
def _repo_root() -> Path:
@@ -34,23 +48,16 @@ def _load_pyproject() -> dict:
return {}
@lru_cache(maxsize=1)
def get_hardware_mark_list() -> frozenset[str]:
"""Return hardware SKU marker names from ``pyproject.toml``.
A marker is treated as a hardware resource marker when its description
contains the ``[hardware-resource]`` tag, e.g.::
"H100: [hardware-resource] Tests that require H100 GPU"
"""
@lru_cache(maxsize=16)
def _marker_names_with_tag(tag: str) -> frozenset[str]:
"""Return pytest marker names whose description contains ``tag``."""
data = _load_pyproject()
entries = data.get("tool", {}).get("pytest", {}).get("ini_options", {}).get("markers", [])
if not entries:
# Fallback when TOML parsing fails: scan tagged marker lines.
text = (_repo_root() / "pyproject.toml").read_text(encoding="utf-8")
return frozenset(
re.findall(
rf'^\s*"([A-Za-z0-9_]+)\s*:\s*{re.escape(_HARDWARE_RESOURCE_MARKER_TAG)}',
rf'^\s*"([A-Za-z0-9_]+)\s*:.*{re.escape(tag)}',
text,
flags=re.M,
)
@@ -59,7 +66,7 @@ def get_hardware_mark_list() -> frozenset[str]:
names: set[str] = set()
for entry in entries:
text = str(entry)
if _HARDWARE_RESOURCE_MARKER_TAG not in text:
if tag not in text:
continue
name = text.split(":", 1)[0].strip()
if name:
@@ -67,138 +74,256 @@ def get_hardware_mark_list() -> frozenset[str]:
return frozenset(names)
def cuda_marks(*, res: str, num_cards: int):
@lru_cache(maxsize=1)
def get_hardware_mark_list() -> frozenset[str]:
"""Return hardware SKU marker names tagged ``[hardware-resource]``."""
return _marker_names_with_tag(_HARDWARE_RESOURCE_MARKER_TAG)
@lru_cache(maxsize=1)
def get_supported_card_counts() -> frozenset[int]:
"""Return ``n`` for each ``cards_{n}`` marker tagged ``[hardware-cards]``."""
counts: set[int] = set()
for name in _marker_names_with_tag(_HARDWARE_CARDS_MARKER_TAG):
matched = _CARDS_MARK_NAME_RE.fullmatch(name)
if matched:
counts.add(int(matched.group(1)))
return frozenset(counts)
@lru_cache(maxsize=1)
def get_supported_platforms() -> frozenset[str]:
"""Return platform marker names tagged ``[hardware-platform]``.
Includes ``cpu`` / ``gpu`` (device-class marks) and accelerator keys.
``hardware_marks`` / ``hardware_test`` ``res`` dicts reject ``cpu`` / ``gpu``.
"""
return _marker_names_with_tag(_HARDWARE_PLATFORM_MARKER_TAG)
@lru_cache(maxsize=1)
def get_level_markers() -> frozenset[str]:
"""Return CI level marker names tagged ``[ci-level]`` (``core_model``, ``slow``, …)."""
return _marker_names_with_tag(_CI_LEVEL_MARKER_TAG)
@lru_cache(maxsize=8)
def _skus_for_platform(platform: str) -> frozenset[str]:
"""SKU names tagged ``[hardware-resource]`` and ``[platform]`` (e.g. ``[cuda]``)."""
return get_hardware_mark_list() & _marker_names_with_tag(f"[{platform}]")
def _require_sku(platform: str, res: str) -> None:
allowed = _skus_for_platform(platform)
if res not in allowed:
supported = ", ".join(sorted(allowed)) or f"(none tagged [hardware-resource] [{platform}])"
raise ValueError(f"Invalid {platform} resource type: {res}. Supported: {supported}")
def _cards_mark(num_cards: int) -> pytest.MarkDecorator:
"""Return ``cards_{n}`` so collection can filter with ``-m cards_2`` (etc.).
Multi-card selection is ``not cards_1`` (or an explicit ``cards_2`` / ``cards_4``).
"""
import pytest
if not isinstance(num_cards, int) or isinstance(num_cards, bool) or num_cards < 1:
raise ValueError(f"num_cards must be a positive int, got {num_cards!r}")
if num_cards not in get_supported_card_counts():
counts = get_supported_card_counts()
supported = ", ".join(f"cards_{n}" for n in sorted(counts))
supported = supported or "(none tagged [hardware-cards])"
raise ValueError(f"num_cards={num_cards} has no registered marker. Supported: {supported}.")
return getattr(pytest.mark, f"cards_{num_cards}")
def _cuda_marks(*, res: str, num_cards: int):
import pytest
from vllm.platforms import current_platform
_require_sku("cuda", res)
test_resource = getattr(pytest.mark, res)
test_platform_detail = pytest.mark.cuda
if res == "L4":
test_resource = pytest.mark.L4
elif res == "H100":
test_resource = pytest.mark.H100
elif res == "H800":
test_resource = pytest.mark.H800
else:
raise ValueError(f"Invalid CUDA resource type: {res}. Supported: L4, H100, H800")
marks = [test_resource, test_platform_detail]
if num_cards == 1:
marks = [test_resource, test_platform_detail, _cards_mark(num_cards)]
if num_cards == 1 or not current_platform.is_cuda():
return marks
test_distributed = pytest.mark.distributed_cuda(num_cards=num_cards)
if not current_platform.is_cuda():
return marks + [test_distributed]
test_skipif = pytest.mark.skipif(
current_platform.device_count() < num_cards,
reason=f"Need at least {num_cards} CUDA GPUs to run the test.",
)
return marks + [test_distributed, test_skipif]
return marks + [
pytest.mark.skipif(
current_platform.device_count() < num_cards,
reason=f"Need at least {num_cards} CUDA GPUs to run the test.",
)
]
def rocm_marks(*, res: str, num_cards: int):
test_platform_detail = pytest.mark.rocm
if res == "MI325":
test_resource = pytest.mark.MI325
else:
raise ValueError(f"Invalid ROCm resource type: {res}. Supported: MI325")
marks = [test_resource, test_platform_detail]
if num_cards == 1:
def _rocm_marks(*, res: str, num_cards: int):
import pytest
_require_sku("rocm", res)
return [getattr(pytest.mark, res), pytest.mark.rocm, _cards_mark(num_cards)]
def _xpu_marks(*, res: str, num_cards: int):
import pytest
from vllm.platforms import current_platform
_require_sku("xpu", res)
marks = [getattr(pytest.mark, res), pytest.mark.xpu, _cards_mark(num_cards)]
if num_cards == 1 or not current_platform.is_xpu():
return marks
test_distributed = pytest.mark.distributed_rocm(num_cards=num_cards)
return marks + [test_distributed]
return marks + [
pytest.mark.skipif(
current_platform.device_count() < num_cards,
reason=f"Need at least {num_cards} XPUs to run the test.",
)
]
def xpu_marks(*, res: str, num_cards: int):
test_platform_detail = pytest.mark.xpu
if res == "B60":
test_resource = pytest.mark.B60
else:
raise ValueError(f"Invalid XPU resource type: {res}. Supported: B60")
marks = [test_resource, test_platform_detail]
if num_cards == 1:
return marks
test_distributed = pytest.mark.distributed_xpu(num_cards=num_cards)
def _musa_marks(*, res: str, num_cards: int):
import pytest
if not current_platform.is_xpu():
return marks + [test_distributed]
test_skipif = pytest.mark.skipif(
current_platform.device_count() < num_cards,
reason=f"Need at least {num_cards} XPUs to run the test.",
)
return marks + [test_distributed, test_skipif]
_require_sku("musa", res)
return [getattr(pytest.mark, res), pytest.mark.musa, _cards_mark(num_cards)]
def musa_marks(*, res: str, num_cards: int):
test_platform_detail = pytest.mark.musa
if res == "S5000":
test_resource = pytest.mark.S5000
else:
raise ValueError(f"Invalid MUSA resource type: {res}. Supported: S5000")
marks = [test_resource, test_platform_detail]
if num_cards == 1:
return marks
test_distributed = pytest.mark.distributed_musa(num_cards=num_cards)
return marks + [test_distributed]
@lru_cache(maxsize=1)
def _gpu_res_platforms() -> frozenset[str]:
"""Accelerator platforms tagged ``[hardware-platform] [gpu]`` (not the ``gpu`` mark)."""
return (get_supported_platforms() & _marker_names_with_tag(_GPU_DEVICE_CLASS_TAG)) - {"gpu"}
def gpu_marks(*, res: str, num_cards: int):
test_platform = pytest.mark.gpu
if res in ("L4", "H100", "H800"):
return [test_platform] + cuda_marks(res=res, num_cards=num_cards)
if res == "MI325":
return [test_platform] + rocm_marks(res=res, num_cards=num_cards)
if res == "B60":
return [test_platform] + xpu_marks(res=res, num_cards=num_cards)
if res == "S5000":
return [test_platform] + musa_marks(res=res, num_cards=num_cards)
raise ValueError(f"Invalid resource type: {res}. Supported: L4, H100, H800, MI325, B60, S5000")
def _npu_marks(*, res: str, num_cards: int):
import pytest
_require_sku("npu", res)
return [pytest.mark.npu, getattr(pytest.mark, res), _cards_mark(num_cards)]
def npu_marks(*, res: str, num_cards: int):
test_platform = pytest.mark.npu
if res == "A2":
test_resource = pytest.mark.A2
elif res == "A3":
test_resource = pytest.mark.A3
else:
test_resource = None
if num_cards == 1:
return [mark for mark in [test_platform, test_resource] if mark is not None]
test_distributed = pytest.mark.distributed_npu(num_cards=num_cards)
return [mark for mark in [test_platform, test_resource, test_distributed] if mark is not None]
def _res_platforms() -> frozenset[str]:
"""Accelerator keys allowed in ``hardware_test(res=...)`` / ``hardware_marks(res=...)``.
A ``[hardware-platform]`` name is a ``res`` key only if some SKU is tagged
with that platform (``[cuda]``, ``[npu]``, …). ``cpu`` / ``gpu`` have no SKUs.
"""
return frozenset(p for p in get_supported_platforms() if _skus_for_platform(p))
def _normalize_num_cards(res: dict[str, str], num_cards: int | dict[str, int]) -> dict[str, int]:
allowed = _res_platforms()
device_class = get_supported_platforms() - allowed
for platform in res:
if platform in device_class:
raise ValueError(
f"{platform!r} is not a res dict key; use pytest.mark.{platform} "
f"or an accelerator key ({', '.join(sorted(allowed))})."
)
if platform not in allowed:
supported = ", ".join(sorted(allowed)) or "(none)"
raise ValueError(f"Unsupported platform: {platform}. res keys: {supported}.")
if isinstance(num_cards, int) and not isinstance(num_cards, bool):
return {platform: num_cards for platform in res}
if not isinstance(num_cards, dict):
raise ValueError(f"num_cards must be a positive int or dict, got {num_cards!r}")
num_cards_dict = dict(num_cards)
for platform in num_cards_dict:
if platform not in res:
raise ValueError(f"Platform '{platform}' in num_cards but not in res.")
for platform in res:
num_cards_dict.setdefault(platform, 1)
return num_cards_dict
def _marks_for_platform(platform: str, resource: str, num_cards: int) -> list[pytest.MarkDecorator]:
import pytest
builders = {
"cuda": _cuda_marks,
"rocm": _rocm_marks,
"xpu": _xpu_marks,
"musa": _musa_marks,
"npu": _npu_marks,
}
builder = builders.get(platform)
if builder is None:
raise ValueError(f"Unsupported platform: {platform}")
marks = builder(res=resource, num_cards=num_cards)
if platform in _gpu_res_platforms():
return [pytest.mark.gpu] + marks
return marks
def _skipif_not_platform(platform: str) -> pytest.MarkDecorator:
import pytest
from vllm_omni.platforms import current_omni_platform
checkers = {
"cuda": current_omni_platform.is_cuda,
"rocm": current_omni_platform.is_rocm,
"xpu": current_omni_platform.is_xpu,
"npu": current_omni_platform.is_npu,
"musa": current_omni_platform.is_musa,
}
return pytest.mark.skipif(not checkers[platform](), reason=f"Requires {platform} platform")
def _apply_marks(func, marks: list[pytest.MarkDecorator]):
for mark in reversed(marks):
func = mark(func)
return func
def hardware_marks(*, res: dict[str, str], num_cards: int | dict[str, int] = 1):
for platform, _ in res.items():
if platform not in ("cuda", "rocm", "xpu", "npu", "musa"):
raise ValueError(f"Unsupported platform: {platform}")
if isinstance(num_cards, int):
num_cards_dict = {platform: num_cards for platform in res.keys()}
else:
num_cards_dict = num_cards
for platform in num_cards_dict.keys():
if platform not in res:
raise ValueError(f"Platform '{platform}' in num_cards but not in res.")
for platform in res.keys():
if platform not in num_cards_dict:
num_cards_dict[platform] = 1
"""Return marks for a **single** pytest item.
Different per-platform ``num_cards`` cannot share one item: ``cards_2`` and
``cards_4`` would both match ``-m cards_2`` / ``-m cards_4``. Use
``@hardware_test`` (one collected variant per platform) or call this once
per platform and attach each list to its own ``pytest.param``.
"""
num_cards_dict = _normalize_num_cards(res, num_cards)
unique_counts = set(num_cards_dict.values())
if len(res) > 1 and len(unique_counts) > 1:
raise ValueError(
"hardware_marks() cannot attach different per-platform num_cards to one pytest item "
f"(got {num_cards_dict}). Use @hardware_test, which collects one variant per platform, "
"or call hardware_marks() once per platform."
)
all_marks: list[pytest.MarkDecorator] = []
for platform, resource in res.items():
cards = num_cards_dict[platform]
if platform in ("cuda", "rocm", "xpu", "musa"):
marks = gpu_marks(res=resource, num_cards=cards)
elif platform == "npu":
marks = npu_marks(res=resource, num_cards=cards)
else:
raise ValueError(f"Unsupported platform: {platform}")
all_marks.extend(marks)
all_marks.extend(_marks_for_platform(platform, resource, num_cards_dict[platform]))
return all_marks
def hardware_test(*, res: dict[str, str], num_cards: int | dict[str, int] = 1):
all_marks = hardware_marks(res=res, num_cards=num_cards)
"""Apply hardware marks; split collected items when card counts differ by platform.
def wrapper(f):
func = f
for mark in reversed(all_marks):
func = mark(func)
return func
A single item cannot carry both ``cards_4`` (CUDA) and ``cards_2`` (ROCm):
``-m "H100 and cards_4 and cuda"`` would otherwise also match
``-m "H100 and cards_2 and cuda"``. When ``num_cards`` differs across
platforms, this decorator parametrizes one variant per platform so each
item only has that platform's SKU and ``cards_{n}``.
"""
import pytest
return wrapper
num_cards_dict = _normalize_num_cards(res, num_cards)
mixed_counts = len(res) > 1 and len(set(num_cards_dict.values())) > 1
if not mixed_counts:
all_marks = hardware_marks(res=res, num_cards=num_cards)
def apply_union_marks(f):
return _apply_marks(f, all_marks)
return apply_union_marks
params = []
for platform, resource in res.items():
marks = _marks_for_platform(platform, resource, num_cards_dict[platform])
marks.append(_skipif_not_platform(platform))
params.append(pytest.param(platform, marks=marks, id=platform))
def apply_split_params(f):
func = pytest.mark.usefixtures(HARDWARE_MARK_NORMALIZATION_FIXTURE)(f)
return pytest.mark.parametrize(HARDWARE_MARK_NORMALIZATION_FIXTURE, params, indirect=True)(func)
return apply_split_params
tests/helpers/tests/test_mark.py
+139
View File
@@ -0,0 +1,139 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import pytest
from tests.helpers.mark import (
_gpu_res_platforms,
hardware_marks,
hardware_test,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _mark_names(marks) -> set[str]:
return {mark.name for mark in marks}
def test_gpu_platforms_come_from_pyproject_gpu_tag():
assert "npu" not in _gpu_res_platforms()
assert "gpu" not in _gpu_res_platforms()
def test_default_num_cards_adds_cards_1():
names = _mark_names(hardware_marks(res={"cuda": "H100"}))
assert "cards_1" in names
assert "H100" in names
assert "cards_2" not in names
def test_h100_two_cards_is_selectable_with_cards_2():
names = _mark_names(hardware_marks(res={"cuda": "H100"}, num_cards=2))
assert names >= {"H100", "cuda", "gpu", "cards_2"}
assert "cards_1" not in names
assert "cards_4" not in names
def test_h100_four_cards_is_selectable_with_cards_4():
names = _mark_names(hardware_marks(res={"cuda": "H100"}, num_cards=4))
assert "cards_4" in names
assert "cards_2" not in names
def test_npu_card_count_mark_is_independent_of_sku():
names = _mark_names(hardware_marks(res={"npu": "A2"}, num_cards=4))
assert "cards_4" in names
assert "npu" in names
assert "gpu" not in names
def test_hardware_marks_rejects_mixed_counts_on_one_item():
with pytest.raises(ValueError, match="different per-platform num_cards"):
hardware_marks(
res={"cuda": "H100", "rocm": "MI325"},
num_cards={"cuda": 4, "rocm": 2},
)
def test_same_count_multi_platform_stays_one_item():
names = _mark_names(hardware_marks(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2))
assert names >= {"H100", "cuda", "MI325", "rocm", "cards_2"}
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
def _probe():
return None
names = {mark.name for mark in _probe.pytestmark}
assert "parametrize" not in names
assert "cards_2" in names
assert "H100" in names
assert "MI325" in names
def test_hardware_test_decorator_applies_cards_mark():
@hardware_test(res={"cuda": "H100"}, num_cards=2)
def _probe():
return None
names = {mark.name for mark in _probe.pytestmark}
assert "cards_2" in names
assert "H100" in names
def _hardware_platform_params(func):
marks = func.pytestmark if isinstance(func.pytestmark, list) else [func.pytestmark]
for mark in marks:
if mark.name != "parametrize":
continue
argnames = mark.args[0]
if argnames == "_normalized_hardware_marks":
return mark.args[1]
raise AssertionError("expected @hardware_test to parametrize _normalized_hardware_marks")
def test_mixed_card_counts_split_so_cuda_filter_does_not_see_rocm_count():
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2})
def _probe():
return None
variants = {param.id: {mark.name for mark in param.marks} for param in _hardware_platform_params(_probe)}
cuda_names = variants["cuda"]
rocm_names = variants["rocm"]
assert cuda_names >= {"H100", "cuda", "cards_4"}
assert "cards_2" not in cuda_names
assert "rocm" not in cuda_names
assert "MI325" not in cuda_names
assert rocm_names >= {"MI325", "rocm", "cards_2"}
assert "cards_4" not in rocm_names
assert "cuda" not in rocm_names
assert "H100" not in rocm_names
def test_unsupported_card_count_raises():
with pytest.raises(ValueError, match="num_cards=16"):
hardware_marks(res={"cuda": "H100"}, num_cards=16)
@pytest.mark.parametrize("platform", ["cpu", "gpu"])
def test_res_rejects_cpu_and_gpu(platform):
with pytest.raises(ValueError, match="not a res dict key"):
hardware_marks(res={platform: "H100"})
def test_npu_rejects_unsupported_sku():
with pytest.raises(ValueError, match="Invalid npu resource type"):
hardware_marks(res={"npu": "A100"})
@pytest.mark.parametrize("sku", ["H200", "B200", "H800"])
def test_cuda_registered_skus_attach_sku_and_cards(sku):
names = _mark_names(hardware_marks(res={"cuda": sku}, num_cards=2))
assert names >= {sku, "cuda", "gpu", "cards_2"}
def test_cuda_rejects_sku_from_another_platform():
with pytest.raises(ValueError, match="Invalid cuda resource type"):
hardware_marks(res={"cuda": "MI325"})
tests/model_executor/models/voxtral_tts/test_cuda_graph_acoustic_transformer.py
+4 -3
View File
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""
Tests for CUDA Graph acoustic transformer wrapper numerical equivalence.
@@ -18,11 +18,12 @@ import pytest
import torch
import torch.nn as nn
from tests.helpers.mark import hardware_marks
pytestmark = [
pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required"),
pytest.mark.core_model,
pytest.mark.cuda,
pytest.mark.L4,
*hardware_marks(res={"cuda": "L4"}),
]
DEVICE = torch.device("cuda:0")
tools/pre_commit/check_test_marks.py
+101 -37
View File
@@ -1,52 +1,45 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Run a pre-commit hook that fails if test files are modified or added
that (probably) never run in the CI. For now, this means that every tests file
needs to have a CI level marker (e.g., core_model, advanced_model, full_model,
local_model, slow, etc) and hardware mark / helper so that we ensure mutated
tests will actually be selected as long as there are pytest commands pointing
at the right paths.
SKU markers (``H100``, ``L4``, … tagged ``[hardware-resource]`` in
``pyproject.toml``) must be applied via ``hardware_test(`` / ``hardware_marks(``
so ``cards_{n}`` is attached. Direct ``pytest.mark.H100`` is rejected.
Platform names allowed as ``pytest.mark.cpu`` / ``cuda`` come from
``get_supported_platforms()`` (``[hardware-platform]`` in ``pyproject.toml``).
CI level names come from ``get_level_markers()`` (``[ci-level]``).
"""
from __future__ import annotations
import importlib.util
import os
import re
import sys
from functools import lru_cache
from pathlib import Path
from types import ModuleType
# CI level markers
LEVEL_MARKERS = ("core_model", "advanced_model", "full_model", "local_model", "slow")
# Hardware markers. These are platforms and resources that CI filters on.
# NOTE: If new hardware marks are added to pyproject.toml etc, we need
# to add them here as well.
HARDWARE_MARKERS = (
"cpu",
"gpu",
"cuda",
"rocm",
"xpu",
"npu",
"musa",
"H100",
"L4",
"MI325",
"B60",
"S5000",
"A2",
"A3",
)
# Helpers from tests/helpers/mark.py that auto-apply hardware marks.
# Helpers from tests/helpers/mark.py that auto-apply hardware + cards_* marks.
HARDWARE_HELPERS = ("hardware_test", "hardware_marks")
# The helper implementation is the only file allowed to write pytest.mark.<SKU>.
_ALLOWED_DIRECT_SKU_FILES = frozenset({"tests/helpers/mark.py"})
# Match mark.X since we could also do `from pytest import mark`.
# \b prevents matching prefixes (e.g., mark.slow vs mark.slow_test).
LEVEL_RE = re.compile(r"mark\.(?:" + "|".join(LEVEL_MARKERS) + r")\b")
HARDWARE_RE = re.compile(r"mark\.(?:" + "|".join(HARDWARE_MARKERS) + r")\b")
HELPER_RE = re.compile(r"(?:" + "|".join(HARDWARE_HELPERS) + r")\s*\(")
MISSING_LEVEL_MARKER = "Level"
MISSING_HARDWARE_MARKER = "Hardware"
DIRECT_SKU_MARKER = "Direct SKU"
# Check if a file is located under tests/ and matches test_<something>.py
# or <something>_test.py, since pytest technically collects on both.
@@ -54,9 +47,68 @@ MISSING_HARDWARE_MARKER = "Hardware"
TEST_FILE_RE = re.compile(r"^tests/(?:.*/)?(?:test_[^/]*\.py$|[^/]*_test\.py$)")
def _normalize_path(path: str) -> str:
return path.replace("\\", "/")
def _repo_root() -> Path:
return Path(__file__).resolve().parents[2]
@lru_cache(maxsize=1)
def _mark_module() -> ModuleType:
"""Load ``mark.py`` by file path (no pytest/vllm; skip helpers ``__init__``)."""
path = _repo_root() / "tests" / "helpers" / "mark.py"
spec = importlib.util.spec_from_file_location("_vllm_omni_mark", path)
if spec is None or spec.loader is None:
raise ImportError(f"Cannot load mark helpers from {path}")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def _mark_name_re(names: tuple[str, ...]) -> re.Pattern[str]:
if not names:
return re.compile(r"(?!)")
return re.compile(r"mark\.(?:" + "|".join(re.escape(n) for n in names) + r")\b")
@lru_cache(maxsize=1)
def level_markers() -> tuple[str, ...]:
"""CI level names tagged ``[ci-level]`` (``core_model``, ``slow``, …)."""
return tuple(sorted(_mark_module().get_level_markers()))
@lru_cache(maxsize=1)
def platform_markers() -> tuple[str, ...]:
"""Names tests may apply as ``pytest.mark.cpu`` / ``cuda`` (not SKUs)."""
return tuple(sorted(_mark_module().get_supported_platforms()))
@lru_cache(maxsize=1)
def sku_markers() -> tuple[str, ...]:
"""SKU marker names tagged ``[hardware-resource]`` in ``pyproject.toml``."""
return tuple(sorted(_mark_module().get_hardware_mark_list()))
@lru_cache(maxsize=1)
def _level_re() -> re.Pattern[str]:
return _mark_name_re(level_markers())
@lru_cache(maxsize=1)
def _platform_re() -> re.Pattern[str]:
return _mark_name_re(platform_markers())
@lru_cache(maxsize=1)
def _sku_mark_re() -> re.Pattern[str]:
return _mark_name_re(sku_markers())
def is_test_file(path: str) -> bool:
"""Determine whether or not a path is pointing at a test file or not."""
return bool(TEST_FILE_RE.match(path))
return bool(TEST_FILE_RE.match(_normalize_path(path)))
def read_test_file(path: str) -> str | None:
@@ -69,22 +121,31 @@ def read_test_file(path: str) -> str | None:
def has_level_marker(contents: str) -> bool:
"""Check if file contents contain at least one CI level marker."""
return bool(LEVEL_RE.search(contents))
return bool(_level_re().search(contents))
def has_hardware_marker(contents: str) -> bool:
"""Check if file contents contain any hardware marker or helpers."""
return bool(HARDWARE_RE.search(contents) or HELPER_RE.search(contents))
"""Check if file contents contain a platform marker or hardware helper."""
return bool(_platform_re().search(contents) or HELPER_RE.search(contents))
def has_direct_sku_marker(path: str, contents: str) -> bool:
"""True when a test applies ``pytest.mark.<SKU>`` instead of the helpers."""
if _normalize_path(path) in _ALLOWED_DIRECT_SKU_FILES:
return False
return bool(_sku_mark_re().search(contents))
def get_files_missing_markers(
staged_files: list[str],
) -> dict[str, list[str]]:
"""Return a dict mapping file path to list of missing marker types."""
"""Return a dict mapping file path to list of missing / invalid marker types."""
results: dict[str, list[str]] = {}
for path in staged_files:
if is_test_file(path) and (contents := read_test_file(path)) is not None:
missing = []
if has_direct_sku_marker(path, contents):
missing.append(DIRECT_SKU_MARKER)
if not has_level_marker(contents):
missing.append(MISSING_LEVEL_MARKER)
if not has_hardware_marker(contents):
@@ -99,14 +160,17 @@ if __name__ == "__main__":
if missing:
file_lines = "\n".join(f" - {path} [{' and '.join(problems)}]" for path, problems in missing.items())
sku = ", ".join(sku_markers())
print(
"\033[91merror:\033[0m test files are missing pytest marks "
"required for Buildkite CI collection.\n\n"
f"Level marks, e.g.: {', '.join(LEVEL_MARKERS[:4])}\n"
f"Hardware marks, e.g.: {', '.join(HARDWARE_MARKERS[:4])}, ...\n"
f" or helpers: {', '.join(HARDWARE_HELPERS)}\n\n"
"required for Buildkite CI collection, or apply SKU marks directly.\n\n"
f"Level marks, e.g.: {', '.join(level_markers()[:4])}\n"
f"Hardware marks, e.g.: {', '.join(platform_markers()[:4])}, ...\n"
f" or helpers: {', '.join(HARDWARE_HELPERS)}\n"
f"Do not write pytest.mark.<SKU> ({sku}). "
"Use hardware_test(...) / hardware_marks(...) so cards_* is attached.\n\n"
"The following files are missing marks:\n"
f"{file_lines}\n\n"
"To skip: SKIP=check-test-ci-coverage git commit ..."
"To skip: SKIP=check-mark git commit ..."
)
sys.exit(1)