[CI]Split GPU jobs by cards_* and reject hand-written SKU marks (#6650)

Signed-off-by: wangyu <410167048@qq.com> Signed-off-by: [Your Name] <your.email@example.com>
2026-08-29 18:42:33 +08:00
1 parent c50b7de167
commit e338f0ece3
@@ -12,16 +12,22 @@ steps:
    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "omni-test"
    steps:
-      - label: ":full_moon: Omni · Function Test with H100"
+      - label: ":full_moon: Omni · Function Test with H100 · Single-GPU"
-        timeout_in_minutes: 120
+        timeout_in_minutes: 90
        commands:
-          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
+          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni and cards_1" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
        mirror_hardwares: h100_1
      - label: ":full_moon: Omni · Function Test with H100 · 2-GPU"
        timeout_in_minutes: 90
        commands:
          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni and cards_2" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py
        mirror_hardwares: h100_2
      - label: ":full_moon: Omni · MiniCPM-o 4.5 Duplex Test"
        timeout_in_minutes: 50
        commands:
-          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and omni" --run-level "full_model"
+          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and omni and cards_1" --run-level "full_model"
        mirror_hardwares: h100_1
      - label: ":full_moon: Omni · Doc Test with L4"
@@ -31,10 +37,10 @@ steps:
          - pytest -s -v tests/examples/ -m "full_model and omni and L4" --run-level "full_model"
        mirror_hardwares: l4_4
-      - label: ":full_moon: Omni · Doc Test with H100"
+      - label: ":full_moon: Omni · Doc Test with H100 · 2-GPU"
        timeout_in_minutes: 90
        commands:
-          - pytest -s -v tests/examples/ -m "full_model and omni and H100" --run-level "full_model"
+          - pytest -s -v tests/examples/ -m "full_model and omni and H100 and cards_2" --run-level "full_model"
        mirror_hardwares: h100_2
      - label: ":full_moon: Omni · Accuracy Test"
@@ -136,15 +142,29 @@ steps:
          - pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy
        mirror_hardwares: l4_1
-      - label: ":full_moon: TTS · Perf Test"
+      - label: ":full_moon: TTS · Perf Test · Single-GPU"
-        key: nightly-tts-performance
+        key: nightly-tts-performance-single
        timeout_in_minutes: 180
        commands:
          - export BENCHMARK_DIR=tests/dfx/perf/results
          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
          - |
            set +e
-            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and cards_1"
            EXIT=$$?
            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
            exit $$EXIT
        mirror_hardwares: h100_1
      - label: ":full_moon: TTS · Perf Test · 2-GPU"
        key: nightly-tts-performance-2gpu
        timeout_in_minutes: 180
        commands:
          - export BENCHMARK_DIR=tests/dfx/perf/results
          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
          - |
            set +e
            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and cards_2"
            EXIT=$$?
            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
            exit $$EXIT
@@ -179,7 +199,6 @@ steps:
      build.pull_request.labels includes "nightly-test" ||
      build.pull_request.labels includes "diffusion-x2iat-test"
    steps:
      # Use explicit file shards so single-card cases do not reserve 2+ GPUs.
      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Single-GPU"
        timeout_in_minutes: 120
@@ -187,24 +206,24 @@ steps:
          - >-
            pytest -sv
            tests/e2e/online_serving/test_qwen_image_expansion.py
-            -m "full_model and diffusion and H100 and not distributed_cuda"
+            -m "full_model and diffusion and H100 and cards_1"
            --run-level "full_model"
        mirror_hardwares: h100_1
-      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Qwen-Image"
+      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · 2-GPU"
        timeout_in_minutes: 120
        commands:
          - >-
            pytest -sv
            tests/e2e/online_serving/test_qwen_image_expansion.py
-            -m "full_model and diffusion and H100 and distributed_cuda"
+            -m "full_model and diffusion and H100 and cards_2"
            --run-level "full_model"
        mirror_hardwares: h100_2
-      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4"
+      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4 · Multi-GPU"
        timeout_in_minutes: 120
        commands:
-          - pytest -sv tests/e2e/ -k "not test_wan and not hunyuan" -m "full_model and diffusion and L4" --run-level "full_model" --ignore=tests/e2e/accuracy
+          - pytest -sv tests/e2e/ -k "not test_wan and not hunyuan" -m "full_model and diffusion and L4 and not cards_1" --run-level "full_model" --ignore=tests/e2e/accuracy
        mirror_hardwares: l4_4
      - label: ":full_moon: Diffusion X2I(&A&T) · Doc Test"
@@ -235,8 +254,8 @@ steps:
          - pytest -s -v tests/e2e/accuracy/test_hidream_o1_image.py -m full_model --run-level full_model
        mirror_hardwares: h100_2
-      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image"
+      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image · Single-GPU"
-        key: nightly-diffusion-x2iat-performance-qwen-image
+        key: nightly-diffusion-x2iat-performance-qwen-image-single
        timeout_in_minutes: 180
        commands:
          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
@@ -244,7 +263,23 @@ steps:
          - export CACHE_DIT_VERSION=1.5.0
          - |
            set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and cards_1"
            EXIT=$$?
            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
            exit $$EXIT
        mirror_hardwares: h100_1
      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image · 4-GPU"
        key: nightly-diffusion-x2iat-performance-qwen-image-4gpu
        timeout_in_minutes: 180
        commands:
          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
          - export CACHE_DIT_VERSION=1.5.0
          - |
            set +e
            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and cards_4"
            EXIT=$$?
            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
@@ -260,16 +295,28 @@ steps:
      build.pull_request.labels includes "diffusion-x2v-test"
    steps:
-      - label: ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test"
+      - label: ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test · Single-GPU"
        timeout_in_minutes: 90
        commands:
-          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda" --run-level "full_model"
+          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda and H100 and cards_1" --run-level "full_model"
        mirror_hardwares: h100_1
      - label: ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test · 2-GPU"
        timeout_in_minutes: 90
        commands:
          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda and H100 and cards_2" --run-level "full_model"
        mirror_hardwares: h100_2
-      - label: ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test"
+      - label: ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test · Single-GPU"
        timeout_in_minutes: 90
        commands:
-          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda" --run-level "full_model"
+          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda and H100 and cards_1" --run-level "full_model"
        mirror_hardwares: h100_1
      - label: ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test · 2-GPU"
        timeout_in_minutes: 90
        commands:
          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda and H100 and cards_2" --run-level "full_model"
        mirror_hardwares: h100_2
      - label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Function Test"
@@ -310,17 +357,32 @@ steps:
          - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m full_model --run-level full_model
        mirror_hardwares: h100_2
-      - label: ":full_moon: Diffusion X2V · Perf Test"
+      - label: ":full_moon: Diffusion X2V · Perf Test · Single-GPU"
-        key: nightly-diffusion-x2v-performance
+        key: nightly-diffusion-x2v-performance-single
        timeout_in_minutes: 180
        commands:
          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
          - |
            set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and cards_1"
            EXIT=$$?
            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
            exit $$EXIT
        mirror_hardwares: h100_1
      - label: ":full_moon: Diffusion X2V · Perf Test · 2-GPU"
        key: nightly-diffusion-x2v-performance-2gpu
        timeout_in_minutes: 180
        commands:
          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
          - |
            set +e
            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and cards_2"
            EXIT1=$$?
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_cosmos3_vllm_omni.json
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_cosmos3_vllm_omni.json -m "H100 and cards_2"
            EXIT2=$$?
            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
@@ -371,10 +433,16 @@ steps:
          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4' --run-level "full_model"
        mirror_hardwares: l4_4
-      - label: ":full_moon: Diffusion · Distributed Test with H100"
+      - label: ":full_moon: Diffusion · Distributed Test with H100 · 2-GPU"
        timeout_in_minutes: 60
        commands:
-          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100' --run-level "full_model"
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100 and cards_2' --run-level "full_model"
        mirror_hardwares: h100_2
      - label: ":full_moon: Diffusion · Distributed Test with H100 · 4-GPU"
        timeout_in_minutes: 60
        commands:
          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100 and cards_4' --run-level "full_model"
        mirror_hardwares: h100_4
      - label: ":full_moon: Diffusion · Offloader Test"
@@ -409,19 +477,25 @@ steps:
    depends_on:
      - nightly-omni-performance-no-async-chunk
      - nightly-omni-performance-async-chunk
-      - nightly-tts-performance
+      - nightly-tts-performance-single
-      - nightly-diffusion-x2iat-performance-qwen-image
+      - nightly-tts-performance-2gpu
-      - nightly-diffusion-x2v-performance
+      - nightly-diffusion-x2iat-performance-qwen-image-single
      - nightly-diffusion-x2iat-performance-qwen-image-4gpu
      - nightly-diffusion-x2v-performance-single
      - nightly-diffusion-x2v-performance-2gpu
    if: build.env("NIGHTLY") == "1" && build.env("EMAIL_DISTRIBUTION") == "1"
    commands:
      - pip install openpyxl
      - export DEFAULT_INPUT_DIR=tests/dfx/perf/results
      - export DEFAULT_OUTPUT_DIR=tests/dfx/perf/results
-      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance-single
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance-2gpu
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-no-async-chunk
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-async-chunk
-      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-single
-      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-4gpu
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance-single
      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance-2gpu
      - python tools/nightly/generate_nightly_perf_excel.py
      - python tools/nightly/generate_nightly_perf_html.py
      - python tools/nightly/send_nightly_email.py --report-file "tests/dfx/perf/results/*.xlsx, tests/dfx/perf/results/*.html"
@@ -44,10 +44,16 @@ steps:
          - pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m "slow"
        mirror_hardwares: h100_4
-      - label: "Reliability Test · Invalid parameters · H100"
+      - label: "Reliability Test · Invalid parameters · H100 · Single-GPU"
-        timeout_in_minutes: 120
+        timeout_in_minutes: 90
        commands:
-          - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100"
+          - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100 and cards_1"
        mirror_hardwares: h100_1
      - label: "Reliability Test · Invalid parameters · H100 · 2-GPU"
        timeout_in_minutes: 90
        commands:
          - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100 and cards_2"
        mirror_hardwares: h100_2
      - label: "Reliability Test · Invalid parameters · L4"
@@ -105,27 +111,37 @@ steps:
    if: build.env("NON_CRITICAL") == "1"
    steps:
-      - label: "Omni · L4"
+      # Qwen2.5 expansion (CUDA 4-card / online 2-card) stays on this l4_4 job.
      - label: "Omni · L4 · Multi-GPU"
        timeout_in_minutes: 90
        commands:
          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-          - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model"
+          - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py -m "slow and L4 and omni and not cards_1" --run-level "core_model"
          - >-
            pytest -s -v tests/e2e/
            --ignore=tests/e2e/online_serving/test_qwen2_5_omni_expansion.py
            --ignore=tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py
-            -m "slow and L4 and omni"
+            -m "slow and L4 and omni and not cards_1"
            --run-level "full_model"
        mirror_hardwares: l4_4
-      - label: "Omni · H100"
+      - label: "Omni · H100 · Single-GPU"
        timeout_in_minutes: 90
        commands:
          - >-
            pytest -sv
-            tests/e2e/ -m "slow and H100 and omni"
+            tests/e2e/ -m "slow and H100 and omni and cards_1"
            --run-level "full_model"
-        mirror_hardwares: h100_2
+        mirror_hardwares: h100_1
      - label: "Omni · H100 · 4-GPU"
        timeout_in_minutes: 90
        commands:
          - >-
            pytest -sv
            tests/e2e/ -m "slow and H100 and omni and cards_4"
            --run-level "full_model"
        mirror_hardwares: h100_4
      - label: "TTS · L4"
        timeout_in_minutes: 120
@@ -169,12 +185,21 @@ steps:
            --run-level "full_model"
        mirror_hardwares: l4_1
-      - label: "Diffusion · L4"
+      - label: "Diffusion · L4 · Single-GPU"
        timeout_in_minutes: 60
        commands:
          - >-
            pytest -sv
-            tests/e2e/ -m "slow and diffusion and L4"
+            tests/e2e/ -m "slow and diffusion and L4 and cards_1"
            --run-level "full_model"
        mirror_hardwares: l4_1
      - label: "Diffusion · L4 · Multi-GPU"
        timeout_in_minutes: 60
        commands:
          - >-
            pytest -sv
            tests/e2e/ -m "slow and diffusion and L4 and not cards_1"
            --run-level "full_model"
        mirror_hardwares: l4_4
@@ -183,15 +208,24 @@ steps:
        commands:
          - >-
            pytest -sv
-            tests/e2e/ -m "slow and diffusion and H100 and not distributed_cuda"
+            tests/e2e/ -m "slow and diffusion and H100 and cards_1"
            --run-level "full_model"
        mirror_hardwares: h100_1
-      - label: "Diffusion · H100 · Multi-GPU"
+      - label: "Diffusion · H100 · 2-GPU"
        timeout_in_minutes: 90
        commands:
          - >-
            pytest -sv
-            tests/e2e/ -m "slow and diffusion and H100 and distributed_cuda"
+            tests/e2e/ -m "slow and diffusion and H100 and cards_2"
            --run-level "full_model"
        mirror_hardwares: h100_2
      - label: "Diffusion · H100 · 4-GPU"
        timeout_in_minutes: 90
        commands:
          - >-
            pytest -sv
            tests/e2e/ -m "slow and diffusion and H100 and cards_4"
            --run-level "full_model"
        mirror_hardwares: h100_4
@@ -338,7 +338,7 @@ Classify the model's **CI priority** first:
 **L2 & L3 online — same file, dual marks on the baseline smoke:** The **first / simplest** case in `test_{slug}.py` (default deploy, minimal steps, single prompt) should carry **both** `@pytest.mark.core_model` and `@pytest.mark.advanced_model` on the **same** function so L2 (`test-ready.yml`) and L3 (`test-merge.yml`) share one smoke test. Heavier deploy variants or API paths in the same file use **`advanced_model` only**. When L3 moves to nightly, migrate those heavier cases into `test_{slug}_expansion.py` with `full_model` and remove the dedicated `test-merge.yml` job (see `test_longcat_image_expansion.py`, `test_qwen_image_expansion.py`).
-**L4 design (high priority):** Combine multiple supported features (Cache-DiT, TP, USP, CFG, HSDP, CPU offload, quantization) into **few parametrized** `OmniServerParams` rows so each feature appears in at least one case without exploding GPU jobs. Shard single-GPU vs multi-GPU cases across the nightly X2I/X2V function steps (1-GPU vs `distributed_cuda`).
+**L4 design (high priority):** Combine multiple supported features (Cache-DiT, TP, USP, CFG, HSDP, CPU offload, quantization) into **few parametrized** `OmniServerParams` rows so each feature appears in at least one case without exploding GPU jobs. Shard single-GPU vs multi-GPU cases across the nightly X2I/X2V function steps (`cards_1` vs `not cards_1`).
 **L4 design (medium / low):** One or two parametrized rows covering the best quality/perf trade-off; skip perf JSON unless the model is high priority.
@@ -34,7 +34,7 @@ but the changes are correct — stage the modified files and re-commit.
 | `check-forbidden-imports` | Stdlib `re`/`base64`, pickle, Hugging Face Hub API, or direct Triton/TileLang | `import regex as re` and `pybase64`; use `vllm.transformers_utils.repo_utils`. Do **not** add the file to `allowed_files` without review |
 | `check-torch-cuda-call` | New `torch.cuda.*` call site | Use `current_omni_platform` / `OmniPlatform`; do not grow `ALLOWED_FILES` without review |
 | `check-tts-adapter-migration` | New `self._tts_model_type` branch in `serving_speech.py` | Put per-model logic in `tts_adapters/`. Lower `MAX_MODEL_TYPE_BRANCHES` when removing branches; do not raise it without review |
-| `check-test-ci-coverage` | New `tests/**/test_*.py` missing level or hardware mark | Add `core_model`/`advanced_model`/… plus `cpu`/`cuda`/`hardware_test(` |
+| `check-mark` | New `tests/**/test_*.py` missing level/hardware mark, or direct `pytest.mark.H100` | Add `core_model`/`advanced_model`/… plus `cpu`/`cuda`/`hardware_test(`; SKU marks only via helpers |
 | `mypy-3.10` | Type error on changed `vllm_omni/` files | Fix the types; model trees are excluded. Extra versions: `pre-commit run --hook-stage manual mypy-3.12` |
 | `markdownlint-cli2` | Docs/README markdown lint | Accept auto-fix under `docs/` / `recipes/` / root README |
 | `check-buildkite` | Invalid `.buildkite/*.yml` | Fix the YAML; do not grow `SKIP_FILES` without review |
@@ -52,8 +52,8 @@ Always attach markers deliberately:
  - `tts` — speech synthesis / TTS-only models (`/v1/audio/speech`, voice clone, etc.)
  - `diffusion` — generative diffusion models (image / audio / text / video from noise)
 - **Cross-cutting area** (when relevant): `parallel`, `cache`, `example`, `benchmark`
- **Hardware**: `cpu`, `gpu`, `cuda`, `rocm`, `npu`, `L4`, `H100`, `distributed_cuda`, …
+- **Hardware**: `cpu`, `gpu`, `cuda`, `rocm`, `npu`, …; SKU (`H100`, `L4`, …) and `cards_{n}` via `@hardware_test(...)` / `hardware_marks(...)` only — never `pytest.mark.H100` (`check-mark` rejects it)
- Optional: `slow`, distributed markers when multi-card is required
+- Optional: `slow`. Multi-card: `num_cards=N` on the helper (filter with `cards_N` / `not cards_1`)
 **Baseline smoke (L2 + L3):** The simplest e2e case per model — default deploy, minimal request — should usually carry **both** `@pytest.mark.core_model` and `@pytest.mark.advanced_model` on the **same** test function so `test-ready.yml` and `test-merge.yml` share one test. `send_*_request` picks validation depth from `--run-level`. References: `test_voxcpm2_tts.py::test_text_to_audio_001`, `test_qwen3_tts_customvoice.py::test_text_to_audio_001`. Heavier scenarios use **`advanced_model` only**; L4 expansion uses **`full_model`**.
@@ -274,7 +274,7 @@ def test_text_to_video_001(omni_server, online_client) -> None:
 **3. Cross-cutting rules**
- New `.py` files need the Omni SPDX header (`Copyright contributors to the vLLM-Omni project`, not `vLLM project`). `tests/` may keep stdlib `re`/`base64`; pickle is still banned unless the file is already on the pickle allowlist. Every collected `tests/**/test_*.py` needs a CI **level** mark and a **hardware** mark/helper (`check-test-ci-coverage` runs locally; GHA skips it). Do not grow `allowed_files` to land a test. Policy: [docs/contributing/README.md](../../../docs/contributing/README.md#linting).
+- New `.py` files need the Omni SPDX header (`Copyright contributors to the vLLM-Omni project`, not `vLLM project`). `tests/` may keep stdlib `re`/`base64`; pickle is still banned unless the file is already on the pickle allowlist. Every collected `tests/**/test_*.py` needs a CI **level** mark and a **hardware** platform mark/helper (`check-mark` runs locally; GHA skips it). Do not write `pytest.mark.H100` / `pytest.mark.L4`; use the helpers so `cards_{n}` is attached. Do not grow `allowed_files` to land a test. Policy: [docs/contributing/README.md](../../../docs/contributing/README.md#linting).
 - Reuse existing fixtures for the chosen scenario; do not mix “online client” assumptions into offline `OmniRunner` tests without a clear reason.
 - Avoid external network dependency in assertions unless the scenario is explicitly “online serving” or doc examples that require a model hub (then align with CI secrets/cache).
 - Keep **one test function = one intent** (one modality combo, one endpoint contract, or one acceleration combo).
@@ -79,7 +79,7 @@ E2E (L2+) should not rely on mocks unless documenting a rare exception; prefer r
   - `full_model` — L4 nightly (`test-nightly.yml`); some expansion tests still carry both `advanced_model` and `full_model` during migration
 2. **Model type** (pick one): `omni`, `tts`, or `diffusion`
 3. **Cross-cutting** when relevant: `parallel`, `cache`, `example`, `benchmark`
-4. **Hardware**: `cpu`, `cuda`, `rocm`, `npu`, `L4`, `H100`, `distributed_cuda`, …
+4. **Hardware**: `cpu`, `cuda`, `rocm`, `npu`, `L4`, `H100`, `cards_1`, …
 5. Multi-card: `@hardware_test(...)` in `tests/helpers/mark.py`
 ## Command Templates
@@ -244,7 +244,7 @@ pytest -s -v dfx/reliability/invalid_param_test/test_invalid_image_generation.py
 ```bash
 cd tests
-pytest -s -v -m "core_model and distributed_cuda and L4" --run-level=core_model
+pytest -s -v -m "core_model and L4 and not cards_1" --run-level=core_model
 ```
 ### Concrete e2e paths (common in-tree)
@@ -27,4 +27,4 @@ jobs:
        # Don't run historical-debt hooks against the whole tree in CI yet.
        # They still run locally on changed files.
        env:
-          SKIP: check-test-ci-coverage,markdownlint-cli2,shellcheck,check-spdx-header,mypy-3.10
+          SKIP: check-mark,markdownlint-cli2,shellcheck,check-spdx-header,mypy-3.10
@@ -105,8 +105,8 @@ repos:
      <<: *mypy_common
      stages: [manual]
-    - id: check-test-ci-coverage
+    - id: check-mark
-      name: Ensure test files have pytestmarks
+      name: Ensure test files have CI marks (no direct SKU)
      entry: python tools/pre_commit/check_test_marks.py
      language: python
      types: [python]
@@ -49,7 +49,7 @@ vLLM-Omni's `pre-commit` hooks will now run automatically every time you commit.
    `--all-files` run stays green while historical debt is cleaned up. The
    current `SKIP` list in
    [`.github/workflows/pre-commit.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.github/workflows/pre-commit.yml)
-    is `check-test-ci-coverage`, `markdownlint-cli2`, `shellcheck`,
+    is `check-mark`, `markdownlint-cli2`, `shellcheck`,
    `check-spdx-header`, and `mypy-3.10`. Those hooks still run on **your
    commit** for changed files. A passing GitHub pre-commit check does not mean
    they passed locally. Hooks **not** on that list (forbidden imports,
@@ -66,7 +66,7 @@ already there). `check-pickle-imports` is gone: pickle is now one rule inside
 | `markdownlint-cli2` | Markdown in `docs/`, `recipes/`, `README.md`, `CONTRIBUTING.md` (not `.claude/` / `.cursor/` / `CLAUDE.md`) | skipped |
 | `mypy-3.10` | Type-check changed `vllm_omni/` files (model trees excluded). `tests/` uses `--follow-imports skip` | skipped |
 | `mypy-3.11` / `3.12` / `3.13` | Same checker, extra Python versions | not installed; `pre-commit run --hook-stage manual mypy-3.12` |
-| `check-test-ci-coverage` | Every `tests/**/test_*.py` has a CI level mark and a hardware mark/helper | skipped |
+| `check-mark` | Every `tests/**/test_*.py` has a CI level mark and a hardware platform mark or helper; no direct `pytest.mark.H100` / SKU | skipped |
 | `check-tts-adapter-migration` | `self._tts_model_type` branches in `serving_speech.py` must not increase | runs |
 | `shellcheck` | `*.sh` quoting / undefined vars | skipped |
 | `check-spdx-header` | Omni SPDX header on `.py` / `.pyi` / `.sh` / `.rs` / `.proto` | skipped |
@@ -144,11 +144,13 @@ pre-commit run --hook-stage manual mypy-3.12
 #### Test CI marks
-`check-test-ci-coverage` requires each collected test module under `tests/` to
+`check-mark` requires each collected test module under `tests/` to
 have at least one CI **level** mark (`core_model`, `advanced_model`,
-`full_model`, `local_model`, or `slow`) and a **hardware** mark (`cpu`, `cuda`,
+`full_model`, `local_model`, or `slow`) and a **hardware** platform mark
-`H100`, …) or helper (`hardware_test(` / `hardware_marks(`). GitHub Actions
+(`cpu`, `cuda`, …) or helper (`hardware_test(` / `hardware_marks(`). SKU marks
-skips this hook; local commit does not. See the
+(`H100`, `L4`, …) must come from those helpers so `cards_{n}` is attached;
 direct `pytest.mark.H100` is rejected. GitHub Actions skips this hook; local
 commit does not. See the
 [test writing guide](./ci/test_writing_guide.md).
 #### TTS adapter ratchet
@@ -402,7 +402,7 @@ loads the model once per mode.
 Before splitting, confirm each half actually collects at least one test under the job's forwarded filters — pytest exits with code 5 ("no tests collected") if a half is empty, which the script reports as a failure, red-ing a job that used to pass as one combined invocation.
 Also check the tests' `num_cards` against the job's `mirror_hardwares` preset.
-`cuda_marks` turns `num_cards > 1` into `skipif(device_count() < num_cards)`, so a
+`_cuda_marks` turns `num_cards > 1` into `skipif(device_count() < num_cards)`, so a
 test asking for more GPUs than the preset provides is silently skipped and its
 coverage artifact comes back empty. Splitting such a job produces a file that
 looks fine and measures nothing — "Diffusion · Z Image Test" is the current
@@ -81,8 +81,8 @@ Supported form:
 | Form | Example | Effect |
 | ---- | ------- | ------ |
-| Array (single platform) | `"mark": [{"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 2}}, "full_model", "diffusion"]` | Hardware marks from `hardware_marks(...)`; `num_cards` > 1 adds `distributed_*` (+ CUDA `skipif_cuda` when applicable). String entries become extra pytest marks. |
+| Array (single platform) | `"mark": [{"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 2}}, "full_model", "diffusion"]` | Hardware marks from `hardware_marks(...)`; `num_cards` adds `cards_{n}` (including `cards_1` when omitted). String entries become extra pytest marks. |
-| Array (multi-platform) | `"mark": [{"hardware_marks": {"res": {"cuda": "H100", "rocm": "MI325", "npu": "A2"}, "num_cards": {"cuda": 2, "rocm": 2, "npu": 1}}}, "full_model", "omni"]` | Same as above, but declares **multiple platforms** in one case. Filter examples: `-m "full_model and H100 and cuda"`, `-m "full_model and MI325 and rocm"`. |
+| Array (multi-platform) | `"mark": [{"hardware_marks": {"res": {"cuda": "H100", "rocm": "MI325"}, "num_cards": 2}}, "full_model", "omni"]` | Same as above, but declares **multiple platforms** on one item. All platforms must share the same `num_cards` (int or identical dict values). Different counts need `@hardware_test` or one `hardware_marks()` per `pytest.param`. Filter examples: `-m "full_model and H100 and cards_2 and cuda"`, `-m "full_model and MI325 and cards_2 and rocm"`. |
 Recommended for L4 perf cases:
@@ -157,7 +157,7 @@ pytest -s -v tests/dfx/perf/scripts/run_benchmark.py \
 ```
 <!-- markdownlint-disable-next-line MD051 -->
-See also [Markers for Tests](#markers-for-tests) for registered hardware markers (`H100`, `L4`, `cuda`, `distributed_cuda`, …).
+See also [Markers for Tests](#markers-for-tests) for registered hardware markers (`H100`, `L4`, `cuda`, `cards_1`, …).
 #### `server_params` Configuration
@@ -88,7 +88,7 @@ Failed jobs: 1/2
    ```bash
    cd tests
    pytest -s -v test_xxxx.py --run-level=core_model
-    pytest -s -v -m "core_model and distributed_cuda and L4" --run-level=core_model
+    pytest -s -v -m "core_model and L4 and not cards_1" --run-level=core_model
    ```
 === "L3 level"
@@ -119,7 +119,7 @@ Failed jobs: 1/2
    ```bash
    pytest -s -v test_xxxx.py --run-level=advanced_model
-    pytest -s -v -m "advanced_model and distributed_cuda and L4" --run-level=advanced_model
+    pytest -s -v -m "advanced_model and L4 and not cards_1" --run-level=advanced_model
    ```
 === "L4 level"
@@ -153,12 +153,12 @@ Failed jobs: 1/2
    ```bash
    cd tests
    pytest -s -v test_xxxx.py --run-level=full_model
-    pytest -s -v -m "full_model and distributed_cuda and L4" --run-level=full_model
+    pytest -s -v -m "full_model and L4 and not cards_1" --run-level=full_model
    pytest -s -v -m "full_model and (omni or tts) and H100" --run-level=full_model
    ```
    If you only want to run specific test cases on a particular platform, you can use:
    ```bash
-    pytest -s -v -m "full_model and distributed_cuda and L4"  --run-level=full_model
+    pytest -s -v -m "full_model and L4 and not cards_1"  --run-level=full_model
    ```
    Note: ``run_benchmark.py`` and ``run_diffusion_benchmark.py`` accept an optional ``--test-config-file``. If omitted, each loads every ``*.json`` under ``tests/dfx/perf/tests/`` (omni/tts vs diffusion split by ``is_diffusion_perf_config``) and pytest ``-m`` filters by each case's JSON ``mark``:
    ```bash
@@ -48,17 +48,12 @@ Defined in `pyproject.toml`:
 | `MI325`            | Tests that require MI325 GPU (AMD/ROCm) *                                                               |
 | `A2`               | Tests that require A2 NPU *                                                                             |
 | `A3`               | Tests that require A3 NPU *                                                                             |
-| `distributed_cuda` | Tests that require multi cards on CUDA platform *                                                       |
+| `cards_{n}`        | Tests that require *n* accelerator cards (`cards_1` … `cards_8`); auto-added from `num_cards` *         |
 | `distributed_rocm` | Tests that require multi cards on ROCm platform  *                                                      |
 | `distributed_npu`  | Tests that require multi cards on NPU platform  *                                                       |
 | `skipif_cuda`      | Skip if the num of CUDA cards is less than the required *                                               |
 | `skipif_rocm`      | Skip if the num of ROCm cards is less than the required *                                               |
 | `skipif_npu`       | Skip if the num of NPU cards is less than the required *                                                |
 | `slow`             | Slow tests (may skip in quick CI)                                                                       |
 | `benchmark`        | Benchmark tests (decorator on runner test functions; perf JSON uses `full_model` + type marker instead) |
 | `local_model`      | Tests requiring local / non-HF-hub model weights                                                        |
-\* Means those markers are auto-added by `@hardware_test` (parametrization decorator) or `hardware_marks` (only returning the list of marks for flexibility).
+\* Means those markers are auto-added by `@hardware_test` (parametrization decorator) or `hardware_marks` (only returning the list of marks for flexibility). Do **not** write `pytest.mark.H100` / `pytest.mark.L4` (or other SKU names) by hand: `check-mark` rejects that so `cards_{n}` cannot be skipped.
 #### Example usage for markers
@@ -81,25 +76,23 @@ def test_video_to_audio()
 This decorator is intended to make hardware-aware, cross-platform test authoring easier and more robust for CI/CD environments. The `hardware_test` decorator in `vllm-omni/tests/helpers/mark.py` performs the following actions:
 1. **Applies platform and resource markers**  
-   Adds the appropriate pytest markers for each specified hardware platform (e.g., `cuda`, `rocm`, `xpu`, `npu`) and resource type (e.g., `L4`, `H100`, `MI325`, `B60`, `A2`, `A3`).
+   Adds the appropriate pytest markers for each specified hardware platform (e.g., `cuda`, `rocm`, `xpu`, `npu`) and resource type (e.g., `L4`, `H100`, `H200`, `B200`, `MI325`, `B60`, `A2`, `A3`).
   ```python
   @pytest.mark.cuda
   @pytest.mark.L4
   ```
-2. **Handles multi-card (distributed) scenarios**  
+2. **Handles card-count filtering**  
-   For tests requiring multiple cards, it automatically adds distributed markers such as `distributed_cuda`, `distributed_rocm`, or `distributed_npu`.
+   Adds a filterable marker `cards_{n}` (including `cards_1` when `num_cards` is omitted / 1). Use this with the SKU and platform, e.g. `-m "H100 and cards_4 and cuda"` vs `-m "H100 and cards_2 and cuda"`. Any multi-card CUDA test is `-m "cuda and not cards_1"`.
-   ```python
+
-   @pytest.mark.distributed_cuda(num_cards=num_cards)
+   When `res` lists more than one platform **and** `num_cards` differs by platform, `@hardware_test` collects **one variant per platform** so those `cards_{n}` marks do not stack. `-m "H100 and cards_4 and cuda"` then selects only the CUDA/H100 4-card variant, not a ROCm 2-card sibling. The same count on every platform stays a single item.
   ```
 3. **Supports flexible card requirements**  
   Accepts `num_cards` as either a single integer for all platforms or as a dictionary with per-platform values. If not specified, defaults to 1 card per platform.
 4. **Integrates resource validation**  
-   On CUDA, adds a skip marker (`skipif_cuda`) if the system does not have the required number of devices.
+   On CUDA and XPU, when `num_cards > 1`, attaches pytest's built-in `skipif` if the machine has fewer devices than requested. This is not a named `skipif_*` marker and is not used for `-m` filtering.
   Support for `skipif_rocm` and `skipif_npu` will be implemented later.
 5. **Works with pytest filtering**  
-   Allows tests to be filtered and selected at runtime using standard pytest marker expressions (e.g., `-m "distributed_cuda and L4"`).
+   Allows tests to be filtered and selected at runtime using standard pytest marker expressions (e.g., `-m "H100 and cards_2"`, `-m "L4 and not cards_1"`).
 ##### Example usage for decorator
@@ -117,13 +110,14 @@ This decorator is intended to make hardware-aware, cross-platform test authoring
        num_cards=2,
    )
    ```
- `res` must be a dict; supported resources: CUDA (L4/H100), ROCm (MI325), XPU (B60), MUSA (S5000), NPU (A2/A3)
+- `res` must be a dict. SKU values come from `pyproject.toml` markers tagged `[hardware-resource]` plus the platform key (`[cuda]`, `[rocm]`, `[xpu]`, `[npu]`, `[musa]`).
 - `num_cards` can be int (all platforms) or dict (per platform); defaults to 1 when missing
 - Distributed markers (`distributed_cuda`, `distributed_rocm`, `distributed_npu`) are auto-added for multi-card cases
 - Filtering examples:
-    - CUDA only: `pytest -m "distributed_cuda and L4"`
+    - CUDA H100 4-card: `pytest -m "H100 and cards_4 and cuda"`
-    - ROCm only: `pytest -m "distributed_rocm and MI325"`
+    - CUDA H100 2-card: `pytest -m "H100 and cards_2 and cuda"`
-    - NPU only: `pytest -m "distributed_npu"`
+    - CUDA multi-card (any count): `pytest -m "L4 and not cards_1"`
    - ROCm multi-card: `pytest -m "MI325 and not cards_1"`
    - NPU multi-card: `pytest -m "npu and not cards_1"`
 #### Function: `hardware_marks`
@@ -179,37 +173,30 @@ If you want to add support for a new platform (e.g., "tpu" for a new accelerator
   [tool.pytest.ini_options]
   markers = [
       # ... existing markers ...
-       "tpu: Tests that require TPU device",
+       "tpu: [hardware-platform] Tests that require TPU device",
-       "TPU_V3: Tests that require TPU v3 hardware",
+       # Add [gpu] next to [hardware-platform] if this platform should also get pytest.mark.gpu.
-       "distributed_tpu: Tests that require multiple TPU devices",
+       "TPU_V3: [hardware-resource] [tpu] Tests that require TPU v3 hardware",
       "cards_1: [hardware-cards] Tests that require 1 accelerator card",
       "cards_2: [hardware-cards] Tests that require 2 accelerator cards",
   ]
   ```
   `check-mark` treats `[hardware-platform]` names (including `cpu`/`gpu`) as allowed `pytest.mark.<name>` values. `hardware_test(res=...)` still rejects `cpu`/`gpu` as `res` keys.
 2. **Implement a marker construction function for your platform** in `vllm-omni/tests/helpers/mark.py`:
   ```python
   # In vllm-omni/tests/helpers/mark.py
-   def tpu_marks(*, res: str, num_cards: int):
+   def _tpu_marks(*, res: str, num_cards: int):
-       test_platform = pytest.mark.tpu
+       _require_sku("tpu", res)
-       if res == "TPU_V3":
+       return [pytest.mark.tpu, getattr(pytest.mark, res), _cards_mark(num_cards)]
           test_resource = pytest.mark.TPU_V3
       else:
           raise ValueError(
               f"Invalid TPU resource type: {res}. Supported: TPU_V3")
       if num_cards == 1:
           return [test_platform, test_resource]
       else:
           test_distributed = pytest.mark.distributed_tpu(num_cards=num_cards)
           # Optionally: add skipif_tpu when implemented
           return [test_platform, test_resource, test_distributed]
   ```
-3. **Update `hardware_marks` to recognize your new platform**:
+3. **Register the builder** in `_marks_for_platform`:
    In the relevant place (see the `hardware_marks` implementation), add:
    ```python
-    if platform == "tpu":
+    builders = {
-        marks = tpu_marks(res=resource, num_cards=cards)
+        # ... existing platforms ...
        "tpu": _tpu_marks,
    }
    ```
-    (`hardware_test` calls `hardware_marks` internally, so both will pick up the change.)
+    (`hardware_test` / `hardware_marks` both go through `_marks_for_platform`.)
 4. **(Recommended) Add a test using your new markers**:
   ```python
   @hardware_test(
@@ -223,11 +210,11 @@ If you want to add support for a new platform (e.g., "tpu" for a new accelerator
 **Summary**:  
 - Add pytest markers for your new platform/resources  
- Implement a marker function (`xxx_marks`)  
+- Implement a private marker function (`_xxx_marks`)
- Plug into `hardware_marks`  
+- Register it in `_marks_for_platform`
- You're done: tests using `@hardware_test` or `hardware_marks` with your platform now automatically get the correct markers, distribution, and isolation!
+- You're done: tests using `@hardware_test` or `hardware_marks` with your platform now automatically get the correct markers and `cards_{n}` filtering.
-See code in `vllm-omni/tests/helpers/mark.py` for existing examples (`cuda_marks`, `rocm_marks`, `npu_marks`).
+See code in `vllm-omni/tests/helpers/mark.py` for existing examples (`_cuda_marks`, `_rocm_marks`, `_npu_marks`).
 ## Test case style
@@ -438,7 +438,7 @@ selects the scheduler and worker family:
 Key topology fields belong to `StagePipelineConfig`:
 | Field | Description |
-|-------|-------------|
+| ------- | ------------- |
 | `model_stage` | Logical stage name (`ar_stage`, `decoder`, etc.). |
 | `execution_type` | Runtime family for this stage. |
 | `input_sources` | Upstream stage IDs that provide input. |
@@ -960,7 +960,7 @@ Common failures and fixes:
 | `check-forbidden-imports` | Stdlib `re`/`base64`, pickle, Hugging Face Hub API, or direct Triton/TileLang | `import regex as re` and `pybase64`; do not grow the allowlist without review |
 | `check-torch-cuda-call` | New `torch.cuda.*` call site | Use `current_omni_platform`; do not grow `ALLOWED_FILES` without review |
 | `check-tts-adapter-migration` | New `_tts_model_type` branch in `serving_speech.py` | Put logic in `tts_adapters/`; lower `MAX_MODEL_TYPE_BRANCHES` when removing branches |
-| `check-test-ci-coverage` | Test file missing level or hardware mark | Add `core_model`/… plus `cpu`/`cuda`/`hardware_test(` |
+| `check-mark` | Test file missing level/hardware mark, or direct `pytest.mark.H100` | Add `core_model`/… plus `cpu`/`cuda`/`hardware_test(`; SKU marks only via helpers |
 | `shellcheck` | Native `shellcheck` missing, or script warning | Install via apt/dnf/brew (or `shellcheck.exe` on Windows). See [Linting](../README.md#linting) |
 | `mypy-3.10` / `markdownlint-cli2` / `check-buildkite` | Types, docs markdown, or Buildkite YAML | See [Linting](../README.md#linting) |
@@ -22,59 +22,60 @@ We use **Qwen-Image** as the reference implementation for decode parallel, and *
 **VAE Patch Parallelism** is an acceleration technique for both **encoding** and **decoding**. Instead of processing the entire tensor at once, the tensor is:
-+ Split into multiple spatial tiles
+- Split into multiple spatial tiles
-+ Distributed across multiple ranks
+- Distributed across multiple ranks
-+ Encoded/Decoded in parallel
+- Encoded/Decoded in parallel
-+ Merged to reconstruct the final output
+- Merged to reconstruct the final output
 This approach:
-+ Distributes computation across multiple devices
+- Distributes computation across multiple devices
-+ Reduces peak memory usage per device
+- Reduces peak memory usage per device
-+ Accelerates encoding/decoding latency
+- Accelerates encoding/decoding latency
 ### When to Use Encode vs Decode Parallel
-| Operation | Use Case | Example |
+| Operation           | Use Case                     | Example                           |
-|-----------|----------|---------|
+| ------------------- | ---------------------------- | --------------------------------- |
-| **Decode Parallel** | Text-to-Image, Text-to-Video | Latent → Image/Video |
+| **Decode Parallel** | Text-to-Image, Text-to-Video | Latent → Image/Video              |
-| **Encode Parallel** | Image-to-Video (I2V) | Image → Latent (for conditioning) |
+| **Encode Parallel** | Image-to-Video (I2V)         | Image → Latent (for conditioning) |
 ### Architecture
 We introduce **DistributedVaeExecutor** as the core component responsible for distributed VAE encoding/decoding.
 The executor is model-agnostic and accepts three function parameters:
-+ split – Partition the latent into tiles
+- split – Partition the latent into tiles
-+ exec – Decode a single tile
+- exec – Decode a single tile
-+ merge – Combine decoded tiles into the final output
+- merge – Combine decoded tiles into the final output
 #### Execution Flow
-+ Call split(z) to generate a list of TileTask and a GridSpec
+- Call split(z) to generate a list of TileTask and a GridSpec
-+ Dispatch tasks across ranks using workload-based balancing
+- Dispatch tasks across ranks using workload-based balancing
-+ Each rank executes exec(task) on its assigned tiles
+- Each rank executes exec(task) on its assigned tiles
-+ Gather decoded tile results to rank 0
+- Gather decoded tile results to rank 0
-+ Rank 0 performs merge(...)
+- Rank 0 performs merge(...)
-+ (Optional) Broadcast final result to all ranks
+- (Optional) Broadcast final result to all ranks
 This design separates:
-+ Distributed execution logic
+- Distributed execution logic
-+ Model-specific tiling and merging logic
+- Model-specific tiling and merging logic
 #### Why split / exec / merge is necessary?
@@ -82,23 +83,23 @@ The latent tensor cannot be arbitrarily partitioned.
 During decoding:
-+ Each output pixel may depend on neighboring pixels
+- Each output pixel may depend on neighboring pixels
-+ The receptive field is model-dependent
+- The receptive field is model-dependent
 Therefore:
-+ Tiles must include overlap
+- Tiles must include overlap
-+ Merge must perform blending to avoid seams
+- Merge must perform blending to avoid seams
 ## Step-by-Step Implementation (Decode)
 ### Step 1: Implement DistributedAutoencoderKLQwenImage
 `QwenImagePipeline` use `AutoencoderKLQwenImage` for vae, so implement a distributed version:
-
+```python
 ```
 class DistributedAutoencoderKLQwenImage(AutoencoderKLQwenImage, DistributedVaeMixin):
    @classmethod
    def from_pretrained(cls, *args: Any, **kwargs: Any):
@@ -106,28 +107,35 @@ class DistributedAutoencoderKLQwenImage(AutoencoderKLQwenImage, DistributedVaeMi
        model.init_distributed()
        return model
 ```
 **Key points**:
-+ Inherit both AutoencoderKLQwenImage and DistributedVaeMixin
+
-+ Call init_distributed() after loading weights
+- Inherit both AutoencoderKLQwenImage and DistributedVaeMixin
 - Call init_distributed() after loading weights
 ### Step 2: Implement split/exec/merge
 Reuse `AutoencoderKLQwenImage.tiled_decode` logic and divide it into three stages. And we need return tiles with `GridSpec` and `TileTask`:
-```
+
 ```python
 class GridSpec:
    split_dims: tuple[int, ...]  # Tensor dimensions being split (e.g., (2, 3) for (B, C, H, W))
    grid_shape: tuple[int, ...]  # Tile grid layout (num_rows, num_cols)
    tile_spec: dict = field(default_factory=dict) # Metadata required for merging
    output_dtype: torch.dtype | None = None # Final output dtype
 ```
-```
+
 ```python
 class TileTask:
    tile_id: int # task id
    grid_coord: tuple[int, ...]  # Tile position in grid
    tensor: torch.Tensor | list[torch.Tensor]  # The tile tensor
    workload: int | float = 1 # Used for load balancing (e.g., tile area)
 ```
 And tiled base split/exec/merge as follow:
-```
+
 ```python
 def tile_split(self, z: torch.Tensor) -> tuple[list[TileTask], GridSpec]:
    # mostly copy from AutoencoderKL
    _, _, num_frames, height, width = z.shape
@@ -210,11 +218,14 @@ def tile_merge(self, coord_tensor_map: dict[tuple[int, ...], torch.Tensor], grid
 ```
 ### Step 3: Override tiled_decode
 We need to override tiled_decode, the main logic is:
-+ check distributed is enabled
+
-+ select split/exec/merge
+- check distributed is enabled
-+ Invoke self.distributed_executor.execute to decode
+- select split/exec/merge
-```
+- Invoke self.distributed_executor.execute to decode
 ```python
 def tiled_decode(self, z: torch.Tensor, return_dict: bool = True):
    if not self.is_distributed_enabled():
        return super().tiled_decode(z, return_dict=return_dict)
@@ -230,11 +241,14 @@ def tiled_decode(self, z: torch.Tensor, return_dict: bool = True):
    return DecoderOutput(sample=result)
 ```
 `broadcast_result` is set to True or False depending on the model; when enabled, the result will be used even on ranks other than 0.
 ### Step 4: Modify Pipeline
 Change vae model from AutoencoderKLQwenImage to DistributedAutoencoderKLQwenImage
-```
+
 ```diff
 class YourModelPipeline(nn.Module):
    def __init__(
        self,
@@ -259,8 +273,8 @@ For models that require VAE encoding (e.g., Image-to-Video), you can also parall
 Similar to decode, split the input tensor into tiles. Key considerations:
-+ **Patchify handling**: If the model uses `patch_size`, scale tile parameters accordingly
+- **Patchify handling**: If the model uses `patch_size`, scale tile parameters accordingly
-+ **Temporal chunking**: Video VAEs may have temporal compression (e.g., 4x)
+- **Temporal chunking**: Video VAEs may have temporal compression (e.g., 4x)
 ```python
 def encode_tile_split(self, x: torch.Tensor) -> tuple[list[TileTask], GridSpec]:
@@ -406,23 +420,28 @@ def tiled_encode(self, x: torch.Tensor) -> torch.Tensor:
 **Key differences from decode parallel:**
 | Aspect | Decode Parallel | Encode Parallel |
-|--------|-----------------|-----------------|
+| -------- | ----------------- | ----------------- |
 | `broadcast_result` | Often `False` (only rank 0 needs output) | `True` (all ranks need latents for diffusion) |
 | Patchify | Applied in merge (unpatchify) | Handled by parent `_encode()` before `tiled_encode()` |
 | Temporal chunking | Frame-by-frame | Chunk-based (e.g., 1 + 4n frames) |
 ## Testing
 Verify numerical consistency between:
 + vae_patch_parallel_size = 1
-+ vae_patch_parallel_size = N
+Verify numerical consistency between:
 - vae_patch_parallel_size = 1
 - vae_patch_parallel_size = N
 Example:
 ```python
 torch.allclose(output_1, output_n, atol=1e-5)
 ```
 Testing requirements:
-+ Fix random seed
+
-+ Use identical tiling strategy
+- Fix random seed
 - Use identical tiling strategy
 ```python
 m = Omni(
@@ -434,6 +453,7 @@ m = Omni(
        ),
    )
 ```
 When vae_patch_parallel_size is larger than the DiT world size, it will automatically fall back to using the DiT world size instead.
 ## Reference Implementations
@@ -441,7 +461,7 @@ When vae_patch_parallel_size is larger than the DiT world size, it will automati
 Complete examples in the codebase:
 | Model | Path | Decode Parallel | Encode Parallel |
-|-------|------|-----------------|-----------------|
+| ------- | ------ | ----------------- | ----------------- |
 | **Z-Image** | `vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl.py` | ✅ | ❌ |
 | **Wan2.2** | `vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_wan.py` | ✅ | ✅ |
 | **Qwen-Image** | `vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_qwenimage.py` | ✅ | ❌ |
@@ -456,7 +476,7 @@ The tile-parallel executor above assigns independent spatial **tiles** to ranks.
 ### How it differs from tile parallel
 | Aspect | Tile parallel (`"tile"`) | Spatially-sharded (`"spatial_shard_height"`/`"spatial_shard_width"`) |
-|--------|--------------------------|-----------------------------------------------|
+| -------- | -------------------------- | ----------------------------------------------- |
 | Unit of work | Independent overlapping tiles | A single global feature map sharded along H or W |
 | Cross-rank communication | Gather tiles to rank 0, stitch + blend | Per-conv **halo exchange** of boundary rows/cols (P2P) |
 | Output assembly | Blend overlapping tiles | All-gather shards on rank 0, trim padding (matches `broadcast_result=False`) |
@@ -481,7 +501,7 @@ serve.py (--vae-parallel-mode) / OmniEngineArgs
 ### Notes
 - The decoder is patched **in place** the first time spatial-shard decode runs and is bound to a single split dimension for the lifetime of the VAE instance.
- Numerical correctness vs. single-GPU decode is covered by `tests/diffusion/distributed/test_wan_spatial_shard.py::test_spatial_shard_decode_matches_reference` (multi-GPU, nightly `full_model` + `distributed_cuda`).
+- Numerical correctness vs. single-GPU decode is covered by `tests/diffusion/distributed/test_wan_spatial_shard.py::test_spatial_shard_decode_matches_reference` (multi-GPU, nightly `full_model` + `cards_2`).
 ---
@@ -248,11 +248,11 @@ addopts = [
    "--dist=loadgroup"
 ]
 markers = [
-    # ci/cd required
+    # ci/cd required — tag with [ci-level] for tests.helpers.mark.get_level_markers().
-    "core_model: L1&L2 tests (run in each PR)",
+    "core_model: [ci-level] L1&L2 tests (run in each PR)",
-    "advanced_model: L3 level tests (run on each merge)",
+    "advanced_model: [ci-level] L3 level tests (run on each merge)",
-    "full_model: L4 level tests (run nightly)",
+    "full_model: [ci-level] L4 level tests (run nightly)",
-    "local_model: tests (run locally)",
+    "local_model: [ci-level] tests (run locally)",
    # function module markers
    "diffusion: Diffusion model tests",
    "omni: Omni model tests",
@@ -262,38 +262,41 @@ markers = [
    "sp: Sequence parallelism tests",
    "example: Doc example code tests",
    # platform markers
-    "cpu: Tests that run on CPU",
+    # Tag with [hardware-platform] for tests.helpers.mark.get_supported_platforms().
-    "gpu: Tests that run on GPU (auto-added)",
+    # Add [gpu] on accelerator platforms that should also get pytest.mark.gpu.
-    "cuda: Tests that run on CUDA (auto-added)",
+    # cpu/gpu are pytest marks; hardware_test(res=...) rejects them as keys.
-    "rocm: Tests that run on AMD/ROCm (auto-added)",
+    "cpu: [hardware-platform] Tests that run on CPU",
-    "xpu: Tests that run on XPU (auto-added)",
+    "gpu: [hardware-platform] Tests that run on GPU (auto-added)",
-    "npu: Tests that run on NPU/Ascend (auto-added)",
+    "cuda: [hardware-platform] [gpu] Tests that run on CUDA (auto-added)",
-    "musa: Tests that run on MUSA/Moore Threads (auto-added)",
+    "rocm: [hardware-platform] [gpu] Tests that run on AMD/ROCm (auto-added)",
    "xpu: [hardware-platform] [gpu] Tests that run on XPU (auto-added)",
    "npu: [hardware-platform] Tests that run on NPU/Ascend (auto-added)",
    "musa: [hardware-platform] [gpu] Tests that run on MUSA/Moore Threads (auto-added)",
    # specified computation resources marks (auto-added)
-    # Tag with [hardware-resource] for tests.helpers.mark.get_hardware_mark_list().
+    # [hardware-resource] + [cuda]/[rocm]/[xpu]/[npu]/[musa] for hardware_test(res=...).
-    "H100: [hardware-resource] Tests that require H100 GPU",
+    # Do not put [gpu] on SKU lines: that would make gpu a res key.
-    "H800: [hardware-resource] Tests that require H800 GPU",
+    "H100: [hardware-resource] [cuda] Tests that require H100 GPU",
-    "H200: [hardware-resource] Tests that require H200 GPU",
+    "H800: [hardware-resource] [cuda] Tests that require H800 GPU",
-    "L4: [hardware-resource] Tests that require L4 GPU",
+    "H200: [hardware-resource] [cuda] Tests that require H200 GPU",
-    "B200: [hardware-resource] Tests that require B200 GPU",
+    "L4: [hardware-resource] [cuda] Tests that require L4 GPU",
-    "MI325: [hardware-resource] Tests that require MI325 GPU (AMD/ROCm)",
+    "B200: [hardware-resource] [cuda] Tests that require B200 GPU",
-    "B60: [hardware-resource] Tests that require Intel Arc Pro B60 XPU",
+    "MI325: [hardware-resource] [rocm] Tests that require MI325 GPU (AMD/ROCm)",
-    "S5000: [hardware-resource] Tests that require S5000 GPU (Moore Threads/MUSA)",
+    "B60: [hardware-resource] [xpu] Tests that require Intel Arc Pro B60 XPU",
-    "A2: [hardware-resource] Tests that require A2 NPU",
+    "S5000: [hardware-resource] [musa] Tests that require S5000 GPU (Moore Threads/MUSA)",
-    "A3: [hardware-resource] Tests that require A3 NPU",
+    "A2: [hardware-resource] [npu] Tests that require A2 NPU",
-    "distributed_cuda: Tests that require multi cards on CUDA platform",
+    "A3: [hardware-resource] [npu] Tests that require A3 NPU",
-    "distributed_rocm: Tests that require multi cards on ROCm platform",
+    # Tag with [hardware-cards] for tests.helpers.mark.get_supported_card_counts().
-    "distributed_xpu: Tests that require multi cards on XPU platform",
+    "cards_1: [hardware-cards] Tests that require 1 accelerator card",
-    "distributed_npu: Tests that require multi cards on NPU platform",
+    "cards_2: [hardware-cards] Tests that require 2 accelerator cards",
-    "distributed_musa: Tests that require multi cards on MUSA platform",
+    "cards_3: [hardware-cards] Tests that require 3 accelerator cards",
-    "skipif_cuda: Skip if the num of CUDA cards is less than the required",
+    "cards_4: [hardware-cards] Tests that require 4 accelerator cards",
-    "skipif_rocm: Skip if the num of ROCm cards is less than the required",
+    "cards_5: [hardware-cards] Tests that require 5 accelerator cards",
-    "skipif_xpu: Skip if the num of XPU cards is less than the required",
+    "cards_6: [hardware-cards] Tests that require 6 accelerator cards",
-    "skipif_npu: Skip if the num of NPU cards is less than the required",
+    "cards_7: [hardware-cards] Tests that require 7 accelerator cards",
-    "skipif_musa: Skip if the num of MUSA cards is less than the required",
+    "cards_8: [hardware-cards] Tests that require 8 accelerator cards",
    # more detailed markers
    "sp: Sequence parallelism tests (multi-GPU)",
-    "slow: Slow tests (may skip in quick CI)",
+    "slow: [ci-level] Slow tests (may skip in quick CI)",
    "benchmark: Benchmark tests",
    "xdist: xdist-parallelizable tests",
 ]
@@ -25,9 +25,9 @@ pytest_plugins = (
    "tests.helpers.fixtures.clean",
    "tests.helpers.fixtures.log",
    "tests.helpers.fixtures.media",
-    "tests.helpers.fixtures.run_args",
+    "tests.helpers.fixtures.pytest_collection",
    "tests.helpers.fixtures.pytest_run_args",
    "tests.helpers.fixtures.runtime",
    "tests.helpers.fixtures.xdist",
 )
@@ -1,3 +1,6 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 """Tests for DFX runner metadata field exclusion."""
 import json
@@ -105,7 +108,7 @@ def test_resolve_pytest_marks_hardware_dict_with_extra():
    assert "H100" in names
    assert "cuda" in names
    assert "gpu" in names
-    assert "distributed_cuda" in names
+    assert "cards_2" in names
    assert "full_model" in names
    assert "diffusion" in names
    assert "local_model" in names
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 """Tests for the Sequence Parallelism (SP) framework.
 These tests verify the SP plan mechanism and hooks work correctly without
@@ -17,6 +17,7 @@ import pytest
 import torch
 import torch.nn as nn
 from tests.helpers.mark import hardware_marks
 from vllm_omni.diffusion.distributed.sp_plan import (
    SequenceParallelInput,
    SequenceParallelOutput,
@@ -419,7 +420,6 @@ class TestHookRegistration:
 # =============================================================================
@pytest.mark.L4
 class TestModelSpPlans:
    """Test that model _sp_plan definitions are valid.
@@ -427,6 +427,8 @@ class TestModelSpPlans:
    May require GPU for model imports.
    """
    pytestmark = hardware_marks(res={"cuda": "L4"})
    def test_zimage_transformer_sp_plan(self):
        """Test ZImageTransformer2DModel _sp_plan structure.
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 """GPU e2e for PersonaPlex elastic batching (bit-exact isolation + recycle).
 Greedy decoding makes every cross-slot leak observable as a bit difference:
@@ -24,12 +24,17 @@ from __future__ import annotations
 import numpy as np
 import pytest
 from tests.helpers.mark import hardware_marks
 torch = pytest.importorskip("torch")
-# slow + H100 + omni routes this module into the weekly Buildkite E2E group
+# slow + H100 + omni + cards_1 routes this module into weekly
-# ("Omni · H100" in .buildkite/cuda/test-weekly.yml: -m "slow and H100 and
+# "Omni · H100 · Single-GPU" (test-weekly.yml).
-# omni" --run-level full_model); the ready/merge CPU sweeps exclude it by mark.
+pytestmark = [
-pytestmark = [pytest.mark.full_model, pytest.mark.omni, pytest.mark.slow, pytest.mark.H100]
+    pytest.mark.omni,
    pytest.mark.slow,
    *hardware_marks(res={"cuda": "H100"}),
 ]
 if not torch.cuda.is_available():  # noqa: E402
    pytest.skip("needs CUDA", allow_module_level=True)
@@ -1,13 +1,13 @@
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 """Smoke test for HunyuanImage-3.0 Image-to-Text (I2T) pipeline."""
 from collections.abc import Generator
 import pytest
 import torch
 from PIL import Image
 from tests.helpers.mark import hardware_test
 from tests.helpers.runtime import OmniRunner
 from tests.helpers.stage_config import get_deploy_config_path
 from vllm_omni import Omni
@@ -44,7 +44,7 @@ EXPECTED_PREFIX_TOKEN_IDS: list[int] = [
 # Decoded form, kept only for human-readable assertion messages.
 EXPECTED_PREFIX_TEXT = "The image is a solid, uniform green color with no variations, objects, or details present. It"
-pytestmark = [pytest.mark.full_model, pytest.mark.diffusion, pytest.mark.cuda]
+pytestmark = [pytest.mark.full_model, pytest.mark.diffusion]
@pytest.fixture(scope="module")
@@ -57,7 +57,7 @@ def omni() -> Generator[Omni, None, None]:
        yield runner.omni
-@pytest.mark.skipif(torch.accelerator.device_count() < 4, reason="Need at least 4 CUDA GPUs.")
+@hardware_test(res={"cuda": "H100"}, num_cards=4)
 def test_i2t_generates_text(omni: Omni) -> None:
    """Verify I2T output's first 20 token IDs match the HF greedy baseline."""
    # Solid-color image keeps the input self-contained and reproducible.
@@ -91,8 +91,8 @@ def test_krea2_cache_dit(offline_client: OfflineOmniClient) -> None:
 def test_krea2_hsdp(offline_client: OfflineOmniClient) -> None:
    """Exercise Hybrid Sharded Data Parallel (HSDP) weight sharding on the DiT across 2 GPUs.
-    Validates the docs' HSDP-support claim for Krea 2. Scheduled on the multi-GPU (distributed_cuda)
+    Validates the docs' HSDP-support claim for Krea 2. Scheduled on the multi-GPU
-    shard by ``num_cards=2``; skipped automatically on boxes with fewer than 2 CUDA devices.
+    shard by ``num_cards=2`` (``not cards_1``); skipped automatically on boxes with fewer than 2 CUDA devices.
    """
    offline_client.send_diffusion_request({"model": MODEL, "prompt": PROMPT, "sampling_params": _sampling()})
@@ -111,7 +111,7 @@ def test_krea2_vae_patch_parallel(offline_client: OfflineOmniClient) -> None:
    """Exercise VAE patch-parallel (decode) across 2 GPUs.
    Validates the docs' VAE-patch-parallel (decode) support claim for Krea 2. Scheduled on the
-    multi-GPU (distributed_cuda) shard by ``num_cards=2``; skipped automatically on boxes with
+    multi-GPU shard by ``num_cards=2`` (``not cards_1``); skipped automatically on boxes with
    fewer than 2 CUDA devices.
    """
    offline_client.send_diffusion_request({"model": MODEL, "prompt": PROMPT, "sampling_params": _sampling()})
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 """
-E2E expansion tests for MiMo-Audio online serving (nightly Omni · Function Test with H100).
+E2E expansion tests for MiMo-Audio online serving (nightly Omni · Function Test with H100 · Single-GPU).
 """
 import os
@@ -0,0 +1,30 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 """Pytest collection-phase hooks.
 All ``pytest_collection_modifyitems`` policies live here so their order is
 explicit. Add a named helper and call it from the hook; do not register a
 second ``pytest_collection_modifyitems`` in another plugin.
 """
 from __future__ import annotations
 import pytest
 def pytest_collection_modifyitems(config, items):
    _pin_xdist_serial(config, items)
 def _pin_xdist_serial(config, items):
    """Pin unmarked tests to one xdist worker.
    Only tests explicitly marked ``pytest.mark.xdist`` run concurrently.
    Others are pinned to a single xdist worker.
    """
    if not config.pluginmanager.hasplugin("xdist"):
        return
    for item in items:
        if not item.get_closest_marker("xdist"):
            item.add_marker(pytest.mark.xdist_group(name="serial"))
@@ -1,3 +1,10 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 """Pytest CLI options and the fixtures that expose them."""
 from __future__ import annotations
 import pytest
@@ -24,6 +24,18 @@ if TYPE_CHECKING:
 omni_fixture_lock = threading.Lock()
@pytest.fixture
 def _normalized_hardware_marks(request: pytest.FixtureRequest):
    """Dummy carrier for mixed-``num_cards`` ``@hardware_test`` mark normalization.
    Pytest has no API to clone an item and only change marks. ``@hardware_test``
    therefore parametrizes this fixture (``indirect=True``) so each collected
    item gets one platform's SKU and ``cards_{n}`` without changing test
    signatures. The returned platform id is unused by test bodies.
    """
    return request.param
@pytest.fixture(scope="function")
 def omni_server_function(
    request: pytest.FixtureRequest,
@@ -1,14 +0,0 @@
 """
 Only tests explicitly marked ``pytest.mark.xdist`` run concurrently.
 Other test are pinned to a single xdist worker.
 """
 import pytest
 def pytest_collection_modifyitems(config, items):
    if not config.pluginmanager.hasplugin("xdist"):
        return
    for item in items:
        if not item.get_closest_marker("xdist"):
            item.add_marker(pytest.mark.xdist_group(name="serial"))
@@ -1,19 +1,33 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
-"""Pytest marks and decorators for hardware / resource selection (CUDA, ROCm, …)."""
+"""Pytest marks and decorators for hardware / resource selection (CUDA, ROCm, …).
 Module import is stdlib-only so ``tools/pre_commit/check_test_marks.py`` can load
 this file without pytest or vllm. Decorator helpers import pytest lazily.
 """
 from __future__ import annotations
 import re
 from functools import lru_cache
 from pathlib import Path
 from typing import TYPE_CHECKING
-import pytest
+if TYPE_CHECKING:
-from vllm.platforms import current_platform
+    import pytest
-# Marker description tag in ``pyproject.toml`` ``tool.pytest.ini_options.markers``.
+# Marker description tags in ``pyproject.toml`` ``tool.pytest.ini_options.markers``.
-# Example: ``"H100: [hardware-resource] Tests that require H100 GPU"``.
+# Example: ``"H100: [hardware-resource] [cuda] Tests that require H100 GPU"``.
 _HARDWARE_RESOURCE_MARKER_TAG = "[hardware-resource]"
 _HARDWARE_CARDS_MARKER_TAG = "[hardware-cards]"
 _HARDWARE_PLATFORM_MARKER_TAG = "[hardware-platform]"
 _CI_LEVEL_MARKER_TAG = "[ci-level]"
 # On ``[hardware-platform]`` lines (not SKUs): also attach ``pytest.mark.gpu``.
 _GPU_DEVICE_CLASS_TAG = "[gpu]"
 _CARDS_MARK_NAME_RE = re.compile(r"^cards_(\d+)$")
 # Indirect fixture in ``tests.helpers.fixtures.runtime``: mixed-count ``@hardware_test``
 # parametrizes it so each collected item carries one platform's SKU and ``cards_{n}``.
 HARDWARE_MARK_NORMALIZATION_FIXTURE = "_normalized_hardware_marks"
 def _repo_root() -> Path:
@@ -34,23 +48,16 @@ def _load_pyproject() -> dict:
        return {}
-@lru_cache(maxsize=1)
+@lru_cache(maxsize=16)
-def get_hardware_mark_list() -> frozenset[str]:
+def _marker_names_with_tag(tag: str) -> frozenset[str]:
-    """Return hardware SKU marker names from ``pyproject.toml``.
+    """Return pytest marker names whose description contains ``tag``."""
    A marker is treated as a hardware resource marker when its description
    contains the ``[hardware-resource]`` tag, e.g.::
        "H100: [hardware-resource] Tests that require H100 GPU"
    """
    data = _load_pyproject()
    entries = data.get("tool", {}).get("pytest", {}).get("ini_options", {}).get("markers", [])
    if not entries:
        # Fallback when TOML parsing fails: scan tagged marker lines.
        text = (_repo_root() / "pyproject.toml").read_text(encoding="utf-8")
        return frozenset(
            re.findall(
-                rf'^\s*"([A-Za-z0-9_]+)\s*:\s*{re.escape(_HARDWARE_RESOURCE_MARKER_TAG)}',
+                rf'^\s*"([A-Za-z0-9_]+)\s*:.*{re.escape(tag)}',
                text,
                flags=re.M,
            )
@@ -59,7 +66,7 @@ def get_hardware_mark_list() -> frozenset[str]:
    names: set[str] = set()
    for entry in entries:
        text = str(entry)
-        if _HARDWARE_RESOURCE_MARKER_TAG not in text:
+        if tag not in text:
            continue
        name = text.split(":", 1)[0].strip()
        if name:
@@ -67,138 +74,256 @@ def get_hardware_mark_list() -> frozenset[str]:
    return frozenset(names)
-def cuda_marks(*, res: str, num_cards: int):
+@lru_cache(maxsize=1)
 def get_hardware_mark_list() -> frozenset[str]:
    """Return hardware SKU marker names tagged ``[hardware-resource]``."""
    return _marker_names_with_tag(_HARDWARE_RESOURCE_MARKER_TAG)
@lru_cache(maxsize=1)
 def get_supported_card_counts() -> frozenset[int]:
    """Return ``n`` for each ``cards_{n}`` marker tagged ``[hardware-cards]``."""
    counts: set[int] = set()
    for name in _marker_names_with_tag(_HARDWARE_CARDS_MARKER_TAG):
        matched = _CARDS_MARK_NAME_RE.fullmatch(name)
        if matched:
            counts.add(int(matched.group(1)))
    return frozenset(counts)
@lru_cache(maxsize=1)
 def get_supported_platforms() -> frozenset[str]:
    """Return platform marker names tagged ``[hardware-platform]``.
    Includes ``cpu`` / ``gpu`` (device-class marks) and accelerator keys.
    ``hardware_marks`` / ``hardware_test`` ``res`` dicts reject ``cpu`` / ``gpu``.
    """
    return _marker_names_with_tag(_HARDWARE_PLATFORM_MARKER_TAG)
@lru_cache(maxsize=1)
 def get_level_markers() -> frozenset[str]:
    """Return CI level marker names tagged ``[ci-level]`` (``core_model``, ``slow``, …)."""
    return _marker_names_with_tag(_CI_LEVEL_MARKER_TAG)
@lru_cache(maxsize=8)
 def _skus_for_platform(platform: str) -> frozenset[str]:
    """SKU names tagged ``[hardware-resource]`` and ``[platform]`` (e.g. ``[cuda]``)."""
    return get_hardware_mark_list() & _marker_names_with_tag(f"[{platform}]")
 def _require_sku(platform: str, res: str) -> None:
    allowed = _skus_for_platform(platform)
    if res not in allowed:
        supported = ", ".join(sorted(allowed)) or f"(none tagged [hardware-resource] [{platform}])"
        raise ValueError(f"Invalid {platform} resource type: {res}. Supported: {supported}")
 def _cards_mark(num_cards: int) -> pytest.MarkDecorator:
    """Return ``cards_{n}`` so collection can filter with ``-m cards_2`` (etc.).
    Multi-card selection is ``not cards_1`` (or an explicit ``cards_2`` / ``cards_4``).
    """
    import pytest
    if not isinstance(num_cards, int) or isinstance(num_cards, bool) or num_cards < 1:
        raise ValueError(f"num_cards must be a positive int, got {num_cards!r}")
    if num_cards not in get_supported_card_counts():
        counts = get_supported_card_counts()
        supported = ", ".join(f"cards_{n}" for n in sorted(counts))
        supported = supported or "(none tagged [hardware-cards])"
        raise ValueError(f"num_cards={num_cards} has no registered marker. Supported: {supported}.")
    return getattr(pytest.mark, f"cards_{num_cards}")
 def _cuda_marks(*, res: str, num_cards: int):
    import pytest
    from vllm.platforms import current_platform
    _require_sku("cuda", res)
    test_resource = getattr(pytest.mark, res)
    test_platform_detail = pytest.mark.cuda
-    if res == "L4":
+    marks = [test_resource, test_platform_detail, _cards_mark(num_cards)]
-        test_resource = pytest.mark.L4
+    if num_cards == 1 or not current_platform.is_cuda():
    elif res == "H100":
        test_resource = pytest.mark.H100
    elif res == "H800":
        test_resource = pytest.mark.H800
    else:
        raise ValueError(f"Invalid CUDA resource type: {res}. Supported: L4, H100, H800")
    marks = [test_resource, test_platform_detail]
    if num_cards == 1:
        return marks
-    test_distributed = pytest.mark.distributed_cuda(num_cards=num_cards)
+    return marks + [
-
+        pytest.mark.skipif(
-    if not current_platform.is_cuda():
+            current_platform.device_count() < num_cards,
-        return marks + [test_distributed]
+            reason=f"Need at least {num_cards} CUDA GPUs to run the test.",
-    test_skipif = pytest.mark.skipif(
+        )
-        current_platform.device_count() < num_cards,
+    ]
        reason=f"Need at least {num_cards} CUDA GPUs to run the test.",
    )
    return marks + [test_distributed, test_skipif]
-def rocm_marks(*, res: str, num_cards: int):
+def _rocm_marks(*, res: str, num_cards: int):
-    test_platform_detail = pytest.mark.rocm
+    import pytest
-    if res == "MI325":
+
-        test_resource = pytest.mark.MI325
+    _require_sku("rocm", res)
-    else:
+    return [getattr(pytest.mark, res), pytest.mark.rocm, _cards_mark(num_cards)]
-        raise ValueError(f"Invalid ROCm resource type: {res}. Supported: MI325")
+
-    marks = [test_resource, test_platform_detail]
+
-    if num_cards == 1:
+def _xpu_marks(*, res: str, num_cards: int):
    import pytest
    from vllm.platforms import current_platform
    _require_sku("xpu", res)
    marks = [getattr(pytest.mark, res), pytest.mark.xpu, _cards_mark(num_cards)]
    if num_cards == 1 or not current_platform.is_xpu():
        return marks
-    test_distributed = pytest.mark.distributed_rocm(num_cards=num_cards)
+    return marks + [
-    return marks + [test_distributed]
+        pytest.mark.skipif(
            current_platform.device_count() < num_cards,
            reason=f"Need at least {num_cards} XPUs to run the test.",
        )
    ]
-def xpu_marks(*, res: str, num_cards: int):
+def _musa_marks(*, res: str, num_cards: int):
-    test_platform_detail = pytest.mark.xpu
+    import pytest
    if res == "B60":
        test_resource = pytest.mark.B60
    else:
        raise ValueError(f"Invalid XPU resource type: {res}. Supported: B60")
    marks = [test_resource, test_platform_detail]
    if num_cards == 1:
        return marks
    test_distributed = pytest.mark.distributed_xpu(num_cards=num_cards)
-    if not current_platform.is_xpu():
+    _require_sku("musa", res)
-        return marks + [test_distributed]
+    return [getattr(pytest.mark, res), pytest.mark.musa, _cards_mark(num_cards)]
    test_skipif = pytest.mark.skipif(
        current_platform.device_count() < num_cards,
        reason=f"Need at least {num_cards} XPUs to run the test.",
    )
    return marks + [test_distributed, test_skipif]
-def musa_marks(*, res: str, num_cards: int):
+@lru_cache(maxsize=1)
-    test_platform_detail = pytest.mark.musa
+def _gpu_res_platforms() -> frozenset[str]:
-    if res == "S5000":
+    """Accelerator platforms tagged ``[hardware-platform] [gpu]`` (not the ``gpu`` mark)."""
-        test_resource = pytest.mark.S5000
+    return (get_supported_platforms() & _marker_names_with_tag(_GPU_DEVICE_CLASS_TAG)) - {"gpu"}
    else:
        raise ValueError(f"Invalid MUSA resource type: {res}. Supported: S5000")
    marks = [test_resource, test_platform_detail]
    if num_cards == 1:
        return marks
    test_distributed = pytest.mark.distributed_musa(num_cards=num_cards)
    return marks + [test_distributed]
-def gpu_marks(*, res: str, num_cards: int):
+def _npu_marks(*, res: str, num_cards: int):
-    test_platform = pytest.mark.gpu
+    import pytest
-    if res in ("L4", "H100", "H800"):
+
-        return [test_platform] + cuda_marks(res=res, num_cards=num_cards)
+    _require_sku("npu", res)
-    if res == "MI325":
+    return [pytest.mark.npu, getattr(pytest.mark, res), _cards_mark(num_cards)]
        return [test_platform] + rocm_marks(res=res, num_cards=num_cards)
    if res == "B60":
        return [test_platform] + xpu_marks(res=res, num_cards=num_cards)
    if res == "S5000":
        return [test_platform] + musa_marks(res=res, num_cards=num_cards)
    raise ValueError(f"Invalid resource type: {res}. Supported: L4, H100, H800, MI325, B60, S5000")
-def npu_marks(*, res: str, num_cards: int):
+def _res_platforms() -> frozenset[str]:
-    test_platform = pytest.mark.npu
+    """Accelerator keys allowed in ``hardware_test(res=...)`` / ``hardware_marks(res=...)``.
-    if res == "A2":
+
-        test_resource = pytest.mark.A2
+    A ``[hardware-platform]`` name is a ``res`` key only if some SKU is tagged
-    elif res == "A3":
+    with that platform (``[cuda]``, ``[npu]``, …). ``cpu`` / ``gpu`` have no SKUs.
-        test_resource = pytest.mark.A3
+    """
-    else:
+    return frozenset(p for p in get_supported_platforms() if _skus_for_platform(p))
-        test_resource = None
+
-    if num_cards == 1:
+
-        return [mark for mark in [test_platform, test_resource] if mark is not None]
+def _normalize_num_cards(res: dict[str, str], num_cards: int | dict[str, int]) -> dict[str, int]:
-    test_distributed = pytest.mark.distributed_npu(num_cards=num_cards)
+    allowed = _res_platforms()
-    return [mark for mark in [test_platform, test_resource, test_distributed] if mark is not None]
+    device_class = get_supported_platforms() - allowed
    for platform in res:
        if platform in device_class:
            raise ValueError(
                f"{platform!r} is not a res dict key; use pytest.mark.{platform} "
                f"or an accelerator key ({', '.join(sorted(allowed))})."
            )
        if platform not in allowed:
            supported = ", ".join(sorted(allowed)) or "(none)"
            raise ValueError(f"Unsupported platform: {platform}. res keys: {supported}.")
    if isinstance(num_cards, int) and not isinstance(num_cards, bool):
        return {platform: num_cards for platform in res}
    if not isinstance(num_cards, dict):
        raise ValueError(f"num_cards must be a positive int or dict, got {num_cards!r}")
    num_cards_dict = dict(num_cards)
    for platform in num_cards_dict:
        if platform not in res:
            raise ValueError(f"Platform '{platform}' in num_cards but not in res.")
    for platform in res:
        num_cards_dict.setdefault(platform, 1)
    return num_cards_dict
 def _marks_for_platform(platform: str, resource: str, num_cards: int) -> list[pytest.MarkDecorator]:
    import pytest
    builders = {
        "cuda": _cuda_marks,
        "rocm": _rocm_marks,
        "xpu": _xpu_marks,
        "musa": _musa_marks,
        "npu": _npu_marks,
    }
    builder = builders.get(platform)
    if builder is None:
        raise ValueError(f"Unsupported platform: {platform}")
    marks = builder(res=resource, num_cards=num_cards)
    if platform in _gpu_res_platforms():
        return [pytest.mark.gpu] + marks
    return marks
 def _skipif_not_platform(platform: str) -> pytest.MarkDecorator:
    import pytest
    from vllm_omni.platforms import current_omni_platform
    checkers = {
        "cuda": current_omni_platform.is_cuda,
        "rocm": current_omni_platform.is_rocm,
        "xpu": current_omni_platform.is_xpu,
        "npu": current_omni_platform.is_npu,
        "musa": current_omni_platform.is_musa,
    }
    return pytest.mark.skipif(not checkers[platform](), reason=f"Requires {platform} platform")
 def _apply_marks(func, marks: list[pytest.MarkDecorator]):
    for mark in reversed(marks):
        func = mark(func)
    return func
 def hardware_marks(*, res: dict[str, str], num_cards: int | dict[str, int] = 1):
-    for platform, _ in res.items():
+    """Return marks for a **single** pytest item.
-        if platform not in ("cuda", "rocm", "xpu", "npu", "musa"):
+
-            raise ValueError(f"Unsupported platform: {platform}")
+    Different per-platform ``num_cards`` cannot share one item: ``cards_2`` and
-    if isinstance(num_cards, int):
+    ``cards_4`` would both match ``-m cards_2`` / ``-m cards_4``. Use
-        num_cards_dict = {platform: num_cards for platform in res.keys()}
+    ``@hardware_test`` (one collected variant per platform) or call this once
-    else:
+    per platform and attach each list to its own ``pytest.param``.
-        num_cards_dict = num_cards
+    """
-        for platform in num_cards_dict.keys():
+    num_cards_dict = _normalize_num_cards(res, num_cards)
-            if platform not in res:
+    unique_counts = set(num_cards_dict.values())
-                raise ValueError(f"Platform '{platform}' in num_cards but not in res.")
+    if len(res) > 1 and len(unique_counts) > 1:
-        for platform in res.keys():
+        raise ValueError(
-            if platform not in num_cards_dict:
+            "hardware_marks() cannot attach different per-platform num_cards to one pytest item "
-                num_cards_dict[platform] = 1
+            f"(got {num_cards_dict}). Use @hardware_test, which collects one variant per platform, "
            "or call hardware_marks() once per platform."
        )
    all_marks: list[pytest.MarkDecorator] = []
    for platform, resource in res.items():
-        cards = num_cards_dict[platform]
+        all_marks.extend(_marks_for_platform(platform, resource, num_cards_dict[platform]))
        if platform in ("cuda", "rocm", "xpu", "musa"):
            marks = gpu_marks(res=resource, num_cards=cards)
        elif platform == "npu":
            marks = npu_marks(res=resource, num_cards=cards)
        else:
            raise ValueError(f"Unsupported platform: {platform}")
        all_marks.extend(marks)
    return all_marks
 def hardware_test(*, res: dict[str, str], num_cards: int | dict[str, int] = 1):
-    all_marks = hardware_marks(res=res, num_cards=num_cards)
+    """Apply hardware marks; split collected items when card counts differ by platform.
-    def wrapper(f):
+    A single item cannot carry both ``cards_4`` (CUDA) and ``cards_2`` (ROCm):
-        func = f
+    ``-m "H100 and cards_4 and cuda"`` would otherwise also match
-        for mark in reversed(all_marks):
+    ``-m "H100 and cards_2 and cuda"``. When ``num_cards`` differs across
-            func = mark(func)
+    platforms, this decorator parametrizes one variant per platform so each
-        return func
+    item only has that platform's SKU and ``cards_{n}``.
    """
    import pytest
-    return wrapper
+    num_cards_dict = _normalize_num_cards(res, num_cards)
    mixed_counts = len(res) > 1 and len(set(num_cards_dict.values())) > 1
    if not mixed_counts:
        all_marks = hardware_marks(res=res, num_cards=num_cards)
        def apply_union_marks(f):
            return _apply_marks(f, all_marks)
        return apply_union_marks
    params = []
    for platform, resource in res.items():
        marks = _marks_for_platform(platform, resource, num_cards_dict[platform])
        marks.append(_skipif_not_platform(platform))
        params.append(pytest.param(platform, marks=marks, id=platform))
    def apply_split_params(f):
        func = pytest.mark.usefixtures(HARDWARE_MARK_NORMALIZATION_FIXTURE)(f)
        return pytest.mark.parametrize(HARDWARE_MARK_NORMALIZATION_FIXTURE, params, indirect=True)(func)
    return apply_split_params
@@ -0,0 +1,139 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 import pytest
 from tests.helpers.mark import (
    _gpu_res_platforms,
    hardware_marks,
    hardware_test,
 )
 pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
 def _mark_names(marks) -> set[str]:
    return {mark.name for mark in marks}
 def test_gpu_platforms_come_from_pyproject_gpu_tag():
    assert "npu" not in _gpu_res_platforms()
    assert "gpu" not in _gpu_res_platforms()
 def test_default_num_cards_adds_cards_1():
    names = _mark_names(hardware_marks(res={"cuda": "H100"}))
    assert "cards_1" in names
    assert "H100" in names
    assert "cards_2" not in names
 def test_h100_two_cards_is_selectable_with_cards_2():
    names = _mark_names(hardware_marks(res={"cuda": "H100"}, num_cards=2))
    assert names >= {"H100", "cuda", "gpu", "cards_2"}
    assert "cards_1" not in names
    assert "cards_4" not in names
 def test_h100_four_cards_is_selectable_with_cards_4():
    names = _mark_names(hardware_marks(res={"cuda": "H100"}, num_cards=4))
    assert "cards_4" in names
    assert "cards_2" not in names
 def test_npu_card_count_mark_is_independent_of_sku():
    names = _mark_names(hardware_marks(res={"npu": "A2"}, num_cards=4))
    assert "cards_4" in names
    assert "npu" in names
    assert "gpu" not in names
 def test_hardware_marks_rejects_mixed_counts_on_one_item():
    with pytest.raises(ValueError, match="different per-platform num_cards"):
        hardware_marks(
            res={"cuda": "H100", "rocm": "MI325"},
            num_cards={"cuda": 4, "rocm": 2},
        )
 def test_same_count_multi_platform_stays_one_item():
    names = _mark_names(hardware_marks(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2))
    assert names >= {"H100", "cuda", "MI325", "rocm", "cards_2"}
    @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
    def _probe():
        return None
    names = {mark.name for mark in _probe.pytestmark}
    assert "parametrize" not in names
    assert "cards_2" in names
    assert "H100" in names
    assert "MI325" in names
 def test_hardware_test_decorator_applies_cards_mark():
    @hardware_test(res={"cuda": "H100"}, num_cards=2)
    def _probe():
        return None
    names = {mark.name for mark in _probe.pytestmark}
    assert "cards_2" in names
    assert "H100" in names
 def _hardware_platform_params(func):
    marks = func.pytestmark if isinstance(func.pytestmark, list) else [func.pytestmark]
    for mark in marks:
        if mark.name != "parametrize":
            continue
        argnames = mark.args[0]
        if argnames == "_normalized_hardware_marks":
            return mark.args[1]
    raise AssertionError("expected @hardware_test to parametrize _normalized_hardware_marks")
 def test_mixed_card_counts_split_so_cuda_filter_does_not_see_rocm_count():
    @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2})
    def _probe():
        return None
    variants = {param.id: {mark.name for mark in param.marks} for param in _hardware_platform_params(_probe)}
    cuda_names = variants["cuda"]
    rocm_names = variants["rocm"]
    assert cuda_names >= {"H100", "cuda", "cards_4"}
    assert "cards_2" not in cuda_names
    assert "rocm" not in cuda_names
    assert "MI325" not in cuda_names
    assert rocm_names >= {"MI325", "rocm", "cards_2"}
    assert "cards_4" not in rocm_names
    assert "cuda" not in rocm_names
    assert "H100" not in rocm_names
 def test_unsupported_card_count_raises():
    with pytest.raises(ValueError, match="num_cards=16"):
        hardware_marks(res={"cuda": "H100"}, num_cards=16)
@pytest.mark.parametrize("platform", ["cpu", "gpu"])
 def test_res_rejects_cpu_and_gpu(platform):
    with pytest.raises(ValueError, match="not a res dict key"):
        hardware_marks(res={platform: "H100"})
 def test_npu_rejects_unsupported_sku():
    with pytest.raises(ValueError, match="Invalid npu resource type"):
        hardware_marks(res={"npu": "A100"})
@pytest.mark.parametrize("sku", ["H200", "B200", "H800"])
 def test_cuda_registered_skus_attach_sku_and_cards(sku):
    names = _mark_names(hardware_marks(res={"cuda": sku}, num_cards=2))
    assert names >= {sku, "cuda", "gpu", "cards_2"}
 def test_cuda_rejects_sku_from_another_platform():
    with pytest.raises(ValueError, match="Invalid cuda resource type"):
        hardware_marks(res={"cuda": "MI325"})
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 """
 Tests for CUDA Graph acoustic transformer wrapper numerical equivalence.
@@ -18,11 +18,12 @@ import pytest
 import torch
 import torch.nn as nn
 from tests.helpers.mark import hardware_marks
 pytestmark = [
    pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required"),
    pytest.mark.core_model,
-    pytest.mark.cuda,
+    *hardware_marks(res={"cuda": "L4"}),
    pytest.mark.L4,
 ]
 DEVICE = torch.device("cuda:0")
@@ -1,52 +1,45 @@
 #!/usr/bin/env python3
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
 """Run a pre-commit hook that fails if test files are modified or added
 that (probably) never run in the CI. For now, this means that every tests file
 needs to have a CI level marker (e.g., core_model, advanced_model, full_model,
 local_model, slow, etc) and hardware mark / helper so that we ensure mutated
 tests will actually be selected as long as there are pytest commands pointing
 at the right paths.
 SKU markers (``H100``, ``L4``, … tagged ``[hardware-resource]`` in
 ``pyproject.toml``) must be applied via ``hardware_test(`` / ``hardware_marks(``
 so ``cards_{n}`` is attached. Direct ``pytest.mark.H100`` is rejected.
 Platform names allowed as ``pytest.mark.cpu`` / ``cuda`` come from
 ``get_supported_platforms()`` (``[hardware-platform]`` in ``pyproject.toml``).
 CI level names come from ``get_level_markers()`` (``[ci-level]``).
 """
 from __future__ import annotations
 import importlib.util
 import os
 import re
 import sys
 from functools import lru_cache
 from pathlib import Path
 from types import ModuleType
-# CI level markers
+# Helpers from tests/helpers/mark.py that auto-apply hardware + cards_* marks.
 LEVEL_MARKERS = ("core_model", "advanced_model", "full_model", "local_model", "slow")
 # Hardware markers. These are platforms and resources that CI filters on.
 # NOTE: If new hardware marks are added to pyproject.toml etc, we need
 # to add them here as well.
 HARDWARE_MARKERS = (
    "cpu",
    "gpu",
    "cuda",
    "rocm",
    "xpu",
    "npu",
    "musa",
    "H100",
    "L4",
    "MI325",
    "B60",
    "S5000",
    "A2",
    "A3",
 )
 # Helpers from tests/helpers/mark.py that auto-apply hardware marks.
 HARDWARE_HELPERS = ("hardware_test", "hardware_marks")
 # The helper implementation is the only file allowed to write pytest.mark.<SKU>.
 _ALLOWED_DIRECT_SKU_FILES = frozenset({"tests/helpers/mark.py"})
 # Match mark.X since we could also do `from pytest import mark`.
 # \b prevents matching prefixes (e.g., mark.slow vs mark.slow_test).
 LEVEL_RE = re.compile(r"mark\.(?:" + "|".join(LEVEL_MARKERS) + r")\b")
 HARDWARE_RE = re.compile(r"mark\.(?:" + "|".join(HARDWARE_MARKERS) + r")\b")
 HELPER_RE = re.compile(r"(?:" + "|".join(HARDWARE_HELPERS) + r")\s*\(")
 MISSING_LEVEL_MARKER = "Level"
 MISSING_HARDWARE_MARKER = "Hardware"
 DIRECT_SKU_MARKER = "Direct SKU"
 # Check if a file is located under tests/ and matches test_<something>.py
 # or <something>_test.py, since pytest technically collects on both.
@@ -54,9 +47,68 @@ MISSING_HARDWARE_MARKER = "Hardware"
 TEST_FILE_RE = re.compile(r"^tests/(?:.*/)?(?:test_[^/]*\.py$|[^/]*_test\.py$)")
 def _normalize_path(path: str) -> str:
    return path.replace("\\", "/")
 def _repo_root() -> Path:
    return Path(__file__).resolve().parents[2]
@lru_cache(maxsize=1)
 def _mark_module() -> ModuleType:
    """Load ``mark.py`` by file path (no pytest/vllm; skip helpers ``__init__``)."""
    path = _repo_root() / "tests" / "helpers" / "mark.py"
    spec = importlib.util.spec_from_file_location("_vllm_omni_mark", path)
    if spec is None or spec.loader is None:
        raise ImportError(f"Cannot load mark helpers from {path}")
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module
 def _mark_name_re(names: tuple[str, ...]) -> re.Pattern[str]:
    if not names:
        return re.compile(r"(?!)")
    return re.compile(r"mark\.(?:" + "|".join(re.escape(n) for n in names) + r")\b")
@lru_cache(maxsize=1)
 def level_markers() -> tuple[str, ...]:
    """CI level names tagged ``[ci-level]`` (``core_model``, ``slow``, …)."""
    return tuple(sorted(_mark_module().get_level_markers()))
@lru_cache(maxsize=1)
 def platform_markers() -> tuple[str, ...]:
    """Names tests may apply as ``pytest.mark.cpu`` / ``cuda`` (not SKUs)."""
    return tuple(sorted(_mark_module().get_supported_platforms()))
@lru_cache(maxsize=1)
 def sku_markers() -> tuple[str, ...]:
    """SKU marker names tagged ``[hardware-resource]`` in ``pyproject.toml``."""
    return tuple(sorted(_mark_module().get_hardware_mark_list()))
@lru_cache(maxsize=1)
 def _level_re() -> re.Pattern[str]:
    return _mark_name_re(level_markers())
@lru_cache(maxsize=1)
 def _platform_re() -> re.Pattern[str]:
    return _mark_name_re(platform_markers())
@lru_cache(maxsize=1)
 def _sku_mark_re() -> re.Pattern[str]:
    return _mark_name_re(sku_markers())
 def is_test_file(path: str) -> bool:
    """Determine whether or not a path is pointing at a test file or not."""
-    return bool(TEST_FILE_RE.match(path))
+    return bool(TEST_FILE_RE.match(_normalize_path(path)))
 def read_test_file(path: str) -> str | None:
@@ -69,22 +121,31 @@ def read_test_file(path: str) -> str | None:
 def has_level_marker(contents: str) -> bool:
    """Check if file contents contain at least one CI level marker."""
-    return bool(LEVEL_RE.search(contents))
+    return bool(_level_re().search(contents))
 def has_hardware_marker(contents: str) -> bool:
-    """Check if file contents contain any hardware marker or helpers."""
+    """Check if file contents contain a platform marker or hardware helper."""
-    return bool(HARDWARE_RE.search(contents) or HELPER_RE.search(contents))
+    return bool(_platform_re().search(contents) or HELPER_RE.search(contents))
 def has_direct_sku_marker(path: str, contents: str) -> bool:
    """True when a test applies ``pytest.mark.<SKU>`` instead of the helpers."""
    if _normalize_path(path) in _ALLOWED_DIRECT_SKU_FILES:
        return False
    return bool(_sku_mark_re().search(contents))
 def get_files_missing_markers(
    staged_files: list[str],
 ) -> dict[str, list[str]]:
-    """Return a dict mapping file path to list of missing marker types."""
+    """Return a dict mapping file path to list of missing / invalid marker types."""
    results: dict[str, list[str]] = {}
    for path in staged_files:
        if is_test_file(path) and (contents := read_test_file(path)) is not None:
            missing = []
            if has_direct_sku_marker(path, contents):
                missing.append(DIRECT_SKU_MARKER)
            if not has_level_marker(contents):
                missing.append(MISSING_LEVEL_MARKER)
            if not has_hardware_marker(contents):
@@ -99,14 +160,17 @@ if __name__ == "__main__":
    if missing:
        file_lines = "\n".join(f"  - {path} [{' and '.join(problems)}]" for path, problems in missing.items())
        sku = ", ".join(sku_markers())
        print(
            "\033[91merror:\033[0m test files are missing pytest marks "
-            "required for Buildkite CI collection.\n\n"
+            "required for Buildkite CI collection, or apply SKU marks directly.\n\n"
-            f"Level marks, e.g.: {', '.join(LEVEL_MARKERS[:4])}\n"
+            f"Level marks, e.g.: {', '.join(level_markers()[:4])}\n"
-            f"Hardware marks, e.g.: {', '.join(HARDWARE_MARKERS[:4])}, ...\n"
+            f"Hardware marks, e.g.: {', '.join(platform_markers()[:4])}, ...\n"
-            f"  or helpers: {', '.join(HARDWARE_HELPERS)}\n\n"
+            f"  or helpers: {', '.join(HARDWARE_HELPERS)}\n"
            f"Do not write pytest.mark.<SKU> ({sku}). "
            "Use hardware_test(...) / hardware_marks(...) so cards_* is attached.\n\n"
            "The following files are missing marks:\n"
            f"{file_lines}\n\n"
-            "To skip: SKIP=check-test-ci-coverage git commit ..."
+            "To skip: SKIP=check-mark git commit ..."
        )
        sys.exit(1)