[Refactor][CI] Unify skip-ci targeting and mirror_hardwares pipeline presets (#5254)

Signed-off-by: wangyu <410167048@qq.com>
Signed-off-by: wangyu <53896905+yenuo26@users.noreply.github.com>
Signed-off-by: [Your Name] [Your Email]
Co-authored-by: Weiming Liao <fayespica@outlook.com>

.buildkite/amd/scripts/bootstrap-amd-omni.sh

@@ -6,6 +6,8 @@
 
 set -euo pipefail
 
+source .buildkite/common/scripts/resolve_skip_ci.sh
+
 if [[ -z "${RUN_ALL:-}" ]]; then
     RUN_ALL=0
 fi
@@ -22,10 +24,6 @@ if [[ -z "${AMD_MIRROR_HW:-}" ]]; then
     AMD_MIRROR_HW="amdproduction"
 fi
 
-if [[ -z "${DOCS_ONLY_DISABLE:-}" ]]; then
-    DOCS_ONLY_DISABLE=0
-fi
-
 fail_fast() {
     DISABLE_LABEL="ci-no-fail-fast"
     # If BUILDKITE_PULL_REQUEST != "false", then we check the PR labels using curl and jq
@@ -56,88 +54,6 @@ check_run_all_label() {
     fi
 }
 
-is_docs_only_change() {
-    local file_path
-    local has_any=0
-
-    while IFS= read -r file_path; do
-        [[ -z "${file_path}" ]] && continue
-        has_any=1
-
-        if [[ "${file_path}" == docs/* ]]; then
-            continue
-        fi
-        if [[ "${file_path}" == *.md ]]; then
-            continue
-        fi
-        if [[ "${file_path}" == "mkdocs.yml" ]]; then
-            continue
-        fi
-        return 1
-    done
-
-    [[ "${has_any}" -eq 1 ]]
-}
-
-resolve_skip_ci() {
-    local is_pr_build=0
-    local files
-    local base_branch base_ref
-
-    if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" && -n "${BUILDKITE_PULL_REQUEST:-}" ]]; then
-        is_pr_build=1
-    fi
-
-    if [[ "${is_pr_build}" -eq 1 ]]; then
-        base_branch="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}"
-        if ! git rev-parse --verify "origin/${base_branch}" >/dev/null 2>&1; then
-            echo "resolve_skip_ci: origin/${base_branch} not found locally; trying fetch" >&2
-            git fetch --depth=200 origin "${base_branch}" >/dev/null 2>&1 || true
-        fi
-
-        base_ref=""
-        if git rev-parse --verify "origin/${base_branch}" >/dev/null 2>&1; then
-            base_ref="origin/${base_branch}"
-        elif git rev-parse --verify "${base_branch}" >/dev/null 2>&1; then
-            base_ref="${base_branch}"
-        else
-            echo "resolve_skip_ci: cannot resolve PR base ${base_branch}; skip-ci=0" >&2
-            echo -n 0
-            return 0
-        fi
-
-        if ! files="$(git diff --name-only "${base_ref}...${BUILDKITE_COMMIT}" 2>/dev/null)"; then
-            echo "resolve_skip_ci: failed to compute PR changed files; skip-ci=0" >&2
-            echo -n 0
-            return 0
-        fi
-    elif [[ "${BUILDKITE_BRANCH:-}" == "main" ]]; then
-        if ! git rev-parse --verify "${BUILDKITE_COMMIT}^" >/dev/null 2>&1; then
-            echo "resolve_skip_ci: commit has no parent on main; skip-ci=0" >&2
-            echo -n 0
-            return 0
-        fi
-        if ! files="$(git diff --name-only "${BUILDKITE_COMMIT}^..${BUILDKITE_COMMIT}" 2>/dev/null)"; then
-            echo "resolve_skip_ci: failed to compute main changed files; skip-ci=0" >&2
-            echo -n 0
-            return 0
-        fi
-    else
-        echo "resolve_skip_ci: not PR/main build; skip-ci=0" >&2
-        echo -n 0
-        return 0
-    fi
-
-    if is_docs_only_change <<< "${files}"; then
-        echo "resolve_skip_ci: docs-only change detected; skip-ci=1" >&2
-        echo -n 1
-        return 0
-    fi
-
-    echo "resolve_skip_ci: non-doc changes detected; skip-ci=0" >&2
-    echo -n 0
-}
-
 if [[ -z "${COV_ENABLED:-}" ]]; then
     COV_ENABLED=0
 fi
@@ -234,23 +150,13 @@ if [[ $BUILDKITE_BRANCH == "main" ]]; then
     file_diff=$(get_diff_main)
 fi
 
-# ----------------------------------------------------------------------
-# Early exit start: skip pipeline if conditions are met
-# ----------------------------------------------------------------------
-
-# Match CUDA Buildkite skip-ci behavior for docs/markdown-only changes.
-if [[ "${DOCS_ONLY_DISABLE}" != "1" ]]; then
-    SKIP_CI="$(resolve_skip_ci)"
-    if [[ "${SKIP_CI}" == "1" ]]; then
-        echo "[docs-only] Docs/markdown-only changes detected. Exiting before pipeline upload."
-        exit 0
-    fi
+# Early exit: unified skip-ci (docs / skip marks) and CI-yaml-only level targeting.
+if [[ $BUILDKITE_BRANCH == "main" ]]; then
+    gate_bootstrap_ci amd l3
+else
+    gate_bootstrap_ci amd l2
 fi
 
-# ----------------------------------------------------------------------
-# Early exit end
-# ----------------------------------------------------------------------
-
 patterns=(
     "docker/Dockerfile"
     "CMakeLists.txt"

.buildkite/common/ci_mirror_hardwares.yml

@@ -0,0 +1,239 @@
+# Uploader-only hardware presets for CUDA / NPU test YAML.
+# Referenced as ``mirror_hardwares: l4_1`` on a step; ``upload_pipeline.py`` expands to
+# ``agents`` (+ optional top-level ``image`` for NPU) + ``plugins``, then strips
+# ``mirror_hardwares`` before Buildkite upload.
+#
+# CUDA naming: {chip}_{count}
+#   l4_1, l4_4, h100_1 … h100_4
+# NPU naming: {queue}_{resource_class}
+#   a2b3_npu_1  — ascend-a2b3, resource_class npu-1 (A2/B3 CI image)
+#   a2b3_npu_4  — ascend-a2b3, resource_class npu-4
+#   a3_npu_2    — ascend-a3, resource_class npu-2 (A3 CI image tag …:$BUILDKITE_COMMIT-a3)
+
+x-npu-k8s-plugins: &npu_k8s_plugins
+  - kubernetes:
+      podSpecPatch:
+        imagePullSecrets:
+          - name: swr-secret
+
+
+mirror_hardwares:
+  l4_1:
+    agents:
+      queue: gpu_1_queue
+    plugins:
+      - docker#v5.2.0:
+          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+          always-pull: true
+          propagate-environment: true
+          shm-size: "8gb"
+          environment:
+            - HF_HOME=/fsx/hf_cache
+            - HF_TOKEN
+          volumes:
+            - /fsx/hf_cache:/fsx/hf_cache
+
+  l4_4:
+    agents:
+      queue: gpu_4_queue
+    plugins:
+      - docker#v5.2.0:
+          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+          always-pull: true
+          propagate-environment: true
+          shm-size: "8gb"
+          environment:
+            - HF_HOME=/fsx/hf_cache
+            - HF_TOKEN
+          volumes:
+            - /fsx/hf_cache:/fsx/hf_cache
+
+  h100_1:
+    agents:
+      queue: mithril-h100-pool
+    plugins:
+      - kubernetes:
+          podSpec:
+            containers:
+              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 1
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: hf-cache
+                    mountPath: /root/.cache/huggingface
+                env:
+                  - name: HF_HOME
+                    value: /root/.cache/huggingface
+                  - name: HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            nodeSelector:
+              node.kubernetes.io/instance-type: gpu-h100-sxm
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+              - name: hf-cache
+                hostPath:
+                  path: /mnt/hf-cache
+                  type: DirectoryOrCreate
+
+  h100_2:
+    agents:
+      queue: mithril-h100-pool
+    plugins:
+      - kubernetes:
+          podSpec:
+            containers:
+              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 2
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: hf-cache
+                    mountPath: /root/.cache/huggingface
+                env:
+                  - name: HF_HOME
+                    value: /root/.cache/huggingface
+                  - name: HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            nodeSelector:
+              node.kubernetes.io/instance-type: gpu-h100-sxm
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+              - name: hf-cache
+                hostPath:
+                  path: /mnt/hf-cache
+                  type: DirectoryOrCreate
+
+  h100_3:
+    agents:
+      queue: mithril-h100-pool
+    plugins:
+      - kubernetes:
+          podSpec:
+            containers:
+              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 3
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: hf-cache
+                    mountPath: /root/.cache/huggingface
+                env:
+                  - name: HF_HOME
+                    value: /root/.cache/huggingface
+                  - name: HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            nodeSelector:
+              node.kubernetes.io/instance-type: gpu-h100-sxm
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+              - name: hf-cache
+                hostPath:
+                  path: /mnt/hf-cache
+                  type: DirectoryOrCreate
+
+  h100_4:
+    agents:
+      queue: mithril-h100-pool
+    plugins:
+      - kubernetes:
+          podSpec:
+            containers:
+              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 4
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: hf-cache
+                    mountPath: /root/.cache/huggingface
+                env:
+                  - name: HF_HOME
+                    value: /root/.cache/huggingface
+                  - name: HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            nodeSelector:
+              node.kubernetes.io/instance-type: gpu-h100-sxm
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+              - name: hf-cache
+                hostPath:
+                  path: /mnt/hf-cache
+                  type: DirectoryOrCreate
+
+  a2b3_npu_1:
+    agents:
+      queue: "ascend-a2b3"
+      resource_class: "npu-1"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}"
+    plugins: *npu_k8s_plugins
+
+  a2b3_npu_4:
+    agents:
+      queue: "ascend-a2b3"
+      resource_class: "npu-4"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}"
+    plugins: *npu_k8s_plugins
+
+
+  a2b3_npu_8:
+    agents:
+      queue: "ascend-a2b3"
+      resource_class: "npu-8"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}"
+    plugins: *npu_k8s_plugins
+
+  a3_npu_2:
+    agents:
+      queue: "ascend-a3"
+      resource_class: "npu-2"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3"
+    plugins: *npu_k8s_plugins
+
+  a3_npu_4:
+    agents:
+      queue: "ascend-a3"
+      resource_class: "npu-4"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3"
+    plugins: *npu_k8s_plugins
+
+  a3_npu_8:
+    agents:
+      queue: "ascend-a3"
+      resource_class: "npu-8"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3"
+    plugins: *npu_k8s_plugins
+
+  a3_npu_16:
+    agents:
+      queue: "ascend-a3"
+      resource_class: "npu-16"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3"
+    plugins: *npu_k8s_plugins

.buildkite/common/scripts/resolve_skip_ci.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Shared Buildkite skip-ci helpers for AMD / Intel bootstrap scripts.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SKIP_CI_PY="${SCRIPT_DIR}/skip_ci.py"
+
+# Resolve git diff once. Exit 0 when this bootstrap should stop uploading.
+# Decision details are logged by skip_ci.py; this helper only annotates skip-all.
+# Usage: gate_bootstrap_ci <platform> <l2|l3>
+gate_bootstrap_ci() {
+    local platform="$1"
+    local level="$2"
+    local kind
+    if kind="$(python3 "${SKIP_CI_PY}" gate "${platform}" "${level}")"; then
+        if [[ "${kind}" == "skip-all" ]]; then
+            buildkite-agent annotate \
+                ":memo: CI skipped — docs or pytest skip-mark changes only" \
+                --style "info" 2>/dev/null || true
+        fi
+        exit 0
+    fi
+}

.buildkite/common/scripts/skip_ci.py

@@ -0,0 +1,617 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+"""Shared Buildkite skip-ci and CI-YAML level targeting logic.
+
+``resolve_ci_decision(changed_files, ...)`` buckets explicit file lists (tests, dry-run).
+``resolve_ci_context_from_git()`` resolves git diff once and returns decision + changed files
+for Buildkite bootstrap / upload scripts.
+
+CLI (for bootstrap scripts; exit code is the signal):
+  python3 skip_ci.py gate <platform> <level>
+      Exit 0 when bootstrap should stop (skip-all, or that L2/L3 target is off).
+      Prints ``skip-all`` or ``skip-l23`` on stdout; logs the decision once.
+  python3 skip_ci.py check-skip-all
+      Exit 0 when only docs/skip-mark changes → bootstrap skips entire CI upload.
+  python3 skip_ci.py check-skip-l2-l3 <platform> <level>
+      Exit 0 when that L2/L3 pipeline should be skipped (CI config YAML-only diffs).
+  python3 skip_ci.py print-annotate
+      Print decision message for Buildkite annotation.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import re
+import subprocess
+import sys
+from dataclasses import dataclass, field, replace
+from pathlib import Path
+
+LOG = "[skip-ci]"
+ROOT = Path(__file__).resolve().parent.parent.parent.parent
+
+PLATFORMS = ("cuda", "amd", "intel", "npu")
+L23_LEVELS = frozenset({"l2", "l3"})
+
+L2_YAML_FILES: dict[str, str] = {
+    ".buildkite/cuda/test-ready.yml": "cuda",
+    ".buildkite/npu/test-npu-ready.yml": "npu",
+    ".buildkite/amd/test-amd-ready.yml": "amd",
+    ".buildkite/intel/pipeline-intel.yml": "intel",
+}
+
+L3_YAML_FILES: dict[str, str] = {
+    ".buildkite/cuda/test-merge.yml": "cuda",
+    ".buildkite/amd/test-amd-merge.yml": "amd",
+}
+
+L45_YAML_FILES: dict[str, str] = {
+    ".buildkite/cuda/test-nightly.yml": "cuda",
+    ".buildkite/cuda/test-weekly.yml": "cuda",
+    ".buildkite/npu/test-npu-nightly.yml": "npu",
+}
+
+
+def _empty_platform_buckets() -> dict[str, list[str]]:
+    return {platform: [] for platform in PLATFORMS}
+
+
+@dataclass(frozen=True)
+class CiDevice:
+    """Per-platform / per-level skip flags for yaml-gated bootstrap (``skip_l2_l3``)."""
+
+    skip_cuda_l2: bool = False
+    skip_cuda_l3: bool = False
+    skip_amd_l2: bool = False
+    skip_amd_l3: bool = False
+    skip_intel_l2: bool = False
+    skip_intel_l3: bool = False
+    skip_npu_l2: bool = False
+    skip_npu_l3: bool = False
+
+    def is_skip(self, platform: str, level: str) -> bool:
+        return bool(getattr(self, f"skip_{platform}_{level}", True))
+
+
+@dataclass
+class DiffBuckets:
+    """Changed files grouped by doc / skip-mark / CI yaml / other."""
+
+    docs: list[str] = field(default_factory=list)
+    skip_tests: list[str] = field(default_factory=list)
+    other: list[str] = field(default_factory=list)
+    l2: dict[str, list[str]] = field(default_factory=_empty_platform_buckets)
+    l3: dict[str, list[str]] = field(default_factory=_empty_platform_buckets)
+    l45: dict[str, list[str]] = field(default_factory=_empty_platform_buckets)
+
+    @property
+    def has_doc_changes(self) -> bool:
+        return bool(self.docs)
+
+    @property
+    def has_skip_test_changes(self) -> bool:
+        return bool(self.skip_tests)
+
+    @property
+    def has_other_changes(self) -> bool:
+        """Product code or paths outside skip-ci scope."""
+        return bool(self.other)
+
+    @property
+    def has_l23_yaml_changes(self) -> bool:
+        """At least one whitelisted L2/L3 CI yaml file changed."""
+        return any(self.l2[platform] or self.l3[platform] for platform in PLATFORMS)
+
+    @property
+    def has_l45_yaml_changes(self) -> bool:
+        """At least one whitelisted L4/L5 (nightly/weekly) CI yaml file changed."""
+        return any(self.l45[platform] for platform in PLATFORMS)
+
+    @property
+    def has_skip_ci_scope_changes(self) -> bool:
+        """At least one doc / skip-mark / whitelisted CI yaml change."""
+        return (
+            self.has_doc_changes or self.has_skip_test_changes or self.has_l23_yaml_changes or self.has_l45_yaml_changes
+        )
+
+    def l2_platforms(self) -> frozenset[str]:
+        return frozenset(platform for platform in PLATFORMS if self.l2[platform])
+
+    def l3_platforms(self) -> frozenset[str]:
+        return frozenset(platform for platform in PLATFORMS if self.l3[platform])
+
+
+@dataclass(frozen=True)
+class SkipL2L3Basis:
+    skip_all_l2: bool
+    skip_all_l3: bool
+
+    @classmethod
+    def from_buckets(cls, buckets: DiffBuckets) -> SkipL2L3Basis | None:
+        """L4/L5 yaml → skip both L2 and L3 (L2/L3 buckets may rescue).
+
+        Docs / skip-mark changes do not block this path; they are ignored for
+        L2/L3 targeting when CI YAML is also present.
+        """
+        if buckets.has_other_changes:
+            return None
+        if not buckets.has_l45_yaml_changes:
+            return None
+        return cls(
+            skip_all_l2=not buckets.l2_platforms(),
+            skip_all_l3=not buckets.l3_platforms(),
+        )
+
+
+@dataclass(frozen=True)
+class SkipL3Basis:
+    enable_l2_platforms: frozenset[str]
+
+    @classmethod
+    def from_buckets(cls, buckets: DiffBuckets) -> SkipL3Basis | None:
+        """L2 yaml changed → skip L3; keep L2 on touched platforms."""
+        platforms = buckets.l2_platforms()
+        if not platforms:
+            return None
+        return cls(enable_l2_platforms=platforms)
+
+
+@dataclass(frozen=True)
+class SkipL2Basis:
+    enable_l3_platforms: frozenset[str]
+
+    @classmethod
+    def from_buckets(cls, buckets: DiffBuckets) -> SkipL2Basis | None:
+        """L3 yaml changed → skip L2; keep L3 on touched platforms."""
+        platforms = buckets.l3_platforms()
+        if not platforms:
+            return None
+        return cls(enable_l3_platforms=platforms)
+
+
+@dataclass(frozen=True)
+class CiDecision:
+    """Final CI skip decision for one diff."""
+
+    skip_all: bool = False
+    skip_l2_l3: bool = False
+    device: CiDevice = field(default_factory=CiDevice)
+    message: str = ""
+
+    @classmethod
+    def skip_all_basis(cls, buckets: DiffBuckets, *, diff_range: str | None) -> bool:
+        """Docs / pytest skip-mark only → suppress entire default CI."""
+        if buckets.has_other_changes:
+            return False
+        if (buckets.has_l23_yaml_changes or buckets.has_l45_yaml_changes) and (
+            buckets.has_doc_changes or buckets.has_skip_test_changes
+        ):
+            return False
+        if not buckets.has_doc_changes and not buckets.has_skip_test_changes:
+            return False
+        if diff_range is None and not buckets.has_doc_changes:
+            return False
+        return True
+
+    @classmethod
+    def from_yaml_basis(
+        cls,
+        l2_l3_basis: SkipL2L3Basis | None,
+        l3_basis: SkipL3Basis | None,
+        l2_basis: SkipL2Basis | None,
+    ) -> CiDecision:
+        """Build a yaml-gated decision by merging L2/L3 skip bases.
+
+        Bases are applied as a union of enables: L2 yaml keeps L2 on touched
+        platforms, L3 yaml keeps L3 on touched platforms. Cross-platform mixes
+        (e.g. CUDA ready + AMD merge) must preserve both enables. The resulting
+        per-platform / per-level matrix is stored on ``device`` as ``skip_*``.
+        """
+        run = {platform: {"l2": True, "l3": True} for platform in PLATFORMS}
+
+        if l2_l3_basis is not None:
+            if l2_l3_basis.skip_all_l2:
+                for platform in PLATFORMS:
+                    run[platform]["l2"] = False
+            if l2_l3_basis.skip_all_l3:
+                for platform in PLATFORMS:
+                    run[platform]["l3"] = False
+
+        if l3_basis is not None:
+            for platform in PLATFORMS:
+                run[platform]["l3"] = False
+                run[platform]["l2"] = platform in l3_basis.enable_l2_platforms
+
+        if l2_basis is not None:
+            for platform in PLATFORMS:
+                if platform in l2_basis.enable_l3_platforms:
+                    run[platform]["l3"] = True
+                    if l3_basis is None or platform not in l3_basis.enable_l2_platforms:
+                        run[platform]["l2"] = False
+                elif l3_basis is None:
+                    # L3-yaml-only: disable platforms that did not change L3.
+                    run[platform]["l2"] = False
+                    run[platform]["l3"] = False
+                # else: leave L2 enables from SkipL3Basis; keep this platform's L3 off.
+
+        return cls(
+            skip_l2_l3=True,
+            device=cls._device_from_run(run),
+        )
+
+    @staticmethod
+    def _device_from_run(run: dict[str, dict[str, bool]]) -> CiDevice:
+        return CiDevice(
+            **{f"skip_{platform}_{level}": not run[platform][level] for platform in PLATFORMS for level in ("l2", "l3")}
+        )
+
+    def is_run(self, platform: str, level: str) -> bool:
+        if self.skip_all:
+            return False
+        if not self.skip_l2_l3:
+            return True
+        if level not in L23_LEVELS:
+            return True
+        return not self.device.is_skip(platform, level)
+
+
+@dataclass(frozen=True)
+class CiContext:
+    """Skip-ci decision plus git diff inputs used to compute it."""
+
+    decision: CiDecision
+    changed_files: list[str] | None
+    diff_range: str | None
+
+
+def _log(message: str) -> None:
+    print(f"{LOG}: {message}", file=sys.stderr)
+
+
+def _git(*args: str) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(
+        ["git", *args],
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+
+
+# --- Diff helpers ---
+
+
+def _resolve_diff_range() -> str | None:
+    """Return a git diff range for PR or main builds, or None when unavailable."""
+    is_pr = os.environ.get("BUILDKITE_PULL_REQUEST", "false") != "false" and os.environ.get(
+        "BUILDKITE_PULL_REQUEST", ""
+    )
+    commit = os.environ.get("BUILDKITE_COMMIT", "")
+
+    if is_pr:
+        base_branch = os.environ.get("BUILDKITE_PULL_REQUEST_BASE_BRANCH", "main")
+        base_ref = f"origin/{base_branch}"
+        if _git("rev-parse", "--verify", base_ref).returncode != 0:
+            _log(f"origin/{base_branch} not found locally; trying fetch")
+            _git("fetch", "--depth=200", "origin", base_branch)
+        if _git("rev-parse", "--verify", base_ref).returncode != 0:
+            if _git("rev-parse", "--verify", base_branch).returncode == 0:
+                base_ref = base_branch
+            else:
+                _log(f"cannot resolve PR base {base_branch}; using safe defaults")
+                return None
+        return f"{base_ref}...{commit}"
+
+    if os.environ.get("BUILDKITE_BRANCH", "") == "main":
+        if _git("rev-parse", "--verify", f"{commit}^").returncode != 0:
+            _log("main commit has no parent; using safe defaults")
+            return None
+        return f"{commit}^..{commit}"
+
+    _log("not PR/main build; using safe defaults")
+    return None
+
+
+def _changed_files_for_diff_range(diff_range: str | None) -> list[str] | None:
+    """Return changed file paths for *diff_range*, or None when unavailable."""
+    if diff_range is None:
+        return None
+
+    result = _git("diff", "--name-only", diff_range)
+    if result.returncode != 0:
+        _log(f"git diff failed ({diff_range}); using safe defaults")
+        return None
+
+    files = [line for line in result.stdout.splitlines() if line.strip()]
+    _log(f"{len(files)} changed file(s)")
+    return files
+
+
+def resolve_ci_context_from_git() -> CiContext:
+    """Resolve git diff once and return skip-ci decision plus changed files."""
+    diff_range = _resolve_diff_range()
+    changed_files = _changed_files_for_diff_range(diff_range)
+    decision = resolve_ci_decision(changed_files, diff_range=diff_range)
+    return CiContext(decision=decision, changed_files=changed_files, diff_range=diff_range)
+
+
+# --- Doc / skip-mark helpers ---
+
+
+_SKIP_MARK_RE = re.compile(r"pytest\.mark\.skip(?:if)?\b|pytest\.skip\s*\(")
+_PYTESTMARK_SKIP_RE = re.compile(r"pytest\.mark\.skip\b")
+_PYTEST_MARK_ONLY_RE = re.compile(r"pytest\.mark\.\w+")
+
+
+def _diff_only_contains_skip_mark_changes(diff_text: str) -> bool:
+    """Return True when a unified diff only *adds* pytest skip marks.
+
+    Deleting a skip mark re-enables the test and must not suppress CI. Editing an
+    existing skip (``-`` old / ``+`` new) still qualifies when at least one skip
+    mark is added.
+    """
+
+    def paren_balance(line: str) -> int:
+        return line.count("(") - line.count(")")
+
+    def is_blank_or_comment(line: str) -> bool:
+        stripped = line.strip()
+        return not stripped or stripped.startswith("#")
+
+    def is_skip_mark_line(line: str) -> bool:
+        return _SKIP_MARK_RE.search(line.strip()) is not None
+
+    def is_pytestmark_adjacent_line(line: str) -> bool:
+        stripped = line.strip().rstrip(",")
+        if not stripped or stripped in {"[", "]"} or stripped.startswith("#"):
+            return True
+        if stripped.startswith("pytestmark"):
+            return True
+        return _PYTEST_MARK_ONLY_RE.search(stripped) is not None
+
+    pending_depth = 0
+    saw_change = False
+    has_added_skip_mark = False
+    for raw_line in diff_text.splitlines():
+        if raw_line.startswith("@@"):
+            pending_depth = 0
+            continue
+        if not (raw_line.startswith("+") or raw_line.startswith("-")):
+            continue
+        if raw_line.startswith("+++") or raw_line.startswith("---"):
+            continue
+
+        saw_change = True
+        is_add = raw_line.startswith("+")
+        content = raw_line[1:]
+
+        if pending_depth > 0:
+            pending_depth += paren_balance(content)
+            if pending_depth < 0:
+                pending_depth = 0
+            continue
+
+        if is_blank_or_comment(content):
+            continue
+
+        if is_skip_mark_line(content):
+            if is_add:
+                has_added_skip_mark = True
+            pending_depth = max(0, paren_balance(content))
+            continue
+
+        if is_pytestmark_adjacent_line(content):
+            continue
+
+        return False
+
+    return saw_change and has_added_skip_mark
+
+
+def _is_skip_testcase_change(file_path: str, *, diff_range: str) -> bool:
+    """Return True when a test file diff is skip-mark-only (including new-file cases)."""
+    diff_result = _git("diff", diff_range, "--", file_path)
+    if diff_result.returncode != 0:
+        _log(f"skip-mark-only: git diff failed for {file_path}")
+        return False
+
+    diff_text = diff_result.stdout
+    if not diff_text.strip():
+        _log(f"skip-mark-only: empty diff for {file_path}")
+        return False
+
+    if _diff_only_contains_skip_mark_changes(diff_text):
+        return True
+
+    if "new file mode" not in diff_text and "--- /dev/null" not in diff_text:
+        _log(f"skip-mark-only: non-skip changes in {file_path}")
+        return False
+
+    commit = os.environ.get("BUILDKITE_COMMIT", "")
+    if not commit:
+        return False
+
+    content_result = _git("show", f"{commit}:{file_path}")
+    if content_result.returncode != 0:
+        _log(f"skip-mark-only: cannot read {file_path} at {commit}")
+        return False
+
+    content = content_result.stdout
+    match = re.search(r"^pytestmark\s*=\s*\[(.*?)^\s*\]", content, re.MULTILINE | re.DOTALL)
+    if match is not None and _PYTESTMARK_SKIP_RE.search(match.group(1)) is not None:
+        _log(f"skip-mark-only: new test file with module-level pytest.mark.skip: {file_path}")
+        return True
+    if re.search(r"^pytestmark\s*=\s*pytest\.mark\.skip\b", content, re.MULTILINE):
+        _log(f"skip-mark-only: new test file with module-level pytest.mark.skip: {file_path}")
+        return True
+
+    _log(f"skip-mark-only: new test file without module-level pytest.mark.skip: {file_path}")
+    return False
+
+
+def _classify_changed_files_into_buckets(changed_files: list[str], *, diff_range: str | None) -> DiffBuckets:
+    """Classify each changed file path into a ``DiffBuckets`` category."""
+    buckets = DiffBuckets()
+    for file_path in changed_files:
+        if not file_path:
+            continue
+        if file_path.startswith("docs/") or file_path.endswith(".md") or file_path == "mkdocs.yml":
+            buckets.docs.append(file_path)
+            continue
+        path = Path(file_path)
+        if path.suffix == ".py" and path.parts and path.parts[0] == "tests":
+            if diff_range is not None and _is_skip_testcase_change(file_path, diff_range=diff_range):
+                buckets.skip_tests.append(file_path)
+            else:
+                buckets.other.append(file_path)
+            continue
+        if file_path in L2_YAML_FILES:
+            platform = L2_YAML_FILES[file_path]
+            buckets.l2[platform].append(file_path)
+        elif file_path in L3_YAML_FILES:
+            platform = L3_YAML_FILES[file_path]
+            buckets.l3[platform].append(file_path)
+        elif file_path in L45_YAML_FILES:
+            platform = L45_YAML_FILES[file_path]
+            buckets.l45[platform].append(file_path)
+        else:
+            buckets.other.append(file_path)
+    return buckets
+
+
+# --- Decision ---
+
+
+def _finish(decision: CiDecision, message: str) -> CiDecision:
+    finished = replace(decision, message=message)
+    _log(message)
+    return finished
+
+
+def _format_yaml_gated_message(buckets: DiffBuckets, decision: CiDecision) -> str:
+    """Human-readable summary of which CI YAML changed and which L2/L3 targets run."""
+    changed_parts: list[str] = []
+    for level_name, by_platform in (("L2", buckets.l2), ("L3", buckets.l3), ("L4/L5", buckets.l45)):
+        files = [path for platform in PLATFORMS for path in by_platform[platform]]
+        if files:
+            changed_parts.append(f"{level_name}=[{', '.join(files)}]")
+
+    run_targets = [
+        f"{platform}/{level}" for platform in PLATFORMS for level in ("l2", "l3") if decision.is_run(platform, level)
+    ]
+    skip_targets = [
+        f"{platform}/{level}"
+        for platform in PLATFORMS
+        for level in ("l2", "l3")
+        if not decision.is_run(platform, level)
+    ]
+
+    changed = "; ".join(changed_parts) if changed_parts else "none"
+    run = ", ".join(run_targets) if run_targets else "none"
+    skip = ", ".join(skip_targets) if skip_targets else "none"
+    if buckets.has_doc_changes or buckets.has_skip_test_changes:
+        return (
+            "CI config YAML + docs/skip-mark changed — follow CI YAML gating; "
+            f"changed: {changed}; run: {run}; skip: {skip}"
+        )
+    return f"only CI config YAML changed — changed: {changed}; run: {run}; skip: {skip}"
+
+
+def resolve_ci_decision(
+    changed_files: list[str] | None = None,
+    *,
+    diff_range: str | None = None,
+) -> CiDecision:
+    if changed_files is None:
+        return _finish(CiDecision(), "could not resolve changed files; run normal CI")
+
+    buckets = _classify_changed_files_into_buckets(changed_files, diff_range=diff_range)
+    if buckets.has_other_changes or not buckets.has_skip_ci_scope_changes:
+        return _finish(CiDecision(), "product or non-whitelisted changes present; run normal CI")
+
+    if CiDecision.skip_all_basis(buckets, diff_range=diff_range):
+        return _finish(
+            CiDecision(skip_all=True),
+            "CI skipped — docs or pytest skip-mark changes only",
+        )
+
+    # Docs / skip-mark mixed with whitelisted CI YAML → same yaml-gated path
+    # as CI-YAML-only (docs/skip-mark do not widen to normal CI).
+    if not buckets.has_l23_yaml_changes and not buckets.has_l45_yaml_changes:
+        if buckets.has_skip_test_changes and diff_range is None:
+            message = (
+                "only pytest skip-mark test changes detected, but no git diff range "
+                "is available to verify them (skip-all requires diff_range); run normal CI"
+            )
+        elif buckets.has_skip_test_changes:
+            message = "pytest skip-mark test changes did not qualify for skip-all and no ci yaml changed; run normal CI"
+        else:
+            message = "doc/skip-mark changes did not qualify for skip-all; run normal CI"
+        return _finish(CiDecision(), message)
+
+    decision = CiDecision.from_yaml_basis(
+        SkipL2L3Basis.from_buckets(buckets),
+        SkipL3Basis.from_buckets(buckets),
+        SkipL2Basis.from_buckets(buckets),
+    )
+    return _finish(decision, _format_yaml_gated_message(buckets, decision))
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    gate_parser = subparsers.add_parser(
+        "gate",
+        help="exit 0 when bootstrap should stop (skip-all or PLATFORM/LEVEL disabled); logs once",
+    )
+    gate_parser.add_argument("platform", help="cuda, amd, intel, or npu")
+    gate_parser.add_argument("level", choices=("l2", "l3"), help="test level")
+
+    subparsers.add_parser(
+        "check-skip-all",
+        help="exit 0 when docs/skip-mark-only diff should skip entire CI upload",
+    )
+
+    skip_parser = subparsers.add_parser(
+        "check-skip-l2-l3",
+        help="exit 0 when PLATFORM l2/l3 pipeline should be skipped (CI config YAML-only diffs)",
+    )
+    skip_parser.add_argument("platform", help="cuda, amd, intel, or npu")
+    skip_parser.add_argument("level", choices=("l2", "l3"), help="test level")
+
+    subparsers.add_parser(
+        "print-annotate",
+        help="print human-readable skip reason for Buildkite annotation",
+    )
+
+    args = parser.parse_args()
+    decision = resolve_ci_context_from_git().decision
+
+    if args.command == "gate":
+        if decision.skip_all:
+            print("skip-all")
+            return 0
+        if not decision.is_run(args.platform, args.level):
+            print("skip-l23")
+            return 0
+        return 1
+
+    if args.command == "check-skip-all":
+        return 0 if decision.skip_all else 1
+
+    if args.command == "check-skip-l2-l3":
+        return 0 if not decision.is_run(args.platform, args.level) else 1
+
+    if args.command == "print-annotate":
+        print(decision.message or "CI will run normally")
+        return 0
+
+    return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.buildkite/common/scripts/upload_pipeline.py

@@ -0,0 +1,449 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+"""Render and optionally upload Buildkite pipeline YAML with diff-aware logic.
+
+Bootstrap mode (``bootstrap-upload-steps.yml``):
+  - Hook uploads the entry YAML (``pipeline.yml`` / ``pipeline-npu.yml``) with one step that runs
+    ``upload_pipeline.py --upload <platform>/bootstrap-upload-steps.yml``.
+  - Injects ``if`` by step ``key`` from skip-ci and uploads child steps (image build, L2–L5 upload).
+  - Detect docs-only, pytest skip-mark-only, or combined skip-ci from git diff.
+  - When only CI level YAML changes, enable **L2/L3** upload steps for affected levels only.
+
+Test pipeline mode (e.g. test-merge.yml):
+  - Drop steps whose ``source_file_dependencies`` do not match changed files.
+  - Expand uploader-only ``mirror_hardwares: l4_1`` into ``agents`` (+ optional ``image``
+    for NPU) + ``plugins`` (see ci_mirror_hardwares.yml).
+
+Usage:
+  python3 upload_pipeline.py [--upload] [--all | --e2e] <pipeline.yml>
+
+Requires PyYAML (``pip install pyyaml``); installs it automatically when missing.
+"""
+
+from __future__ import annotations
+
+import argparse
+import copy
+import subprocess
+import sys
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+try:
+    import yaml
+except ModuleNotFoundError:
+    subprocess.run(
+        [sys.executable, "-m", "pip", "install", "-q", "pyyaml"],
+        check=True,
+    )
+    import yaml
+
+from skip_ci import (
+    ROOT,
+    resolve_ci_context_from_git,
+)
+
+# --- Constants ---
+
+LOG = "upload_pipeline"
+BOOTSTRAP_STEPS_FILENAME = "bootstrap-upload-steps.yml"
+BOOTSTRAP_IMAGE_BUILD_KEYS = frozenset({"image-build", "image-build-a2", "image-build-a3"})
+BOOTSTRAP_UPLOAD_IF_KEYS = {
+    "upload-ready-pipeline": "ready",
+    "upload-merge-pipeline": "merge",
+    "upload-nightly-pipeline": "nightly",
+    "upload-weekly-pipeline": "weekly",
+}
+E2E_GROUP_MARKER = "E2E Test"
+CI_MIRROR_HARDWARES_PATH = ROOT / ".buildkite/common/ci_mirror_hardwares.yml"
+
+CUDA_NIGHTLY_ONLY = (
+    '(build.pull_request.labels includes "nightly-test") || (build.branch == "main" && build.env("NIGHTLY") == "1")'
+)
+NPU_NIGHTLY_ONLY = (
+    '(build.branch == "main" && build.env("NIGHTLY") == "1") || '
+    '(build.branch != "main" && ('
+    'build.pull_request.labels includes "nightly-test" || '
+    'build.pull_request.labels includes "omni-test" || '
+    'build.pull_request.labels includes "tts-test" || '
+    'build.pull_request.labels includes "diffusion-x2iat-test" || '
+    'build.pull_request.labels includes "diffusion-x2v-test"'
+    "))"
+)
+
+
+# --- Logging ---
+
+
+def _log(message: str) -> None:
+    print(f"{LOG}: {message}", file=sys.stderr)
+
+
+# --- Bootstrap pipeline (bootstrap-upload-steps.yml) ---
+
+
+def _get_bootstrap_platform(path: Path) -> str:
+    parts = path.as_posix().split("/")
+    return "npu" if "npu" in parts else "cuda"
+
+
+def _load_bootstrap_steps(path: Path) -> str:
+    if path.name != BOOTSTRAP_STEPS_FILENAME:
+        raise ValueError(f"expected {BOOTSTRAP_STEPS_FILENAME}, got {path.name}")
+    return path.read_text(encoding="utf-8")
+
+
+def _format_bootstrap_if(expr: str) -> str:
+    """Return a Buildkite ``if`` string. Buildkite rejects YAML bool ``if`` values."""
+    if expr in ("true", "false"):
+        return expr
+    return f"({expr})"
+
+
+def _compute_bootstrap_if_exprs(*, decision, platform: str) -> dict[str, str]:
+    disabled = "false"
+    nightly_main = 'build.branch == "main" && build.env("NIGHTLY") == "1"'
+    nightly_only = NPU_NIGHTLY_ONLY if platform == "npu" else CUDA_NIGHTLY_ONLY
+
+    if platform == "npu":
+        ready_pr = (
+            'build.branch != "main" && ('
+            'build.pull_request.labels includes "npu-test" || '
+            'build.pull_request.labels includes "ready"'
+            ")"
+        )
+        nightly_label_if = nightly_only
+        weekly_label_if = disabled
+        merge_base = disabled
+    else:
+        ready_pr = 'build.branch != "main" && build.pull_request.labels includes "ready"'
+        merge_main = 'build.branch == "main" && build.env("NIGHTLY") != "1" && build.env("WEEKLY") != "1"'
+        merge_pr = 'build.branch != "main" && build.pull_request.labels includes "merge-test"'
+        nightly_label_if = (
+            '(build.branch == "main" && build.env("NIGHTLY") == "1") || '
+            '(build.branch != "main" && ('
+            'build.pull_request.labels includes "nightly-test" || '
+            'build.pull_request.labels includes "omni-test" || '
+            'build.pull_request.labels includes "tts-test" || '
+            'build.pull_request.labels includes "diffusion-x2iat-test" || '
+            'build.pull_request.labels includes "diffusion-x2v-test"'
+            "))"
+        )
+        weekly_label_if = (
+            '(build.branch == "main" && build.env("WEEKLY") == "1") || '
+            '(build.branch != "main" && build.pull_request.labels includes "weekly-test")'
+        )
+        merge_base = f"({nightly_main}) || (({merge_main}) || ({merge_pr}))"
+
+    ready_base = f"({nightly_main}) || ({ready_pr})"
+    weekly_main = 'build.branch == "main" && build.env("WEEKLY") == "1"'
+
+    if decision.skip_all:
+        # Docs / skip-mark only: no PR-label escape hatch. Main scheduled
+        # NIGHTLY=1 still runs L4 + L2/L3 (--e2e); WEEKLY=1 still runs L5.
+        image_expr = f"({nightly_main}) || ({weekly_main})" if platform == "cuda" else nightly_main
+        ready_expr = nightly_main
+        merge_expr = nightly_main if platform == "cuda" else disabled
+        nightly_expr = nightly_main
+        weekly_expr = weekly_main if platform == "cuda" else disabled
+    elif decision.skip_l2_l3:
+        l2_enabled = decision.is_run("npu", "l2") if platform == "npu" else decision.is_run("cuda", "l2")
+        l3_enabled = platform == "cuda" and decision.is_run("cuda", "l3")
+
+        ready_expr = ready_base if l2_enabled else disabled
+        merge_expr = merge_base if l3_enabled else disabled
+        nightly_expr = nightly_label_if
+        weekly_expr = weekly_label_if if platform == "cuda" else disabled
+
+        image_parts = [f"({nightly_label_if})"]
+        if platform == "cuda":
+            image_parts.append(f"({weekly_label_if})")
+        if l2_enabled:
+            image_parts.insert(0, f"({ready_base})")
+        if l3_enabled:
+            image_parts.insert(1 if l2_enabled else 0, f"({merge_base})")
+        image_expr = " || ".join(image_parts)
+    else:
+        image_expr = "true"
+        ready_expr = ready_base
+        merge_expr = merge_base if platform == "cuda" else disabled
+        nightly_expr = nightly_label_if
+        weekly_expr = weekly_label_if if platform == "cuda" else disabled
+
+    return {
+        "image": _format_bootstrap_if(image_expr),
+        "ready": _format_bootstrap_if(ready_expr),
+        "merge": _format_bootstrap_if(merge_expr),
+        "nightly": _format_bootstrap_if(nightly_expr),
+        "weekly": _format_bootstrap_if(weekly_expr),
+    }
+
+
+def _apply_bootstrap_if(steps: list[Any], if_exprs: dict[str, str]) -> list[Any]:
+    """Inject ``if`` by step key; drop steps that are unconditionally disabled."""
+    kept: list[Any] = []
+    for step in steps:
+        if not isinstance(step, dict):
+            kept.append(step)
+            continue
+        nested = step.get("steps")
+        if isinstance(nested, list):
+            nested_kept = _apply_bootstrap_if(nested, if_exprs)
+            if nested_kept:
+                kept.append({**step, "steps": nested_kept})
+            continue
+        key = step.get("key")
+        if key in BOOTSTRAP_IMAGE_BUILD_KEYS:
+            step["if"] = if_exprs["image"]
+        elif key in BOOTSTRAP_UPLOAD_IF_KEYS:
+            step["if"] = if_exprs[BOOTSTRAP_UPLOAD_IF_KEYS[key]]
+        if_expr = step.get("if")
+        if if_expr == "false":
+            _log(f"omit disabled bootstrap step {key!r}")
+            continue
+        if if_expr == "true":
+            # Unconditional step: omit ``if`` (Buildkite requires string ``if``, not YAML bool).
+            step.pop("if", None)
+        kept.append(step)
+    return kept
+
+
+def _render_bootstrap_pipeline(
+    steps_yaml: str,
+    *,
+    decision,
+    path: Path,
+) -> str:
+    """Load bootstrap steps YAML and inject ``if`` expressions from skip-ci decision."""
+    doc = yaml.safe_load(steps_yaml)
+    if not isinstance(doc, dict):
+        raise ValueError(f"invalid bootstrap steps YAML for {path}")
+    steps = doc.get("steps")
+    if not isinstance(steps, list):
+        raise ValueError(f"bootstrap steps YAML must contain steps: list in {path}")
+
+    platform = _get_bootstrap_platform(path)
+    if_exprs = _compute_bootstrap_if_exprs(decision=decision, platform=platform)
+    doc["steps"] = _apply_bootstrap_if(steps, if_exprs)
+    return yaml.safe_dump(doc, sort_keys=False)
+
+
+# --- Test pipeline (test-ready.yml, test-merge.yml) ---
+
+
+@lru_cache(maxsize=1)
+def _load_mirror_hardwares() -> dict[str, dict[str, Any]]:
+    if not CI_MIRROR_HARDWARES_PATH.is_file():
+        raise FileNotFoundError(f"missing CI mirror_hardwares registry: {CI_MIRROR_HARDWARES_PATH}")
+    doc = yaml.safe_load(CI_MIRROR_HARDWARES_PATH.read_text(encoding="utf-8"))
+    if not isinstance(doc, dict):
+        raise ValueError(f"invalid CI mirror_hardwares registry: {CI_MIRROR_HARDWARES_PATH}")
+    presets = doc.get("mirror_hardwares")
+    if not isinstance(presets, dict):
+        raise ValueError(f"mirror_hardwares must be a mapping in {CI_MIRROR_HARDWARES_PATH}")
+    return presets
+
+
+def _expand_mirror_hardwares(step: dict[str, Any]) -> dict[str, Any]:
+    """Replace uploader-only ``mirror_hardwares`` with preset fields from ci_mirror_hardwares.yml."""
+    hardware = step.get("mirror_hardwares")
+    if hardware is None:
+        return step
+
+    if not isinstance(hardware, str) or not hardware.strip():
+        raise ValueError(
+            f"mirror_hardwares must be a non-empty string in step {_get_step_label(step)!r}",
+        )
+
+    preset = _load_mirror_hardwares().get(hardware)
+    if preset is None:
+        known = ", ".join(sorted(_load_mirror_hardwares()))
+        raise ValueError(
+            f"unknown mirror_hardwares {hardware!r} in step {_get_step_label(step)!r}; known: {known}",
+        )
+
+    if step.get("agents") is not None or step.get("plugins") is not None or step.get("image") is not None:
+        raise ValueError(
+            f"step {_get_step_label(step)!r} sets mirror_hardwares together with agents/plugins/image; "
+            "use mirror_hardwares only",
+        )
+
+    expanded = copy.deepcopy(preset)
+    return {key: value for key, value in step.items() if key != "mirror_hardwares"} | expanded
+
+
+def _match_source_file(changed_files: list[str], prefixes: list[str]) -> bool:
+    for path in changed_files:
+        for prefix in prefixes:
+            normalized = prefix.rstrip("/")
+            if path == normalized or path.startswith(f"{normalized}/"):
+                return True
+    return False
+
+
+def _get_step_label(step: dict[str, Any]) -> str:
+    return str(step.get("group") or step.get("label") or "<step>")
+
+
+def _process_test_steps(
+    steps: list[Any],
+    changed_files: list[str] | None,
+) -> list[Any]:
+    """Drop steps by ``source_file_dependencies`` when *changed_files* is set; always strip that field."""
+    processed: list[Any] = []
+    for step in steps:
+        if not isinstance(step, dict):
+            processed.append(step)
+            continue
+
+        deps = step.get("source_file_dependencies")
+        if deps is not None and not isinstance(deps, list):
+            raise ValueError(
+                f"source_file_dependencies must be a list in step {_get_step_label(step)!r}",
+            )
+        if changed_files is not None and deps is not None and not _match_source_file(changed_files, deps):
+            _log(f"skip {_get_step_label(step)!r} (no changes under {deps})")
+            continue
+
+        nested = step.get("steps")
+        if nested is not None:
+            kept_nested = _process_test_steps(nested, changed_files)
+            if changed_files is not None and not kept_nested:
+                _log(f"omit empty group {_get_step_label(step)!r}")
+                continue
+            new_step = {key: value for key, value in step.items() if key != "source_file_dependencies"}
+            new_step["steps"] = kept_nested
+            processed.append(new_step)
+            continue
+
+        if deps is not None:
+            processed.append(
+                _expand_mirror_hardwares(
+                    {key: value for key, value in step.items() if key != "source_file_dependencies"},
+                ),
+            )
+        else:
+            processed.append(_expand_mirror_hardwares(step))
+
+    return processed
+
+
+def _select_e2e_group_steps(steps: list[Any]) -> list[Any]:
+    """Keep only top-level groups whose name contains ``E2E_GROUP_MARKER``."""
+    selected = [
+        step
+        for step in steps
+        if isinstance(step, dict) and isinstance(step.get("group"), str) and E2E_GROUP_MARKER in step["group"]
+    ]
+    if not selected:
+        _log(f"no group matching {E2E_GROUP_MARKER!r} found")
+    else:
+        _log(f"keep {len(selected)} group(s) matching {E2E_GROUP_MARKER!r}")
+    return selected
+
+
+def _render_test_pipeline(
+    doc: dict[str, Any],
+    changed_files: list[str] | None,
+    *,
+    e2e_only: bool = False,
+) -> dict[str, Any]:
+    """Filter steps by PR diff and strip uploader-only ``source_file_dependencies`` metadata."""
+    steps = doc.get("steps")
+    if not isinstance(steps, list):
+        return doc
+    if e2e_only:
+        steps = _select_e2e_group_steps(steps)
+    steps = _process_test_steps(steps, changed_files)
+    return {**doc, "steps": steps}
+
+
+# --- Entry (read file → bootstrap or test render → YAML string) ---
+
+
+def _render_pipeline(
+    path: Path,
+    *,
+    force_all: bool = False,
+    e2e_only: bool = False,
+) -> str:
+    if path.name == BOOTSTRAP_STEPS_FILENAME:
+        ctx = resolve_ci_context_from_git()
+        continuation = _load_bootstrap_steps(path)
+        return _render_bootstrap_pipeline(
+            continuation,
+            decision=ctx.decision,
+            path=path,
+        )
+
+    text = path.read_text(encoding="utf-8")
+    ctx = resolve_ci_context_from_git()
+    if force_all or e2e_only:
+        changed_files = None
+    else:
+        changed_files = ctx.changed_files
+
+    doc = yaml.safe_load(text)
+    if not isinstance(doc, dict):
+        raise ValueError(f"invalid pipeline YAML: {path}")
+
+    doc = _render_test_pipeline(doc, changed_files, e2e_only=e2e_only)
+    return yaml.safe_dump(doc, sort_keys=False)
+
+
+def _upload_to_buildkite(content: str) -> None:
+    subprocess.run(
+        ["buildkite-agent", "pipeline", "upload"],
+        input=content,
+        text=True,
+        check=True,
+    )
+
+
+# --- CLI ---
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "pipeline",
+        nargs="?",
+        default=".buildkite/cuda/bootstrap-upload-steps.yml",
+        help="Pipeline YAML path (default: .buildkite/cuda/bootstrap-upload-steps.yml)",
+    )
+    parser.add_argument(
+        "--upload",
+        action="store_true",
+        help="Pipe rendered YAML to buildkite-agent pipeline upload",
+    )
+    mode = parser.add_mutually_exclusive_group()
+    mode.add_argument(
+        "--all",
+        action="store_true",
+        help="Keep all steps (disable diff-aware skipping)",
+    )
+    mode.add_argument(
+        "--e2e",
+        action="store_true",
+        help="Keep only the E2E Test group",
+    )
+    args = parser.parse_args()
+
+    path = Path(args.pipeline)
+    if not path.is_absolute():
+        path = ROOT / path
+    if not path.is_file():
+        _log(f"missing pipeline file: {path}")
+        return 1
+
+    rendered = _render_pipeline(path, force_all=args.all, e2e_only=args.e2e)
+    if args.upload:
+        _upload_to_buildkite(rendered)
+    else:
+        sys.stdout.write(rendered)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.buildkite/cuda/bootstrap-upload-steps.yml

@@ -0,0 +1,55 @@
+# Uploaded by upload_pipeline.py --upload (not the Buildkite hook).
+# ``if`` is injected by step ``key`` from skip-ci at upload time.
+steps:
+  - label: ":docker: Build image"
+    key: image-build
+    commands:
+      - "bash .buildkite/cuda/scripts/build-ci-image.sh"
+    agents:
+      queue: "cpu_queue_premerge"
+
+  # L2 Test — PR ready label (diff-aware) or main+NIGHTLY=1 (--e2e, full E2E group)
+  - label: "Upload Ready Pipeline"
+    depends_on: image-build
+    key: upload-ready-pipeline
+    commands:
+      - |
+        EXTRA=""
+        if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then
+          EXTRA="--e2e"
+        fi
+        python3 .buildkite/common/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-ready.yml
+    agents:
+      queue: "cpu_queue_premerge"
+
+  # L3 Test — PR merge-test label (diff-aware) or main+NIGHTLY=1 (--e2e, full E2E group)
+  - label: "Upload Merge Pipeline"
+    depends_on: image-build
+    key: upload-merge-pipeline
+    commands:
+      - |
+        EXTRA=""
+        if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then
+          EXTRA="--e2e"
+        fi
+        python3 .buildkite/common/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-merge.yml
+    agents:
+      queue: "cpu_queue_premerge"
+
+  # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
+  - label: "Upload Nightly Pipeline"
+    depends_on: image-build
+    key: upload-nightly-pipeline
+    commands:
+      - python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/cuda/test-nightly.yml
+    agents:
+      queue: "cpu_queue_premerge"
+
+  # L5 Test — main+WEEKLY=1 (scheduled), or PR with weekly-test label
+  - label: "Upload Weekly Pipeline"
+    depends_on: image-build
+    key: upload-weekly-pipeline
+    commands:
+      - python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/cuda/test-weekly.yml
+    agents:
+      queue: "cpu_queue_premerge"

.buildkite/cuda/pipeline.yml

@@ -1,84 +1,9 @@
-# Document 1: Buildkite loads only this block on first parse. The next step resolves docs-only,
-# pytest skip-mark-only, or combined skip-ci from git diff, then uploads document 2. When skip-ci
-# applies, image-build still runs for the nightly exception in upload_pipeline.py (nightly-test
-# label or main NIGHTLY=1); otherwise upload-nightly can be skipped together with test-ready/test-merge.
-# Weekly uploads also depend on image-build.
-
-# Document 2: appended after `---`; same file, read by upload_pipeline.py (not evaluated as a second pipeline by Buildkite). Child uploads include test-ready.yml, test-merge.yml, test-nightly.yml, and test-weekly.yml.
+# Hook uploads this entry YAML; upload_pipeline.py --upload reads bootstrap-upload-steps.yml
+# and injects ``if`` by step ``key`` from skip-ci.
 steps:
-  - label: ":github: Resolve skip-ci (docs / skip marks) & upload pipeline"
+  - label: ":github: Resolve skip-ci & upload pipeline"
     key: upload-ci-pipeline
     commands:
-      - "python3 .buildkite/cuda/scripts/upload_pipeline.py --upload .buildkite/cuda/pipeline.yml"
-    agents:
-      queue: "cpu_queue_premerge"
-
----
-steps:
-  - label: ":docker: Build image"
-    key: image-build
-    if: __IMAGE_BUILD_IF__
-    commands:
-      - "bash .buildkite/cuda/scripts/build-ci-image.sh"
-    agents:
-      queue: "cpu_queue_premerge"
-
-  # L2 Test — PR ready label (diff-aware) or main+NIGHTLY=1 (--e2e, full E2E group)
-  - label: "Upload Ready Pipeline"
-    depends_on: image-build
-    key: upload-ready-pipeline
-    if: __UPLOAD_READY_IF__
-    commands:
-      - |
-        EXTRA=""
-        if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then
-          EXTRA="--e2e"
-        fi
-        python3 .buildkite/cuda/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-ready.yml
-    agents:
-      queue: "cpu_queue_premerge"
-
-  # L3 Test — PR merge-test label (diff-aware) or main+NIGHTLY=1 (--e2e, full E2E group)
-  - label: "Upload Merge Pipeline"
-    depends_on: image-build
-    key: upload-merge-pipeline
-    if: __UPLOAD_MERGE_IF__
-    commands:
-      - |
-        EXTRA=""
-        if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then
-          EXTRA="--e2e"
-        fi
-        python3 .buildkite/cuda/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-merge.yml
-    agents:
-      queue: "cpu_queue_premerge"
-
-  # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
-  - label: "Upload Nightly Pipeline"
-    depends_on: image-build
-    key: upload-nightly-pipeline
-    if: >-
-      (build.branch == "main" && build.env("NIGHTLY") == "1") ||
-      (build.branch != "main" && (
-        build.pull_request.labels includes "nightly-test" ||
-        build.pull_request.labels includes "omni-test" ||
-        build.pull_request.labels includes "tts-test" ||
-        build.pull_request.labels includes "diffusion-x2iat-test" ||
-        build.pull_request.labels includes "diffusion-x2v-test"
-      ))
-    commands:
-      - buildkite-agent pipeline upload .buildkite/cuda/test-nightly.yml
-    agents:
-      queue: "cpu_queue_premerge"
-
-  # L5 Test — main+WEEKLY=1 (scheduled), or PR with weekly-test label
-  - label: "Upload Weekly Pipeline"
-    depends_on: image-build
-    key: upload-weekly-pipeline
-    if: >-
-      (build.branch == "main" && build.env("WEEKLY") == "1") ||
-      (build.branch != "main" && build.pull_request.labels includes "weekly-test")
-    commands:
-      - buildkite-agent pipeline upload .buildkite/cuda/test-weekly.yml
+      - "python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/cuda/bootstrap-upload-steps.yml"
     agents:
       queue: "cpu_queue_premerge"

.buildkite/cuda/rebase-pipeline.yml

@@ -11,7 +11,7 @@ steps:
     depends_on: image-build
     key: upload-ready-pipeline
     commands:
-      - python3 .buildkite/cuda/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-ready.yml
+      - python3 .buildkite/common/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-ready.yml
     agents:
       queue: "cpu_queue_premerge"
 
@@ -20,7 +20,7 @@ steps:
     depends_on: image-build
     key: upload-merge-pipeline
     commands:
-      - python3 .buildkite/cuda/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-merge.yml
+      - python3 .buildkite/common/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-merge.yml
     agents:
       queue: "cpu_queue_premerge"
 
@@ -30,6 +30,6 @@ steps:
     key: upload-nightly-pipeline
     if: build.env("NIGHTLY") == "1"
     commands:
-      - python3 .buildkite/cuda/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-nightly.yml
+      - python3 .buildkite/common/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-nightly.yml
     agents:
       queue: "cpu_queue_premerge"

.buildkite/cuda/scripts/upload_pipeline.py

@@ -1,547 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: Apache-2.0
-"""Render and optionally upload Buildkite pipeline YAML with diff-aware logic.
-
-Bootstrap mode (pipeline.yml with __IMAGE_BUILD_IF__ placeholders):
-  - Detect docs-only, pytest skip-mark-only, or combined docs+skip-mark PR/main changes and
-    substitute skip-ci ``if`` expressions.
-
-Test pipeline mode (e.g. test-merge.yml):
-  - Drop steps or groups whose ``source_file_dependencies`` do not match changed files.
-  - ``source_file_dependencies`` may be set on a leaf step or on a group step.
-  - Always strip ``source_file_dependencies`` from uploaded YAML (Buildkite does not
-    recognize this uploader-only key).
-
-Usage:
-  python3 upload_pipeline.py [--upload] [--all | --e2e] <pipeline.yml>
-
-  # Bootstrap (replaces upload_pipeline_with_skip_ci.sh):
-  python3 upload_pipeline.py --upload .buildkite/cuda/pipeline.yml
-
-  # Test pipeline (replaces upload_test_pipeline_with_diff_skip.py):
-  python3 upload_pipeline.py --upload .buildkite/cuda/test-merge.yml
-
-  # Full suite (rebase pipeline): keep all steps, still strip the uploader-only key:
-  python3 upload_pipeline.py --upload --all .buildkite/cuda/test-merge.yml
-
-  # Nightly L2 E2E only: keep the E2E Test group, ignore source_file_dependencies:
-  python3 upload_pipeline.py --upload --e2e .buildkite/cuda/test-merge.yml
-"""
-
-from __future__ import annotations
-
-import argparse
-import os
-import re
-import subprocess
-import sys
-from pathlib import Path
-from typing import Any
-
-try:
-    import yaml
-except ImportError:
-    subprocess.check_call(
-        [sys.executable, "-m", "pip", "install", "-q", "pyyaml"],
-    )
-    import yaml
-
-LOG = "upload_pipeline"
-ROOT = Path(__file__).resolve().parent.parent.parent.parent
-DOC_SEP = "\n---\n"
-BOOTSTRAP_MARKER = "__IMAGE_BUILD_IF__"
-E2E_GROUP_MARKER = "E2E Test"
-
-
-def _log(message: str) -> None:
-    print(f"{LOG}: {message}", file=sys.stderr)
-
-
-def _git(*args: str) -> subprocess.CompletedProcess[str]:
-    return subprocess.run(
-        ["git", *args],
-        check=False,
-        capture_output=True,
-        text=True,
-    )
-
-
-def resolve_diff_range() -> str | None:
-    """Return a git diff range for PR or main builds, or None when unavailable."""
-    is_pr = os.environ.get("BUILDKITE_PULL_REQUEST", "false") != "false" and os.environ.get(
-        "BUILDKITE_PULL_REQUEST", ""
-    )
-    commit = os.environ.get("BUILDKITE_COMMIT", "")
-
-    if is_pr:
-        base_branch = os.environ.get("BUILDKITE_PULL_REQUEST_BASE_BRANCH", "main")
-        base_ref = f"origin/{base_branch}"
-        if _git("rev-parse", "--verify", base_ref).returncode != 0:
-            _log(f"origin/{base_branch} not found locally; trying fetch")
-            _git("fetch", "--depth=200", "origin", base_branch)
-        if _git("rev-parse", "--verify", base_ref).returncode != 0:
-            if _git("rev-parse", "--verify", base_branch).returncode == 0:
-                base_ref = base_branch
-            else:
-                _log(f"cannot resolve PR base {base_branch}; using safe defaults")
-                return None
-        return f"{base_ref}...{commit}"
-
-    if os.environ.get("BUILDKITE_BRANCH", "") == "main":
-        if _git("rev-parse", "--verify", f"{commit}^").returncode != 0:
-            _log("main commit has no parent; using safe defaults")
-            return None
-        return f"{commit}^..{commit}"
-
-    _log("not PR/main build; using safe defaults")
-    return None
-
-
-def resolve_changed_files() -> list[str] | None:
-    """Return changed file paths, or None when diff cannot be resolved."""
-    diff_range = resolve_diff_range()
-    if diff_range is None:
-        return None
-
-    result = _git("diff", "--name-only", diff_range)
-    if result.returncode != 0:
-        _log(f"git diff failed ({diff_range}); using safe defaults")
-        return None
-
-    files = [line for line in result.stdout.splitlines() if line.strip()]
-    _log(f"{len(files)} changed file(s)")
-    return files
-
-
-def _is_doc_file(file_path: str) -> bool:
-    if not file_path:
-        return False
-    if file_path.startswith("docs/"):
-        return True
-    if file_path.endswith(".md"):
-        return True
-    return file_path == "mkdocs.yml"
-
-
-def is_docs_only_change(changed_files: list[str]) -> bool:
-    has_any = False
-    for file_path in changed_files:
-        if not file_path:
-            continue
-        has_any = True
-        if not _is_doc_file(file_path):
-            return False
-    return has_any
-
-
-def _is_test_python_file(file_path: str) -> bool:
-    path = Path(file_path)
-    return path.suffix == ".py" and bool(path.parts) and path.parts[0] == "tests"
-
-
-_SKIP_MARK_RE = re.compile(r"pytest\.mark\.skip(?:if)?\b|pytest\.skip\s*\(")
-_PYTESTMARK_SKIP_RE = re.compile(r"pytest\.mark\.skip\b")
-_PYTEST_MARK_ONLY_RE = re.compile(r"pytest\.mark\.\w+")
-
-
-def _paren_balance(line: str) -> int:
-    return line.count("(") - line.count(")")
-
-
-def _is_skip_mark_related_content(line: str) -> bool:
-    stripped = line.strip()
-    if not stripped:
-        return True
-    if stripped.startswith("#"):
-        return True
-    return _SKIP_MARK_RE.search(stripped) is not None
-
-
-def _is_pytestmark_adjacent_content(line: str) -> bool:
-    """Allow pytestmark list refactors that only add skip/skipif alongside existing marks."""
-    stripped = line.strip().rstrip(",")
-    if not stripped:
-        return True
-    if stripped in {"[", "]"}:
-        return True
-    if stripped.startswith("#"):
-        return True
-    if stripped.startswith("pytestmark"):
-        return True
-    return _PYTEST_MARK_ONLY_RE.search(stripped) is not None
-
-
-def diff_only_contains_skip_mark_changes(diff_text: str) -> bool:
-    """True when diff only edits skip marks or reformats pytestmark to add them."""
-    pending_depth = 0
-    saw_change = False
-    has_skip_mark_edit = False
-    for raw_line in diff_text.splitlines():
-        if raw_line.startswith("@@"):
-            pending_depth = 0
-            continue
-        if not (raw_line.startswith("+") or raw_line.startswith("-")):
-            continue
-        if raw_line.startswith("+++") or raw_line.startswith("---"):
-            continue
-
-        saw_change = True
-        content = raw_line[1:]
-
-        if pending_depth > 0:
-            pending_depth += _paren_balance(content)
-            if pending_depth < 0:
-                pending_depth = 0
-            continue
-
-        if _is_skip_mark_related_content(content):
-            has_skip_mark_edit = True
-            pending_depth = max(0, _paren_balance(content))
-            continue
-
-        if _is_pytestmark_adjacent_content(content):
-            continue
-
-        return False
-
-    return saw_change and has_skip_mark_edit
-
-
-def _git_file_diff(file_path: str, diff_range: str) -> str | None:
-    result = _git("diff", diff_range, "--", file_path)
-    if result.returncode != 0:
-        _log(f"skip-mark-only: git diff failed for {file_path}")
-        return None
-    return result.stdout
-
-
-def _is_new_file_diff(diff_text: str) -> bool:
-    return "new file mode" in diff_text or "--- /dev/null" in diff_text
-
-
-def _file_has_module_level_pytest_skip(content: str) -> bool:
-    """True when ``pytestmark`` applies unconditional ``pytest.mark.skip`` to the module."""
-    match = re.search(r"^pytestmark\s*=\s*\[(.*?)^\s*\]", content, re.MULTILINE | re.DOTALL)
-    if match is not None:
-        return _PYTESTMARK_SKIP_RE.search(match.group(1)) is not None
-    match = re.search(r"^pytestmark\s*=\s*pytest\.mark\.skip\b", content, re.MULTILINE)
-    return match is not None
-
-
-def _file_content_at_commit(file_path: str, commit: str) -> str | None:
-    result = _git("show", f"{commit}:{file_path}")
-    if result.returncode != 0:
-        _log(f"skip-mark-only: cannot read {file_path} at {commit}")
-        return None
-    return result.stdout
-
-
-def _qualifies_as_skip_mark_test_change(file_path: str, diff_range: str) -> bool:
-    diff_text = _git_file_diff(file_path, diff_range)
-    if diff_text is None:
-        return False
-    if not diff_text.strip():
-        _log(f"skip-mark-only: empty diff for {file_path}")
-        return False
-
-    if diff_only_contains_skip_mark_changes(diff_text):
-        return True
-
-    if not _is_new_file_diff(diff_text):
-        _log(f"skip-mark-only: non-skip changes in {file_path}")
-        return False
-
-    commit = os.environ.get("BUILDKITE_COMMIT", "")
-    if not commit:
-        return False
-    content = _file_content_at_commit(file_path, commit)
-    if content is None:
-        return False
-    if _file_has_module_level_pytest_skip(content):
-        _log(f"skip-mark-only: new test file with module-level pytest.mark.skip: {file_path}")
-        return True
-
-    _log(f"skip-mark-only: new test file without module-level pytest.mark.skip: {file_path}")
-    return False
-
-
-def is_skip_mark_only_change(changed_files: list[str], *, diff_range: str) -> bool:
-    """True when every changed file is a tests/ module with only skip-mark edits."""
-    has_any = False
-    for file_path in changed_files:
-        if not file_path:
-            continue
-        has_any = True
-        if not _is_test_python_file(file_path):
-            return False
-        if not _qualifies_as_skip_mark_test_change(file_path, diff_range):
-            return False
-    return has_any
-
-
-def is_docs_or_skip_mark_only_change(changed_files: list[str], *, diff_range: str) -> bool:
-    """True when each changed file is a doc path or a qualifying skip-mark test change."""
-    has_any = False
-    for file_path in changed_files:
-        if not file_path:
-            continue
-        has_any = True
-        if _is_doc_file(file_path):
-            continue
-        if _is_test_python_file(file_path) and _qualifies_as_skip_mark_test_change(file_path, diff_range):
-            continue
-        _log(f"docs/skip-mark-only: rejecting {file_path}")
-        return False
-    return has_any
-
-
-def resolve_skip_ci(changed_files: list[str] | None, *, diff_range: str | None = None) -> bool:
-    if changed_files is None:
-        _log("skip-ci=0 (could not resolve changed files)")
-        return False
-
-    if diff_range is not None and is_docs_or_skip_mark_only_change(changed_files, diff_range=diff_range):
-        if is_docs_only_change(changed_files):
-            _log("docs-only change detected; skip-ci=1")
-        elif is_skip_mark_only_change(changed_files, diff_range=diff_range):
-            _log("pytest skip-mark-only change detected; skip-ci=1")
-        else:
-            _log("docs + pytest skip-mark-only change detected; skip-ci=1")
-        return True
-
-    if diff_range is None and is_docs_only_change(changed_files):
-        _log("docs-only change detected; skip-ci=1")
-        return True
-
-    _log("non-doc/non-skip-mark changes detected; skip-ci=0")
-    return False
-
-
-def render_bootstrap_pipeline(text: str, *, skip_ci: bool) -> str:
-    if DOC_SEP in text:
-        _, continuation = text.split(DOC_SEP, 1)
-    else:
-        continuation = text
-
-    nightly_only = (
-        '(build.pull_request.labels includes "nightly-test") || (build.branch == "main" && build.env("NIGHTLY") == "1")'
-    )
-    nightly_main = 'build.branch == "main" && build.env("NIGHTLY") == "1"'
-    ready_pr = 'build.branch != "main" && build.pull_request.labels includes "ready"'
-    merge_main = 'build.branch == "main" && build.env("NIGHTLY") != "1" && build.env("WEEKLY") != "1"'
-    merge_pr = 'build.branch != "main" && build.pull_request.labels includes "merge-test"'
-    if skip_ci:
-        image_if = f"'{nightly_only}'"
-        ready_if = f"'({nightly_main})'"
-        merge_if = f"'({nightly_main})'"
-    else:
-        image_if = "'true'"
-        ready_if = f"'(({nightly_main}) || ({ready_pr}))'"
-        merge_if = f"'(({nightly_main}) || (({merge_main}) || ({merge_pr})))'"
-
-    return (
-        continuation.replace("__IMAGE_BUILD_IF__", image_if)
-        .replace("__UPLOAD_READY_IF__", ready_if)
-        .replace("__UPLOAD_MERGE_IF__", merge_if)
-    )
-
-
-def _matches_dependencies(changed_files: list[str], prefixes: list[str]) -> bool:
-    for path in changed_files:
-        for prefix in prefixes:
-            normalized = prefix.rstrip("/")
-            if path == normalized or path.startswith(f"{normalized}/"):
-                return True
-    return False
-
-
-def _strip_source_file_dependencies(step: dict[str, Any]) -> dict[str, Any]:
-    return {key: value for key, value in step.items() if key != "source_file_dependencies"}
-
-
-def _step_label(step: dict[str, Any]) -> str:
-    return str(step.get("group") or step.get("label") or "<step>")
-
-
-def _filter_steps(steps: list[Any], changed_files: list[str]) -> list[Any]:
-    filtered: list[Any] = []
-    for step in steps:
-        if not isinstance(step, dict):
-            filtered.append(step)
-            continue
-
-        deps = step.get("source_file_dependencies")
-        if deps is not None and not isinstance(deps, list):
-            raise ValueError(
-                f"source_file_dependencies must be a list in step {_step_label(step)!r}",
-            )
-        if deps is not None and not _matches_dependencies(changed_files, deps):
-            _log(
-                f"skip {_step_label(step)!r} (no changes under {deps})",
-            )
-            continue
-
-        nested = step.get("steps")
-        if nested is not None:
-            kept_nested = _filter_steps(nested, changed_files)
-            if not kept_nested:
-                _log(f"omit empty group {_step_label(step)!r}")
-                continue
-            new_step = _strip_source_file_dependencies(step)
-            new_step["steps"] = kept_nested
-            filtered.append(new_step)
-            continue
-
-        if deps is not None:
-            filtered.append(_strip_source_file_dependencies(step))
-        else:
-            filtered.append(step)
-
-    return filtered
-
-
-def _is_e2e_group(step: dict[str, Any]) -> bool:
-    group = step.get("group")
-    return isinstance(group, str) and E2E_GROUP_MARKER in group
-
-
-def _select_e2e_group_steps(steps: list[Any]) -> list[Any]:
-    selected = [step for step in steps if isinstance(step, dict) and _is_e2e_group(step)]
-    if not selected:
-        _log(f"no group matching {E2E_GROUP_MARKER!r} found")
-    else:
-        _log(f"keep {len(selected)} group(s) matching {E2E_GROUP_MARKER!r}")
-    return selected
-
-
-def _strip_uploader_metadata_from_steps(steps: list[Any]) -> list[Any]:
-    """Remove uploader-only keys while keeping all steps (no diff filtering)."""
-    stripped: list[Any] = []
-    for step in steps:
-        if not isinstance(step, dict):
-            stripped.append(step)
-            continue
-
-        deps = step.get("source_file_dependencies")
-        if deps is not None and not isinstance(deps, list):
-            raise ValueError(
-                f"source_file_dependencies must be a list in step {_step_label(step)!r}",
-            )
-
-        nested = step.get("steps")
-        new_step = _strip_source_file_dependencies(step)
-        if nested is not None:
-            new_step["steps"] = _strip_uploader_metadata_from_steps(nested)
-        stripped.append(new_step)
-
-    return stripped
-
-
-def render_test_pipeline(
-    doc: dict[str, Any],
-    changed_files: list[str] | None,
-    *,
-    e2e_only: bool = False,
-) -> dict[str, Any]:
-    steps = doc.get("steps")
-    if not isinstance(steps, list):
-        return doc
-    if e2e_only:
-        steps = _select_e2e_group_steps(steps)
-        steps = _strip_uploader_metadata_from_steps(steps)
-    elif changed_files is not None:
-        steps = _filter_steps(steps, changed_files)
-    else:
-        steps = _strip_uploader_metadata_from_steps(steps)
-    return {**doc, "steps": steps}
-
-
-def resolve_pipeline_path(arg: str) -> Path:
-    path = Path(arg)
-    if path.is_absolute():
-        return path
-    return ROOT / path
-
-
-def render_pipeline(
-    path: Path,
-    *,
-    force_all: bool = False,
-    e2e_only: bool = False,
-) -> str:
-    text = path.read_text(encoding="utf-8")
-    diff_range = resolve_diff_range()
-    changed_files = resolve_changed_files()
-
-    # ``--all`` forces the keep-all-steps path (no diff-aware skipping) while still
-    # stripping ``source_file_dependencies``. Used by the rebase pipeline so main builds
-    # run the full e2e suite (see .buildkite/cuda/rebase-pipeline.yml).
-    if force_all or e2e_only:
-        changed_files = None
-
-    if BOOTSTRAP_MARKER in text:
-        rendered = render_bootstrap_pipeline(
-            text,
-            skip_ci=resolve_skip_ci(changed_files, diff_range=diff_range),
-        )
-        return rendered
-
-    doc = yaml.safe_load(text)
-    if not isinstance(doc, dict):
-        raise ValueError(f"invalid pipeline YAML: {path}")
-
-    doc = render_test_pipeline(doc, changed_files, e2e_only=e2e_only)
-
-    return yaml.safe_dump(doc, sort_keys=False)
-
-
-def upload_to_buildkite(content: str) -> None:
-    subprocess.run(
-        ["buildkite-agent", "pipeline", "upload"],
-        input=content,
-        text=True,
-        check=True,
-    )
-
-
-def main() -> int:
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument(
-        "pipeline",
-        nargs="?",
-        default=".buildkite/cuda/pipeline.yml",
-        help="Pipeline YAML path (default: .buildkite/cuda/pipeline.yml)",
-    )
-    parser.add_argument(
-        "--upload",
-        action="store_true",
-        help="Pipe rendered YAML to buildkite-agent pipeline upload",
-    )
-    mode = parser.add_mutually_exclusive_group()
-    mode.add_argument(
-        "--all",
-        action="store_true",
-        help="Keep all steps (disable diff-aware skipping); still strips source_file_dependencies",
-    )
-    mode.add_argument(
-        "--e2e",
-        action="store_true",
-        help="Keep only the E2E Test group; disable diff-aware skipping for those steps",
-    )
-    args = parser.parse_args()
-
-    path = resolve_pipeline_path(args.pipeline)
-    if not path.is_file():
-        _log(f"missing pipeline file: {path}")
-        return 1
-
-    rendered = render_pipeline(path, force_all=args.all, e2e_only=args.e2e)
-    if args.upload:
-        upload_to_buildkite(rendered)
-    else:
-        sys.stdout.write(rendered)
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

.buildkite/cuda/test-merge.yml

@@ -11,34 +11,12 @@ steps:
       - label: "Simple · Diffusion & Model Executor Test"
         commands:
           - "timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
+        mirror_hardwares: l4_1
 
       - label: "Simple · Other Test"
         commands:
           - "timeout 20m pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
+        mirror_hardwares: l4_1
 
   - group: ":card_index_dividers: Diffusion Test"
     depends_on: upload-merge-pipeline
@@ -47,35 +25,13 @@ steps:
         timeout_in_minutes: 30
         commands:
           - pytest -sv tests/diffusion/lora/ -m 'advanced_model and cuda' --run-level "advanced_model"
-        agents:
-          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
+        mirror_hardwares: l4_1
 
       - label: "Diffusion · Offloader Test"
         timeout_in_minutes: 40
         commands:
           - timeout 40m pytest -s -v tests/diffusion/offloader -m "advanced_model and cuda"
-        agents:
-          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
+        mirror_hardwares: l4_1
 
   - label: "Distributed Test"
     depends_on: upload-merge-pipeline
@@ -87,39 +43,7 @@ steps:
           export VLLM_IMAGE_FETCH_TIMEOUT=60
           pytest -s -v tests/distributed -m 'advanced_model and cuda and H100' --run-level 'advanced_model'
         "
-    agents:
-      queue: "mithril-h100-pool"
-    plugins:
-      - kubernetes:
-          podSpec:
-            containers:
-              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                resources:
-                  limits:
-                    nvidia.com/gpu: 1
-                volumeMounts:
-                  - name: devshm
-                    mountPath: /dev/shm
-                  - name: hf-cache
-                    mountPath: /root/.cache/huggingface
-                env:
-                  - name: HF_HOME
-                    value: /root/.cache/huggingface
-                  - name: HF_TOKEN
-                    valueFrom:
-                      secretKeyRef:
-                        name: hf-token-secret
-                        key: token
-            nodeSelector:
-              node.kubernetes.io/instance-type: gpu-h100-sxm
-            volumes:
-              - name: devshm
-                emptyDir:
-                  medium: Memory
-              - name: hf-cache
-                hostPath:
-                  path: /mnt/hf-cache
-                  type: DirectoryOrCreate
+    mirror_hardwares: h100_1
 
   - group: ":card_index_dividers: Entrypoints Test"
     depends_on: upload-merge-pipeline
@@ -128,56 +52,13 @@ steps:
         timeout_in_minutes: 60
         commands:
           - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and H100" --run-level "advanced_model"
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 2
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
+        mirror_hardwares: h100_2
 
       - label: "Entrypoint Test with L4"
         timeout_in_minutes: 30
         commands:
           - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and L4" --run-level "advanced_model"
-        agents:
-          queue: "gpu_1_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
+        mirror_hardwares: l4_1
 
   - group: ":card_index_dividers: E2E Test"
     depends_on: upload-merge-pipeline
@@ -196,18 +77,7 @@ steps:
               export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
               pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m 'advanced_model and cuda' --run-level 'advanced_model'
             "
-        agents:
-          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
+        mirror_hardwares: l4_1
 
       - label: "TTS · Qwen3-TTS Base Test"
         source_file_dependencies:
@@ -223,18 +93,7 @@ steps:
               export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
               pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model'
             "
-        agents:
-          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
+        mirror_hardwares: l4_1
 
       - label: "TTS · Higgs-Audio-V3 Test"
         source_file_dependencies:
@@ -250,19 +109,7 @@ steps:
               export VLLM_MOE_USE_DEEP_GEMM=0
               pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py -m 'advanced_model' --run-level 'advanced_model'
             "
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
+        mirror_hardwares: l4_1
 
       - label: "Omni · Qwen3-Omni Test"
         source_file_dependencies:
@@ -274,39 +121,7 @@ steps:
         timeout_in_minutes: 50
         commands:
           - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 2
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
+        mirror_hardwares: h100_2
 
       - label: "TTS · VoxCPM2 Test"
         source_file_dependencies:
@@ -322,19 +137,7 @@ steps:
               export VLLM_WORKER_MULTIPROC_METHOD=spawn
               pytest -s -v tests/e2e/offline_inference/test_voxcpm2_tts.py tests/e2e/online_serving/test_voxcpm2_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model'
             "
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
+        mirror_hardwares: l4_1
 
       - label: "TTS · SoulX-Singer Test"
         source_file_dependencies:
@@ -350,19 +153,7 @@ steps:
               export VLLM_WORKER_MULTIPROC_METHOD=spawn
               pytest -s -v tests/e2e/offline_inference/test_soulxsinger.py -k 'svc or precomputed' -m 'advanced_model and cuda' --run-level 'advanced_model'
             "
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
+        mirror_hardwares: l4_1
 
       - label: "Diffusion · Qwen Image Edit Test"
         source_file_dependencies:
@@ -371,40 +162,7 @@ steps:
         timeout_in_minutes: 20
         commands:
           - pytest -s -v tests/e2e/online_serving/test_qwen_image_edit.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
+        mirror_hardwares: h100_1
       - label: "Diffusion · Bagel Test"
         source_file_dependencies:
           - tests/e2e/offline_inference/test_bagel.py
@@ -416,40 +174,7 @@ steps:
         timeout_in_minutes: 20
         commands:
           - pytest -s -v tests/e2e/offline_inference/test_bagel.py tests/e2e/online_serving/test_bagel.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
+        mirror_hardwares: h100_1
 
       - label: "Diffusion · Qwen Image Test"
         source_file_dependencies:
@@ -461,39 +186,7 @@ steps:
               export VLLM_IMAGE_FETCH_TIMEOUT=60
               pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'advanced_model and cuda' --run-level 'advanced_model'
             '
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
+        mirror_hardwares: h100_1
 
       - label: "Diffusion · Qwen Image Layered Test"
         source_file_dependencies:
@@ -505,39 +198,7 @@ steps:
               export VLLM_IMAGE_FETCH_TIMEOUT=60
               pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'advanced_model and cuda' --run-level 'advanced_model'
             "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
+        mirror_hardwares: h100_1
 
       - label: "Diffusion · Wan22 Test"
         source_file_dependencies:
@@ -550,39 +211,7 @@ steps:
               export VLLM_IMAGE_FETCH_TIMEOUT=60
               pytest -s -v tests/e2e/offline_inference/test_wan22_t2v.py tests/e2e/online_serving/test_wan22_t2v.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
             "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
+        mirror_hardwares: h100_1
 
       - label: "Diffusion · Z Image Test"
         source_file_dependencies:
@@ -592,19 +221,7 @@ steps:
         timeout_in_minutes: 30
         commands:
           - pytest -s -v tests/e2e/offline_inference/test_z_image.py tests/e2e/online_serving/test_z_image.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
-        agents:
-          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
+        mirror_hardwares: l4_1
       - label: "Diffusion · FLUX.2 Klein Test"
         source_file_dependencies:
           - tests/e2e/offline_inference/test_flux2_klein.py
@@ -612,15 +229,4 @@ steps:
         timeout_in_minutes: 30
         commands:
           - pytest -s -v tests/e2e/offline_inference/test_flux2_klein.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
-        agents:
-          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
+        mirror_hardwares: l4_1

.buildkite/cuda/test-ready.yml

@@ -8,104 +8,41 @@ steps:
   - group: ":card_index_dividers: Simple Test"
     depends_on: upload-ready-pipeline
     steps:
+
       - label: "Simple · Diffusion & Model Executor Test"
         commands:
           - "timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
+        mirror_hardwares: l4_1
 
       - label: "Simple · Other Test"
         commands:
           - "timeout 20m pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
+        mirror_hardwares: l4_1
 
   - label: "Custom Pipeline Test"
     depends_on: upload-ready-pipeline
     commands:
       - timeout 20m pytest -s -v tests/e2e/offline_inference/custom_pipeline/ -m "core_model"
-    agents:
-      queue: "gpu_1_queue"
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
+    mirror_hardwares: l4_1
 
   - group: ":card_index_dividers: Diffusion Test"
     depends_on: upload-ready-pipeline
     steps:
+
       - label: "Diffusion · Offloader Test"
         commands:
           - timeout 40m pytest -s -v tests/diffusion/offloader -m "core_model and cuda" --run-level "core_model"
-        agents:
-          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
+        mirror_hardwares: l4_1
 
       - label: "Diffusion · Cache Test"
         commands:
           - timeout 15m pytest -sv tests/diffusion/cache -m 'core_model and cuda' --run-level "core_model"
-        agents:
-          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
+        mirror_hardwares: l4_1
 
       - label: "Diffusion · Batch Test"
         commands:
           - timeout 15m pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model"
-        agents:
-          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
+        mirror_hardwares: l4_1
 
       - label: "Diffusion · Distributed Test"
         source_file_dependencies:
@@ -116,73 +53,26 @@ steps:
           - vllm_omni/diffusion/models/
         commands:
           - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda' --run-level "core_model"
-        agents:
-          queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
+        mirror_hardwares: l4_4
 
       - label: "Diffusion · Model Test"
         commands:
           - timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model"
-        agents:
-          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
+        mirror_hardwares: l4_1
 
       - label: "Diffusion · Other Test"
         commands:
           - timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda' --run-level "core_model"
-        agents:
-          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
+        mirror_hardwares: l4_1
 
   - group: ":card_index_dividers: Tiny Model Tests [Single GPU]"
     depends_on: upload-ready-pipeline
     steps:
+
       - label: "Tiny Model Tests · Diffusion (base)"
         commands:
           - timeout 15m pytest -sv tests/model_tests/diffusion -m 'core_model and cuda' --run-level "core_model"
-        agents:
-          # Ready label only runs the cases that don't need parallelism for tiny
-          # model tests, which is usually just a smoke test for TTI and I2I
-          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
+        mirror_hardwares: l4_1
 
   - label: "Engine&Model Executor Test"
     depends_on: upload-ready-pipeline
@@ -191,20 +81,7 @@ steps:
         timeout 15m bash -c "
                   pytest -sv tests/engine/ tests/model_executor/ -m 'core_model and cuda' --run-level 'core_model'
         "
-    agents:
-      queue: "gpu_1_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          shm-size: "8gb"
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-            - "HF_TOKEN"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
+    mirror_hardwares: l4_1
 
   - label: "Distributed Test"
     depends_on: upload-ready-pipeline
@@ -213,19 +90,7 @@ steps:
         timeout 20m bash -c "
           pytest -sv tests/distributed/ -m 'core_model and cuda and L4' --run-level 'core_model'
         "
-    agents:
-      queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          shm-size: "8gb"
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-            - "HF_TOKEN"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
+    mirror_hardwares: l4_1
 
   - label: "Entrypoints Test"
     depends_on: upload-ready-pipeline
@@ -235,23 +100,12 @@ steps:
           export VLLM_WORKER_MULTIPROC_METHOD=spawn
           pytest -sv tests/entrypoints/ -m 'core_model and cuda' --run-level 'core_model'
         "
-    agents:
-      queue: "gpu_1_queue"
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          shm-size: "8gb"
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-            - "HF_TOKEN"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
+    mirror_hardwares: l4_1
 
   - group: ":card_index_dividers: E2E Test"
     depends_on: upload-ready-pipeline
     steps:
+
       - label: "RLHF · VeRL-Omni E2E Test"
         source_file_dependencies:
           - tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py
@@ -266,19 +120,7 @@ steps:
             timeout 20m bash -c "
               pytest -s -v tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py
             "
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
+        mirror_hardwares: l4_1
 
       - label: "Omni · Qwen3-Omni Test"
         source_file_dependencies:
@@ -291,34 +133,7 @@ steps:
             timeout 20m bash -c "
               pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m 'core_model' --run-level 'core_model'
             "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 2
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
+        mirror_hardwares: h100_2
 
       - label: "TTS · VoxCPM2 Test"
         source_file_dependencies:
@@ -333,19 +148,7 @@ steps:
               export VLLM_WORKER_MULTIPROC_METHOD=spawn
               pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts.py -m 'core_model' --run-level 'core_model'
             "
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
+        mirror_hardwares: l4_1
 
       - label: "TTS · Qwen3-TTS CustomVoice Test"
         source_file_dependencies:
@@ -359,19 +162,7 @@ steps:
               export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
               pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'
             "
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
+        mirror_hardwares: l4_1
 
       - label: "TTS · Qwen3-TTS Base Test"
         source_file_dependencies:
@@ -387,19 +178,7 @@ steps:
               export VLLM_OMNI_USE_V2_RUNNER=1
               pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py -m 'core_model' --run-level 'core_model'
             "
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
+        mirror_hardwares: l4_1
 
       - label: "TTS · Higgs-Audio-V3 Test"
         source_file_dependencies:
@@ -415,19 +194,7 @@ steps:
               export VLLM_MOE_USE_DEEP_GEMM=0
               pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py -m 'core_model' --run-level 'core_model'
             "
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
+        mirror_hardwares: l4_1
 
       - label: "Diffusion · Qwen Image Edit Test"
         source_file_dependencies:
@@ -438,34 +205,7 @@ steps:
             timeout 20m bash -c "
               pytest -s -v tests/e2e/online_serving/test_qwen_image_edit.py -m 'core_model' --run-level 'core_model'
             "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
+        mirror_hardwares: h100_1
 
       - label: "Diffusion · Bagel Test"
         source_file_dependencies:
@@ -480,39 +220,7 @@ steps:
               export VLLM_IMAGE_FETCH_TIMEOUT=60
               pytest -s -v tests/e2e/online_serving/test_bagel.py -m 'core_model' --run-level 'core_model'
             "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
+        mirror_hardwares: h100_1
 
       - label: "Diffusion · Wan22 Test"
         source_file_dependencies:
@@ -524,39 +232,7 @@ steps:
               export VLLM_IMAGE_FETCH_TIMEOUT=60
               pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model' --run-level 'core_model'
             "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
+        mirror_hardwares: h100_1
 
       - label: "Diffusion · Cosmos3 Test"
         source_file_dependencies:
@@ -568,40 +244,7 @@ steps:
               export VLLM_IMAGE_FETCH_TIMEOUT=60
               pytest -s -v tests/e2e/online_serving/test_cosmos3.py -m 'core_model' --run-level 'core_model'
             "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
+        mirror_hardwares: h100_1
 
       - label: "Diffusion · Qwen Image Test"
         source_file_dependencies:
@@ -613,39 +256,7 @@ steps:
               export VLLM_IMAGE_FETCH_TIMEOUT=60
               pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model' --run-level 'core_model'
             "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
+        mirror_hardwares: h100_1
 
       - label: "Diffusion · Qwen Image Layered Test"
         source_file_dependencies:
@@ -657,39 +268,7 @@ steps:
               export VLLM_IMAGE_FETCH_TIMEOUT=60
               pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'core_model' --run-level 'core_model'
             "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
+        mirror_hardwares: h100_1
 
       - label: "Diffusion · GGUF Plugin Test"
         soft_fail: true
@@ -699,15 +278,4 @@ steps:
         commands:
           - pip install "vllm-gguf-plugin>=0.0.3"
           - timeout 30m pytest -sv tests/plugin/test_gguf_diffusion.py
-        agents:
-          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
+        mirror_hardwares: l4_1

.buildkite/cuda/test-weekly.yml

@@ -5,45 +5,14 @@ env:
   HF_HUB_ETAG_TIMEOUT: 60
 
 steps:
+
   - label: "Reliability Test - qwen3-omni"
     timeout_in_minutes: 180
     depends_on: upload-weekly-pipeline
     if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
     commands:
       - pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m "slow"
-    agents:
-      queue: "mithril-h100-pool"
-    plugins:
-      - kubernetes:
-          podSpec:
-            containers:
-              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                resources:
-                  limits:
-                    nvidia.com/gpu: 2
-                volumeMounts:
-                  - name: devshm
-                    mountPath: /dev/shm
-                  - name: hf-cache
-                    mountPath: /root/.cache/huggingface
-                env:
-                  - name: HF_HOME
-                    value: /root/.cache/huggingface
-                  - name: HF_TOKEN
-                    valueFrom:
-                      secretKeyRef:
-                        name: hf-token-secret
-                        key: token
-            nodeSelector:
-              node.kubernetes.io/instance-type: gpu-h100-sxm
-            volumes:
-              - name: devshm
-                emptyDir:
-                  medium: Memory
-              - name: hf-cache
-                hostPath:
-                  path: /mnt/hf-cache
-                  type: DirectoryOrCreate
+    mirror_hardwares: h100_2
 
   - label: "Reliability Test - wan22"
     timeout_in_minutes: 180
@@ -51,39 +20,7 @@ steps:
     if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
     commands:
       - pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m "slow"
-    agents:
-      queue: "mithril-h100-pool"
-    plugins:
-      - kubernetes:
-          podSpec:
-            containers:
-              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                resources:
-                  limits:
-                    nvidia.com/gpu: 2
-                volumeMounts:
-                  - name: devshm
-                    mountPath: /dev/shm
-                  - name: hf-cache
-                    mountPath: /root/.cache/huggingface
-                env:
-                  - name: HF_HOME
-                    value: /root/.cache/huggingface
-                  - name: HF_TOKEN
-                    valueFrom:
-                      secretKeyRef:
-                        name: hf-token-secret
-                        key: token
-            nodeSelector:
-              node.kubernetes.io/instance-type: gpu-h100-sxm
-            volumes:
-              - name: devshm
-                emptyDir:
-                  medium: Memory
-              - name: hf-cache
-                hostPath:
-                  path: /mnt/hf-cache
-                  type: DirectoryOrCreate
+    mirror_hardwares: h100_2
 
   - label: "Reliability Test - hunyuan-image"
     timeout_in_minutes: 180
@@ -91,39 +28,7 @@ steps:
     if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
     commands:
       - pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m "slow"
-    agents:
-      queue: "mithril-h100-pool"
-    plugins:
-      - kubernetes:
-          podSpec:
-            containers:
-              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                resources:
-                  limits:
-                    nvidia.com/gpu: 4
-                volumeMounts:
-                  - name: devshm
-                    mountPath: /dev/shm
-                  - name: hf-cache
-                    mountPath: /root/.cache/huggingface
-                env:
-                  - name: HF_HOME
-                    value: /root/.cache/huggingface
-                  - name: HF_TOKEN
-                    valueFrom:
-                      secretKeyRef:
-                        name: hf-token-secret
-                        key: token
-            nodeSelector:
-              node.kubernetes.io/instance-type: gpu-h100-sxm
-            volumes:
-              - name: devshm
-                emptyDir:
-                  medium: Memory
-              - name: hf-cache
-                hostPath:
-                  path: /mnt/hf-cache
-                  type: DirectoryOrCreate
+    mirror_hardwares: h100_4
 
   - label: "Reliability Test - voxcpm2"
     timeout_in_minutes: 180
@@ -131,86 +36,30 @@ steps:
     if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
     commands:
       - pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m "slow"
-    agents:
-      queue: "gpu_1_queue"
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          shm-size: "8gb"
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-            - "HF_TOKEN"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
+    mirror_hardwares: l4_1
 
-  # Invalid HTTP / WS payloads vs live Omni servers (Issue #3649 tracks known skips).
   - group: "Reliability Test - Invalid parameters Test"
     depends_on: upload-weekly-pipeline
     if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
     steps:
+
       - label: "Invalid parameters Test · H100"
         timeout_in_minutes: 120
         commands:
           - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100"
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 2
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
+        mirror_hardwares: h100_2
 
-      # Stable Audio Open, Qwen3-TTS speech/batch/voices paths marked ``L4``.
       - label: "Invalid parameters Test · L4"
         timeout_in_minutes: 120
         commands:
           - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and L4"
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
+        mirror_hardwares: l4_1
 
   - group: ":card_index_dividers: E2E Tests"
     depends_on: upload-weekly-pipeline
     if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
     steps:
+
       - label: "Omni · L4"
         timeout_in_minutes: 90
         commands:
@@ -222,19 +71,7 @@ steps:
             --ignore=tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py
             -m "slow and L4 and omni"
             --run-level "full_model"
-        agents:
-          queue: "gpu_4_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
+        mirror_hardwares: l4_4
 
       - label: "Omni · H100"
         timeout_in_minutes: 90
@@ -243,39 +80,7 @@ steps:
             pytest -sv
             tests/e2e/ -m "slow and H100 and omni"
             --run-level "full_model"
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 2
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
+        mirror_hardwares: h100_2
 
       - label: "TTS · L4"
         timeout_in_minutes: 90
@@ -289,19 +94,7 @@ steps:
               pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1
             done
             exit $$failed
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
+        mirror_hardwares: l4_1
 
       - label: "TTS · H100"
         timeout_in_minutes: 90
@@ -313,39 +106,7 @@ steps:
             pytest -sv
             tests/e2e/ -m "slow and H100 and tts"
             --run-level "full_model"
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
+        mirror_hardwares: h100_1
 
       - label: "Diffusion · L4"
         timeout_in_minutes: 60
@@ -354,19 +115,7 @@ steps:
             pytest -sv
             tests/e2e/ -m "slow and diffusion and L4"
             --run-level "full_model"
-        agents:
-          queue: "gpu_4_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
+        mirror_hardwares: l4_4
 
       - label: "Diffusion · H100 · Single-GPU"
         timeout_in_minutes: 120
@@ -375,39 +124,7 @@ steps:
             pytest -sv
             tests/e2e/ -m "slow and diffusion and H100 and not distributed_cuda"
             --run-level "full_model"
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
+        mirror_hardwares: h100_1
 
       - label: "Diffusion · H100 · Multi-GPU"
         timeout_in_minutes: 90
@@ -416,36 +133,4 @@ steps:
             pytest -sv
             tests/e2e/ -m "slow and diffusion and H100 and distributed_cuda"
             --run-level "full_model"
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 2
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
+        mirror_hardwares: h100_2

.buildkite/intel/scripts/bootstrap-intel-omni.sh

@@ -1,51 +1,16 @@
 #!/bin/bash
 # vllm-omni Intel bootstrap
-# Uses static pipeline-intel.yaml for Intel XPU tests
-# Based on: bootstrap-amd-omni.sh
+# Uses static pipeline-intel.yml for Intel XPU tests
 
 set -euo pipefail
 
-if [[ -z "${DOCS_ONLY_DISABLE:-}" ]]; then
-    DOCS_ONLY_DISABLE=0
-fi
+source .buildkite/common/scripts/resolve_skip_ci.sh
 
 upload_pipeline() {
     echo "--- 🛠 Preparing Intel pipeline"
     buildkite-agent pipeline upload .buildkite/intel/pipeline-intel.yml
 }
 
-get_diff() {
-    $(git add .)
-    echo $(git diff --name-only --diff-filter=ACMDR $(git merge-base origin/main HEAD))
-}
-
-get_diff_main() {
-    $(git add .)
-    echo $(git diff --name-only --diff-filter=ACMDR HEAD~1)
-}
-
-file_diff=$(get_diff)
-if [[ $BUILDKITE_BRANCH == "main" ]]; then
-    file_diff=$(get_diff_main)
-fi
-
-# ----------------------------------------------------------------------
-# Early exit: skip pipeline if all changed files are under docs/
-# ----------------------------------------------------------------------
-if [[ "${DOCS_ONLY_DISABLE}" != "1" ]] && [[ -n "${file_diff:-}" ]]; then
-    docs_only=1
-    while IFS= read -r f; do
-        [[ -z "$f" ]] && continue
-        if [[ "$f" != docs/* ]]; then
-            docs_only=0
-            break
-        fi
-    done < <(printf '%s\n' "$file_diff" | tr ' ' '\n' | tr -d '\r')
-
-    if [[ "$docs_only" -eq 1 ]]; then
-        buildkite-agent annotate ":memo: CI skipped — docs only" --style "info"
-        exit 0
-    fi
-fi
+gate_bootstrap_ci intel l2
 
 upload_pipeline

.buildkite/npu/bootstrap-upload-steps.yml

@@ -0,0 +1,181 @@
+# Uploaded by upload_pipeline.py --upload (not the Buildkite hook).
+# ``if`` is injected by step ``key`` from skip-ci at upload time.
+steps:
+  - label: ":buildkit: Build and Push NPU Test Image - A2"
+    key: image-build-a2
+    agents:
+      queue: "ascend-a2b3"
+      resource_class: "npu-1"
+    plugins:
+      - kubernetes:
+          metadata:
+            annotations:
+              vault.hashicorp.com/agent-init-first: "true"
+              vault.hashicorp.com/agent-inject: "true"
+              vault.hashicorp.com/agent-inject-perms-ca.pem: "0400"
+              vault.hashicorp.com/agent-inject-perms-cert.pem: "0400"
+              vault.hashicorp.com/agent-inject-perms-config.json: "0400"
+              vault.hashicorp.com/agent-inject-perms-key.pem: "0400"
+              vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd
+              vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd
+              vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd
+              vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd
+              vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}"
+              vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}"
+              vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}"
+              vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}"
+              vault.hashicorp.com/agent-pre-populate-only: "true"
+              vault.hashicorp.com/agent-run-as-group: "1000"
+              vault.hashicorp.com/agent-run-as-user: "1000"
+              vault.hashicorp.com/agent-service-account-token-volume-name: token-vol
+              vault.hashicorp.com/role: ascend-gha-runners
+              vault.hashicorp.com/secret-volume-path: /home/user/.docker/
+              vault.hashicorp.com/tls-skip-verify: "true"
+          podSpecPatch:
+            volumes:
+            - name: token-vol
+              projected:
+                defaultMode: 420
+                sources:
+                - serviceAccountToken:
+                    audience: api
+                    expirationSeconds: 600
+                    path: token
+    env:
+      VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
+      IMAGE_NAME: "vllm-omni-ci-npu"
+      IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
+      BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
+    command: |
+      set -ex
+
+      echo "--- Building and pushing NPU Test Image"
+      echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
+      echo "buildkitd address: $${BUILDKITD_ADDR}"
+
+      if ! command -v buildctl &> /dev/null; then
+        echo "Installing buildctl..."
+        mkdir -p /tmp/buildkit
+        BUILDKIT_VERSION="v0.29.0"
+        wget -q "https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz
+        tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
+        cp /tmp/buildkit/bin/buildctl /usr/local/bin/
+      fi
+
+      export DOCKER_CONFIG=/home/user/.docker
+      buildctl \
+        --addr="$${BUILDKITD_ADDR}" \
+        --tlscacert=/home/user/.docker/ca.pem \
+        --tlscert=/home/user/.docker/cert.pem \
+        --tlskey=/home/user/.docker/key.pem \
+        build \
+        --frontend dockerfile.v0 \
+        --local context=. \
+        --local dockerfile=./docker \
+        --opt filename=Dockerfile.npu.ci \
+        --secret id=dockerconfig,src=/home/user/.docker/config.json \
+        --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \
+        --progress=plain
+
+      echo "--- Image pushed successfully"
+      echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
+
+  - label: ":buildkit: Build and Push NPU Test Image - A3"
+    key: image-build-a3
+    agents:
+      queue: "ascend-a2b3"
+      resource_class: "npu-1"
+    plugins:
+      - kubernetes:
+          metadata:
+            annotations:
+              vault.hashicorp.com/agent-init-first: "true"
+              vault.hashicorp.com/agent-inject: "true"
+              vault.hashicorp.com/agent-inject-perms-ca.pem: "0400"
+              vault.hashicorp.com/agent-inject-perms-cert.pem: "0400"
+              vault.hashicorp.com/agent-inject-perms-config.json: "0400"
+              vault.hashicorp.com/agent-inject-perms-key.pem: "0400"
+              vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd
+              vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd
+              vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd
+              vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd
+              vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}"
+              vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}"
+              vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}"
+              vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}"
+              vault.hashicorp.com/agent-pre-populate-only: "true"
+              vault.hashicorp.com/agent-run-as-group: "1000"
+              vault.hashicorp.com/agent-run-as-user: "1000"
+              vault.hashicorp.com/agent-service-account-token-volume-name: token-vol
+              vault.hashicorp.com/role: ascend-gha-runners
+              vault.hashicorp.com/secret-volume-path: /home/user/.docker/
+              vault.hashicorp.com/tls-skip-verify: "true"
+          podSpecPatch:
+            volumes:
+            - name: token-vol
+              projected:
+                defaultMode: 420
+                sources:
+                - serviceAccountToken:
+                    audience: api
+                    expirationSeconds: 600
+                    path: token
+    env:
+      VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}-a3"
+      IMAGE_NAME: "vllm-omni-ci-npu"
+      IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
+      BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
+    command: |
+      set -ex
+
+      echo "--- Building and pushing NPU Test Image"
+      echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
+      echo "buildkitd address: $${BUILDKITD_ADDR}"
+
+      if ! command -v buildctl &> /dev/null; then
+        echo "Installing buildctl..."
+        mkdir -p /tmp/buildkit
+        BUILDKIT_VERSION="v0.29.0"
+        wget -q "https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz
+        tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
+        cp /tmp/buildkit/bin/buildctl /usr/local/bin/
+      fi
+
+      export DOCKER_CONFIG=/home/user/.docker
+      buildctl \
+        --addr="$${BUILDKITD_ADDR}" \
+        --tlscacert=/home/user/.docker/ca.pem \
+        --tlscert=/home/user/.docker/cert.pem \
+        --tlskey=/home/user/.docker/key.pem \
+        build \
+        --frontend dockerfile.v0 \
+        --local context=. \
+        --local dockerfile=./docker \
+        --opt filename=Dockerfile.npu.ci.a3 \
+        --secret id=dockerconfig,src=/home/user/.docker/config.json \
+        --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \
+        --progress=plain
+
+      echo "--- Image pushed successfully"
+      echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
+
+  - label: "Upload Ready Pipeline"
+    depends_on:
+      - image-build-a2
+      - image-build-a3
+    key: upload-ready-pipeline
+    agents:
+      queue: "cpu_queue_premerge"
+    command: |
+      python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/npu/test-npu-ready.yml
+
+  # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
+  - label: "Upload Nightly Pipeline"
+    depends_on:
+      - image-build-a2
+      - image-build-a3
+    key: upload-nightly-pipeline
+    commands:
+      - python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/npu/test-npu-nightly.yml
+    agents:
+      queue: "cpu_queue_premerge"

.buildkite/npu/pipeline-npu.yml

@@ -1,195 +1,9 @@
+# Hook uploads this entry YAML; upload_pipeline.py --upload reads bootstrap-upload-steps.yml
+# and injects ``if`` by step ``key`` from skip-ci.
 steps:
-  - label: ":buildkit: Build and Push NPU Test Image - A2"
-    key: image-build
-    agents:
-      queue: "ascend-a2b3"
-      resource_class: "npu-1"
-    plugins:
-      - kubernetes:
-          metadata:
-            annotations:
-              vault.hashicorp.com/agent-init-first: "true"
-              vault.hashicorp.com/agent-inject: "true"
-              vault.hashicorp.com/agent-inject-perms-ca.pem: "0400"
-              vault.hashicorp.com/agent-inject-perms-cert.pem: "0400"
-              vault.hashicorp.com/agent-inject-perms-config.json: "0400"
-              vault.hashicorp.com/agent-inject-perms-key.pem: "0400"
-              vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}"
-              vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}"
-              vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}"
-              vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}"
-              vault.hashicorp.com/agent-pre-populate-only: "true"
-              vault.hashicorp.com/agent-run-as-group: "1000"
-              vault.hashicorp.com/agent-run-as-user: "1000"
-              vault.hashicorp.com/agent-service-account-token-volume-name: token-vol
-              vault.hashicorp.com/role: ascend-gha-runners
-              vault.hashicorp.com/secret-volume-path: /home/user/.docker/
-              vault.hashicorp.com/tls-skip-verify: "true"
-          podSpecPatch:
-            volumes:
-            - name: token-vol
-              projected:
-                defaultMode: 420
-                sources:
-                - serviceAccountToken:
-                    audience: api
-                    expirationSeconds: 600
-                    path: token
-    env:
-      VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
-      IMAGE_NAME: "vllm-omni-ci-npu"
-      IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
-      BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
-    command: |
-      set -ex
-
-      echo "--- Building and pushing NPU Test Image"
-      echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
-      echo "buildkitd address: $${BUILDKITD_ADDR}"
-
-      if ! command -v buildctl &> /dev/null; then
-        echo "Installing buildctl..."
-        mkdir -p /tmp/buildkit
-        BUILDKIT_VERSION="v0.29.0"
-        wget -q "https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz
-        tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
-        cp /tmp/buildkit/bin/buildctl /usr/local/bin/
-      fi
-
-      export DOCKER_CONFIG=/home/user/.docker
-      buildctl \
-        --addr="$${BUILDKITD_ADDR}" \
-        --tlscacert=/home/user/.docker/ca.pem \
-        --tlscert=/home/user/.docker/cert.pem \
-        --tlskey=/home/user/.docker/key.pem \
-        build \
-        --frontend dockerfile.v0 \
-        --local context=. \
-        --local dockerfile=./docker \
-        --opt filename=Dockerfile.npu.ci \
-        --secret id=dockerconfig,src=/home/user/.docker/config.json \
-        --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \
-        --progress=plain
-
-      echo "--- Image pushed successfully"
-      echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
-
-  - label: ":buildkit: Build and Push NPU Test Image - A3"
-    key: image-build-a3
-    agents:
-      queue: "ascend-a2b3"
-      resource_class: "npu-1"
-    plugins:
-      - kubernetes:
-          metadata:
-            annotations:
-              vault.hashicorp.com/agent-init-first: "true"
-              vault.hashicorp.com/agent-inject: "true"
-              vault.hashicorp.com/agent-inject-perms-ca.pem: "0400"
-              vault.hashicorp.com/agent-inject-perms-cert.pem: "0400"
-              vault.hashicorp.com/agent-inject-perms-config.json: "0400"
-              vault.hashicorp.com/agent-inject-perms-key.pem: "0400"
-              vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}"
-              vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}"
-              vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}"
-              vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}"
-              vault.hashicorp.com/agent-pre-populate-only: "true"
-              vault.hashicorp.com/agent-run-as-group: "1000"
-              vault.hashicorp.com/agent-run-as-user: "1000"
-              vault.hashicorp.com/agent-service-account-token-volume-name: token-vol
-              vault.hashicorp.com/role: ascend-gha-runners
-              vault.hashicorp.com/secret-volume-path: /home/user/.docker/
-              vault.hashicorp.com/tls-skip-verify: "true"
-          podSpecPatch:
-            volumes:
-            - name: token-vol
-              projected:
-                defaultMode: 420
-                sources:
-                - serviceAccountToken:
-                    audience: api
-                    expirationSeconds: 600
-                    path: token
-    env:
-      VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}-a3"
-      IMAGE_NAME: "vllm-omni-ci-npu"
-      IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
-      BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
-    command: |
-      set -ex
-
-      echo "--- Building and pushing NPU Test Image"
-      echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
-      echo "buildkitd address: $${BUILDKITD_ADDR}"
-
-      if ! command -v buildctl &> /dev/null; then
-        echo "Installing buildctl..."
-        mkdir -p /tmp/buildkit
-        BUILDKIT_VERSION="v0.29.0"
-        wget -q "https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz
-        tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
-        cp /tmp/buildkit/bin/buildctl /usr/local/bin/
-      fi
-
-      export DOCKER_CONFIG=/home/user/.docker
-      buildctl \
-        --addr="$${BUILDKITD_ADDR}" \
-        --tlscacert=/home/user/.docker/ca.pem \
-        --tlscert=/home/user/.docker/cert.pem \
-        --tlskey=/home/user/.docker/key.pem \
-        build \
-        --frontend dockerfile.v0 \
-        --local context=. \
-        --local dockerfile=./docker \
-        --opt filename=Dockerfile.npu.ci.a3 \
-        --secret id=dockerconfig,src=/home/user/.docker/config.json \
-        --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \
-        --progress=plain
-
-      echo "--- Image pushed successfully"
-      echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
-
-  - label: "Upload Ready Pipeline"
-    depends_on:
-      - image-build
-      - image-build-a3
-    key: upload-ready-pipeline
-    if: >-
-      build.branch != "main" && (
-        build.pull_request.labels includes "npu-test" ||
-        build.pull_request.labels includes "ready"
-      )
-    agents:
-      queue: "ascend-a2b3"
-      resource_class: "npu-1"
-    command: |
-      buildkite-agent pipeline upload .buildkite/npu/test-npu-ready.yml
-
-  # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
-  - label: "Upload Nightly Pipeline"
-    depends_on:
-      - image-build
-      - image-build-a3
-    key: upload-nightly-pipeline
-    if: >-
-      (build.branch == "main" && build.env("NIGHTLY") == "1") ||
-      (build.branch != "main" && (
-        build.pull_request.labels includes "nightly-test" ||
-        build.pull_request.labels includes "omni-test" ||
-        build.pull_request.labels includes "tts-test" ||
-        build.pull_request.labels includes "diffusion-x2iat-test" ||
-        build.pull_request.labels includes "diffusion-x2v-test"
-      ))
+  - label: ":github: Resolve skip-ci & upload pipeline"
+    key: upload-ci-pipeline
     commands:
-      - buildkite-agent pipeline upload .buildkite/npu/test-npu-nightly.yml
+      - "python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/npu/bootstrap-upload-steps.yml"
     agents:
-      queue: "ascend-a2b3"
-      resource_class: "npu-1"
+      queue: "cpu_queue_premerge"

.buildkite/npu/test-npu-nightly.yml

@@ -8,15 +8,7 @@ steps:
       build.pull_request.labels includes "diffusion-x2v-test"
     steps:
       - label: ":full_moon: Diffusion X2V · Function Test"
-        agents:
-          queue: "ascend-a2b3"
-          resource_class: "npu-4"
-        image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}"
-        plugins:
-          - kubernetes:
-              podSpecPatch:
-                imagePullSecrets:
-                  - name: swr-secret
+        mirror_hardwares: a2b3_npu_4
         env:
           VLLM_WORKER_MULTIPROC_METHOD: spawn
           HF_TOKEN: "${HF_TOKEN}"
@@ -28,15 +20,7 @@ steps:
     if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test"
     steps:
       - label: ":full_moon: TTS · Perf Test"
-        agents:
-          queue: "ascend-a3"
-          resource_class: "npu-2"
-        image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3"
-        plugins:
-          - kubernetes:
-              podSpecPatch:
-                imagePullSecrets:
-                  - name: swr-secret
+        mirror_hardwares: a3_npu_2
         env:
           VLLM_WORKER_MULTIPROC_METHOD: spawn
           VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"

.buildkite/npu/test-npu-ready.yml

@@ -4,15 +4,7 @@ steps:
     depends_on: upload-ready-pipeline
     steps:
       - label: ":sound: TTS · Qwen3-TTS CustomVoice Test"
-        agents:
-          queue: "ascend-a2b3"
-          resource_class: "npu-1"
-        image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}"
-        plugins:
-          - kubernetes:
-              podSpecPatch:
-                imagePullSecrets:
-                  - name: swr-secret
+        mirror_hardwares: a2b3_npu_1
         env:
           VLLM_WORKER_MULTIPROC_METHOD: spawn
           VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"

.claude/skills/add-diffusion-model/SKILL.md

@@ -317,7 +317,7 @@ Required updates:
 
 ### Step 8: Add E2E Tests
 
-**Follow the [vllm-omni-test skill](../vllm-omni-test/SKILL.md)** for markers, file naming, Buildkite wiring, and run commands. Also read [l4_functionality_tests.inc.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/test_examples/l4_functionality_tests.inc.md) and [CI_5levels.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/CI_5levels.md).
+**Follow the [vllm-omni-test skill](../vllm-omni-test/SKILL.md)** for markers, file naming, Buildkite wiring, and run commands. Also read [l4_functionality_tests.inc.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/test_examples/l4_functionality_tests.inc.md), [test_system_overview.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/test_system_overview.md), and [test_writing_guide.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/test_writing_guide.md).
 
 Classify the model's **CI priority** first:
 

.claude/skills/add-tts-model/SKILL.md

@@ -307,7 +307,7 @@ def build_voice_clone_prompt(ref_audio_path: str, text: str, codec) -> list:
 
 ### Test Case Writing (CI Levels)
 
-**Follow the [vllm-omni-test skill](../vllm-omni-test/SKILL.md)** for markers, file naming (`test_{slug}.py` / `test_{slug}_expansion.py`), Buildkite wiring, and copy-paste run commands. Also read [CI_5levels.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/CI_5levels.md) and [tests_style.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/tests_style.md).
+**Follow the [vllm-omni-test skill](../vllm-omni-test/SKILL.md)** for markers, file naming (`test_{slug}.py` / `test_{slug}_expansion.py`), Buildkite wiring, and copy-paste run commands. Also read [test_system_overview.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/test_system_overview.md) and [test_writing_guide.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/test_writing_guide.md).
 
 Classify the model's **CI priority** first (high / medium / low). High-priority TTS models are typically those on the integration hot path or listed in tracking issues such as [#1832](https://github.com/vllm-project/vllm-omni/issues/1832); medium and low tiers cover the long tail. When unsure, ask the reviewer which tier applies.
 

.claude/skills/vllm-omni-test/SKILL.md

@@ -9,7 +9,7 @@ description: Generate and run tests for vllm-project/vllm-omni with CI-aligned l
 
 Use this skill to generate minimal, stable test cases and run them with the correct marker/level strategy for [vllm-project/vllm-omni](https://github.com/vllm-project/vllm-omni).
 
-**Link convention:** Paths such as `.buildkite/` and `docs/contributing/` live at the **vllm-omni repo root**. Markdown links use repo-relative paths from this skill file (e.g. `../../../.buildkite/cuda/test-ready.yml`, `../../../docs/contributing/ci/CI_5levels.md`).
+**Link convention:** Paths such as `.buildkite/` and `docs/contributing/` live at the **vllm-omni repo root**. Markdown links use repo-relative paths from this skill file (e.g. `../../../.buildkite/cuda/test-ready.yml`, `../../../docs/contributing/ci/test_system_overview.md`).
 
 Default priorities:
 
@@ -559,7 +559,7 @@ L1 tests under `tests/entrypoints/` may keep **minimal** asserts next to the han
 
 #### Omni Test Writing Guidance (L1-L4 Layering)
 
-When the goal is a general Omni/multimodal test case, prioritize mapping the test to the correct purpose, directory, and resource assumptions aligned with the layers (see [CI_5levels.md](../../../docs/contributing/ci/CI_5levels.md)):
+When the goal is a general Omni/multimodal test case, prioritize mapping the test to the correct purpose, directory, and resource assumptions aligned with the layers (see [test_system_overview.md](../../../docs/contributing/ci/test_system_overview.md) and [test_writing_guide.md](../../../docs/contributing/ci/test_writing_guide.md)):
 
 - **L1**: Unit/logic validation on CPU (`core_model and cpu`). Cover input validation, branches, and exception paths (`tests/<component>/test_*.py`). **Mock with `mocker` / `monkeypatch` only — not `unittest.mock`.**
 - **L2**: Basic e2e (online/offline basic scenarios). Prefer dummy/lightweight models to validate the end-to-end request-to-output-structure/streaming chain (typically `tests/e2e/online_serving/` and `tests/e2e/offline_inference/`).

.claude/skills/vllm-omni-test/references/test-routing.md

@@ -2,7 +2,7 @@
 
 Use this reference to map testing goals to levels, markers, and runnable commands.
 
-**Repo paths** (`.buildkite/`, `docs/contributing/…`): link with repo-relative paths from this file (e.g. `../../../../.buildkite/cuda/test-ready.yml`, `../../../../docs/contributing/ci/CI_5levels.md`).
+**Repo paths** (`.buildkite/`, `docs/contributing/…`): link with repo-relative paths from this file (e.g. `../../../../.buildkite/cuda/test-ready.yml`, `../../../../docs/contributing/ci/test_system_overview.md`).
 
 ## Model-centric e2e filename convention (L2–L4)
 

docs/contributing/README.md

@@ -71,7 +71,7 @@ For additional features and advanced configurations, refer to the:
 ### Testing
 
 vLLM-Omni uses `pytest` to test the codebase.
-Please refer to the [test instructions](./ci/test_guide.md) for detailed testing information.
+Please refer to the [test instructions](./ci/test_execution_guide.md) for detailed testing information.
 
 !!! warning
     Currently, not all unit tests pass when run on CPU platforms. If you don't have access to a GPU platform to run unit tests locally, rely on the continuous integration system to run the tests for now.
@@ -127,11 +127,11 @@ The skill offers two modes:
 The precheck covers five PR types: Bug Fix, Performance, New Model, Diffusion Model, and General. Each type has a tailored checklist that validates evidence quality (repro steps, A/B benchmarks, registry entries, etc.). See the [precheck-pr skill](https://github.com/vllm-project/vllm-omni/blob/main/.claude/skills/precheck-pr/SKILL.md) for the full checklist.
 
 ### Local Test
-Please run the L1 and L2 test cases locally first and attach the results before contacting us to add the "ready" label. Please refer to the [test instructions](./ci/test_guide.md) for running the test cases.
+Please run the L1 and L2 test cases locally first and attach the results before contacting us to add the "ready" label. Please refer to the [test instructions](./ci/test_execution_guide.md) for running the test cases.
 
 ### Automatic skip-ci (docs and pytest skip marks)
 
-On pull requests and `main` pushes, the bootstrap step in [`.buildkite/cuda/pipeline.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/pipeline.yml) runs [`.buildkite/cuda/scripts/upload_pipeline.py`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/scripts/upload_pipeline.py) against the git diff. When every changed file qualifies, **L2 (`ready`) and L3 (`merge-test`) pipelines are not uploaded**, so the default GPU CI jobs are skipped.
+On pull requests and `main` pushes, the bootstrap step in [`.buildkite/cuda/pipeline.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/pipeline.yml) runs [`.buildkite/common/scripts/upload_pipeline.py`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/scripts/upload_pipeline.py) against the git diff. When every changed file qualifies, **L2 (`ready`) and L3 (`merge-test`) pipelines are not uploaded**, so the default GPU CI jobs are skipped.
 
 | Change per file | Examples |
 | --- | --- |
@@ -148,7 +148,7 @@ These PR shapes all trigger skip-ci:
 Skip-ci does **not** apply when the diff also touches product code (for example `vllm_omni/`), or when test files change assertions, imports, fixtures, or other non-skip logic. If the diff cannot be resolved (non-PR branches outside `main`), CI runs as usual.
 
 !!! note
-    Skipping L2/L3 does **not** disable the Docker image build step. Nightly (L4) upload can still run when the PR has a `nightly-test` label or on scheduled `main` builds with `NIGHTLY=1`. See [CI levels](./ci/CI_5levels.md) for how bootstrap skip-ci relates to diff-gated E2E jobs.
+    Skipping L2/L3 does **not** disable the Docker image build step. Nightly (L4) upload can still run when the PR has a `nightly-test` label or on scheduled `main` builds with `NIGHTLY=1`. Bootstrap child steps live in `bootstrap-upload-steps.yml`; the hook entry step runs `upload_pipeline.py --upload <platform>/bootstrap-upload-steps.yml`, which injects `if` by step `key` from skip-ci before upload. See [CI Settings — Diff-aware CI](./ci/ci_settings.md#diff-aware-ci) and [Test System Overview](./ci/test_system_overview.md).
 
 ### Code Quality
 

docs/contributing/ci/.nav.yaml

@@ -1,6 +1,6 @@
 nav:
-  - CI_5levels.md
+  - test_system_overview.md
+  - ci_settings.md
+  - test_writing_guide.md
   - failures.md
-  - test_guide.md
-  - test_markers.md
-  - test_style.md
+  - test_execution_guide.md

docs/contributing/ci/ci_settings.md

@@ -0,0 +1,344 @@
+# CI Settings
+
+This document describes **where** Buildkite YAML lives in the repo, **how each platform organizes CI**, and **how to add a new job**. It does not document agent queues, GPU types, or container plugin details—those belong in infra / preset files (for example `.buildkite/common/ci_mirror_hardwares.yml`).
+
+For CI levels (L1–L5) and triggers, see [Test System Overview](./test_system_overview.md). For test authoring, see [Test Writing Guide](./test_writing_guide.md). For running tests locally or replaying CI jobs, see [Test Execution Guide](./test_execution_guide.md).
+
+## Directory layout
+
+Canonical layout (prefer these paths for new changes):
+
+```
+.buildkite/
+├── common/                          # Shared across platforms
+│   ├── scripts/
+│   │   ├── skip_ci.py               # skip-ci decision (docs / skip-mark / CI YAML paths)
+│   │   ├── upload_pipeline.py       # Bootstrap + test-pipeline uploader (CUDA/NPU)
+│   │   └── resolve_skip_ci.sh       # Shell helpers for AMD/Intel bootstrap
+│   └── ci_mirror_hardwares.yml      # CUDA uploader presets (referenced by name only)
+├── cuda/                            # Primary NVIDIA CUDA CI
+│   ├── pipeline.yml                 # Bootstrap entry (hook upload)
+│   ├── bootstrap-upload-steps.yml   # Bootstrap child steps (upload_pipeline --upload)
+│   ├── test-ready.yml               # L2
+│   ├── test-merge.yml               # L3
+│   ├── test-nightly.yml             # L4
+│   ├── test-weekly.yml              # L5
+│   └── rebase-pipeline.yml
+├── npu/
+│   ├── pipeline-npu.yml             # Bootstrap entry (hook upload)
+│   ├── bootstrap-upload-steps.yml   # Bootstrap child steps (upload_pipeline --upload)
+│   ├── pipeline-npu-a3.yml          # A3 variant (when used)
+│   ├── test-npu-ready.yml           # L2
+│   ├── test-npu-nightly.yml         # L4
+│   └── scripts/
+├── amd/
+│   ├── test-amd-ready.yml           # L2 job definitions (template input)
+│   ├── test-amd-merge.yml           # L3 job definitions
+│   ├── test-template-amd-omni.j2    # Renders final pipeline.yaml
+│   └── scripts/
+│       ├── bootstrap-amd-omni.sh    # Entry: skip-ci → Jinja → upload
+│       └── run-amd-test.sh          # Wraps pytest inside ROCm docker
+├── intel/
+│   ├── pipeline-intel.yml           # Static Intel XPU pipeline
+│   └── scripts/
+│       ├── bootstrap-intel-omni.sh
+│       └── run-xpu-test.sh
+└── release/
+    ├── release-pipeline.yml
+    └── scripts/
+```
+
+**Placement rules**
+
+| Rule | Detail |
+| ---- | ------ |
+| Platform code under platform dir | New CUDA jobs go in `.buildkite/cuda/`; do not add new top-level `.buildkite/test-*.yml` files. |
+| Shared logic in `common/` | Skip-ci and CUDA upload rendering stay in `.buildkite/common/scripts/`. |
+| Bootstrap vs test YAML | **Bootstrap** (`pipeline*.yml`) builds images and uploads **child** test pipelines. **Test** YAML (`test-*.yml`) lists pytest steps only. |
+| Register CI YAML in `skip_ci.py` | If you add a new whitelisted test pipeline file, update `L2_YAML_FILES`, `L3_YAML_FILES`, or `L45_YAML_FILES` in `.buildkite/common/scripts/skip_ci.py` so skip-ci paths stay correct. |
+
+There are still **legacy copies** at `.buildkite/*.yaml` (without the `cuda/` prefix). Treat `.buildkite/cuda/*` as source of truth.
+
+## Platform comparison
+
+| Platform | Bootstrap entry | Test job files | Upload mechanism | Job hardware in YAML |
+| -------- | ---------------- | -------------- | ---------------- | -------------------- |
+| **CUDA** | `cuda/pipeline.yml` | `test-ready.yml`, `test-merge.yml`, `test-nightly.yml`, `test-weekly.yml` | `upload_pipeline.py --upload` (expands uploader-only keys) | `mirror_hardwares: <preset>` (string) |
+| **NPU** | `npu/pipeline-npu.yml` | `test-npu-ready.yml`, `test-npu-nightly.yml` | `upload_pipeline.py --upload` | `mirror_hardwares: a2b3_npu_1` / `a2b3_npu_4` / `a3_npu_2` |
+| **AMD** | `amd/scripts/bootstrap-amd-omni.sh` | `test-amd-ready.yml`, `test-amd-merge.yml` | Jinja (`test-template-amd-omni.j2`) → `pipeline upload` | `agent_pool` + `mirror_hardwares: [amdproduction]` (array, template filter) |
+| **Intel** | `intel/scripts/bootstrap-intel-omni.sh` | `intel/pipeline-intel.yml` (steps inline) | Direct `pipeline upload` | Inline `agents.queue` on each step |
+
+## Platform configuration style
+
+=== "CUDA"
+
+    **Bootstrap:** [`cuda/pipeline.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/pipeline.yml) (hook upload) + [`cuda/bootstrap-upload-steps.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/bootstrap-upload-steps.yml) (child steps). Document 1 runs [`upload_pipeline.py --upload .buildkite/cuda/bootstrap-upload-steps.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/scripts/upload_pipeline.py), which injects `if` by step `key` from skip-ci and uploads image build plus L2–L5 child pipeline upload steps (see [Diff-aware CI — Bootstrap skip](#bootstrap-skip)).
+
+    **Test YAML:** `cuda/test-ready.yml` (L2), `test-merge.yml` (L3), `test-nightly.yml` (L4), `test-weekly.yml` (L5). Each file starts with shared `env:` then `steps:`.
+
+    | CI level | File | Typical trigger |
+    | -------- | ---- | ----------------- |
+    | L2 | `cuda/test-ready.yml` | `ready` label |
+    | L3 | `cuda/test-merge.yml` | `merge-test` label / main merge |
+    | L4 | `cuda/test-nightly.yml` | `nightly-test` label or `NIGHTLY=1` |
+    | L5 | `cuda/test-weekly.yml` | `weekly-test` label or `WEEKLY=1` |
+
+    **Upload:** `upload_pipeline.py --upload` expands uploader-only keys before Buildkite upload.
+
+    **Hardware in YAML:** `mirror_hardwares: <preset>` (string)—preset names in [`common/ci_mirror_hardwares.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/ci_mirror_hardwares.yml). Do **not** set `agents` / `plugins` on the same step.
+
+    **Conventions**
+
+    - **`depends_on`:** leaf jobs depend on `upload-ready-pipeline`, `upload-merge-pipeline`, etc.
+    - **`group` / `label`:** `:card_index_dividers:` groups; labels like `Diffusion · Qwen Image Test`.
+    - **`commands`:** `timeout … pytest …` with markers and `--run-level` for the pipeline level.
+    - **`source_file_dependencies`:** required on **E2E Test** leaf jobs in L2/L3; see [Step filtering](#step-filtering).
+
+    **Adding a job**
+
+    1. Pick the level file (ready / merge / nightly / weekly).
+    2. Add a step under the right **group** (usually **E2E Test** for model pytest).
+    3. Set `label`, `commands`, `mirror_hardwares`, `depends_on: upload-<level>-pipeline`.
+    4. For L2/L3 E2E, add `source_file_dependencies` (pytest + model + deploy YAML prefixes).
+    5. Dry-run:
+
+    ```bash
+    python3 .buildkite/common/scripts/upload_pipeline.py .buildkite/cuda/test-ready.yml
+    ```
+
+=== "NPU"
+
+    **Bootstrap:** [`npu/pipeline-npu.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/npu/pipeline-npu.yml) + [`npu/bootstrap-upload-steps.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/npu/bootstrap-upload-steps.yml)—same split as CUDA; builds A2/B3 and A3 CI images, then uploads child test pipelines.
+
+    **Test YAML:** `npu/test-npu-ready.yml` (L2), `test-npu-nightly.yml` (L4).
+
+    **Upload:** same as CUDA—`upload_pipeline.py --upload`.
+
+    **Hardware in YAML:** `mirror_hardwares` preset (string), expanded to `agents`, top-level `image`, and `plugins`. Presets: `a2b3_npu_1`, `a2b3_npu_4`, `a3_npu_2` in `common/ci_mirror_hardwares.yml`.
+
+    **Conventions**
+
+    - **`depends_on: upload-ready-pipeline`** (or `upload-nightly-pipeline`) ties jobs to bootstrap upload keys.
+    - Do not duplicate `agents` / `image` / `plugins` when using `mirror_hardwares`.
+
+    **Adding a job**
+
+    1. Edit `test-npu-ready.yml` (L2) or `test-npu-nightly.yml` (L4).
+    2. Add a step with `mirror_hardwares` (add a new preset in `ci_mirror_hardwares.yml` first if needed).
+    3. Set `commands` to your pytest file and markers.
+    4. Dry-run:
+
+    ```bash
+    python3 .buildkite/common/scripts/upload_pipeline.py .buildkite/npu/test-npu-ready.yml
+    ```
+
+    5. Register new pipeline paths in `skip_ci.py` (`L2_YAML_FILES` or `L45_YAML_FILES`) when applicable.
+
+=== "AMD"
+
+    **Bootstrap:** [`amd/scripts/bootstrap-amd-omni.sh`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/amd/scripts/bootstrap-amd-omni.sh)—skip-ci, diff filtering, Jinja render, then `buildkite-agent pipeline upload`.
+
+    **Test YAML (data):** `amd/test-amd-ready.yml` (L2 / PR), `test-amd-merge.yml` (L3 / main).
+
+    **Rendering:** [`test-template-amd-omni.j2`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/amd/test-template-amd-omni.j2) wraps data steps with `amd-build` image build and `amd_<agent_pool>` queues. Do **not** hand-edit generated `pipeline.yaml`.
+
+    **Hardware in YAML:** `agent_pool` (for example `mi325_1`) plus `mirror_hardwares: [amdproduction]` (array—Buildkite template filter, not the CUDA/NPU uploader preset mechanism).
+
+    **Data file fields**
+
+    | Field | Purpose |
+    | ----- | ------- |
+    | `agent_pool` | ROCm pool; template maps to `queue: amd_<pool>`. |
+    | `mirror_hardwares` | Which mirror HW runs the step (for example `[amdproduction]`). |
+    | `commands` | Passed into `run-amd-test.sh` via `TEST_COMMAND`. |
+
+    **Adding a job**
+
+    1. Edit `test-amd-ready.yml` or `test-amd-merge.yml`.
+    2. Copy a neighboring block: `label`, `agent_pool`, `mirror_hardwares`, `commands`, optional `grade`.
+    3. Regenerate via bootstrap / Jinja; update `skip_ci.py` if you add a new YAML path.
+
+=== "Intel"
+
+    **Bootstrap:** [`intel/scripts/bootstrap-intel-omni.sh`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/intel/scripts/bootstrap-intel-omni.sh)—skip-ci, then direct `pipeline upload`.
+
+    **Test YAML:** steps live inline in [`intel/pipeline-intel.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/intel/pipeline-intel.yml); runners under `intel/scripts/` (for example `run-xpu-test.sh`).
+
+    **Upload:** `buildkite-agent pipeline upload` (no `upload_pipeline.py` mirror expansion).
+
+    **Hardware in YAML:** inline `agents.queue` (for example `intel-gpu-omni`) on each step.
+
+    **Adding a job**
+
+    1. Add a step to `pipeline-intel.yml`, or extend an existing runner script.
+    2. Match `agents`, `env`, `timeout_in_minutes`, and `command`/`commands` with sibling steps.
+    3. Keep `pipeline-intel.yml` listed in `L2_YAML_FILES` in `skip_ci.py`.
+
+## Cross-cutting conventions
+
+### Diff-aware CI {#diff-aware-ci}
+
+PR diffs drive **two independent skip layers**. Both read changed files from git, but at different pipeline stages and with different granularity:
+
+| Layer | Script | When | What is skipped | Where you configure |
+| ----- | ------ | ---- | --------------- | ------------------- |
+| **Bootstrap** | [`skip_ci.py`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/scripts/skip_ci.py) + [`upload_pipeline.py`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/scripts/upload_pipeline.py) | Before child test pipelines upload (`cuda/pipeline.yml`, `npu/pipeline-npu.yml`, AMD/Intel bootstraps) | Entire default CI, or whole L2/L3 upload for a platform | Whitelists in `skip_ci.py`; bootstrap `if` injected by step `key` in `upload_pipeline.py` |
+| **Step filter** | [`upload_pipeline.py`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/scripts/upload_pipeline.py) | While uploading CUDA L2/L3 YAML | Individual Buildkite steps inside `test-ready.yml` / `test-merge.yml` | `source_file_dependencies` on each step or group |
+
+**Changed files** (both layers):
+
+| Build context | Diff command |
+| --- | --- |
+| Pull request | `git diff --name-only origin/<base>...<BUILDKITE_COMMIT>` |
+| `main` push | `git diff --name-only <commit>^..<commit>` |
+| Local dry-run / non-PR | Diff unavailable → bootstrap and step filter both keep all steps; uploader-only keys are still stripped |
+
+Label triggers (`ready`, `merge-test`) are unchanged—diff-aware logic only reduces what runs **after** a pipeline is already scheduled.
+
+#### Bootstrap skip {#bootstrap-skip}
+
+[`skip_ci.py`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/scripts/skip_ci.py) classifies the git diff into buckets (docs / qualifying skip-mark / whitelisted L2·L3·L4/L5 YAML / other), then picks one decision path. CUDA/NPU apply that decision by injecting Buildkite `if` on `bootstrap-upload-steps.yml` step keys; AMD/Intel call `gate_bootstrap_ci <platform> <l2|l3>`.
+
+##### Decision overview
+
+| Diff shape | Path | Default L2/L3 |
+| --- | --- | --- |
+| Product code / non-whitelisted paths | normal CI | all on |
+| Docs and/or qualifying skip-mark only | `skip_all` | all off (scheduled `main` NIGHTLY/WEEKLY exceptions below) |
+| Whitelisted CI YAML only | yaml-gated (`skip_l2_l3`) | per platform/level matrix |
+| Docs/skip-mark **+** whitelisted CI YAML | yaml-gated (`skip_l2_l3`) | same as CI-YAML-only (does **not** widen to normal CI) |
+| Non-qualifying skip-mark and no CI YAML | normal CI | all on |
+| Diff unavailable | normal CI | all on |
+
+**`skip_all` exceptions (CUDA/NPU bootstrap only):** PR labels (`nightly-test`, `merge-test`, `npu-test`, `weekly-test`, …) do **not** revive jobs. On `main`, scheduled `NIGHTLY=1` still builds the image and uploads L4 plus L2/L3 (with `--e2e`); `WEEKLY=1` still builds the image and uploads L5 (CUDA).
+
+**Yaml-gated nightly/weekly:** L4/L5 upload steps keep their normal label / `NIGHTLY` / `WEEKLY` conditions (for example PR `nightly-test` still uploads nightly). Only L2/L3 upload steps are matrix-gated.
+
+##### Whitelisted CI YAML → platform
+
+Register new files in `L2_YAML_FILES`, `L3_YAML_FILES`, or `L45_YAML_FILES` in `skip_ci.py`.
+
+| Level | File | Platform |
+| --- | --- | --- |
+| L2 | `.buildkite/cuda/test-ready.yml` | cuda |
+| L2 | `.buildkite/npu/test-npu-ready.yml` | npu |
+| L2 | `.buildkite/amd/test-amd-ready.yml` | amd |
+| L2 | `.buildkite/intel/pipeline-intel.yml` | intel |
+| L3 | `.buildkite/cuda/test-merge.yml` | cuda |
+| L3 | `.buildkite/amd/test-amd-merge.yml` | amd |
+| L4/L5 | `.buildkite/cuda/test-nightly.yml`, `test-weekly.yml` | cuda |
+| L4/L5 | `.buildkite/npu/test-npu-nightly.yml` | npu |
+
+##### Yaml-gated category branches
+
+`L2?` / `L3?` / `L45?` = whether any file in that whitelist changed. Untouched platforms/levels end up off.
+
+| ID | L2? | L3? | L45? | Enabled L2/L3 |
+| --- | --- | --- | --- | --- |
+| G1 | | | ✓ | none (all L2/L3 off) |
+| G2 | ✓ | | | L2 only on platforms whose L2 YAML changed |
+| G3 | | ✓ | | L3 only on platforms whose L3 YAML changed |
+| G4 | ✓ | ✓ | | union of G2 and G3 (same platform ready+merge → both on) |
+| G5 | ✓ | | ✓ | same as G2 (L45 rescued by L2 YAML) |
+| G6 | | ✓ | ✓ | same as G3 (L45 rescued by L3 YAML) |
+| G7 | ✓ | ✓ | ✓ | same as G4 |
+
+##### Yaml-gated examples (platform-level)
+
+Docs/skip-mark mixed with any row below follows the same matrix.
+
+| Diff (whitelist) | Branch | Enabled L2/L3 |
+| --- | --- | --- |
+| Only `cuda/test-ready.yml` | G2 | `cuda/l2` |
+| Only `amd/test-amd-ready.yml` | G2 | `amd/l2` |
+| Only `npu/test-npu-ready.yml` | G2 | `npu/l2` |
+| Only `intel/pipeline-intel.yml` | G2 | `intel/l2` |
+| Only `cuda/test-merge.yml` | G3 | `cuda/l3` |
+| Only `amd/test-amd-merge.yml` | G3 | `amd/l3` |
+| CUDA ready + CUDA merge | G4 | `cuda/l2` + `cuda/l3` |
+| CUDA ready + AMD merge | G4 | `cuda/l2` + `amd/l3` |
+| CUDA ready + AMD ready | G2 | `cuda/l2` + `amd/l2` |
+| CUDA merge + AMD merge | G3 | `cuda/l3` + `amd/l3` |
+| Only nightly / weekly / npu-nightly YAML | G1 | (none) |
+| CUDA ready + nightly | G5 | `cuda/l2` |
+| CUDA merge + nightly | G6 | `cuda/l3` |
+| CUDA ready + merge + nightly | G7 | `cuda/l2` + `cuda/l3` |
+
+##### How platforms consume the decision
+
+| Platform | Mechanism |
+| --- | --- |
+| **CUDA** | `upload_pipeline.py` injects `if` by step `key` (`image-build`, `upload-ready-pipeline`, `upload-merge-pipeline`, …) from `is_run(cuda, l2/l3)` |
+| **NPU** | Same injection; no merge upload step (L3 always off in bootstrap) |
+| **AMD / Intel** | `gate_bootstrap_ci <platform> <l2\|l3>` exits 0 when `skip_all` or that target is off |
+
+Unit coverage: `tests/buildkite/test_skip_ci.py`.
+
+#### Step filtering {#step-filtering}
+
+CUDA **L2** (`.buildkite/cuda/test-ready.yml`) and **L3** (`.buildkite/cuda/test-merge.yml`) only. Bootstrap upload entry: `upload_pipeline.py --upload .buildkite/cuda/bootstrap-upload-steps.yml`.
+
+**Uploader-only keys** — removed before Buildkite sees the YAML; never used at runtime on agents:
+
+| Key | Purpose |
+| --- | ------- |
+| `source_file_dependencies` | List of path **prefixes**. If any changed file equals a prefix or starts with `prefix/`, keep the step (or group); otherwise omit it. |
+| `mirror_hardwares` | Expand to `agents` + `plugins` (+ optional `image`) from `ci_mirror_hardwares.yml`. |
+
+**Policy**
+
+- **Always uploaded** (no key): groups outside **E2E Test**—Simple Test, Diffusion unit tests, Engine/Model Executor, Distributed, Custom Pipeline, Entrypoints (L2), LoRA / Entrypoints (L3).
+- **Diff-gated**: every **E2E Test** leaf job. List the smallest prefix set per step—pytest file(s), model code under `vllm_omni/model_executor/models/` or `vllm_omni/diffusion/models/`, plus `stage_input_processors/` and `vllm_omni/deploy/*.yaml` when applicable. A **group** may define the key instead; the whole group drops if no prefix matches.
+
+**YAML examples**
+
+```yaml
+      - label: "Diffusion · Qwen Image Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen_image.py
+          - vllm_omni/diffusion/models/qwen_image/
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model' ...
+        mirror_hardwares: h100_1
+
+      - label: "TTS · Qwen3-TTS CustomVoice Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen3_tts_customvoice.py
+          - vllm_omni/model_executor/models/qwen3_tts/
+          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
+          - vllm_omni/deploy/qwen3_tts.yaml
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py ...
+        mirror_hardwares: l4_4
+```
+
+**Local dry-run**
+
+```bash
+python3 .buildkite/common/scripts/upload_pipeline.py .buildkite/cuda/test-ready.yml
+python3 .buildkite/common/scripts/upload_pipeline.py .buildkite/cuda/test-merge.yml | grep source_file_dependencies
+# (no output expected)
+```
+
+On PR builds, `upload_pipeline.py` logs `skip '…' (no changes under …)` for omitted steps.
+
+### Labels and grouping
+
+- Use **groups** for dashboard readability; keep **E2E Test** as the group name CUDA diff filtering expects for `--e2e` nightly runs on main.
+- Prefix labels by model domain: **Omni ·**, **TTS ·**, **Diffusion ·**, **Simple ·**, etc., matching existing steps.
+
+### Validation checklist
+
+| Check | Command / location |
+| ----- | ------------------ |
+| CUDA render | `python3 .buildkite/common/scripts/upload_pipeline.py .buildkite/cuda/test-<level>.yml` |
+| No leaked uploader keys | `… \| grep -E 'mirror_hardwares|source_file_dependencies'` → empty |
+| Skip-ci / upload unit tests | `pytest tests/buildkite/` |
+| Local job replay (CUDA L2+) | `tools/run_ready_jobs.sh`, `tools/run_merge_jobs.sh`, `tools/nightly/run_nightly_jobs.sh` (read YAML from `cuda/`) |
+
+## Related documentation
+
+- [Test System Overview](./test_system_overview.md) — L1–L5 scope, triggers, test pyramid
+- [Test Writing Guide](./test_writing_guide.md) — markers, directories, L1–L5 examples
+- [Test Execution Guide](./test_execution_guide.md) — running CI-aligned jobs locally
+- Implementation: `.buildkite/common/scripts/upload_pipeline.py`, `.buildkite/common/scripts/skip_ci.py`

docs/contributing/ci/failures.md

@@ -79,7 +79,7 @@ For resilience behavior under injected process signals/OOM pressure (validation
 
 ### Running Tests Locally
 
-For instructions on running tests locally (including specific test files, functions, and markers), see the [Running Tests](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/test_guide/#running-tests) section in the Test Guide.
+For instructions on running tests locally (including specific test files, functions, and markers), see the [Running Tests](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/test_execution_guide/#running-tests) section in the Test Execution guide.
 
 ## Step 4: Raise an Issue or Fix It
 

docs/contributing/ci/test_examples/l4_functionality_tests.inc.md

@@ -37,7 +37,7 @@ Currently all the features are available in online serving mode. Hence, only nee
 **Code Style**
 
 - Validation: test that the multimodal output files of your model have the correct shapes. `OpenAIClientHandler.send_diffusion_request` should have taken care of this.
-- Test marks: always add `full_model` and `diffusion` for L4 nightly `test_*_expansion.py` cases. Add GPU-related marks if needed. Ref: [Markers for Tests](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/tests_markers/).
+- Test marks: always add `full_model` and `diffusion` for L4 nightly `test_*_expansion.py` cases. Add GPU-related marks if needed. Ref: [Markers for Tests](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/test_writing_guide/#markers-for-tests).
 - To maximize code reuse, you may refer to
     - `tests/conftest.py` for `omni_server` (running server in subprocess) and `openai_client` fixtures (sending requests and validating output), `generate_synthetic_image` and `assert_XXX_valid` helper.
     - `tests/helpers/mark.py` for `@hardware_test(...)` and `hardware_marks`.

docs/contributing/ci/test_examples/l4_performance_tests.inc.md

@@ -148,7 +148,7 @@ pytest -sv tests/dfx/perf/scripts/run_diffusion_benchmark.py --assert-baseline \
   --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
 ```
 
-See also [Markers for Tests](./tests_markers.md) for registered hardware markers (`H100`, `L4`, `cuda`, `distributed_cuda`, …).
+See also [Markers for Tests](#markers-for-tests) for registered hardware markers (`H100`, `L4`, `cuda`, `distributed_cuda`, …).
 
 **`server_params` Configuration**
 

docs/contributing/ci/test_execution_guide.md

@@ -1,4 +1,4 @@
-# Test Guide
+# Test Execution Guide
 ## Setting Up the Test Environment
 ### Creating a Container
 vLLM-Omni provides an official Docker image for deployment. These images are built upon vLLM Docker images and are available on [Docker Hub](https://hub.docker.com/r/vllm/vllm-omni/tags). The version of vLLM-Omni indicates which vLLM release it is based on.
@@ -191,7 +191,7 @@ Failed jobs: 1/2
 
     The latest L5 CI jobs (reliability + invalid-parameter weekly steps) are in [test-weekly.yml](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/cuda/test-weekly.yml).
 
-You can find more information about markers in the documentation: [marker doc](./tests_markers.md)
+You can find more information about markers in [Test Writing Guide — Markers for Tests](./test_writing_guide.md#markers-for-tests).
 
 ## Adding New Test Cases
-Please refer to the [L5 Layering Specification document](./CI_5levels.md).
+Please refer to the [Test System Overview](./test_system_overview.md) and [Test Writing Guide](./test_writing_guide.md).

docs/contributing/ci/test_system_overview.md

@@ -0,0 +1,210 @@
+# Multi-Level Automated Testing System Overview
+
+## Document Overview
+
+This testing system aims to build a complete, efficient, and well-structured quality assurance framework for the development, integration, and release of model services. It draws on the concept of the test pyramid from modern software engineering, progressively expanding testing activities from basic code logic verification to complex end-to-end (E2E) functionality, performance, accuracy, and even long-term stability validation.
+
+Through five levels (L1-L5) and common (Common) specifications, the system clarifies the testing objectives, scope, execution frequency, and required resources for different development stages (e.g., each commit, PR merge, daily build, pre-release). This ensures that models meet high standards for functionality, performance, and reliability across various deployment scenarios (online serving and offline inference).
+
+<table>
+  <thead>
+    <tr>
+      <th>Level</th>
+      <th>Scope & Focus</th>
+      <th>Model Coverage Strategy</th>
+      <th>Feature Coverage Strategy</th>
+      <th>Interface Coverage Strategy</th>
+      <th>Tags</th>
+      <th>Time Cost</th>
+      <th>Test Dir</th>
+      <th>Doc</th>
+      <th>Frequency</th>
+      <th>Hardware</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td rowspan="2"><strong>Common</strong></td>
+      <td>Contribution Guideline & PR checklist</td>
+      <td>/</td>
+      <td>/</td>
+      <td>/</td>
+      <td>/</td>
+      <td>/</td>
+      <td>/</td>
+      <td><a href="https://github.com/vllm-project/vllm-omni/blob/main/.github/PULL_REQUEST_TEMPLATE.md">PR Checklist</a></td>
+      <td>/</td>
+      <td>/</td>
+    </tr>
+    <tr>
+      <td>CI Failure Description</td>
+      <td>/</td>
+      <td>/</td>
+      <td>/</td>
+      <td>/</td>
+      <td>/</td>
+      <td>/</td>
+      <td><a href="./failures.md">CI Failures</a></td>
+      <td>/</td>
+      <td>/</td>
+    </tr>
+    <tr>
+      <td><strong>L1</strong><br>(Unit & Logic)</td>
+      <td>Unit tests for components like entrypoints, models</td>
+      <td>/</td>
+      <td>/</td>
+      <td>/</td>
+      <td><code>core_model and cpu</code></td>
+      <td rowspan="2">&lt;15min</td>
+      <td>/tests/{component_name}/test_xxx</td>
+      <td>
+        <a href="test_writing_guide.md#l1--l2-level-testing-unit-testing-and-basic-end-to-end-verification">L1 &amp; L2</a><br>
+        Section 1 L1&amp;L2: Purpose, Test Content, Directory Location, Example
+      </td>
+      <td>PR with ready label (also can run locally)</td>
+      <td>CPU</td>
+    </tr>
+    <tr>
+      <td><strong>L2</strong><br>(E2E across models & GPU-required UT)</td>
+      <td>Online (basic deployment scenarios):<br>dummy, normal inference function (output format, stream), some instance startup UT</td>
+      <td>High-priority models + online basic scenarios; request success, non-empty output, format match (no Whisper/accuracy)</td>
+      <td>High-priority features (using random lightweight models)</td>
+      <td>High-priority interfaces (using random lightweight models)</td>
+      <td><code>core_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion</code></td>
+      <td>
+        <strong>Model tests:</strong><br>
+        /tests/e2e/online_serving/test_{model_name}.py<br>
+        <strong>Feature tests:</strong><br>
+        /tests/{component_name}/test_xxx<br>
+        <strong>Interface tests:</strong><br>
+        /tests/entrypoints/test_xxx
+      </td>
+      <td>
+        <a href="test_writing_guide.md#l1--l2-level-testing-unit-testing-and-basic-end-to-end-verification">L1 &amp; L2</a><br>
+        L1&amp;L2: Purpose, Test Content, Directory Location, Example
+      </td>
+      <td>PR with ready label</td>
+      <td>GPU</td>
+    </tr>
+    <tr>
+      <td><strong>L3</strong><br>(Important Perf & Integration & Accuracy)</td>
+      <td>Online & Offline (multiple deployment scenarios):<br>real model, normal inference function, normal accuracy</td>
+      <td>High/medium-priority models + key online/offline scenarios; real weights, Whisper/similarity, preset voice gender, basic accuracy</td>
+      <td>Medium-priority features (using random lightweight models)</td>
+      <td>Medium-priority interfaces (using random lightweight models)</td>
+      <td><code>advanced_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion</code></td>
+      <td>&lt;30min</td>
+      <td>
+        <strong>Model tests:</strong><br>
+        /tests/e2e/online_serving/test_{model_name}.py<br>
+        /tests/e2e/offline_inference/test_{model_name}.py<br>
+        <strong>Feature tests:</strong><br>
+        /tests/{component_name}/test_xxx<br>
+        <strong>Interface tests:</strong><br>
+        /tests/entrypoints/test_xxx
+      </td>
+      <td>
+        <a href="test_writing_guide.md#l3-level-testing-core-integration-performance-and-accuracy-verification">L3</a><br>
+        L3: Purpose, Test Content, Directory Location, Example
+      </td>
+      <td>PR Merged (Also run L1&L2 Tests)</td>
+      <td>GPU</td>
+    </tr>
+    <tr>
+      <td><strong>L4</strong><br>(Perf & Integration & Accuracy)</td>
+      <td>Online: full functional scenarios + performance test + doc test + accuracy test</td>
+      <td>High-priority models: function, performance, accuracy, and doc testing<br>Medium-priority models: function and doc testing</td>
+      <td>Low-priority features (using real weights)</td>
+      <td>Low-priority interfaces (using real weights)</td>
+      <td><code>full_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion</code></td>
+      <td>&lt;3 hour</td>
+      <td>
+        <strong>Model tests:</strong><br>
+        /tests/e2e/online_serving/test_{model_name}_expansion.py<br>
+        <strong>Feature tests:</strong><br>
+        /tests/{component_name}/test_xxx<br>
+        <strong>Interface tests:</strong><br>
+        /tests/entrypoints/test_xxx<br>
+        <strong>Performance:</strong><br>
+        /tests/dfx/perf/tests/test_qwen3_omni_*.json (Omni), test_tts.json (TTS),<br>
+        test_voxcpm2.json, test_higgs_audio_v3.json, and<br>
+        /tests/dfx/perf/tests/test_{diffusion_model}_vllm_omni.json (Diffusion)<br>
+        <strong>Doc Test:</strong><br>
+        tests/examples/online_serving/test_{model_name}.py<br>
+        tests/examples/offline_inference/test_{model_name}.py<br>
+        <strong>Accuracy Test:</strong><br>
+        /tests/e2e/accuracy/test_{model_name}.py
+      </td>
+      <td>
+        <a href="test_writing_guide.md#l4-level-testing-full-functionality-performance-and-documentation-testing">L4</a><br>
+        L4: Purpose, Test Content, Directory Location, Example
+      </td>
+      <td>Nightly</td>
+      <td>GPU</td>
+    </tr>
+    <tr>
+      <td><strong>L5</strong><br>(Stability & Reliability)</td>
+      <td>Online: long-term stability test + reliability test</td>
+      <td>Long-term stability and reliability testing for high-priority models<br>Low-priority models: function and doc testing</td>
+      <td>/</td>
+      <td>Invalid-parameter validation for high-priority interfaces</td>
+      <td><code>slow and hardware_test(H100, L4, etc.) and omni/tts/diffusion</code></td>
+      <td> Depends on reality </td>
+      <td>
+        <strong>Stability:</strong><br>
+        /tests/dfx/stability/tests/test_qwen3_omni.json<br>
+        /tests/dfx/stability/tests/test_wan22.json<br>
+        <strong>Reliability:</strong><br>
+        tests/dfx/reliability/test_reliability_{model_key}.py<br>
+        (e.g. <code>test_reliability_qwen3_omni.py</code>, <code>test_reliability_wan22.py</code>, <code>test_reliability_hunyuan_image.py</code>, <code>test_reliability_voxcpm2.py</code>)
+      </td>
+      <td>
+        <a href="test_writing_guide.md#l5-level-testing-stability-and-reliability-testing">L5</a><br>
+        L5: Purpose, Test Content, Directory Location, Example
+      </td>
+      <td>Weekly / Days before Release</td>
+      <td>GPU</td>
+    </tr>
+  </tbody>
+</table>
+
+For per-level test authoring (directories, markers, examples), see [Test Writing Guide](./test_writing_guide.md).
+
+## Common Specifications
+
+Before entering specific testing levels, the project establishes two common specifications aimed at standardizing the development process and quickly locating issues.
+
+1.  ***PR Checklist ([`.github/PULL_REQUEST_TEMPLATE.md`](https://github.com/vllm-project/vllm-omni/blob/main/.github/PULL_REQUEST_TEMPLATE.md))***: This template defines the self-check items that must be completed before submitting a code review (Pull Request). It ensures that each code change meets basic requirements such as code style, dependency updates, and documentation synchronization before entering the automated testing pipeline, serving as the first manual line of defense for quality assurance.
+2.  ***CI Failure Explanation ([CI Failures](./failures.md))***: This document archives and explains common failure patterns in the Continuous Integration (CI) pipeline, error log interpretation, and preliminary troubleshooting steps. It helps developers and testers quickly diagnose the causes of automated test failures, improving problem-solving efficiency.
+
+## Notes
+
+### L2 / L3 diff-aware CI (CUDA)
+
+On CUDA **L2** and **L3**, **E2E Test** Buildkite jobs may be omitted at pipeline upload when the PR diff does not touch their path prefixes; other groups still always upload. Full two-layer mechanics and YAML examples: [CI Settings — Diff-aware CI](./ci_settings.md#diff-aware-ci) ([step filtering](./ci_settings.md#step-filtering)).
+
+### Test helper environment variables
+
+Some shared helpers under `tests/helpers/` honor optional environment variables for local debugging. These are **not** set in CI by default.
+
+| Variable | Accepted values | Description |
+| -------- | --------------- | ----------- |
+| `VLLM_OMNI_KEEP_REQUEST_MEDIA` | `1`, `true`, `yes` (case-insensitive) | When enabled, temporary WAV files created by `tests.helpers.media.convert_audio_bytes_to_text` are **not** deleted when the pytest process exits. By default, each call writes a unique file under the system temp directory via `tempfile.mkstemp` and registers `atexit` cleanup. Use this when debugging audio output validation (Whisper transcription, keyword checks, text–audio similarity). The saved path is logged as `audio data is saved: <path>`. |
+
+Example (Linux / macOS):
+
+```bash
+export VLLM_OMNI_KEEP_REQUEST_MEDIA=1
+pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -k test_mix_to_text_audio
+```
+
+Example (Windows PowerShell):
+
+```powershell
+$env:VLLM_OMNI_KEEP_REQUEST_MEDIA = "1"
+pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -k test_mix_to_text_audio
+```
+
+## Summary
+
+This multi-level testing system achieves continuous, progressive validation of model service quality by tightly integrating testing activities with the development workflow (commit, review, merge, release). From rapid unit testing to comprehensive end-to-end testing, and further to in-depth performance, stability, and reliability verification, each level has clear objectives, collectively building a robust quality protection net. By following this system, teams can deliver high-quality, highly reliable model services more efficiently.

docs/contributing/ci/test_writing_guide.md

@@ -1,177 +1,9 @@
-# Multi-Level Automated Testing System Documentation
+# Test Writing Guide
 
-## Document Overview
+This document describes how to author tests for each CI level (L1–L5). For the level matrix, CI triggers, and Buildkite upload behavior, see [Test System Overview](./test_system_overview.md).
 
-This testing system aims to build a complete, efficient, and well-structured quality assurance framework for the development, integration, and release of model services. It draws on the concept of the test pyramid from modern software engineering, progressively expanding testing activities from basic code logic verification to complex end-to-end (E2E) functionality, performance, accuracy, and even long-term stability validation.
+## Test Directory Structure
 
-Through five levels (L1-L5) and common (Common) specifications, the system clarifies the testing objectives, scope, execution frequency, and required resources for different development stages (e.g., each commit, PR merge, daily build, pre-release). This ensures that models meet high standards for functionality, performance, and reliability across various deployment scenarios (online serving and offline inference).
-
-<table>
-  <thead>
-    <tr>
-      <th>Level</th>
-      <th>Scope & Focus</th>
-      <th>Model Coverage Strategy</th>
-      <th>Feature Coverage Strategy</th>
-      <th>Interface Coverage Strategy</th>
-      <th>Tags</th>
-      <th>Time Cost</th>
-      <th>Test Dir</th>
-      <th>Doc</th>
-      <th>Frequency</th>
-      <th>Hardware</th>
-    </tr>
-  </thead>
-  <tbody>
-    <tr>
-      <td rowspan="2"><strong>Common</strong></td>
-      <td>Contribution Guideline & PR checklist</td>
-      <td>/</td>
-      <td>/</td>
-      <td>/</td>
-      <td>/</td>
-      <td>/</td>
-      <td>/</td>
-      <td>.github/PULL_REQUEST_TEMPLATE.md <a href="../tests_style/"> Test Style (PR Checklist)</a></td>
-      <td>/</td>
-      <td>/</td>
-    </tr>
-    <tr>
-      <td>CI Failure Description</td>
-      <td>/</td>
-      <td>/</td>
-      <td>/</td>
-      <td>/</td>
-      <td>/</td>
-      <td>/</td>
-      <td><a href="../failures/"> CI Failures</a></td>
-      <td>/</td>
-      <td>/</td>
-    </tr>
-    <tr>
-      <td><strong>L1</strong><br>(Unit & Logic)</td>
-      <td>Unit tests for components like entrypoints, models</td>
-      <td>/</td>
-      <td>/</td>
-      <td>/</td>
-      <td><code>core_model and cpu</code></td>
-      <td rowspan="2">&lt;15min</td>
-      <td>/tests/{component_name}/test_xxx</td>
-      <td>
-        <a href="#chapter-1-l1-l2-level-testing-unit-testing-and-basic-end-to-end-verification">Chapter 1</a><br>
-        Section 1 L1&amp;L2: Purpose, Test Content, Directory Location, Example
-      </td>
-      <td>PR with ready label (also can run locally)</td>
-      <td>CPU</td>
-    </tr>
-    <tr>
-      <td><strong>L2</strong><br>(E2E across models & GPU-required UT)</td>
-      <td>Online (basic deployment scenarios):<br>dummy, normal inference function (output format, stream), some instance startup UT</td>
-      <td>High-priority models + online basic scenarios; request success, non-empty output, format match (no Whisper/accuracy)</td>
-      <td>High-priority features (using random lightweight models)</td>
-      <td>High-priority interfaces (using random lightweight models)</td>
-      <td><code>core_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion</code></td>
-      <td>
-        <strong>Model tests:</strong><br>
-        /tests/e2e/online_serving/test_{model_name}.py<br>
-        <strong>Feature tests:</strong><br>
-        /tests/{component_name}/test_xxx<br>
-        <strong>Interface tests:</strong><br>
-        /tests/entrypoints/test_xxx
-      </td>
-      <td>
-        <a href="#chapter-1-l1-l2-level-testing-unit-testing-and-basic-end-to-end-verification">Chapter 1</a><br>
-        L1&amp;L2: Purpose, Test Content, Directory Location, Example
-      </td>
-      <td>PR with ready label</td>
-      <td>GPU</td>
-    </tr>
-    <tr>
-      <td><strong>L3</strong><br>(Important Perf & Integration & Accuracy)</td>
-      <td>Online & Offline (multiple deployment scenarios):<br>real model, normal inference function, normal accuracy</td>
-      <td>High/medium-priority models + key online/offline scenarios; real weights, Whisper/similarity, preset voice gender, basic accuracy</td>
-      <td>Medium-priority features (using random lightweight models)</td>
-      <td>Medium-priority interfaces (using random lightweight models)</td>
-      <td><code>advanced_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion</code></td>
-      <td>&lt;30min</td>
-      <td>
-        <strong>Model tests:</strong><br>
-        /tests/e2e/online_serving/test_{model_name}.py<br>
-        /tests/e2e/offline_inference/test_{model_name}.py<br>
-        <strong>Feature tests:</strong><br>
-        /tests/{component_name}/test_xxx<br>
-        <strong>Interface tests:</strong><br>
-        /tests/entrypoints/test_xxx
-      </td>
-      <td>
-        <a href="#chapter-2-l3-level-testing-core-integration-performance-and-accuracy-verification">Chapter 2</a><br>
-        L3: Purpose, Test Content, Directory Location, Example
-      </td>
-      <td>PR Merged (Also run L1&L2 Tests)</td>
-      <td>GPU</td>
-    </tr>
-    <tr>
-      <td><strong>L4</strong><br>(Perf & Integration & Accuracy)</td>
-      <td>Online: full functional scenarios + performance test + doc test + accuracy test</td>
-      <td>High-priority models: function, performance, accuracy, and doc testing<br>Medium-priority models: function and doc testing</td>
-      <td>Low-priority features (using real weights)</td>
-      <td>Low-priority interfaces (using real weights)</td>
-      <td><code>full_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion</code></td>
-      <td>&lt;3 hour</td>
-      <td>
-        <strong>Model tests:</strong><br>
-        /tests/e2e/online_serving/test_{model_name}_expansion.py<br>
-        <strong>Feature tests:</strong><br>
-        /tests/{component_name}/test_xxx<br>
-        <strong>Interface tests:</strong><br>
-        /tests/entrypoints/test_xxx<br>
-        <strong>Performance:</strong><br>
-        /tests/dfx/perf/tests/test_qwen3_omni_*.json (Omni), test_tts.json (TTS),<br>
-        test_voxcpm2.json, test_higgs_audio_v3.json, and<br>
-        /tests/dfx/perf/tests/test_{diffusion_model}_vllm_omni.json (Diffusion)<br>
-        <strong>Doc Test:</strong><br>
-        tests/examples/online_serving/test_{model_name}.py<br>
-        tests/examples/offline_inference/test_{model_name}.py<br>
-        <strong>Accuracy Test:</strong><br>
-        /tests/e2e/accuracy/test_{model_name}.py
-      </td>
-      <td>
-        <a href="#chapter-3-l4-level-testing-full-functionality-performance-and-documentation-testing">Chapter 3</a><br>
-        L4: Purpose, Test Content, Directory Location, Example
-      </td>
-      <td>Nightly</td>
-      <td>GPU</td>
-    </tr>
-    <tr>
-      <td><strong>L5</strong><br>(Stability & Reliability)</td>
-      <td>Online: long-term stability test + reliability test</td>
-      <td>Long-term stability and reliability testing for high-priority models<br>Low-priority models: function and doc testing</td>
-      <td>/</td>
-      <td>Invalid-parameter validation for high-priority interfaces</td>
-      <td><code>slow and hardware_test(H100, L4, etc.) and omni/tts/diffusion</code></td>
-      <td> Depends on reality </td>
-      <td>
-        <strong>Stability:</strong><br>
-        /tests/dfx/stability/tests/test_qwen3_omni.json<br>
-        /tests/dfx/stability/tests/test_wan22.json<br>
-        <strong>Reliability:</strong><br>
-        tests/dfx/reliability/test_reliability_{model_key}.py<br>
-        (e.g. <code>test_reliability_qwen3_omni.py</code>, <code>test_reliability_wan22.py</code>, <code>test_reliability_hunyuan_image.py</code>, <code>test_reliability_voxcpm2.py</code>)
-      </td>
-      <td>
-        <a href="#chapter-4-l5-level-testing-stability-and-reliability-testing">Chapter 4</a><br>
-        L5: Purpose, Test Content, Directory Location, Example
-      </td>
-      <td>Weekly / Days before Release</td>
-      <td>GPU</td>
-    </tr>
-  </tbody>
-</table>
-
-
----
-<details>
-<summary> The folder structure for tests file based on the 5 levels design</summary>
 Legend: `✅` = test exists, `⬜` = suggested to add.
 ```
 vllm_omni/                                    tests/
@@ -304,134 +136,219 @@ vllm_omni/                                    tests/
 # vllm_omni/deploy/ instead — see docs/configuration/stage_configs.md.
 ```
 
+## Markers for Tests
 
-</details>
+By adding markers before test functions, tests can later be executed uniformly by simply declaring the corresponding marker type.
 
+### Current Markers
+Defined in `pyproject.toml`:
 
-## Common Specifications
+| Marker             | Description                                               |
+| ------------------ | --------------------------------------------------------- |
+| `core_model`       | L1&L2 tests (run in each PR)                              |
+| `advanced_model`   | L3 tests (run on each merge to main)                 |
+| `full_model`       | L4 tests (run nightly) |
+| `diffusion`        | Diffusion model tests                                     |
+| `omni`             | Omni multimodal model tests |
+| `tts`              | TTS model tests     |
+| `cache`            | Cache backend tests                                       |
+| `parallel`         | Parallelism/distributed tests                             |
+| `cpu`              | Tests that run on CPU                                     |
+| `gpu`              | Tests that run on GPU *                                   |
+| `cuda`             | Tests that run on CUDA *                                  |
+| `rocm`             | Tests that run on AMD/ROCm *                              |
+| `xpu`              | Tests that run on Intel XPU *                             |
+| `npu`              | Tests that run on NPU/Ascend *                            |
+| `H100`             | Tests that require H100 GPU  *                            |
+| `L4`               | Tests that require L4 GPU *                               |
+| `MI325`            | Tests that require MI325 GPU (AMD/ROCm) *                 |
+| `A2`               | Tests that require A2 NPU *                               |
+| `A3`               | Tests that require A3 NPU *                               |
+| `distributed_cuda` | Tests that require multi cards on CUDA platform *         |
+| `distributed_rocm` | Tests that require multi cards on ROCm platform  *        |
+| `distributed_npu`  | Tests that require multi cards on NPU platform  *         |
+| `skipif_cuda`      | Skip if the num of CUDA cards is less than the required * |
+| `skipif_rocm`      | Skip if the num of ROCm cards is less than the required * |
+| `skipif_npu`       | Skip if the num of NPU cards is less than the required *  |
+| `slow`             | Slow tests (may skip in quick CI)                         |
+| `benchmark`        | Benchmark tests (decorator on runner test functions; perf JSON uses `full_model` + type marker instead) |
+| `local_model`      | Tests requiring local / non-HF-hub model weights          |
 
-Before entering specific testing levels, the project establishes two common specifications aimed at standardizing the development process and quickly locating issues.
+\* Means those markers are auto-added by `@hardware_test` (parametrization decorator) or `hardware_marks` (only returning the list of marks for flexibility).
 
-1.  ***PR Checklist ([Tests Style](../ci/tests_style.md))***: This template defines the self-check items that must be completed before submitting a code review (Pull Request). It ensures that each code change meets basic requirements such as code style, dependency updates, and documentation synchronization before entering the automated testing pipeline, serving as the first manual line of defense for quality assurance.
-2.  ***CI Failure Explanation ([CI Failures](../ci/failures.md))***: This document archives and explains common failure patterns in the Continuous Integration (CI) pipeline, error log interpretation, and preliminary troubleshooting steps. It helps developers and testers quickly diagnose the causes of automated test failures, improving problem-solving efficiency.
+#### Example usage for markers
 
-## Notes
+```python
+from tests.helpers.mark import hardware_test
 
-### Diff-aware Buildkite uploads (`source_file_dependencies`)
-
-L2 (`.buildkite/cuda/test-ready.yml`) and L3 (`.buildkite/cuda/test-merge.yml`) pipelines can **skip unrelated GPU jobs at upload time** based on the PR diff. This is implemented by `.buildkite/cuda/scripts/upload_pipeline.py`, which filters steps before calling `buildkite-agent pipeline upload`.
-
-#### What `source_file_dependencies` is
-
-- A **uploader-only** YAML key on a Buildkite step or group. **Buildkite does not understand it**; `upload_pipeline.py` always **removes** it from the YAML that is uploaded.
-- A list of path **prefixes** (directories or individual files). If **any** changed file in the diff equals a prefix or starts with `prefix/`, the step is kept; otherwise the step (or entire group) is **omitted** from the uploaded pipeline.
-
-#### When filtering runs
-
-| Build context | Changed files used for matching |
-| --- | --- |
-| Pull request | `git diff --name-only origin/<base>...<BUILDKITE_COMMIT>` |
-| `main` branch push | `git diff --name-only <commit>^..<commit>` |
-| Other (e.g. local dry-run, non-PR branch) | Diff cannot be resolved → **no filtering**; all steps are uploaded and `source_file_dependencies` is still stripped |
-
-Docs-only PRs are handled earlier in bootstrap (`.buildkite/cuda/pipeline.yml`) via skip-ci logic; `source_file_dependencies` applies only to the **uploaded** L2/L3 test pipelines.
-
-#### Where it is configured
-
-| Level | Pipeline file | Upload entry |
-| --- | --- | --- |
-| L2 | `.buildkite/cuda/test-ready.yml` | `upload_pipeline.py --upload .buildkite/cuda/test-ready.yml` (from `cuda/pipeline.yml`) |
-| L3 | `.buildkite/cuda/test-merge.yml` | `upload_pipeline.py --upload .buildkite/cuda/test-merge.yml` (from `cuda/pipeline.yml`) |
-
-Steps **without** `source_file_dependencies` are always uploaded (subject to the usual label conditions: `ready` for L2, `merge-test` for L3).
-
-#### Current skip policy (L2 / L3)
-
-To balance CI cost and coverage:
-
-- **Always run** (no `source_file_dependencies`): baseline groups outside E2E Test—e.g. Simple Test, Diffusion unit tests, Engine/Model Executor, Distributed, Custom Pipeline, Entrypoints (L2), LoRA / Entrypoints (L3).
-- **Diff-gated** (`source_file_dependencies` set): **every leaf job under the E2E Test group** in `test-ready.yml` and `test-merge.yml`, regardless of queue (`mithril-h100-pool`, `gpu_1_queue`, or `gpu_4_queue`). Each step lists the smallest set of prefixes that should trigger it—typically:
-  - pytest file(s) exercised by the job (online and/or offline);
-  - model code under `vllm_omni/model_executor/models/` or `vllm_omni/diffusion/models/`;
-  - related `vllm_omni/model_executor/stage_input_processors/` and `vllm_omni/deploy/*.yaml` when applicable.
-
-Adding a new E2E step: add `source_file_dependencies` on the leaf job with those prefixes. Prefer **per-step** deps rather than a broad group-level list unless every child shares the same paths.
-
-#### YAML examples
-
-H100 E2E (kubernetes / `mithril-h100-pool`):
-
-```yaml
-      - label: "Diffusion · Qwen Image Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_qwen_image.py
-          - vllm_omni/diffusion/models/qwen_image/
-        commands:
-          - pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model' ...
-        agents:
-          queue: "mithril-h100-pool"
+@pytest.mark.core_model
+@pytest.mark.omni
+@hardware_test(
+   res={"cuda": "L4", "rocm": "MI325", "npu": "A2"},
+   num_cards=2,
+)
+@pytest.mark.parametrize("omni_server", test_params, indirect=True)
+def test_video_to_audio()
+    ...
 ```
 
-Docker E2E (`gpu_1_queue` / `gpu_4_queue`)—same key, same upload-time filtering:
+#### Decorator: `@hardware_test`
 
-```yaml
-      - label: "TTS · Qwen3-TTS CustomVoice Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_qwen3_tts_customvoice.py
-          - vllm_omni/model_executor/models/qwen3_tts/
-          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
-          - vllm_omni/deploy/qwen3_tts.yaml
-        commands:
-          - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py ...
-        agents:
-          queue: "gpu_4_queue"
+This decorator is intended to make hardware-aware, cross-platform test authoring easier and more robust for CI/CD environments. The `hardware_test` decorator in `vllm-omni/tests/helpers/mark.py` performs the following actions:
+
+1. **Applies platform and resource markers**  
+   Adds the appropriate pytest markers for each specified hardware platform (e.g., `cuda`, `rocm`, `xpu`, `npu`) and resource type (e.g., `L4`, `H100`, `MI325`, `B60`, `A2`, `A3`).
+   ```
+   @pytest.mark.cuda
+   @pytest.mark.L4
+   ```
+2. **Handles multi-card (distributed) scenarios**  
+   For tests requiring multiple cards, it automatically adds distributed markers such as `distributed_cuda`, `distributed_rocm`, or `distributed_npu`.
+   ```
+   @pytest.mark.distributed_cuda(num_cards=num_cards)
+   ```
+3. **Supports flexible card requirements**  
+   Accepts `num_cards` as either a single integer for all platforms or as a dictionary with per-platform values. If not specified, defaults to 1 card per platform.
+
+4. **Integrates resource validation**  
+   On CUDA, adds a skip marker (`skipif_cuda`) if the system does not have the required number of devices.
+   Support for `skipif_rocm` and `skipif_npu` will be implemented later.
+
+
+5. **Works with pytest filtering**  
+   Allows tests to be filtered and selected at runtime using standard pytest marker expressions (e.g., `-m "distributed_cuda and L4"`).
+
+##### Example usage for decorator
+- Single call for multiple platforms:
+    ```python
+    @hardware_test(
+        res={"cuda": "L4", "rocm": "MI325", "xpu": "B60", "npu": "A2"},
+        num_cards={"cuda": 2, "rocm": 2, "xpu": 2, "npu": 2},
+    )
+    ```
+    or
+    ```python
+    @hardware_test(
+        res={"cuda": "L4", "rocm": "MI325", "xpu": "B60", "npu": "A2"},
+        num_cards=2,
+    )
+    ```
+- `res` must be a dict; supported resources: CUDA (L4/H100), ROCm (MI325), XPU (B60), MUSA (S5000), NPU (A2/A3)
+- `num_cards` can be int (all platforms) or dict (per platform); defaults to 1 when missing
+- Distributed markers (`distributed_cuda`, `distributed_rocm`, `distributed_npu`) are auto-added for multi-card cases
+- Filtering examples:
+    - CUDA only: `pytest -m "distributed_cuda and L4"`
+    - ROCm only: `pytest -m "distributed_rocm and MI325"`
+    - NPU only: `pytest -m "distributed_npu"`
+
+#### Function: `hardware_marks`
+
+`hardware_marks` returns a list of pytest mark objects with the same signature as `@hardware_test`. Use it when you need more flexibility, such as attaching hardware marks to individual `pytest.param` entries rather than an entire test function.
+
+```python
+from tests.helpers.mark import hardware_marks
+
+MULTI_CARD_MARKS = hardware_marks(
+    res={"cuda": "H100", "rocm": "MI325", "npu": "A2"}, num_cards=2
+)
+
+@pytest.mark.parametrize("omni_server", [
+    pytest.param(OmniServerParams(...), id="case_001", marks=MULTI_CARD_MARKS),
+], indirect=True)
+def test_feature(omni_server):
+    ...
 ```
 
-A **group** may also define `source_file_dependencies`; nested steps inherit filtering as a unit—the whole group is dropped if no prefix matches.
+#### JSON `mark` field (L4 perf configs)
 
-#### Local dry-run
+Perf JSON under `tests/dfx/perf/tests/` can attach pytest marks per **case** (`test_name` block). Parsed by `tests.dfx.conftest.resolve_pytest_marks` and applied to each parametrized run in `run_benchmark.py` / `run_diffusion_benchmark.py`.
 
-```bash
-# Render filtered YAML to stdout (no upload)
-python3 .buildkite/cuda/scripts/upload_pipeline.py .buildkite/cuda/test-ready.yml
+When `mark` is present, it must be an **array** with exactly one ``hardware_marks`` object (same semantics as `hardware_marks()` above), followed by registered pytest marker name strings such as `full_model`, `omni`, `tts`, or `diffusion`.
 
-# Confirm uploader-only keys are stripped
-python3 .buildkite/cuda/scripts/upload_pipeline.py .buildkite/cuda/test-merge.yml | grep source_file_dependencies
-# (no output expected)
+```json
+{
+  "test_name": "test_cosmos3_t2i_official_demo_2gpu",
+  "mark": [
+    {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 2}},
+    "full_model",
+    "diffusion"
+  ],
+  "server_type": "vllm-omni",
+  "server_params": { "...": "..." },
+  "benchmark_params": [{ "name": "1024x1024_steps4", "...": "..." }]
+}
 ```
 
-On a PR build, Buildkite logs from `upload_pipeline.py` include lines such as `skip '…' (no changes under …)` for omitted steps.
+- Local bulk load: `pytest -sv tests/dfx/perf/scripts/run_diffusion_benchmark.py -m "full_model and diffusion and H100"`
+- Nightly CI perf steps: `--test-config-file tests/dfx/perf/tests/test_<model>_vllm_omni.json` (file selects cases; no `-m`)
+- Result filenames use **runtime** GPU detection (`get_runtime_resource_label`); `H100` is omitted on the default CI pool
 
-#### Related
+See [L4 performance test examples](./test_examples/l4_performance_tests.inc.md) for the full schema.
 
-- Implementation: `.buildkite/cuda/scripts/upload_pipeline.py`
-- L2/L3 diff skip does **not** replace label-based triggers (`ready`, `merge-test`); it only reduces which steps appear **after** the pipeline is already scheduled.
+### Add Support for a New Platform
 
-### Test helper environment variables
+If you want to add support for a new platform (e.g., "tpu" for a new accelerator), follow these steps:
 
-Some shared helpers under `tests/helpers/` honor optional environment variables for local debugging. These are **not** set in CI by default.
+1. **Extend the marker list in your pytest config** so that platform/resource markers are defined:
+   ```toml
+   # In pyproject.toml or pytest.ini
+   [tool.pytest.ini_options]
+   markers = [
+       # ... existing markers ...
+       "tpu: Tests that require TPU device",
+       "TPU_V3: Tests that require TPU v3 hardware",
+       "distributed_tpu: Tests that require multiple TPU devices",
+   ]
+   ```
+2. **Implement a marker construction function for your platform** in `vllm-omni/tests/helpers/mark.py`:
+   ```python
+   # In vllm-omni/tests/helpers/mark.py
 
-| Variable | Accepted values | Description |
-| -------- | --------------- | ----------- |
-| `VLLM_OMNI_KEEP_REQUEST_MEDIA` | `1`, `true`, `yes` (case-insensitive) | When enabled, temporary WAV files created by `tests.helpers.media.convert_audio_bytes_to_text` are **not** deleted when the pytest process exits. By default, each call writes a unique file under the system temp directory via `tempfile.mkstemp` and registers `atexit` cleanup. Use this when debugging audio output validation (Whisper transcription, keyword checks, text–audio similarity). The saved path is logged as `audio data is saved: <path>`. |
+   def tpu_marks(*, res: str, num_cards: int):
+       test_platform = pytest.mark.tpu
+       if res == "TPU_V3":
+           test_resource = pytest.mark.TPU_V3
+       else:
+           raise ValueError(
+               f"Invalid TPU resource type: {res}. Supported: TPU_V3")
 
-Example (Linux / macOS):
+       if num_cards == 1:
+           return [test_platform, test_resource]
+       else:
+           test_distributed = pytest.mark.distributed_tpu(num_cards=num_cards)
+           # Optionally: add skipif_tpu when implemented
+           return [test_platform, test_resource, test_distributed]
+   ```
+3. **Update `hardware_marks` to recognize your new platform**:
+    In the relevant place (see the `hardware_marks` implementation), add:
+    ```python
+    if platform == "tpu":
+        marks = tpu_marks(res=resource, num_cards=cards)
+    ```
+    (`hardware_test` calls `hardware_marks` internally, so both will pick up the change.)
+4. **(Recommended) Add a test using your new markers**:
+   ```python
+   @hardware_test(
+       res={"tpu": "TPU_V3"},
+       num_cards=2,
+   )
+   def test_my_tpu_feature():
+       ...
+   ```
 
-```bash
-export VLLM_OMNI_KEEP_REQUEST_MEDIA=1
-pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -k test_mix_to_text_audio
-```
+**Summary**:  
+- Add pytest markers for your new platform/resources  
+- Implement a marker function (`xxx_marks`)  
+- Plug into `hardware_marks`  
+- You're done: tests using `@hardware_test` or `hardware_marks` with your platform now automatically get the correct markers, distribution, and isolation!
 
-Example (Windows PowerShell):
+See code in `vllm-omni/tests/helpers/mark.py` for existing examples (`cuda_marks`, `rocm_marks`, `npu_marks`).
 
-```powershell
-$env:VLLM_OMNI_KEEP_REQUEST_MEDIA = "1"
-pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -k test_mix_to_text_audio
-```
+## Test case style
 
-## Chapter 1: L1 & L2 Level Testing - Unit Testing and Basic End-to-End Verification
+### L1 & L2 Level Testing - Unit Testing and Basic End-to-End Verification
 
-### 1.1 Testing Purpose
+#### 1.1 Testing Purpose
 
 L1 and L2 level testing form the foundation of the quality assurance system. L1 level testing focuses on verifying the internal logic correctness of code units (e.g., functions, classes), ensuring each independent component behaves as designed.
 
@@ -439,7 +356,7 @@ L2 level testing builds upon L1 by introducing GPU resources and verifying that
 
 
 
-### 1.2 Testing Content and Scope
+#### 1.2 Testing Content and Scope
 
 -   ***L1 (Unit & Logic Testing)***:
 -   -   ***Scope***: Tests internal functions and methods of core components such as `entrypoints`, `models`.
@@ -454,7 +371,7 @@ L2 level testing builds upon L1 by introducing GPU resources and verifying that
         -   **Omni multimodal** (`assert_omni_response`): L2 asserts successful completion; keyword, transcript, and cross-modal similarity checks run only at L3+.
     -   ***Characteristic***: Requires ***GPU*** resources to perform model computations.
 
-### 1.3 Test Directory and Execution Files
+#### 1.3 Test Directory and Execution Files
 
 A clear directory structure is key to managing test cases efficiently.
 
@@ -464,10 +381,10 @@ A clear directory structure is key to managing test cases efficiently.
 -   -   Online Serving: `/tests/e2e/online_serving/test_{model_name}.py`
     -   Offline Inference: `/tests/e2e/offline_inference/test_{model_name}.py`
 
-### 1.4 Execution Method and Example
+#### 1.4 Execution Method and Example
 
 -   ***Trigger Timing***: **`PR with ready label`**. That is, when a developer adds a "ready for review" or similar label to a PR on platforms like GitHub, L1 and L2 tests are automatically triggered.
--   ***Diff-aware step skipping***: On L2, **E2E Test** jobs may be omitted at pipeline upload when the PR diff does not touch their [`source_file_dependencies`](#diff-aware-buildkite-uploads-source_file_dependencies) prefixes (H100 and docker queues alike); non-E2E groups still always upload. See [Diff-aware Buildkite uploads](#diff-aware-buildkite-uploads-source_file_dependencies).
+-   ***Diff-aware step skipping***: On L2, **E2E Test** jobs may be omitted at pipeline upload when the PR diff does not touch their [`source_file_dependencies`](ci_settings.md#step-filtering) prefixes; non-E2E groups still always upload. See [CI Settings — Diff-aware CI](ci_settings.md#diff-aware-ci).
 -   ***Execution Environment***: L1 uses ***CPU*** environment; L2 requires ***GPU*** environment.
 -   ***Script Example***:
 
@@ -535,7 +452,7 @@ def test_cap_and_align_mel_length_no_mismatch(repeats, code_len, max_mel_frames)
 
 <details>
 <summary> L2 Test Examples</summary>
-You can refer to Test Examples in Chapter 2 to see example test cases that incorporate both L2 and L3 testing logic.
+You can refer to Test Examples in the L3 section to see example test cases that incorporate both L2 and L3 testing logic.
 </details>
 
 -   -   ***Run Command***:
@@ -543,9 +460,9 @@ You can refer to Test Examples in Chapter 2 to see example test cases that incor
     `pytest -s -v /tests/e2e/online_serving/test_{model_name}.py`
     `pytest -s -v -m 'core_model and cpu' --run-level=core_model`
 
-## Chapter 2: L3 Level Testing - Core Integration, Performance, and Accuracy Verification
+### L3 Level Testing - Core Integration, Performance, and Accuracy Verification
 
-### 2.1 Testing Purpose
+#### 2.1 Testing Purpose
 
 L3 level testing executes after code is merged into the main branch. Its core purpose is to verify the integration effect, key performance indicators, and output accuracy of ***real models*** in ***multiple deployment scenarios***
 
@@ -553,30 +470,30 @@ L3 level testing executes after code is merged into the main branch. Its core pu
 
 
 
-### 2.2 Testing Content and Scope
+#### 2.2 Testing Content and Scope
 
 -   ***Deployment Scenarios***: Covers richer `online` and `offline` deployment configurations, which may include different hardware configurations, batch sizes, concurrency levels, etc.
 -   ***Core Verification***:
 -   1.  ***Inference Functionality***: Ensures real models can perform forward computation normally and return results.
     2.  ***Accuracy Compliance***: Verifies that the model's evaluation metrics (e.g., accuracy) meet the expected baseline, preventing code changes from introducing accuracy issues.
     3.  ***Important Performance***: Verifies whether performance (e.g., P99 latency, throughput) in core scenarios meets preset thresholds.
--   ***L3 response validation (`advanced_model`)***: At `--run-level advanced_model`, `tests/helpers/assertions.py` adds semantic checks on top of L2 payload gates (see Chapter 1):
+-   ***L3 response validation (`advanced_model`)***: At `--run-level advanced_model`, `tests/helpers/assertions.py` adds semantic checks on top of L2 payload gates (see L1 & L2 above):
     -   **Speech / TTS** (`assert_audio_speech_response`): Whisper transcript of returned audio vs `request_config["input"]` (cosine similarity &gt; 0.9 when input text is set); optional `min_audio_bytes` floor; PCM harmonic-to-noise ratio when `response_format` is `pcm`; **preset voice gender** via `_assert_preset_voice_gender_from_audio` when `voice` matches a known preset in `_PRESET_VOICE_GENDER_MAP` (pitch/F0-based classifier; skipped for unknown voices or `pcm` output).
     -   **Omni multimodal** (`assert_omni_response`): non-empty text/audio outputs; `key_words` in transcript or text; text–audio similarity / containment; preset **`speaker`** gender check (same helper as TTS).
     -   **Diffusion** (`assert_*_diffusion_response`): image/video dimension and frame-count parity with request parameters where configured.
     -   **Accuracy suites**: pixel/video similarity and metric baselines under `/tests/e2e/accuracy/` (separate from inline helper assertions).
 
-### 2.3 Test Directory and Execution Files
+#### 2.3 Test Directory and Execution Files
 
 -   ***Functional Testing***:
 -   -   Online Serving: `/tests/e2e/online_serving/test_{model_name}_expansion.py`
     -   Offline Inference: `/tests/e2e/offline_inference/test_{model_name}_expansion.py`
     -   (Note: `_expansion.py` likely means it contains more comprehensive scenario cases compared to L2 tests).
 
-### 2.4 Execution Method and Example
+#### 2.4 Execution Method and Example
 
 -   ***Trigger Timing***: **`PR Merged`**. Automatically triggered after code review is approved and merged into the main branch (typically via `merge-test` label on the PR before merge).
--   ***Diff-aware step skipping***: Same [`source_file_dependencies`](#diff-aware-buildkite-uploads-source_file_dependencies) mechanism as L2—**all E2E Test** leaf steps are diff-gated; other L3 groups always upload. See [Diff-aware Buildkite uploads](#diff-aware-buildkite-uploads-source_file_dependencies).
+-   ***Diff-aware step skipping***: Same [`source_file_dependencies`](ci_settings.md#step-filtering) mechanism as L2—**all E2E Test** leaf steps are diff-gated; other L3 groups always upload. See [CI Settings — Diff-aware CI](ci_settings.md#diff-aware-ci).
 -   ***Execution Environment***: ***GPU*** servers.
 -   ***Script Example***:
 
@@ -692,27 +609,27 @@ L3 level testing executes after code is merged into the main branch. Its core pu
 
     **Implicit Validation**: `send_omni_request`, `send_audio_speech_request`, and `send_diffusion_request` call `assert_*_response` helpers with the session `--run-level`. At `core_model`, checks are limited to success plus cheap payload sanity (non-empty audio/media, format/content-type, optional `min_audio_bytes`). At `advanced_model` and `full_model`, deep validation adds Whisper transcripts, keyword/similarity, diffusion dimension checks, PCM HNR, etc.
 
-    **Audio output debugging**: Deep validation may transcribe returned audio via `convert_audio_bytes_to_text` (Whisper). If an audio keyword or text–audio similarity assertion fails, set `VLLM_OMNI_KEEP_REQUEST_MEDIA=1` before running pytest to keep the intermediate WAV files for inspection (see [Test helper environment variables](#test-helper-environment-variables)).
+    **Audio output debugging**: Deep validation may transcribe returned audio via `convert_audio_bytes_to_text` (Whisper). If an audio keyword or text–audio similarity assertion fails, set `VLLM_OMNI_KEEP_REQUEST_MEDIA=1` before running pytest to keep the intermediate WAV files for inspection (see [Test helper environment variables](test_system_overview.md#test-helper-environment-variables)).
 
 -   ***Run Command (L3 merge)***: `pytest -s -v /tests/e2e/online_serving/test_{model_name}.py -m advanced_model --run-level=advanced_model`
 
 -   ***Run Command (L4 nightly expansion)***: `pytest -s -v /tests/e2e/online_serving/test_{model_name}_expansion.py -m full_model --run-level=full_model`
 
-## Chapter 3: L4 Level Testing - Full Functionality, Performance, and Documentation Testing
+### L4 Level Testing - Full Functionality, Performance, and Documentation Testing
 
-### 3.1 Testing Purpose
+#### 3.1 Testing Purpose
 
 L4 level testing is a comprehensive quality audit before a version release. It expands upon L3, executing ***full*** functional scenarios, conducting systematic ***performance stress tests***, and simultaneously verifying the correctness of accompanying ***example documentation***. Its purpose is to perform deep validation of the system during off-peak nighttime hours, providing quality trend reports for daytime development and data support for release decisions.
 
 
 
-### 3.2 Testing Content and Scope
+#### 3.2 Testing Content and Scope
 
 -   ***Full Functionality Testing***: Executes all test cases defined in `test_{model_name}_expansion.py`, covering all implemented features, positive flows, boundary conditions, and exception handling.
 -   ***Performance Testing***: Uses `tests/dfx/perf/tests/test_qwen3_omni_*.json` (Omni), `test_tts.json` / `test_voxcpm2.json` / `test_higgs_audio_v3.json` (TTS), and diffusion configs `tests/dfx/perf/tests/test_*_vllm_omni.json` (passed to `run_benchmark.py` or `run_diffusion_benchmark.py` via `--test-config-file` in nightly **Perf Test** steps) to drive throughput, latency, and memory benchmarks. Each JSON **case** may declare an optional top-level **`mark`** array: exactly one ``hardware_marks`` object plus pytest marker name strings (`full_model`, `omni` / `tts` / `diffusion`, …). Runners attach those marks to each parametrized `(server, benchmark index)` pair so **local** bulk runs can filter with `-m` (for example `-m "full_model and H100 and diffusion"`). Nightly CI perf jobs select workloads by **`--test-config-file`**, not `-m`. Details are in the Performance Tests example below.
 -   ***Documentation Testing***: Verifies whether the example code provided to users is runnable and its results match the description.
 
-### 3.3 Test Directory and Execution Files
+#### 3.3 Test Directory and Execution Files
 
 -   ***Functional Testing***: Same directories as L3.
 -   ***Performance Test Configuration***: `tests/dfx/perf/tests/test_qwen3_omni_*.json`, `test_tts.json`, `test_voxcpm2.json`, `test_higgs_audio_v3.json`, and diffusion configs `tests/dfx/perf/tests/test_*_vllm_omni.json` (e.g. `test_qwen_image_vllm_omni.json`, `test_cosmos3_vllm_omni.json`). Optional per-case `mark` for local `-m` filtering is documented in Section 3.4 Performance Tests.
@@ -720,7 +637,7 @@ L4 level testing is a comprehensive quality audit before a version release. It e
     -   `tests/examples/online_serving/test_{model_name}.py`
     -   `tests/examples/offline_inference/test_{model_name}.py`
 
-### 3.4 Execution Method and Example
+#### 3.4 Execution Method and Example
 
 -   ***Trigger Timing***: **`Nightly`**, automatically executed every night.
 -   ***Execution Environment***: ***GPU*** server clusters to meet the resource demands of performance testing.
@@ -740,20 +657,20 @@ L4 level testing is a comprehensive quality audit before a version release. It e
 
 -   ***Run Command***: (Specific commands would depend on the performance testing tool and configuration defined in `nightly.json`).
 
-## Chapter 4: L5 Level Testing - Stability and Reliability Testing
+### L5 Level Testing - Stability and Reliability Testing
 
-### 4.1 Testing Purpose
+#### 4.1 Testing Purpose
 
 L5 level testing focuses on the performance of model services under ***long-running*** and ***abnormal fault*** scenarios. It aims to uncover deep-seated issues that only manifest under sustained pressure or extreme conditions, such as memory leaks, resource contention, gradual performance degradation, and lack of fault tolerance mechanisms. This is the final, yet crucial, line of defense for ensuring service high availability and production environment robustness.
 
 
 
-### 4.2 Testing Content and Scope
+#### 4.2 Testing Content and Scope
 
 -   ***Long-term Stability (Stability) Testing***: Uses JSON under `tests/dfx/stability/tests/` (for example `test_qwen3_omni.json` and `test_wan22.json`) to run the service under moderate load for an extended period (e.g., over 12 hours), monitoring whether metrics like memory/VRAM usage, response time, and throughput degrade over time, and whether the service process remains stable.
 -   ***Reliability Testing***: Uses pytest suites under `tests/dfx/reliability/` to inject controlled faults against a **live** `vllm_omni serve` instance (same **`omni_server` / `omni_server_function`** fixture style as E2E). Current suites emphasize **GPU memory pressure** (CUDA sidecar “memory hog”), **worker / runtime process kill** (`SIGKILL` on `VLLM::Worker` for Qwen3-Omni, `multiprocessing.spawn` for Wan2.2 video workers, or `vLLM-Omni::` for HunyuanImage DiT workers), **large multimodal chat**, **`/v1/videos`**, **`/v1/images/generations`**, or **`/v1/audio/speech`** jobs under OOM, **`/health` → 503** and **fast-fail / non-hanging concurrent** requests after kill, and **OpenAI-style 5xx error contracts** (e.g. text vs text+audio under OOM). **Post-fault recovery** checks exist where enabled (some cases may be `skip` while issues are tracked). See the Reliability `<details>` block in Section 4.4 for file-level responsibilities and CI markers (`slow`, `hardware_test`, POSIX-only kill).
 
-### 4.3 Test Directory and Execution Files
+#### 4.3 Test Directory and Execution Files
 
 -   ***Stability Test Configuration***: `tests/dfx/stability/tests/test_qwen3_omni.json`, `tests/dfx/stability/tests/test_wan22.json` (one JSON per model / runner family)
 -   ***Reliability Test Suite*** (`tests/dfx/reliability/`):
@@ -765,7 +682,7 @@ L5 level testing focuses on the performance of model services under ***long-runn
     -   `conftest.py` — `fault_injector` and `omni_server_after_fault` / `omni_server_after_fault_function` fixtures to run a callable **after** the server is ready.
     -   `README.md` — Short local run commands for this directory.
 
-### 4.4 Execution Method and Example
+#### 4.4 Execution Method and Example
 
 -   ***Trigger Timing***: **`Weekly`** (weekly) or **`Days before Release`** (several days before a major release). Due to long execution times, the frequency is lower.
 -   ***Execution Environment***: ***GPU*** servers, requiring a stable and exclusive testing environment.
@@ -799,7 +716,7 @@ When you want to add L5-level stability test cases, add or extend the appropriat
 }
 ```
 
-#### Parameter Explanation
+##### Parameter Explanation
 
 ***Overview***
 
@@ -809,16 +726,16 @@ When you want to add L5-level stability test cases, add or extend the appropriat
 | server_params    | Yes      | Server-side configuration parameters (model, stage configuration, etc.)     |
 | benchmark_params | Yes      | Stability benchmark running parameters (supports multiple configurations)   |
 
-#### server_params Configuration
+##### server_params Configuration
 
-##### Basic Parameters
+###### Basic Parameters
 
 | Parameter         | Required | Example                            | Description                         |
 | ----------------- | -------- | ---------------------------------- | ----------------------------------- |
 | model             | Yes      | "Qwen/Qwen3-Omni-30B-A3B-Instruct" | Model name or path                  |
 | stage_config_name | Yes      | "qwen3_omni.yaml"                  | Stage configuration file name       |
 
-##### Dynamic Configuration (update/delete)
+###### Dynamic Configuration (update/delete)
 
 Supports incremental modifications based on the basic configuration:
 
@@ -828,9 +745,9 @@ Supports incremental modifications based on the basic configuration:
 | delete    | Delete specified configuration items |
 
 ***Example***:
-You can refer to Test Examples in Chapter 3.4
+You can refer to Test Examples in L4 §3.4
 
-#### benchmark_params Configuration
+##### benchmark_params Configuration
 
 For stability testing, the key parameters are:
 
@@ -840,18 +757,18 @@ For stability testing, the key parameters are:
     -   `max_concurrency`: Maximum number of concurrent requests. When this is used, `request_rate` is set to `inf` internally.
 -   **num_prompts_per_batch**: Number of prompts sent in each batch. Multiple batches will be executed sequentially within `duration_sec`.
 
-All other optional parameters follow the same rules as the in Chapter 3.4.
+All other optional parameters follow the same rules as in L4 §3.4.
 
 </details>
 
 <details>
 <summary> Reliability test suite (<code>tests/dfx/reliability</code>)</summary>
 
-#### Purpose and relationship to stability
+##### Purpose and relationship to stability
 
 Reliability tests are **short fault-injection** integration runs (L5 **(b)** in `tests/dfx/reliability/README.md`). They complement **stability** JSON-driven long runs: instead of hours of steady traffic, they **perturb** the server (GPU OOM sidecar, fatal signals on selected processes) and check **failure mode** and **latency bounds** (e.g. chat or `/v1/videos` must not hang under concurrent fault-time load).
 
-#### Directory layout
+##### Directory layout
 
 | Path | Responsibility |
 | ---- | -------------- |
@@ -863,13 +780,13 @@ Reliability tests are **short fault-injection** integration runs (L5 **(b)** in
 | `test_reliability_voxcpm2.py` | VoxCPM2: `/v1/audio/speech` under OOM (error contract), **SIGKILL** on `VLLM::` workers and serve/tree targets, health / fast-fail / concurrent speech requests; some OOM cases may be skipped while tracked in issues. |
 | `README.md` | Minimal run / collect examples. |
 
-#### Parametrization and markers
+##### Parametrization and markers
 
 - Each test module defines a **`RELIABILITY_SCENARIOS`** list (`test_name`, `server_params`: model, `stage_config_name` or diffusion `server_args`, etc.). **`create_reliability_omni_server_params()`** in `tests/dfx/conftest.py` resolves stage paths (including XPU substitutions where applicable) and builds **`OmniServerParams`** lists consumed by **`@pytest.mark.parametrize(..., indirect=True)`** on `omni_server` or `omni_server_function`.
 - Cases are tagged **`@pytest.mark.slow`** for weekly / selective CI. GPU-heavy suites use **`@hardware_test(res={"cuda": "H100"}, num_cards=...)`** or **`@hardware_test(res={"cuda": "L4"}, num_cards=1)`** (Qwen3-Omni **2**× H100; Wan2.2 **1**× H100; HunyuanImage DiT **4**× H100; VoxCPM2 **1**× L4).
 - **Process-kill** tests use **`@pytest.mark.skipif(os.name == "nt", ...)`** because injection uses POSIX **`pgrep` / `kill`**.
 
-#### CI trigger
+##### CI trigger
 
 Weekly Buildkite (`.buildkite/cuda/test-weekly.yml`) runs one step per model suite (trigger: `WEEKLY=1` or PR label `weekly-test`), for example:
 
@@ -887,7 +804,7 @@ pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m "slow"
 pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m "slow"
 ```
 
-#### Local commands
+##### Local commands
 
 ```bash
 pytest --collect-only tests/dfx/reliability
@@ -897,7 +814,7 @@ pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m slow
 pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m slow
 ```
 
-#### Adding a new model suite
+##### Adding a new model suite
 
 1. Add `test_reliability_<model>.py` under `tests/dfx/reliability/`.
 2. Define **`RELIABILITY_SCENARIOS`** and pass them through **`create_reliability_omni_server_params()`** with the correct deploy or e2e stage-config directory (same pattern as existing files).
@@ -908,7 +825,3 @@ pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m slow
 
 -   -   ***Stability***: `pytest -s -v tests/dfx/stability/scripts/test_stability_qwen3_omni.py` or `pytest -s -v tests/dfx/stability/scripts/test_stability_wan22.py` (or add `test_stability_<model>.py` alongside a matching JSON config)
     -   ***Reliability***: `pytest -s -v tests/dfx/reliability/test_reliability_<model>.py -m slow` (current suites: `qwen3_omni`, `wan22`, `hunyuan_image`, `voxcpm2`; add `test_reliability_<suite>.py` and a matching step in `.buildkite/cuda/test-weekly.yml` for new models)
-
-## Summary
-
-This multi-level testing system achieves continuous, progressive validation of model service quality by tightly integrating testing activities with the development workflow (commit, review, merge, release). From rapid unit testing to comprehensive end-to-end testing, and further to in-depth performance, stability, and reliability verification, each level has clear objectives, collectively building a robust quality protection net. By following this system, teams can deliver high-quality, highly reliable model services more efficiently.

docs/contributing/ci/tests_markers.md

@@ -1,207 +0,0 @@
-# Markers for Tests
-
-By adding markers before test functions, tests can later be executed uniformly by simply declaring the corresponding marker type.
-
-## Current Markers
-Defined in `pyproject.toml`:
-
-| Marker             | Description                                               |
-| ------------------ | --------------------------------------------------------- |
-| `core_model`       | L1&L2 tests (run in each PR)                              |
-| `advanced_model`   | L3 tests (run on each merge to main)                 |
-| `full_model`       | L4 tests (run nightly) |
-| `diffusion`        | Diffusion model tests                                     |
-| `omni`             | Omni multimodal model tests |
-| `tts`              | TTS model tests     |
-| `cache`            | Cache backend tests                                       |
-| `parallel`         | Parallelism/distributed tests                             |
-| `cpu`              | Tests that run on CPU                                     |
-| `gpu`              | Tests that run on GPU *                                   |
-| `cuda`             | Tests that run on CUDA *                                  |
-| `rocm`             | Tests that run on AMD/ROCm *                              |
-| `xpu`              | Tests that run on Intel XPU *                             |
-| `npu`              | Tests that run on NPU/Ascend *                            |
-| `H100`             | Tests that require H100 GPU  *                            |
-| `L4`               | Tests that require L4 GPU *                               |
-| `MI325`            | Tests that require MI325 GPU (AMD/ROCm) *                 |
-| `A2`               | Tests that require A2 NPU *                               |
-| `A3`               | Tests that require A3 NPU *                               |
-| `distributed_cuda` | Tests that require multi cards on CUDA platform *         |
-| `distributed_rocm` | Tests that require multi cards on ROCm platform  *        |
-| `distributed_npu`  | Tests that require multi cards on NPU platform  *         |
-| `skipif_cuda`      | Skip if the num of CUDA cards is less than the required * |
-| `skipif_rocm`      | Skip if the num of ROCm cards is less than the required * |
-| `skipif_npu`       | Skip if the num of NPU cards is less than the required *  |
-| `slow`             | Slow tests (may skip in quick CI)                         |
-| `benchmark`        | Benchmark tests (decorator on runner test functions; perf JSON uses `full_model` + type marker instead) |
-| `local_model`      | Tests requiring local / non-HF-hub model weights          |
-
-\* Means those markers are auto-added by `@hardware_test` (parametrization decorator) or `hardware_marks` (only returning the list of marks for flexibility).
-
-### Example usage for markers
-
-```python
-from tests.helpers.mark import hardware_test
-
-@pytest.mark.core_model
-@pytest.mark.omni
-@hardware_test(
-   res={"cuda": "L4", "rocm": "MI325", "npu": "A2"},
-   num_cards=2,
-)
-@pytest.mark.parametrize("omni_server", test_params, indirect=True)
-def test_video_to_audio()
-    ...
-```
-
-### Decorator: `@hardware_test`
-
-This decorator is intended to make hardware-aware, cross-platform test authoring easier and more robust for CI/CD environments. The `hardware_test` decorator in `vllm-omni/tests/helpers/mark.py` performs the following actions:
-
-1. **Applies platform and resource markers**  
-   Adds the appropriate pytest markers for each specified hardware platform (e.g., `cuda`, `rocm`, `xpu`, `npu`) and resource type (e.g., `L4`, `H100`, `MI325`, `B60`, `A2`, `A3`).
-   ```
-   @pytest.mark.cuda
-   @pytest.mark.L4
-   ```
-2. **Handles multi-card (distributed) scenarios**  
-   For tests requiring multiple cards, it automatically adds distributed markers such as `distributed_cuda`, `distributed_rocm`, or `distributed_npu`.
-   ```
-   @pytest.mark.distributed_cuda(num_cards=num_cards)
-   ```
-3. **Supports flexible card requirements**  
-   Accepts `num_cards` as either a single integer for all platforms or as a dictionary with per-platform values. If not specified, defaults to 1 card per platform.
-
-4. **Integrates resource validation**  
-   On CUDA, adds a skip marker (`skipif_cuda`) if the system does not have the required number of devices.
-   Support for `skipif_rocm` and `skipif_npu` will be implemented later.
-
-
-5. **Works with pytest filtering**  
-   Allows tests to be filtered and selected at runtime using standard pytest marker expressions (e.g., `-m "distributed_cuda and L4"`).
-
-#### Example usage for decorator
-- Single call for multiple platforms:
-    ```python
-    @hardware_test(
-        res={"cuda": "L4", "rocm": "MI325", "xpu": "B60", "npu": "A2"},
-        num_cards={"cuda": 2, "rocm": 2, "xpu": 2, "npu": 2},
-    )
-    ```
-    or
-    ```python
-    @hardware_test(
-        res={"cuda": "L4", "rocm": "MI325", "xpu": "B60", "npu": "A2"},
-        num_cards=2,
-    )
-    ```
-- `res` must be a dict; supported resources: CUDA (L4/H100), ROCm (MI325), XPU (B60), MUSA (S5000), NPU (A2/A3)
-- `num_cards` can be int (all platforms) or dict (per platform); defaults to 1 when missing
-- Distributed markers (`distributed_cuda`, `distributed_rocm`, `distributed_npu`) are auto-added for multi-card cases
-- Filtering examples:
-    - CUDA only: `pytest -m "distributed_cuda and L4"`
-    - ROCm only: `pytest -m "distributed_rocm and MI325"`
-    - NPU only: `pytest -m "distributed_npu"`
-
-### Function: `hardware_marks`
-
-`hardware_marks` returns a list of pytest mark objects with the same signature as `@hardware_test`. Use it when you need more flexibility, such as attaching hardware marks to individual `pytest.param` entries rather than an entire test function.
-
-```python
-from tests.helpers.mark import hardware_marks
-
-MULTI_CARD_MARKS = hardware_marks(
-    res={"cuda": "H100", "rocm": "MI325", "npu": "A2"}, num_cards=2
-)
-
-@pytest.mark.parametrize("omni_server", [
-    pytest.param(OmniServerParams(...), id="case_001", marks=MULTI_CARD_MARKS),
-], indirect=True)
-def test_feature(omni_server):
-    ...
-```
-
-### JSON `mark` field (L4 perf configs)
-
-Perf JSON under `tests/dfx/perf/tests/` can attach pytest marks per **case** (`test_name` block). Parsed by `tests.dfx.conftest.resolve_pytest_marks` and applied to each parametrized run in `run_benchmark.py` / `run_diffusion_benchmark.py`.
-
-When `mark` is present, it must be an **array** with exactly one ``hardware_marks`` object (same semantics as `hardware_marks()` above), followed by registered pytest marker name strings such as `full_model`, `omni`, `tts`, or `diffusion`.
-
-```json
-{
-  "test_name": "test_cosmos3_t2i_official_demo_2gpu",
-  "mark": [
-    {"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 2}},
-    "full_model",
-    "diffusion"
-  ],
-  "server_type": "vllm-omni",
-  "server_params": { "...": "..." },
-  "benchmark_params": [{ "name": "1024x1024_steps4", "...": "..." }]
-}
-```
-
-- Local bulk load: `pytest -sv tests/dfx/perf/scripts/run_diffusion_benchmark.py -m "full_model and diffusion and H100"`
-- Nightly CI perf steps: `--test-config-file tests/dfx/perf/tests/test_<model>_vllm_omni.json` (file selects cases; no `-m`)
-- Result filenames use **runtime** GPU detection (`get_runtime_resource_label`); `H100` is omitted on the default CI pool
-
-See [L4 performance test examples](./test_examples/l4_performance_tests.inc.md) for the full schema.
-
-## Add Support for a New Platform
-
-If you want to add support for a new platform (e.g., "tpu" for a new accelerator), follow these steps:
-
-1. **Extend the marker list in your pytest config** so that platform/resource markers are defined:
-   ```toml
-   # In pyproject.toml or pytest.ini
-   [tool.pytest.ini_options]
-   markers = [
-       # ... existing markers ...
-       "tpu: Tests that require TPU device",
-       "TPU_V3: Tests that require TPU v3 hardware",
-       "distributed_tpu: Tests that require multiple TPU devices",
-   ]
-   ```
-2. **Implement a marker construction function for your platform** in `vllm-omni/tests/helpers/mark.py`:
-   ```python
-   # In vllm-omni/tests/helpers/mark.py
-
-   def tpu_marks(*, res: str, num_cards: int):
-       test_platform = pytest.mark.tpu
-       if res == "TPU_V3":
-           test_resource = pytest.mark.TPU_V3
-       else:
-           raise ValueError(
-               f"Invalid TPU resource type: {res}. Supported: TPU_V3")
-
-       if num_cards == 1:
-           return [test_platform, test_resource]
-       else:
-           test_distributed = pytest.mark.distributed_tpu(num_cards=num_cards)
-           # Optionally: add skipif_tpu when implemented
-           return [test_platform, test_resource, test_distributed]
-   ```
-3. **Update `hardware_marks` to recognize your new platform**:
-    In the relevant place (see the `hardware_marks` implementation), add:
-    ```python
-    if platform == "tpu":
-        marks = tpu_marks(res=resource, num_cards=cards)
-    ```
-    (`hardware_test` calls `hardware_marks` internally, so both will pick up the change.)
-4. **(Recommended) Add a test using your new markers**:
-   ```python
-   @hardware_test(
-       res={"tpu": "TPU_V3"},
-       num_cards=2,
-   )
-   def test_my_tpu_feature():
-       ...
-   ```
-
-**Summary**:  
-- Add pytest markers for your new platform/resources  
-- Implement a marker function (`xxx_marks`)  
-- Plug into `hardware_marks`  
-- You're done: tests using `@hardware_test` or `hardware_marks` with your platform now automatically get the correct markers, distribution, and isolation!
-
-See code in `vllm-omni/tests/helpers/mark.py` for existing examples (`cuda_marks`, `rocm_marks`, `npu_marks`).

docs/contributing/ci/tests_style.md

@@ -1,466 +0,0 @@
-# Test File Structure and Style Guide
-
-To ensure project maintainability and sustainable development, we encourage contributors to submit test code (unit tests, system tests, or end-to-end tests) alongside their code changes. This document outlines the guidelines for organizing and naming test files.
-
-## Checklist before submitting your test files
-
-1. The file is saved in an appropriate place and the file name is clear.
-2. The coding style follows the requirements outlined below.
-3. All test functions have appropriate pytest markers.
-4. For tests that need run in CI, please ensure it labeled as ``@pytest.mark.core_model` the test is configured under the `./buildkite/` folder.
-
-
-## Test Types
-
-For more details about our [Five Levels Tests design](../ci/CI_5levels.md).
-
-### Unit Tests and System Tests
-For unit tests and system tests, we strongly recommend placing test files in the same directory structure as the source code being tested, using the naming convention `test_*.py`.
-
-### End-to-End (E2E) Tests for Models
-End-to-end tests verify the complete functionality of a system or component. For our project, the E2E tests for different omni models are organized into two subdirectories:
-
-- **`tests/e2e/offline_inference/`**: Tests for offline inference modes (e.g., Qwen3Omni offline inference)
-
-- **`tests/e2e/online_serving/`**: Tests for online serving scenarios (e.g., API server tests)
-
-## Test Directory Structure
-
-The ideal directory structure mirrors the source code organization. Legend: `✅` = test exists, `⬜` = suggested to add.
-
-```
-vllm_omni/                                    tests/
-├── config/                             →     ├── config/
-│   ├── model.py                              │   └── test_model.py                    ⬜
-│   └── lora.py                               │   └── test_lora.py                      ⬜
-│
-├── core/                               →     ├── core/
-│   └── sched/                                 │   └── sched/
-│       ├── omni_ar_scheduler.py               │       ├── test_omni_ar_scheduler.py    ⬜
-│       ├── omni_generation_scheduler.py       │       ├── test_omni_generation_scheduler.py  ⬜
-│       └── output.py                          │       └── test_output.py               ✅ currently in entrypoints/test_omni_new_request_data.py (tests output.OmniNewRequestData)
-│
-├── diffusion/                          →     ├── diffusion/
-│   ├── diffusion_engine.py                    │   ├── test_diffusion_engine.py          ⬜
-│   ├── attention/                             │   ├── attention/
-│   │   ├── layer.py                            │   │   ├── test_attention_sp.py         ✅
-│   │   └── backends/                           │   │   └── test_flash_attn.py           ✅
-│   ├── distributed/                           │   ├── distributed/
-│   │   └── ...                                 │   │   ├── test_comm.py                 ✅
-│   │                                            │   │   ├── test_cfg_parallel.py        ✅
-│   │                                            │   │   └── test_sp_plan_hooks.py       ✅
-│   ├── lora/                                   │   ├── lora/
-│   │   └── ...                                 │   │   ├── test_base_linear.py          ✅
-│   │                                            │   │   └── test_lora_manager.py        ✅
-│   ├── models/                                 │   ├── models/
-│   │   ├── qwen_image/                         │   │   ├── qwen_image/                 (e2e coverage)
-│   │   ├── ovis_image/                         │   │   ├── ovis_image/
-│   │   │   └── ...                             │   │   │   └── test_ovis_image.py     ✅
-│   │   ├── z_image/                            │   │   └── z_image/
-│   │   └── ...                                 │   │       └── test_zimage_tp_constraints.py  ✅
-│   └── worker/                                 │   └── worker/
-│       ├── diffusion_worker.py                 │       └── test_diffusion_worker.py   ✅ file at diffusion/test_diffusion_worker.py
-│       └── diffusion_model_runner.py            │
-│
-├── distributed/                         →     ├── distributed/
-│   └── omni_connectors/                         │   └── omni_connectors/
-│       ├── adapter.py                           │       ├── test_adapter_and_flow.py   ✅
-│       ├── kv_transfer_manager.py               │       ├── test_basic_connectors.py   ✅
-│       ├── connectors/                           │       ├── test_kv_flow.py             ✅
-│       └── utils/                               │       └── test_omni_connector_configs.py  ✅
-│
-├── engine/                             →     ├── engine/
-│   ├── input_processor.py                      │   ├── test_input_processor.py         ⬜  (no processor.py in source)
-│   ├── output_processor.py                     │   └── test_output_processor.py         ⬜
-│   └── arg_utils.py                            │   └── test_arg_utils.py               ⬜
-│
-├── entrypoints/                        →     ├── entrypoints/
-│   ├── stage_utils.py                          │   ├── test_stage_utils.py            ✅
-│   ├── cli/                                     │   ├── cli/                           (benchmarks/test_serve_cli.py covers CLI serve)
-│   │   └── ...                                  │   │   └── test_*.py                  ⬜
-│   └── openai/                                  │   └── openai_api/                    # maps to entrypoints/openai/
-│       ├── api_server.py                        │       ├── test_api_server.py         ⬜  (e2e indirect coverage)
-│       ├── serving_chat.py                       │       ├── test_serving_chat_sampling_params.py  ✅
-│       ├── serving_speech.py                     │       ├── test_serving_speech.py     ✅
-│       └── image_api_utils.py                   │       └── test_image_server.py      ✅
-│
-├── inputs/                             →     ├── inputs/
-│   ├── data.py                                 │   ├── test_data.py                   ⬜
-│   ├── parse.py                                │   ├── test_parse.py                 ⬜
-│   └── preprocess.py                            │   └── test_preprocess.py            ✅ currently in entrypoints/test_omni_input_preprocessor.py
-│
-├── model_executor/                     →     ├── model_executor/
-│   ├── layers/                                  │   ├── layers/
-│   │   └── mrope.py                             │   │   └── test_mrope.py              ⬜
-│   ├── model_loader/                            │   ├── model_loader/
-│   │   └── weight_utils.py                      │   │   └── test_weight_utils.py      ⬜
-│   ├── models/                                  │   ├── models/
-│   │   ├── qwen2_5_omni/                         │   │   ├── qwen2_5_omni/
-│   │   │   ├── qwen2_5_omni_thinker.py           │   │   │   ├── test_audio_length.py  ✅
-│   │   │   ├── qwen2_5_omni_talker.py            │   │   │   ├── test_qwen2_5_omni_thinker.py  ⬜
-│   │   │   └── qwen2_5_omni_token2wav.py         │   │   │   ├── test_qwen2_5_omni_talker.py  ⬜
-│   │   └── qwen3_omni/                          │   │   │   └── test_qwen2_5_omni_token2wav.py  ⬜
-│   │       └── ...                               │   │   └── qwen3_omni/
-│   ├── stage_configs/                           │   │       └── test_*.py              ⬜
-│   │   └── *.yaml                               │   └── stage_configs/                 (used by e2e, test_*.py can be added)  ⬜
-│   └── stage_input_processors/                  │   └── stage_input_processors/
-│       └── ...                                  │       └── test_*.py                 ⬜
-│
-├── sample/                             →     ├── sample/
-│   └── __init__.py                             │   └── test_*.py                      ⬜
-│
-├── utils/                              →     ├── utils/
-│   └── __init__.py                             │   └── test_*.py                       ⬜  (no platform_utils.py currently)
-│
-├── worker/                             →     ├── worker/
-│   ├── gpu_ar_model_runner.py                  │   ├── test_gpu_ar_model_runner.py    ⬜
-│   ├── gpu_ar_worker.py                        │   ├── test_gpu_ar_worker.py           ⬜
-│   ├── gpu_generation_model_runner.py          │   ├── test_gpu_generation_model_runner.py  ✅
-│   ├── gpu_generation_worker.py                │   ├── test_gpu_generation_worker.py  ⬜
-│   ├── gpu_model_runner.py                     │   ├── test_omni_gpu_model_runner.py   ✅
-│   └── mixins.py                               │   └── (npu under platforms/npu/worker/)  # not worker/npu/
-│
-├── platforms/                          →     (no tests/platforms/, e2e and stage_configs provide indirect coverage)
-│   ├── cuda/
-│   ├── npu/worker/                             # NPU worker here, not vllm_omni/worker/npu/
-│   ├── rocm/
-│   └── xpu/worker/
-│
-├── outputs.py                          →     test_outputs.py                         ✅ (at tests root)
-├── (logger, patch, request, version)    →     (no corresponding unit test)
-│
-└── e2e (tests side only)               →     ├── e2e/
-                                               ├── online_serving/                     ✅ non-empty
-                                               │   ├── test_qwen2_5_omni.py
-                                               │   ├── test_async_omni.py
-                                               │   ├── test_qwen3_omni.py
-                                               │   ├── test_qwen3_omni_expansion.py
-                                               │   ├── test_mimo_audio.py
-                                               │   └── test_images_generations_lora.py
-                                               └── offline_inference/                  ✅
-                                                   ├── test_qwen2_5_omni.py
-                                                   ├── test_qwen3_omni.py
-                                                   ├── test_bagel_text2img.py
-                                                   ├── test_z_image.py
-                                                   ├── test_wan22.py
-                                                   ├── test_zimage_tensor_parallel.py
-                                                   ├── test_cache_dit.py
-                                                   ├── test_teacache.py
-                                                   ├── test_stable_audio_expansion.py
-                                                   ├── test_diffusion_cpu_offload.py
-                                                   ├── test_diffusion_layerwise_offload.py
-                                                   ├── test_diffusion_lora.py
-                                                   ├── test_sequence_parallel.py
-                                                   ├── test_qwen_image_edit_expansion.py
-                                                   └── stage_configs/                  (legacy schema, still present
-                                                       ├── bagel_*.yaml                 for unmigrated models)
-                                                       └── npu/, rocm/, etc.
-
-# Migrated models (qwen3_omni_moe, qwen2_5_omni, qwen3_tts) live under
-# vllm_omni/deploy/ instead — see docs/configuration/stage_configs.md.
-examples/                                     tests
-│                                             └── examples
-├── online_serving/                     →         ├── online_serving/
-│   └── {doc_page_title}/README.md                │   └── test_{doc_page_title}.py  ⬜
-└── offline_inference/                  →         └── offline_inference/
-    └── {doc_page_title}/README.md                    └── test_{doc_page_title}.py  ⬜
-```
-
-
-
-### Naming Conventions
-
-- **Unit Tests**: Use `test_<module_name>.py` format. Example: `stage_utils.py` → `test_stage_utils.py`
-
-- **E2E Tests**: Place in `tests/e2e/offline_inference/` or `tests/e2e/online_serving/` with descriptive names. Example: `tests/e2e/offline_inference/test_qwen3_omni.py`, `tests/e2e/offline_inference/test_diffusion_model.py`
-
-- **Expansion Tests**
-
-### Best Practices
-
-1. **Mirror Source Structure**: Test directories should mirror the source code structure
-2. **Test Type Indicators**: Use comments to indicate test types (UT for unit tests, E2E for end-to-end tests)
-3. **Shared Resources**: Place shared test configurations (e.g., CI configs) in appropriate subdirectories
-4. **Consistent Naming**: Follow the `test_*.py` naming convention consistently across all test files
-
-
-## Test codes requirements
-
-### Coding style
-
-1. **File header**: Add SPDX license header to all test files
-2. **Imports**: Pls don't use manual `sys.path` modifications, use standard imports instead.
-3. **Test type differentiation**:
-
-      - Unit tests: Maintain mock style
-      - E2E tests for models: Consider using OmniRunner uniformly, avoid decorators
-
-4. **Documentation**: Add docstrings to all test functions
-5. **Environment variables**: Set uniformly in `conftest.py` or at the top of files
-6. **Type annotations**: Add type annotations to all test function parameters
-7. **Pytest Markers**: Add necessary markers like `@pytest.mark.core_model` and use `@hardware_test` to declare hardware requirements (check detailed in [Markers for Tests](../ci/tests_markers.md)).
-
-### Template
-#### E2E - Online serving
-
-E2E Online tests for Qwen3-Omni model with mix input and audio+text output. Based on `tests/e2e/online_serving/test_qwen3_omni.py`.
-
-```python
-"""
-E2E Online tests for Qwen3-Omni model with mix input and audio+text output.
-"""
-
-import os
-
-os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
-
-import threading
-from pathlib import Path
-
-import openai
-import pytest
-
-from tests.helpers.media import (
-    convert_audio_bytes_to_text,
-    cosine_similarity_text,
-    generate_synthetic_video,
-)
-from tests.helpers.runtime import OmniServer, dummy_messages_from_mix_data
-from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
-from vllm_omni.platforms import current_omni_platform
-
-# Edit: model name and stage config path
-models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
-
-#If you use the default configuration file, you can directly use the following address.
-def get_default_config():
-    return get_deploy_config_path("ci/qwen3_omni_moe.yaml")
-
-#If you need to modify the configuration file, you can use modify_stage_config.
-def get_chunk_config():
-    path = modify_stage_config(
-        get_default_config(),
-        updates={
-            "async_chunk": True,
-            "stage_args": {
-                0: {
-                    "engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker_async_chunk"
-                },
-                1: {
-                    "engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav_async_chunk"
-                },
-            },
-        },
-        deletes={"stage_args": {2: ["custom_process_input_func"]}},
-    )
-    return path
-
-stage_configs = [get_default_config(), CHUNK_CONFIG_PATH]
-
-test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
-
-
-#Please use this method to launch the online instance.
-_omni_server_lock = threading.Lock()
-
-@pytest.fixture(scope="module")
-def omni_server(request):
-    """Start vLLM-Omni server as a subprocess. Use module scope for multi-stage init (10-20+ min)."""
-    with _omni_server_lock:
-        model, stage_config_path = request.param
-        with OmniServer(
-            model,
-            ["--stage-configs-path", stage_config_path, "--stage-init-timeout", "120"],
-        ) as server:
-            yield server
-
-
-@pytest.fixture
-def client(omni_server):
-    """OpenAI client for the running vLLM-Omni server."""
-    return openai.OpenAI(
-        base_url=f"http://{omni_server.host}:{omni_server.port}/v1",
-        api_key="EMPTY",
-    )
-
-#Please use function definitions above the test function to define the prompts and other parameters you need.
-def get_system_prompt():
-    return {
-        "role": "system",
-        "content": [
-            {
-                "type": "text",
-                "text": (
-                    "You are Qwen, a virtual human developed by the Qwen Team, "
-                    "Alibaba Group, capable of perceiving auditory and visual inputs, "
-                    "as well as generating text and speech."
-                ),
-            }
-        ],
-    }
-
-...
-
-#Please define test case tags according to the instructions in the marker documentation.
-@pytest.mark.core_model
-@pytest.mark.omni
-@pytest.mark.parametrize("omni_server", test_params, indirect=True)
-def test_mix_to_text_audio_001(client: openai.OpenAI, omni_server, request) -> None:
-    # PLEASE FOLLOW THESE TEMPLATE INSTRUCTIONS:
-    # ============================================================================
-    # TEMPLATE USAGE GUIDE:
-    # 1. Copy this entire function as a starting point for multi-modal tests
-    # 2. Update the test name to reflect your specific test scenario
-    # 3. Modify input/output modalities as needed (see OPTIONS section below)
-    # 4. Adjust assertions based on your expected outcomes
-    # 5. Add custom validation logic for your specific use case
-    # ============================================================================
-
-    #Please list the relevant test points.
-    """
-    Test multi-modal input processing and text/audio output generation via OpenAI API.
-    Deploy Setting: default yaml
-    Input Modal: text + audio + video + image
-    Output Modal: text + audio
-    Input Setting: stream=True
-    Datasets: single request
-    """
-    # SECTION 1: TEST SETUP AND INITIALIZATION
-    # =========================================
-    # INSTRUCTIONS: Initialize test variables and prepare test environment
-    # MODIFY: Add any additional test setup required for your scenario
-    e2e_list = list()
-    # SECTION 2: TEST DATA GENERATION
-    # ================================
-    # INSTRUCTIONS: Generate or load test data for each input modality
-    # MODIFY: Replace synthetic generators with your actual data sources
-    # VIDEO DATA - Generate synthetic video for testing
-    # FORMAT: data:video/mp4;base64,{base64_encoded_video}
-    # PARAMETERS: width, height, duration_frames
-    video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(224, 224, 300)['base64']}"
-    # IMAGE DATA - Generate synthetic image for testing
-    # FORMAT: data:image/jpeg;base64,{base64_encoded_image}
-    # PARAMETERS: width, height
-    image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(224, 224)['base64']}"
-    # AUDIO DATA - Generate synthetic audio for testing
-    # FORMAT: data:audio/wav;base64,{base64_encoded_audio}
-    # PARAMETERS: duration_seconds, channels
-    audio_data_url = f"data:audio/wav;base64,{generate_synthetic_audio(5, 1)['base64']}"
-
-    # SECTION 3: MESSAGE CONSTRUCTION
-    # ================================
-    # INSTRUCTIONS: Assemble the complete message payload for API request
-    # MODIFY: Add/remove modalities or change prompt structure as needed
-
-    # USAGE: Construct a message containing all input modalities
-    # IMPORTANT: Ensure the message structure matches OpenAI API expectations
-    # CUSTOMIZATION POINTS:
-    #   - system_prompt: Controls the assistant's behavior
-    #   - content_text: The user's text prompt/question
-    #   - *_data_url: URLs for media content (video/image/audio)
-    messages = dummy_messages_from_mix_data(
-        system_prompt=get_system_prompt(),
-        video_data_url=video_data_url,
-        image_data_url=image_data_url,
-        audio_data_url=audio_data_url,
-        content_text=get_prompt("mix"),
-    )
-
-    # SECTION 4: API REQUEST EXECUTION
-    # =================================
-    # INSTRUCTIONS: Make the API call and measure performance
-    # MODIFY: Add timeout, retry logic, or additional parameters
-    start_time = time.perf_counter()
-    chat_completion = client.chat.completions.create(model=omni_server.model, messages=messages, stream=True)
-
-    #Call using your preferred method and obtain the final audio and text outputs.
-    ...
-
-    # SECTION 5: OUTPUT VALIDATION
-    # =============================
-    # INSTRUCTIONS: Verify that outputs meet expected criteria
-    # MODIFY: Adjust validation logic for your specific requirements
-
-    # ASSERTION 1: E2E Validation
-    # PURPOSE: Verify that the E2E latency is less than the baseline.
-    current_e2e = time.perf_counter() - start_time
-    print(f"the request e2e is: {current_e2e}")
-    e2e_list.append(current_e2e)
-
-    print(f"the avg e2e is: {sum(e2e_list) / len(e2e_list)}")
-
-
-
-    # ASSERTION 2: Text Output Validation
-    # PURPOSE: Verify that text output was generated with keyword content
-    assert text_content is not None and len(text_content) >= 2, "No text output is generated"
-    assert any(
-        keyword in text_content.lower() for keyword in ["square", "quadrate", "sphere", "globe", "circle", "round"]
-    ), "The output does not contain any of the keywords."
-
-
-    # ASSERTION 3: Cross-Modal Consistency
-    # PURPOSE: Verify text and audio outputs convey the same information
-    # CUSTOMIZATION: Adjust similarity threshold (0.9) based on accuracy requirements
-    assert audio_data is not None, "No audio output is generated"
-    audio_content = convert_audio_bytes_to_text(audio_data)
-    print(f"text content is: {text_content}")
-    print(f"audio content is: {audio_content}")
-    similarity = cosine_similarity_text(audio_content.lower(), text_content.lower())
-    print(f"similarity is: {similarity}")
-    assert similarity > 0.9, "The audio content is not same as the text"
-```
-
-
-#### E2E - Offline inference
-```python
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""
-Offline E2E smoke test for an omni model (video → audio).
-"""
-
-import os
-from pathlib import Path
-
-import pytest
-from vllm.assets.video import VideoAsset
-
-from tests.helpers.mark import hardware_test
-from ..multi_stages.conftest import OmniRunner
-
-# Optional: set process start method for workers
-os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
-
-models = ["{your model name}"] #Edit here to load your model
-stage_configs = [str(Path(__file__).parent / "stage_configs" / {your model yaml})] #Edit here to load your model yaml
-
-# Create parameter combinations for model and stage config
-test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
-
-# function name: test_{input_modality}_to_{output_modality}
-# modality candidate: text, image, audio, video, mixed_modalities
-@pytest.mark.core_model
-@pytest.mark.omni
-@hardware_test(
-    res={"cuda": "L4", "rocm": "MI325", "npu": "A2"},
-    num_cards=2,
-)
-@pytest.mark.parametrize("test_config", test_params)
-def test_video_to_audio(omni_runner: type[OmniRunner], model: str) -> None:
-    """Offline inference: video input, audio output."""
-    model, stage_config_path = test_config
-    with omni_runner(model, seed=42, stage_configs_path=stage_config_path) as runner:
-        # Prepare inputs
-        video = VideoAsset(name="sample", num_frames=4).np_ndarrays
-
-        outputs = runner.generate_multimodal(
-            prompts="Describe this video briefly.",
-            videos=video,
-        )
-
-        # Minimal assertions: got outputs and at least one audio result
-        assert outputs
-        has_audio = any(o.final_output_type == "audio" for o in outputs)
-        assert has_audio
-```

docs/contributing/model/adding_diffusion_model.md

@@ -685,7 +685,7 @@ For a fair comparison, keep the same **prompt**, **seed**, **resolution**, **num
 
 To ensure project maintainability and sustainable development, please submit test code (unit tests, system tests, or end-to-end tests) alongside their code changes.
 
-For comprehensive testing guidelines and the definition of test levels (L1-L5), please refer to the [Multi-Level Automated Testing System Documentation](../ci/CI_5levels.md). You are at least required to add an L4 *functionality* test described in that document.
+For the definition of test levels (L1-L5), see [Test System Overview](../ci/test_system_overview.md). You are at least required to add an L4 *functionality* test as described in [Test Writing Guide](../ci/test_writing_guide.md#l4-level-testing-full-functionality-performance-and-documentation-testing).
 
 ---
 

docs/contributing/model/adding_omni_model.md

@@ -540,7 +540,7 @@ Do **not** add a no-suffix `thinker2talker` when a `sync_process_input_func` is
 
 ## Testing
 
-For comprehensive testing guidelines, please refer to the [Test File Structure and Style Guide](../ci/tests_style.md).
+For comprehensive testing guidelines, please refer to the [Test Writing Guide](../ci/test_writing_guide.md).
 
 ## Adding a Model Recipe
 

docs/contributing/model/adding_tts_model.md

@@ -637,7 +637,7 @@ Key points:
 
 ## Testing
 
-For general testing conventions, see [tests_style.md](../ci/tests_style.md).
+For general testing conventions, see [Test Writing Guide](../ci/test_writing_guide.md).
 
 Recommended test cases for a new TTS model:
 

tests/buildkite/test_skip_ci.py

@@ -0,0 +1,370 @@
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+import sys
+from dataclasses import asdict
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2] / ".buildkite" / "common" / "scripts"))
+
+from skip_ci import (  # noqa: E402
+    CiDecision,
+    _diff_only_contains_skip_mark_changes,
+    resolve_ci_decision,
+)
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+
+def _decision(changed_files: list[str]) -> CiDecision:
+    return resolve_ci_decision(changed_files)
+
+
+def _is_yaml_gated(decision: CiDecision) -> bool:
+    return decision.skip_l2_l3 and not decision.skip_all
+
+
+def _all_l23_skipped(decision: CiDecision) -> bool:
+    checks = (
+        ("cuda", "l2"),
+        ("cuda", "l3"),
+        ("amd", "l2"),
+        ("amd", "l3"),
+        ("intel", "l2"),
+        ("npu", "l2"),
+    )
+    return all(not decision.is_run(platform, level) for platform, level in checks)
+
+
+def test_docs_only_change() -> None:
+    assert _decision(["docs/foo.md", "README.md", "mkdocs.yml"]).skip_all
+
+
+def test_docs_only_rejects_product_code() -> None:
+    assert not _decision(["docs/foo.md", "vllm_omni/foo.py"]).skip_all
+
+
+def test_skip_mark_diff_only() -> None:
+    diff = """\
+diff --git a/tests/foo/test_bar.py b/tests/foo/test_bar.py
+--- a/tests/foo/test_bar.py
++++ b/tests/foo/test_bar.py
+@@ -1,3 +1,4 @@
+ import pytest
++
++@pytest.mark.skip(reason="temp")
+ def test_x():
+"""
+    assert _diff_only_contains_skip_mark_changes(diff)
+
+
+def test_skip_mark_removal_does_not_qualify() -> None:
+    """Removing a skip decorator re-enables the test; must not suppress CI."""
+    diff = """\
+diff --git a/tests/foo/test_bar.py b/tests/foo/test_bar.py
+--- a/tests/foo/test_bar.py
++++ b/tests/foo/test_bar.py
+@@ -1,5 +1,4 @@
+ import pytest
+-
+-@pytest.mark.skip(reason="temp")
+ def test_x():
+     assert True
+"""
+    assert not _diff_only_contains_skip_mark_changes(diff)
+
+
+def test_skip_mark_reason_edit_still_qualifies() -> None:
+    diff = """\
+diff --git a/tests/foo/test_bar.py b/tests/foo/test_bar.py
+--- a/tests/foo/test_bar.py
++++ b/tests/foo/test_bar.py
+@@ -1,4 +1,4 @@
+ import pytest
+-@pytest.mark.skip(reason="old")
++@pytest.mark.skip(reason="new")
+ def test_x():
+"""
+    assert _diff_only_contains_skip_mark_changes(diff)
+
+
+def test_skip_mark_diff_rejects_assertion_change() -> None:
+    diff = """\
+diff --git a/tests/foo/test_bar.py b/tests/foo/test_bar.py
+--- a/tests/foo/test_bar.py
++++ b/tests/foo/test_bar.py
+@@ -1,3 +1,3 @@
+ def test_x():
+-    assert 1 == 1
++    assert 2 == 2
+"""
+    assert not _diff_only_contains_skip_mark_changes(diff)
+
+
+def test_cuda_ready_only_runs_cuda_l2() -> None:
+    decision = _decision([".buildkite/cuda/test-ready.yml"])
+    assert _is_yaml_gated(decision)
+    assert decision.is_run("cuda", "l2")
+    assert not decision.is_run("cuda", "l3")
+    assert _all_l23_skipped(decision) is False
+
+
+def test_cuda_merge_only_runs_cuda_l3() -> None:
+    decision = _decision([".buildkite/cuda/test-merge.yml"])
+    assert _is_yaml_gated(decision)
+    assert not decision.is_run("cuda", "l2")
+    assert decision.is_run("cuda", "l3")
+    assert not decision.is_run("amd", "l3")
+
+
+def test_l4_only_skips_all_l23() -> None:
+    decision = _decision([".buildkite/cuda/test-nightly.yml"])
+    assert _is_yaml_gated(decision)
+    assert _all_l23_skipped(decision)
+    assert not decision.is_run("cuda", "l2")
+    assert not decision.is_run("cuda", "l3")
+
+
+def test_weekly_only_skips_all_l23() -> None:
+    decision = _decision([".buildkite/cuda/test-weekly.yml"])
+    assert _is_yaml_gated(decision)
+    assert _all_l23_skipped(decision)
+    assert not decision.is_run("cuda", "l2")
+    assert not decision.is_run("cuda", "l3")
+
+
+def test_npu_nightly_only_skips_all_l23() -> None:
+    decision = _decision([".buildkite/npu/test-npu-nightly.yml"])
+    assert _is_yaml_gated(decision)
+    assert _all_l23_skipped(decision)
+    assert not decision.is_run("npu", "l2")
+    assert not decision.is_run("npu", "l3")
+
+
+def test_nightly_and_weekly_skips_all_l23() -> None:
+    decision = _decision(
+        [
+            ".buildkite/cuda/test-nightly.yml",
+            ".buildkite/cuda/test-weekly.yml",
+        ],
+    )
+    assert _is_yaml_gated(decision)
+    assert _all_l23_skipped(decision)
+    assert not decision.is_run("cuda", "l2")
+    assert not decision.is_run("cuda", "l3")
+
+
+def test_l2_and_l4_skips_l3() -> None:
+    files = [
+        ".buildkite/cuda/test-ready.yml",
+        ".buildkite/cuda/test-nightly.yml",
+    ]
+    decision = _decision(files)
+    assert _is_yaml_gated(decision)
+    assert decision.is_run("cuda", "l2")
+    assert not decision.is_run("cuda", "l3")
+
+
+def test_l3_and_l4_skips_l2() -> None:
+    files = [
+        ".buildkite/cuda/test-merge.yml",
+        ".buildkite/cuda/test-nightly.yml",
+    ]
+    decision = _decision(files)
+    assert _is_yaml_gated(decision)
+    assert not decision.is_run("cuda", "l2")
+    assert decision.is_run("cuda", "l3")
+
+
+def test_ready_and_merge_runs_cuda_l2_l3() -> None:
+    decision = _decision(
+        [
+            ".buildkite/cuda/test-ready.yml",
+            ".buildkite/cuda/test-merge.yml",
+        ],
+    )
+    assert _is_yaml_gated(decision)
+    assert decision.is_run("cuda", "l2")
+    assert decision.is_run("cuda", "l3")
+
+
+def test_cross_platform_cuda_l2_and_amd_l3_preserves_both() -> None:
+    """CUDA ready + AMD merge must keep cuda/l2 and amd/l3 (not wipe L2 outside L3 platforms)."""
+    decision = _decision(
+        [
+            ".buildkite/cuda/test-ready.yml",
+            ".buildkite/amd/test-amd-merge.yml",
+        ],
+    )
+    assert _is_yaml_gated(decision)
+    assert decision.is_run("cuda", "l2")
+    assert not decision.is_run("cuda", "l3")
+    assert not decision.is_run("amd", "l2")
+    assert decision.is_run("amd", "l3")
+    assert not decision.is_run("intel", "l2")
+    assert not decision.is_run("npu", "l2")
+    assert not decision.device.skip_cuda_l2
+    assert decision.device.skip_cuda_l3
+    assert decision.device.skip_amd_l2
+    assert not decision.device.skip_amd_l3
+
+
+def test_amd_ready_only_runs_amd_l2() -> None:
+    decision = _decision([".buildkite/amd/test-amd-ready.yml"])
+    assert _is_yaml_gated(decision)
+    assert decision.is_run("amd", "l2")
+    assert not decision.is_run("cuda", "l2")
+
+
+def test_npu_ready_only_runs_npu_l2() -> None:
+    decision = _decision([".buildkite/npu/test-npu-ready.yml"])
+    assert _is_yaml_gated(decision)
+    assert decision.is_run("npu", "l2")
+    assert not decision.is_run("cuda", "l2")
+    assert not decision.is_run("npu", "l3")
+
+
+def test_amd_merge_only_runs_amd_l3() -> None:
+    decision = _decision([".buildkite/amd/test-amd-merge.yml"])
+    assert _is_yaml_gated(decision)
+    assert decision.is_run("amd", "l3")
+    assert not decision.is_run("amd", "l2")
+    assert not decision.is_run("cuda", "l3")
+
+
+def test_intel_pipeline_runs_intel_l2() -> None:
+    decision = _decision([".buildkite/intel/pipeline-intel.yml"])
+    assert _is_yaml_gated(decision)
+    assert decision.is_run("intel", "l2")
+    assert not decision.is_run("cuda", "l2")
+
+
+def test_cuda_and_amd_ready_runs_both_l2() -> None:
+    decision = _decision(
+        [
+            ".buildkite/cuda/test-ready.yml",
+            ".buildkite/amd/test-amd-ready.yml",
+        ],
+    )
+    assert _is_yaml_gated(decision)
+    assert decision.is_run("cuda", "l2")
+    assert decision.is_run("amd", "l2")
+    assert not decision.is_run("cuda", "l3")
+    assert not decision.is_run("amd", "l3")
+    assert not decision.is_run("intel", "l2")
+    assert not decision.is_run("npu", "l2")
+
+
+def test_cuda_and_amd_merge_runs_both_l3() -> None:
+    decision = _decision(
+        [
+            ".buildkite/cuda/test-merge.yml",
+            ".buildkite/amd/test-amd-merge.yml",
+        ],
+    )
+    assert _is_yaml_gated(decision)
+    assert decision.is_run("cuda", "l3")
+    assert decision.is_run("amd", "l3")
+    assert not decision.is_run("cuda", "l2")
+    assert not decision.is_run("amd", "l2")
+
+
+def test_ready_merge_and_nightly_runs_cuda_l2_l3() -> None:
+    """G7: L2+L3+L45 → same matrix as ready+merge (L45 rescued by L2/L3 YAML)."""
+    decision = _decision(
+        [
+            ".buildkite/cuda/test-ready.yml",
+            ".buildkite/cuda/test-merge.yml",
+            ".buildkite/cuda/test-nightly.yml",
+        ],
+    )
+    assert _is_yaml_gated(decision)
+    assert decision.is_run("cuda", "l2")
+    assert decision.is_run("cuda", "l3")
+    assert not decision.is_run("amd", "l2")
+    assert not decision.is_run("amd", "l3")
+
+
+def test_non_whitelist_buildkite_runs_normal_ci() -> None:
+    assert not _is_yaml_gated(_decision([".buildkite/common/scripts/skip_ci.py"]))
+    assert not _is_yaml_gated(_decision([".buildkite/cuda/pipeline.yml"]))
+
+
+def test_whitelist_plus_non_whitelist_buildkite_runs_normal_ci() -> None:
+    assert not _is_yaml_gated(
+        _decision(
+            [
+                ".buildkite/cuda/test-ready.yml",
+                ".buildkite/cuda/pipeline.yml",
+            ],
+        ),
+    )
+
+
+def test_mixed_product_code_disables_targeting() -> None:
+    assert not _is_yaml_gated(
+        _decision([".buildkite/cuda/test-ready.yml", "vllm_omni/x.py"]),
+    )
+
+
+def test_docs_mixed_with_ci_yaml_follows_yaml_gating() -> None:
+    decision = _decision(["docs/a.md", ".buildkite/cuda/test-ready.yml"])
+    assert _is_yaml_gated(decision)
+    assert not decision.skip_all
+    assert decision.is_run("cuda", "l2")
+    assert not decision.is_run("cuda", "l3")
+    assert not decision.is_run("amd", "l2")
+    assert "follow CI YAML gating" in decision.message
+
+
+def test_docs_mixed_with_l45_yaml_skips_all_l23() -> None:
+    decision = _decision(["docs/a.md", ".buildkite/cuda/test-nightly.yml"])
+    assert _is_yaml_gated(decision)
+    assert not decision.skip_all
+    assert not decision.is_run("cuda", "l2")
+    assert not decision.is_run("cuda", "l3")
+    assert not decision.is_run("amd", "l2")
+
+
+def test_non_whitelist_path_disables_targeting() -> None:
+    assert not _is_yaml_gated(_decision([".buildkite/unknown/foo.yml"]))
+    assert not _is_yaml_gated(_decision(["vllm_omni/x.py"]))
+
+
+def test_is_run_l23_when_not_yaml_gated() -> None:
+    decision = _decision(["vllm_omni/x.py"])
+    assert decision.is_run("cuda", "l2")
+    assert decision.is_run("amd", "l3")
+
+
+def test_is_run_l23_respects_decision() -> None:
+    decision = _decision([".buildkite/cuda/test-ready.yml"])
+    assert _is_yaml_gated(decision)
+    assert decision.is_run("cuda", "l2")
+    assert not decision.is_run("cuda", "l3")
+    assert not decision.is_run("amd", "l2")
+
+
+def test_docs_or_skip_mark_only_without_diff_range_helper() -> None:
+    assert _decision(["docs/a.md"]).skip_all
+    assert not _decision(["vllm_omni/a.py"]).skip_all
+
+
+def test_decision_serializes_to_dict() -> None:
+    decision = _decision([".buildkite/cuda/test-ready.yml"])
+    assert _is_yaml_gated(decision)
+    payload = asdict(decision)
+    assert payload["skip_l2_l3"] is True
+    assert payload["device"]["skip_cuda_l2"] is False
+    assert payload["device"]["skip_cuda_l3"] is True
+
+
+def test_yaml_gated_message_lists_changed_and_targets() -> None:
+    decision = _decision([".buildkite/cuda/test-ready.yml"])
+    assert decision.message.startswith("only CI config YAML changed")
+    assert "changed: L2=[.buildkite/cuda/test-ready.yml]" in decision.message
+    assert "run: cuda/l2" in decision.message
+    assert "skip: cuda/l3" in decision.message
+    assert "amd/l2" in decision.message

tests/buildkite/test_upload_pipeline.py

@@ -0,0 +1,141 @@
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2] / ".buildkite" / "common" / "scripts"))
+
+from skip_ci import resolve_ci_decision  # noqa: E402
+from upload_pipeline import (  # noqa: E402
+    _expand_mirror_hardwares,
+    _load_bootstrap_steps,
+    _render_bootstrap_pipeline,
+    _render_test_pipeline,
+)
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+CUDA_BOOTSTRAP_STEPS = Path(".buildkite/cuda/bootstrap-upload-steps.yml")
+BOOTSTRAP_STEPS_TEMPLATE = """steps:
+  - key: image-build
+  - key: upload-ready-pipeline
+  - key: upload-merge-pipeline
+  - key: upload-nightly-pipeline
+  - key: upload-weekly-pipeline
+"""
+
+
+def _render(changed_files: list[str]) -> str:
+    decision = resolve_ci_decision(changed_files)
+    return _render_bootstrap_pipeline(
+        BOOTSTRAP_STEPS_TEMPLATE,
+        decision=decision,
+        path=CUDA_BOOTSTRAP_STEPS,
+    )
+
+
+def test_bootstrap_if_injected_by_step_key() -> None:
+    import yaml
+
+    rendered = _render_bootstrap_pipeline(
+        BOOTSTRAP_STEPS_TEMPLATE,
+        decision=resolve_ci_decision([]),
+        path=Path(".buildkite/npu/bootstrap-upload-steps.yml"),
+    )
+    doc = yaml.safe_load(rendered)
+    by_key = {step["key"]: step for step in doc["steps"]}
+    assert "image-build" in by_key
+    # Unconditional image has no ``if`` (Buildkite rejects YAML bool if: true).
+    assert "if" not in by_key["image-build"]
+    assert isinstance(by_key["upload-ready-pipeline"]["if"], str)
+    assert isinstance(by_key["upload-nightly-pipeline"]["if"], str)
+
+
+def test_bootstrap_steps_loaded_from_file() -> None:
+    steps = _load_bootstrap_steps(CUDA_BOOTSTRAP_STEPS)
+    assert "key: image-build" in steps
+    assert "key: upload-ready-pipeline" in steps
+    assert "placeholder:" not in steps
+
+
+def test_docs_only_allows_main_scheduled_nightly_weekly_only() -> None:
+    """skip_all: no PR labels; main + NIGHTLY=1 / WEEKLY=1 still gates scheduled CI."""
+    rendered = _render(["docs/foo.md"])
+    assert "key: image-build" in rendered
+    assert "key: upload-nightly-pipeline" in rendered
+    assert "key: upload-weekly-pipeline" in rendered
+    # Scheduled nightly also uploads L2/L3 with --e2e
+    assert "key: upload-ready-pipeline" in rendered
+    assert "key: upload-merge-pipeline" in rendered
+    assert 'build.env("NIGHTLY") == "1"' in rendered
+    assert 'build.env("WEEKLY") == "1"' in rendered
+    assert "nightly-test" not in rendered
+    assert "weekly-test" not in rendered
+    assert "merge-test" not in rendered
+    assert 'labels includes "ready"' not in rendered
+    assert "if: false" not in rendered
+
+
+def test_yaml_gated_l45_only_does_not_unconditionally_build_image() -> None:
+    rendered = _render([".buildkite/cuda/test-nightly.yml"])
+    assert "if: true" not in rendered
+    assert 'build.pull_request.labels includes "nightly-test"' in rendered
+    assert 'build.pull_request.labels includes "weekly-test"' in rendered
+    # L2/L3 upload steps are unconditionally disabled → omitted from pipeline
+    assert "key: upload-ready-pipeline" not in rendered
+    assert "key: upload-merge-pipeline" not in rendered
+    assert "key: upload-weekly-pipeline" in rendered
+
+
+def test_yaml_gated_l2_still_enables_image_via_ready_base() -> None:
+    rendered = _render([".buildkite/cuda/test-ready.yml"])
+    assert 'build.pull_request.labels includes "ready"' in rendered
+    assert "if: true" not in rendered
+
+
+def test_mirror_hardwares_l4_1_expands_to_agents_and_plugins() -> None:
+    doc = {
+        "steps": [
+            {
+                "label": "Simple Test",
+                "mirror_hardwares": "l4_1",
+                "commands": ["pytest -sv tests/example"],
+            },
+        ],
+    }
+    rendered = _render_test_pipeline(doc, changed_files=None)
+    step = rendered["steps"][0]
+    assert "mirror_hardwares" not in step
+    assert step["agents"]["queue"] == "gpu_1_queue"
+    assert step["plugins"][0]["docker#v5.2.0"]["image"].endswith("$BUILDKITE_COMMIT")
+
+
+def test_mirror_hardwares_conflicts_with_explicit_agents() -> None:
+    with pytest.raises(ValueError, match="agents/plugins/image"):
+        _expand_mirror_hardwares(
+            {"label": "bad", "mirror_hardwares": "l4_1", "agents": {"queue": "gpu_1_queue"}},
+        )
+
+
+def test_mirror_hardwares_a2b3_npu_4_expands_agents_image_and_plugins() -> None:
+    doc = {
+        "steps": [
+            {
+                "label": "NPU X2V Test",
+                "mirror_hardwares": "a2b3_npu_4",
+                "commands": ["pytest -sv tests/example"],
+            },
+        ],
+    }
+    rendered = _render_test_pipeline(doc, changed_files=None)
+    step = rendered["steps"][0]
+    assert "mirror_hardwares" not in step
+    assert step["agents"]["queue"] == "ascend-a2b3"
+    assert step["agents"]["resource_class"] == "npu-4"
+    assert step["image"].endswith("${BUILDKITE_COMMIT}")
+    assert step["plugins"][0]["kubernetes"]["podSpecPatch"]["imagePullSecrets"] == [
+        {"name": "swr-secret"},
+    ]