Revert "[bugfix][MiniCPM-o] Fix offline_inference/test_minicpmo_4_5.py and on…"

This reverts commit 6d84409a68.

.buildkite/amd/scripts/bootstrap-amd-omni.sh

@@ -1,4 +1,7 @@
 #!/bin/bash
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
 # vllm-omni customized version
 # Based on: https://github.com/vllm-project/ci-infra/blob/main/buildkite/bootstrap-amd.sh
 # Last synced: 2025-12-15
@@ -6,6 +9,11 @@
 
 set -euo pipefail
 
+# The bootstrap runs from the repository root; ShellCheck does not follow
+# sourced files unless invoked with -x.
+# shellcheck disable=SC1091
+source .buildkite/common/scripts/resolve_skip_ci.sh
+
 if [[ -z "${RUN_ALL:-}" ]]; then
     RUN_ALL=0
 fi
@@ -22,10 +30,6 @@ if [[ -z "${AMD_MIRROR_HW:-}" ]]; then
     AMD_MIRROR_HW="amdproduction"
 fi
 
-if [[ -z "${DOCS_ONLY_DISABLE:-}" ]]; then
-    DOCS_ONLY_DISABLE=0
-fi
-
 fail_fast() {
     DISABLE_LABEL="ci-no-fail-fast"
     # If BUILDKITE_PULL_REQUEST != "false", then we check the PR labels using curl and jq
@@ -56,88 +60,6 @@ check_run_all_label() {
     fi
 }
 
-is_docs_only_change() {
-    local file_path
-    local has_any=0
-
-    while IFS= read -r file_path; do
-        [[ -z "${file_path}" ]] && continue
-        has_any=1
-
-        if [[ "${file_path}" == docs/* ]]; then
-            continue
-        fi
-        if [[ "${file_path}" == *.md ]]; then
-            continue
-        fi
-        if [[ "${file_path}" == "mkdocs.yml" ]]; then
-            continue
-        fi
-        return 1
-    done
-
-    [[ "${has_any}" -eq 1 ]]
-}
-
-resolve_skip_ci() {
-    local is_pr_build=0
-    local files
-    local base_branch base_ref
-
-    if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" && -n "${BUILDKITE_PULL_REQUEST:-}" ]]; then
-        is_pr_build=1
-    fi
-
-    if [[ "${is_pr_build}" -eq 1 ]]; then
-        base_branch="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}"
-        if ! git rev-parse --verify "origin/${base_branch}" >/dev/null 2>&1; then
-            echo "resolve_skip_ci: origin/${base_branch} not found locally; trying fetch" >&2
-            git fetch --depth=200 origin "${base_branch}" >/dev/null 2>&1 || true
-        fi
-
-        base_ref=""
-        if git rev-parse --verify "origin/${base_branch}" >/dev/null 2>&1; then
-            base_ref="origin/${base_branch}"
-        elif git rev-parse --verify "${base_branch}" >/dev/null 2>&1; then
-            base_ref="${base_branch}"
-        else
-            echo "resolve_skip_ci: cannot resolve PR base ${base_branch}; skip-ci=0" >&2
-            echo -n 0
-            return 0
-        fi
-
-        if ! files="$(git diff --name-only "${base_ref}...${BUILDKITE_COMMIT}" 2>/dev/null)"; then
-            echo "resolve_skip_ci: failed to compute PR changed files; skip-ci=0" >&2
-            echo -n 0
-            return 0
-        fi
-    elif [[ "${BUILDKITE_BRANCH:-}" == "main" ]]; then
-        if ! git rev-parse --verify "${BUILDKITE_COMMIT}^" >/dev/null 2>&1; then
-            echo "resolve_skip_ci: commit has no parent on main; skip-ci=0" >&2
-            echo -n 0
-            return 0
-        fi
-        if ! files="$(git diff --name-only "${BUILDKITE_COMMIT}^..${BUILDKITE_COMMIT}" 2>/dev/null)"; then
-            echo "resolve_skip_ci: failed to compute main changed files; skip-ci=0" >&2
-            echo -n 0
-            return 0
-        fi
-    else
-        echo "resolve_skip_ci: not PR/main build; skip-ci=0" >&2
-        echo -n 0
-        return 0
-    fi
-
-    if is_docs_only_change <<< "${files}"; then
-        echo "resolve_skip_ci: docs-only change detected; skip-ci=1" >&2
-        echo -n 1
-        return 0
-    fi
-
-    echo "resolve_skip_ci: non-doc changes detected; skip-ci=0" >&2
-    echo -n 0
-}
-
 if [[ -z "${COV_ENABLED:-}" ]]; then
     COV_ENABLED=0
 fi
@@ -147,6 +69,8 @@ upload_pipeline() {
     # Install minijinja
     ls .buildkite || buildkite-agent annotate --style error 'Please merge upstream main branch for buildkite CI'
     curl -sSfL https://github.com/mitsuhiko/minijinja/releases/download/2.3.1/minijinja-cli-installer.sh | sh
+    # Installed by the minijinja bootstrap above and only present on the CI agent.
+    # shellcheck disable=SC1091
     source /var/lib/buildkite-agent/.cargo/env
 
     if [[ $BUILDKITE_PIPELINE_SLUG == "fastcheck" ]]; then
@@ -154,14 +78,14 @@ upload_pipeline() {
     fi
 
     # Use local template file for vllm-omni
-    cp .buildkite/test-template-amd-omni.j2 .buildkite/test-template.j2
+    cp .buildkite/amd/test-template-amd-omni.j2 .buildkite/amd/test-template.j2
 
 
     # (WIP) Use pipeline generator instead of jinja template
-    if [ -e ".buildkite/pipeline_generator/pipeline_generator.py" ]; then
+    if [ -e ".buildkite/amd/pipeline_generator/pipeline_generator.py" ]; then
         python -m pip install click pydantic
-        python .buildkite/pipeline_generator/pipeline_generator.py --run_all=$RUN_ALL --list_file_diff="$LIST_FILE_DIFF" --nightly="$NIGHTLY" --mirror_hw="$AMD_MIRROR_HW"
-        buildkite-agent pipeline upload .buildkite/pipeline.yaml
+        python .buildkite/amd/pipeline_generator/pipeline_generator.py --run_all=$RUN_ALL --list_file_diff="$LIST_FILE_DIFF" --nightly="$NIGHTLY" --mirror_hw="$AMD_MIRROR_HW"
+        buildkite-agent pipeline upload .buildkite/amd/pipeline.yaml
         exit 0
     fi
     echo "List file diff: $LIST_FILE_DIFF"
@@ -171,29 +95,89 @@ upload_pipeline() {
 
     FAIL_FAST=$(fail_fast)
 
-    cd .buildkite
+    cd .buildkite/amd
 
     # Select test definition file: merge suite for main, ready suite for PRs.
-    # For debugging, DEBUG_TEST_YAML can override the selection — accepts
-    # "merge" or "ready" (case-insensitive).
+    # For debugging, DEBUG_TEST_YAML accepts a comma-separated list containing
+    # "merge" and/or "ready" (case-insensitive). Multiple suites are combined
+    # before rendering so they share one amd-build step.
     if [[ -n "${DEBUG_TEST_YAML:-}" ]]; then
-        case "${DEBUG_TEST_YAML,,}" in
-            merge)
-                TEST_YAML="test-amd-merge.yml"
-                ;;
-            ready)
-                TEST_YAML="test-amd-ready.yaml"
-                ;;
-            *)
-                echo "ERROR: DEBUG_TEST_YAML must be 'merge' or 'ready', got '$DEBUG_TEST_YAML'" >&2
+        declare -a DEBUG_TEST_SPECS=()
+        declare -A SEEN_DEBUG_TESTS=()
+        IFS=',' read -ra REQUESTED_DEBUG_TESTS <<< "${DEBUG_TEST_YAML,,}"
+
+        for requested_test in "${REQUESTED_DEBUG_TESTS[@]}"; do
+            # Permit readable values such as "ready, merge," and ignore the
+            # empty item produced by a trailing comma.
+            requested_test="${requested_test//[[:space:]]/}"
+            [[ -z "$requested_test" ]] && continue
+
+            if [[ -n "${SEEN_DEBUG_TESTS[$requested_test]:-}" ]]; then
+                echo "ERROR: duplicate DEBUG_TEST_YAML suite '$requested_test'" >&2
                 exit 1
-                ;;
-        esac
-        echo "DEBUG_TEST_YAML override: using $TEST_YAML"
+            fi
+            SEEN_DEBUG_TESTS[$requested_test]=1
+
+            case "$requested_test" in
+                ready)
+                    DEBUG_TEST_SPECS+=("READY_TESTS:test-amd-ready.yml")
+                    ;;
+                merge)
+                    DEBUG_TEST_SPECS+=("MERGE_TESTS:test-amd-merge.yml")
+                    ;;
+                *)
+                    echo "ERROR: DEBUG_TEST_YAML entries must be 'merge' or 'ready', got '$requested_test'" >&2
+                    exit 1
+                    ;;
+            esac
+        done
+
+        if [[ ${#DEBUG_TEST_SPECS[@]} -eq 0 ]]; then
+            echo "ERROR: DEBUG_TEST_YAML did not contain a test suite" >&2
+            exit 1
+        elif [[ ${#DEBUG_TEST_SPECS[@]} -eq 1 ]]; then
+            TEST_YAML="${DEBUG_TEST_SPECS[0]#*:}"
+        else
+            TEST_YAML=$(mktemp "${TMPDIR:-/tmp}/amd-debug-tests.XXXXXX.yml")
+            python - "$TEST_YAML" "${DEBUG_TEST_SPECS[@]}" <<'PY'
+import sys
+
+import yaml
+
+
+output_path, *suite_specs = sys.argv[1:]
+combined = {"env": {}, "steps": []}
+
+for suite_spec in suite_specs:
+    group_name, input_path = suite_spec.split(":", 1)
+    with open(input_path, encoding="utf-8") as test_file:
+        suite = yaml.safe_load(test_file)
+
+    for name, value in (suite.get("env") or {}).items():
+        previous = combined["env"].get(name, value)
+        if previous != value:
+            raise ValueError(
+                f"Conflicting environment value for {name}: {previous!r} != {value!r}"
+            )
+        combined["env"][name] = value
+
+    suite_steps = []
+    for entry in suite.get("steps") or []:
+        if "group" in entry:
+            suite_steps.extend(entry.get("steps") or [])
+        else:
+            suite_steps.append(entry)
+    combined["steps"].append({"group": group_name, "steps": suite_steps})
+
+with open(output_path, "w", encoding="utf-8") as output_file:
+    yaml.safe_dump(combined, output_file, sort_keys=False)
+PY
+        fi
+        echo "DEBUG_TEST_YAML override: using ${DEBUG_TEST_SPECS[*]}"
     elif [[ $BUILDKITE_BRANCH == "main" ]]; then
         TEST_YAML="test-amd-merge.yml"
     else
-        TEST_YAML="test-amd-ready.yaml"
+        TEST_YAML="test-amd-ready.yml"
     fi
 
     (
@@ -214,19 +198,20 @@ upload_pipeline() {
             > pipeline.yaml
     )
     cat pipeline.yaml
+    if [[ "$TEST_YAML" == "${TMPDIR:-/tmp}/amd-debug-tests."*.yml ]]; then
+        rm -f -- "$TEST_YAML"
+    fi
     buildkite-agent artifact upload pipeline.yaml
     buildkite-agent pipeline upload pipeline.yaml
     exit 0
 }
 
 get_diff() {
-    $(git add .)
-    echo $(git diff --name-only --diff-filter=ACMDR $(git merge-base origin/main HEAD))
+    git diff --name-only --diff-filter=ACMDR "$(git merge-base origin/main HEAD)"
 }
 
 get_diff_main() {
-    $(git add .)
-    echo $(git diff --name-only --diff-filter=ACMDR HEAD~1)
+    git diff --name-only --diff-filter=ACMDR HEAD~1
 }
 
 file_diff=$(get_diff)
@@ -234,23 +219,13 @@ if [[ $BUILDKITE_BRANCH == "main" ]]; then
     file_diff=$(get_diff_main)
 fi
 
-# ----------------------------------------------------------------------
-# Early exit start: skip pipeline if conditions are met
-# ----------------------------------------------------------------------
-
-# Match CUDA Buildkite skip-ci behavior for docs/markdown-only changes.
-if [[ "${DOCS_ONLY_DISABLE}" != "1" ]]; then
-    SKIP_CI="$(resolve_skip_ci)"
-    if [[ "${SKIP_CI}" == "1" ]]; then
-        echo "[docs-only] Docs/markdown-only changes detected. Exiting before pipeline upload."
-        exit 0
-    fi
+# Early exit: unified skip-ci (docs / skip marks) and CI-yaml-only level targeting.
+if [[ $BUILDKITE_BRANCH == "main" ]]; then
+    gate_bootstrap_ci amd l3
+else
+    gate_bootstrap_ci amd l2
 fi
 
-# ----------------------------------------------------------------------
-# Early exit end
-# ----------------------------------------------------------------------
-
 patterns=(
     "docker/Dockerfile"
     "CMakeLists.txt"
@@ -275,7 +250,7 @@ for file in $file_diff; do
     # First check if file matches any pattern
     matches_pattern=0
     for pattern in "${patterns[@]}"; do
-        if [[ $file == $pattern* ]] || [[ $file == $pattern ]]; then
+        if [[ $file == "$pattern"* ]] || [[ $file == "$pattern" ]]; then
             matches_pattern=1
             break
         fi
@@ -285,7 +260,7 @@ for file in $file_diff; do
     if [[ $matches_pattern -eq 1 ]]; then
         matches_ignore=0
         for ignore in "${ignore_patterns[@]}"; do
-            if [[ $file == $ignore* ]] || [[ $file == $ignore ]]; then
+            if [[ $file == "$ignore"* ]] || [[ $file == "$ignore" ]]; then
                 matches_ignore=1
                 break
             fi

.buildkite/amd/scripts/run-amd-test.sh

@@ -0,0 +1,103 @@
+#!/bin/bash
+
+# Run vLLM-Omni ROCm tests directly in the MI300 Kubernetes pod. The pod's
+# container image is selected by test-template-amd-omni.j2; MI300 has no DinD.
+set -euo pipefail
+
+: "${PYTHONFAULTHANDLER:=1}"
+: "${HF_HOME:=/home/buildkite-agent/huggingface}"
+: "${HF_HUB_DOWNLOAD_TIMEOUT:=300}"
+: "${HF_HUB_ETAG_TIMEOUT:=60}"
+: "${MIOPEN_DEBUG_CONV_DIRECT:=0}"
+: "${MIOPEN_DEBUG_CONV_GEMM:=0}"
+: "${VLLM_ROCM_USE_AITER:=0}"
+export PYTHONFAULTHANDLER HF_HOME HF_HUB_DOWNLOAD_TIMEOUT HF_HUB_ETAG_TIMEOUT
+export MIOPEN_DEBUG_CONV_DIRECT MIOPEN_DEBUG_CONV_GEMM VLLM_ROCM_USE_AITER
+export PYTORCH_ROCM_ARCH=""
+export PYTHONPATH="${PYTHONPATH:-..}"
+
+if [[ "${VLLM_CI_DOCKER_DISABLED:-0}" != "1" ]]; then
+    echo "Error: MI300 CI must run natively with Docker disabled." >&2
+    exit 1
+fi
+
+if [[ -n "${TEST_COMMANDS:-}" ]]; then
+    commands="${TEST_COMMANDS}"
+else
+    commands="$*"
+fi
+if [[ -z "${commands}" ]]; then
+    echo "Error: No test commands provided." >&2
+    exit 1
+fi
+
+echo "--- Native in-pod ROCm CI"
+
+job_id="${BUILDKITE_JOB_ID:-${BUILDKITE_PARALLEL_JOB:-local}}"
+job_id="${job_id//[^A-Za-z0-9_.-]/_}"
+native_cache_root="/tmp/vllm-omni-native-${job_id}"
+TMPDIR="${native_cache_root}/tmp"
+TORCHINDUCTOR_CACHE_DIR="${native_cache_root}/torchinductor"
+TRITON_CACHE_DIR="${native_cache_root}/triton"
+VLLM_CACHE_ROOT="${native_cache_root}/vllm"
+XDG_CACHE_HOME="${native_cache_root}/xdg"
+HF_DATASETS_CACHE="${native_cache_root}/huggingface/datasets"
+export TMPDIR TORCHINDUCTOR_CACHE_DIR TRITON_CACHE_DIR VLLM_CACHE_ROOT
+export XDG_CACHE_HOME HF_DATASETS_CACHE
+mkdir -p "${TMPDIR}" "${TORCHINDUCTOR_CACHE_DIR}" "${TRITON_CACHE_DIR}" \
+    "${VLLM_CACHE_ROOT}" "${XDG_CACHE_HOME}" "${HF_HOME}" \
+    "${HF_DATASETS_CACHE}"
+
+if [[ "${VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE:-0}" == "1" ]]; then
+    if ! command -v findmnt >/dev/null 2>&1; then
+        echo "Error: findmnt is required to verify the Hugging Face cache mount." >&2
+        exit 1
+    fi
+    hf_mount=$(findmnt -n -T "${HF_HOME}" -o TARGET 2>/dev/null || true)
+    if [[ -z "${hf_mount}" || "${hf_mount}" == "/" ]]; then
+        echo "Error: MI300 CI requires a persistent volume at or above ${HF_HOME}." >&2
+        exit 1
+    fi
+fi
+
+rocminfo
+
+expected_gpus="${VLLM_CI_EXPECTED_GPU_COUNT:-1}"
+python3 - <<'PY'
+import os
+
+import torch
+
+expected = int(os.environ.get("VLLM_CI_EXPECTED_GPU_COUNT", "1"))
+assert torch.version.hip, "PyTorch is not a ROCm build"
+assert torch.cuda.is_available(), "ROCm GPU is not available to PyTorch"
+actual = torch.cuda.device_count()
+assert actual == expected, f"Expected {expected} ROCm GPU(s), found {actual}"
+PY
+
+echo "Commands:${commands}"
+if /bin/bash -o pipefail -c '
+set -E
+test_status=0
+trap '\''
+    command_status=$?
+    # Keep running subsequent commands, but retain a failure for the final exit.
+    # Prefer a test failure over pytest "no tests collected" when both occur.
+    if (( test_status == 0 || (test_status == 5 && command_status != 5) )); then
+        test_status=${command_status}
+    fi
+'\'' ERR
+eval "$1"
+exit "${test_status}"
+' _ "${commands}"; then
+    exit 0
+else
+    exit_code=$?
+fi
+
+if [[ ${exit_code} -eq 5 && "${VLLM_CI_ALLOW_NO_TESTS:-0}" == "1" ]]; then
+    echo "Pytest collected no tests; VLLM_CI_ALLOW_NO_TESTS=1, treating exit code 5 as success."
+    exit 0
+fi
+
+exit "${exit_code}"

.buildkite/amd/test-amd-merge.yml

@@ -7,80 +7,106 @@ steps:
 
 - group: ":card_index_dividers: Simple Test"
   steps:
-    - label: "Simple · Diffusion & Model Executor Test"
-      agent_pool: mi325_1
+    - label: "Simple · Model Executor Test · Shard %N/%t"
+      agent_pool: mi300_1
+      parallelism: 2
       depends_on: amd-build
-      timeout_in_minutes: 50
       mirror_hardwares: [amdproduction]
       grade: Blocking
       commands:
         - export VLLM_ROCM_USE_AITER=0
         # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
         # that is not supported on CPU
-        - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+        - "pytest -sv tests/model_executor -m 'core_model and cpu' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+
+    - label: "Simple · Diffusion Test · Shard %N/%t"
+      agent_pool: mi300_1
+      parallelism: 4
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
+        # that is not supported on CPU
+        - "pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+
+    - label: "Simple · Engine&Entrypoints Test"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        - "pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu'"
 
     - label: "Simple · Other Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
       depends_on: amd-build
-      timeout_in_minutes: 50
       mirror_hardwares: [amdproduction]
       grade: Blocking
       commands:
         - export VLLM_ROCM_USE_AITER=0
         # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
         # that is not supported on CPU
-        - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+        - "pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine  --ignore=tests/diffusion/cache/test_teacache_extractors.py"
 
 - group: ":card_index_dividers: Diffusion Test"
   steps:
     - label: "Diffusion · LoRA Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
+      artifact_paths:
+        - lora-server.log
       commands:
-        - timeout 20m pytest -s -v tests/diffusion/lora/test_images_generations_lora.py
+        - pytest -sv tests/diffusion/lora/ -m 'advanced_model and cuda' --run-level "advanced_model" 2>&1 | tee lora-server.log
 
     - label: "Diffusion Model CPU offloading Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
       commands:
         - export VLLM_LOGGING_LEVEL=DEBUG
         - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-        - |
-          timeout 20m bash -c '
-            set +e
-            pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
-            EXIT1=\$?
-            pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
-            EXIT2=\$?
-            exit \$((EXIT1 | EXIT2))
-          '
+        - pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
+        - pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
 
-    - label: "Diffusion Sequence Parallelism Test (Need 4 GPUs)"
-      agent_pool: mi325_4
+    - label: "Diffusion Sequence Parallelism Test (Need 4 GPUs) · Shard 1/2"
+      agent_pool: mi300_4
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
+      timeout_in_minutes: 40
       grade: Blocking
       commands:
         - export VLLM_LOGGING_LEVEL=DEBUG
         - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-        - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py
-        - timeout 20m pytest -s -v tests/diffusion/distributed/test_ulysses_uaa_perf.py
+        - pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py
+
+    - label: "Diffusion Sequence Parallelism Test (Need 4 GPUs) · Shard 2/2"
+      agent_pool: mi300_4
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      timeout_in_minutes: 40
+      grade: Blocking
+      commands:
+        - export VLLM_LOGGING_LEVEL=DEBUG
+        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+        - pytest -s -v tests/diffusion/distributed/test_ulysses_uaa_perf.py
 
     # merge-only tests
     - label: "Diffusion Tensor Parallelism Test"
-      agent_pool: mi325_2
+      agent_pool: mi300_2
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
       commands:
-        - pytest -s -v tests/diffusion/distributed/test_zimage_parallelism.py
+        - pytest -s -v tests/diffusion/distributed/test_tensor_parallel.py
 
     - label: "Diffusion Model Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
@@ -88,7 +114,7 @@ steps:
         - timeout 30m pytest -s -v tests/e2e/offline_inference/test_z_image.py
 
     - label: "Diffusion GPU Worker Test"
-      agent_pool: mi325_2
+      agent_pool: mi300_2
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
@@ -96,7 +122,7 @@ steps:
         - timeout 20m pytest -s -v tests/diffusion/test_diffusion_worker.py
 
     - label: "Diffusion · Wan22 Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
@@ -109,7 +135,7 @@ steps:
 
 ## ISSUE depends on `diffusers` package: https://github.com/huggingface/diffusers/issues/13274
 # - label: "Audio Generation Model Test"
-#   agent_pool: mi325_1
+#   agent_pool: mi300_1
 #   depends_on: amd-build
 #   mirror_hardwares: [amdproduction]
 #   grade: Blocking
@@ -120,7 +146,7 @@ steps:
 #     - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_expansion.py -m "advanced_model and diffusion and L4" --run-level advanced_model
 
 - label: "Engine Test"
-  agent_pool: mi325_1
+  agent_pool: mi300_1
   depends_on: amd-build
   mirror_hardwares: [amdproduction]
   grade: Blocking
@@ -135,7 +161,7 @@ steps:
 - group: ":card_index_dividers: E2E Test"
   steps:
     - label: "Omni Model Test Qwen2-5-Omni"
-      agent_pool: mi325_4
+      agent_pool: mi300_4
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       commands:
@@ -143,7 +169,7 @@ steps:
         - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model"
 
     - label: "Omni Model Test Qwen3-Omni"
-      agent_pool: mi325_2
+      agent_pool: mi300_2
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
@@ -151,52 +177,65 @@ steps:
         - export VLLM_WORKER_MULTIPROC_METHOD=spawn
         - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model'
 
-    - label: "Qwen3-TTS CustomVoice E2E Test"
-      agent_pool: mi325_1
+    - label: "Qwen3-TTS CustomVoice E2E Test · Shard %N/%t"
+      agent_pool: mi300_1
+      parallelism: 3
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
+      timeout_in_minutes: 80
       commands:
         - |
-          timeout 20m bash -c '
+          timeout --signal=TERM --kill-after=2m 80m bash -c '
             export VLLM_LOGGING_LEVEL=DEBUG
             export VLLM_WORKER_MULTIPROC_METHOD=spawn
             export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-            pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m "advanced_model" --run-level "advanced_model"
+            TESTS=(
+              "tests/e2e/online_serving/test_qwen3_tts_customvoice.py::test_default_cuda_graph_startup[async_chunk]"
+              "tests/e2e/online_serving/test_qwen3_tts_customvoice.py::test_text_to_audio_002[async_chunk]"
+              "tests/e2e/offline_inference/test_qwen3_tts_customvoice.py::test_text_to_audio_001[no_cuda_graph]"
+            )
+            TEST="$${TESTS[$$BUILDKITE_PARALLEL_JOB]}"
+            pytest -s -v "$$TEST" -m "advanced_model" --run-level "advanced_model"
+          '
+
+    # TODO: Restore this case to the blocking sharded step once its long-running
+    # Whisper validation on AMD CI is resolved.
+    - label: "Qwen3-TTS CustomVoice async_chunk Test (quarantined)"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: NonBlocking
+      timeout_in_minutes: 80
+      commands:
+        - |
+          timeout --signal=TERM --kill-after=2m 75m bash -c '
+            export VLLM_LOGGING_LEVEL=DEBUG
+            export VLLM_WORKER_MULTIPROC_METHOD=spawn
+            export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+            pytest -s -v "tests/e2e/online_serving/test_qwen3_tts_customvoice.py::test_text_to_audio_001[async_chunk]" -m "advanced_model" --run-level "advanced_model"
           '
 
     - label: "Qwen3-TTS Base E2E Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
+      timeout_in_minutes: 50
       commands:
         - |
-          timeout 30m bash -c '
+          timeout 40m bash -c '
             export VLLM_LOGGING_LEVEL=DEBUG
             export VLLM_WORKER_MULTIPROC_METHOD=spawn
             export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-            pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m "advanced_model" --run-level "advanced_model"
-          '
-
-    - label: "Voxtral-TTS E2E Test"
-      agent_pool: mi325_1
-      depends_on: amd-build
-      mirror_hardwares: [amdproduction]
-      grade: Blocking
-      commands:
-        - |
-          timeout 20m bash -c '
-            export VLLM_LOGGING_LEVEL=DEBUG
-            export VLLM_WORKER_MULTIPROC_METHOD=spawn
-            pytest -s -v tests/e2e/online_serving/test_voxtral_tts.py tests/e2e/offline_inference/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model"
+            pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model'
           '
 
 # TODO: Bagel test on ROCm is very unstable. @tjtanaa
 # Need to debug before reneable numerical changes across large PRs
 # # split Bagel Model Test with H100 (Real Weights) into three tests
 # - label: "Bagel Text2Img Model Test (1/3)"
-#   agent_pool: mi325_1
+#   agent_pool: mi300_1
 #   depends_on: amd-build
 #   mirror_hardwares: [amdproduction]
 #   grade: Blocking
@@ -208,7 +247,7 @@ steps:
 #     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "advanced_model" --run-level "advanced_model" -k "shared_memory" -k "rocm"
 
 # - label: "Bagel Img2Img Model Test (2/3)"
-#   agent_pool: mi325_1
+#   agent_pool: mi300_1
 #   depends_on: amd-build
 #   mirror_hardwares: [amdproduction]
 #   grade: Blocking
@@ -220,7 +259,7 @@ steps:
 #     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "advanced_model" --run-level "advanced_model" -k "rocm"
 
 # - label: "Bagel Online Serving Test (3/3)"
-#   agent_pool: mi325_1
+#   agent_pool: mi300_1
 #   depends_on: amd-build
 #   mirror_hardwares: [amdproduction]
 #   grade: Blocking

.buildkite/amd/test-amd-ready.yml

@@ -7,8 +7,9 @@ steps:
 
 - group: ":card_index_dividers: Simple Test"
   steps:
-    - label: "Simple · Diffusion & Model Executor Test"
-      agent_pool: mi325_1
+    - label: "Simple · Model Executor Test · Shard %N/%t"
+      agent_pool: mi300_1
+      parallelism: 2
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
@@ -16,10 +17,30 @@ steps:
         - export VLLM_ROCM_USE_AITER=0
         # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
         # that is not supported on CPU
-        - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+        - "pytest -sv tests/model_executor -m 'core_model and cpu' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+
+    - label: "Simple · Diffusion Test"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
+        # that is not supported on CPU
+        - "pytest -sv tests/diffusion -m 'core_model and cpu'  --ignore=tests/diffusion/cache/test_teacache_extractors.py"
+
+    - label: "Simple · Engine&Entrypoints Test"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        - "pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu'"
 
     - label: "Simple · Other Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
@@ -27,19 +48,19 @@ steps:
         - export VLLM_ROCM_USE_AITER=0
         # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
         # that is not supported on CPU
-        - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+        - "pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine  --ignore=tests/diffusion/cache/test_teacache_extractors.py"
 
 - label: "Custom Pipeline Test"
-  agent_pool: mi325_1
+  agent_pool: mi300_1
   depends_on: amd-build
   mirror_hardwares: [amdproduction]
   grade: Blocking
   commands:
-    - timeout 20m pytest -s -v tests/e2e/offline_inference/custom_pipeline/ -m "core_model"
+    - pytest -s -v tests/e2e/features/custom_pipeline/ -m "core_model"
 
 
 - label: "Voxtral TTS CUDA Unit Test"
-  agent_pool: mi325_1
+  agent_pool: mi300_1
   depends_on: amd-build
   mirror_hardwares: [amdproduction]
   grade: Blocking
@@ -49,33 +70,25 @@ steps:
 - group: ":card_index_dividers: Diffusion Test"
   steps:
     - label: "Diffusion · Batch Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
       commands:
-        - timeout 20m pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model"
+        - pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model"
 
     - label: "Diffusion · Offloader Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
       commands:
         - export VLLM_LOGGING_LEVEL=DEBUG
         - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-        - |
-          timeout 20m bash -c '
-            set +e
-            pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
-            EXIT1=\$?
-            pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
-            EXIT2=\$?
-            exit \$((EXIT1 | EXIT2))
-          '
+        - pytest -s -v tests/diffusion/offloader -m "core_model and cuda" --run-level "core_model"
 
     - label: "Diffusion · Cache Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
@@ -83,10 +96,10 @@ steps:
         - export GPU_ARCHS=gfx942
         - export VLLM_LOGGING_LEVEL=DEBUG
         - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-        - timeout 15m pytest -s -v -m "core_model and cache and diffusion and not distributed_cuda and L4"
+        - pytest -sv tests/diffusion/cache -m 'core_model and cuda' --run-level "core_model"
 
     - label: "Diffusion Sequence Parallelism Test"
-      agent_pool: mi325_2
+      agent_pool: mi300_2
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
@@ -96,7 +109,7 @@ steps:
         - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py -m core_model
 
     - label: "Diffusion GPU Worker Test"
-      agent_pool: mi325_2
+      agent_pool: mi300_2
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
@@ -104,7 +117,7 @@ steps:
         - timeout 20m pytest -s -v tests/diffusion/test_diffusion_worker.py
 
     - label: "Diffusion · Model Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
@@ -112,7 +125,7 @@ steps:
         - timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model"
 
     - label: "Diffusion · Wan22 Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
@@ -126,7 +139,7 @@ steps:
 
 ## ISSUE depends on `diffusers` package: https://github.com/huggingface/diffusers/issues/13274
 # - label: "Audio Generation Model Test"
-#   agent_pool: mi325_1
+#   agent_pool: mi300_1
 #   depends_on: amd-build
 #   mirror_hardwares: [amdproduction]
 #   grade: Blocking
@@ -137,7 +150,7 @@ steps:
 #     - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_expansion.py -m "advanced_model and diffusion and L4" --run-level advanced_model
 
 - label: "Engine Test"
-  agent_pool: mi325_1
+  agent_pool: mi300_1
   depends_on: amd-build
   mirror_hardwares: [amdproduction]
   grade: Blocking
@@ -149,7 +162,7 @@ steps:
       '
 
 # - label: "Omni Model Test Qwen3-Omni"
-#   agent_pool: mi325_2
+#   agent_pool: mi300_2
 #   depends_on: amd-build
 #   mirror_hardwares: [amdproduction]
 #   grade: Blocking
@@ -161,21 +174,8 @@ steps:
 
 - group: ":card_index_dividers: E2E Test"
   steps:
-    - label: "MiMo-Audio E2E Test with H100"
-      agent_pool: mi325_1
-      depends_on: amd-build
-      mirror_hardwares: [amdproduction]
-      grade: Blocking
-      commands:
-        - |
-          timeout 30m bash -c '
-            export VLLM_LOGGING_LEVEL=DEBUG
-            export VLLM_WORKER_MULTIPROC_METHOD=spawn
-            pytest -s -v tests/e2e/online_serving/test_mimo_audio.py -m "core_model" --run-level "core_model"
-          '
-
     - label: "Qwen3-TTS E2E Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
@@ -185,35 +185,18 @@ steps:
         - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
         - timeout 30m pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m "core_model" --run-level "core_model"
 
-    - label: "Voxtral-TTS E2E Test"
-      agent_pool: mi325_1
-      depends_on: amd-build
-      mirror_hardwares: [amdproduction]
-      grade: Blocking
-      commands:
-        - |
-          timeout 20m bash -c '
-            export VLLM_LOGGING_LEVEL=DEBUG
-            export VLLM_WORKER_MULTIPROC_METHOD=spawn
-            pytest -s -v tests/e2e/online_serving/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model"
-            pytest -s -v tests/e2e/offline_inference/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model"
-          '
-
     - label: "CosyVoice3-TTS E2E Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
       commands:
-        - |
-          timeout 20m bash -c '
-            pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "full_model" --run-level "core_model"
-          '
+        - pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "slow" --run-level "core_model"
 
 # TODO: Bagel test on ROCm is very unstable. @tjtanaa
 # Need to debug before reneable numerical changes across large PRs
 # - label: "Bagel Text2Img Model Test"
-#   agent_pool: mi325_1
+#   agent_pool: mi300_1
 #   depends_on: amd-build
 #   mirror_hardwares: [amdproduction]
 #   grade: Blocking
@@ -225,7 +208,7 @@ steps:
 #     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"
 
 # - label: "Bagel Img2Img Model Test"
-#   agent_pool: mi325_1
+#   agent_pool: mi300_1
 #   depends_on: amd-build
 #   mirror_hardwares: [amdproduction]
 #   grade: Blocking
@@ -237,7 +220,7 @@ steps:
 #     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"
 
 # - label: "Bagel Online Serving Test"
-#   agent_pool: mi325_1
+#   agent_pool: mi300_1
 #   depends_on: amd-build
 #   mirror_hardwares: [amdproduction]
 #   grade: Blocking

.buildkite/amd/test-template-amd-omni.j2

@@ -0,0 +1,237 @@
+{# vllm-omni customized version
+   Based on: https://github.com/vllm-project/ci-infra/blob/main/buildkite/test-template-amd.j2
+   Last synced: 2026-08-04
+   Modifications: Use the shared MI300 queues and Omni command-passing contract
+#}
+{% set docker_image_amd = "rocm/vllm-omni:$BUILDKITE_COMMIT" %}
+{% set default_working_dir = "/app/vllm-omni" %}
+
+{# MI300 runs tests natively in Kubernetes; Docker-in-Docker is unavailable. #}
+{% macro amd_native_k8s_plugin(indent, step) -%}
+{{ indent }}plugins:
+{{ indent }}  - kubernetes:
+{{ indent }}      podSpecPatch:
+{{ indent }}        automountServiceAccountToken: false
+{{ indent }}        securityContext:
+{{ indent }}          seccompProfile:
+{{ indent }}            type: RuntimeDefault
+{{ indent }}        imagePullSecrets:
+{{ indent }}          - name: docker-config
+{{ indent }}        containers:
+{{ indent }}          - name: container-0
+{{ indent }}            image: {{ docker_image_amd }}
+{{ indent }}            imagePullPolicy: Always
+{{ indent }}            securityContext:
+{{ indent }}              allowPrivilegeEscalation: false
+{{ indent }}              capabilities:
+{{ indent }}                add:
+{{ indent }}                  - IPC_LOCK
+{{ indent }}            resources:
+{{ indent }}              limits:
+{{ indent }}                amd.com/gpu: "{{ step.agent_pool[-1:] }}"
+{{ indent }}              requests:
+{{ indent }}                amd.com/gpu: "{{ step.agent_pool[-1:] }}"
+{{ indent }}            volumeMounts:
+{{ indent }}              - name: devshm
+{{ indent }}                mountPath: /dev/shm
+{{ indent }}            env:
+{{ indent }}              - name: AMD_CI_RUNTIME
+{{ indent }}                value: native
+{{ indent }}              - name: NATIVE_CI
+{{ indent }}                value: "true"
+{{ indent }}              - name: VLLM_CI_DOCKER_DISABLED
+{{ indent }}                value: "1"
+{{ indent }}              - name: VLLM_CI_EXPECTED_GPU_COUNT
+{{ indent }}                value: "{{ step.agent_pool[-1:] }}"
+{{ indent }}              - name: HF_HOME
+{{ indent }}                value: /home/buildkite-agent/huggingface
+{{ indent }}              - name: MIOPEN_DEBUG_CONV_DIRECT
+{{ indent }}                value: "0"
+{{ indent }}              - name: MIOPEN_DEBUG_CONV_GEMM
+{{ indent }}                value: "0"
+{{ indent }}              - name: VLLM_ROCM_USE_AITER
+{{ indent }}                value: "0"
+{{ indent }}              - name: PYTORCH_ROCM_ARCH
+{{ indent }}                value: ""
+{{ indent }}        volumes:
+{{ indent }}          - name: devshm
+{{ indent }}            emptyDir:
+{{ indent }}              medium: Memory
+{{ indent }}              sizeLimit: 16Gi
+{%- endmacro %}
+
+{% if env %}
+env:
+{% for name in env %}
+  {{ name }}: "{{ env[name] }}"
+{% endfor %}
+{% endif %}
+steps:
+  - group: "AMD Tests"
+    depends_on: ~
+    steps:
+      - label: "AMD: :docker: build image"
+        depends_on: ~
+        soft_fail: false
+        commands:
+          - "docker build -f docker/Dockerfile.rocm -t {{ docker_image_amd }} --target test --build-arg ARG_PYTORCH_ROCM_ARCH=gfx942 --progress plain ."
+          - "docker push {{ docker_image_amd }}"
+        key: "amd-build"
+        env:
+          DOCKER_BUILDKIT: "1"
+        retry:
+          automatic:
+            - exit_status: -1  # Agent was lost
+              limit: 2
+            - exit_status: -10  # Agent was lost
+              limit: 2
+            - exit_status: 128  # Git connectivity issues
+              limit: 2
+            - exit_status: 1  # Machine occasionally fail
+              limit: 1
+        agents:
+          queue: amd-cpu
+
+{% for step_group in steps %}
+{% if step_group.group %}
+  - group: "{{ step_group.group }}"
+{% if step_group.depends_on %}
+    depends_on: {{ step_group.depends_on }}
+{% endif %}
+    steps:
+{% for step in step_group.steps %}
+{% if step.mirror_hardwares and mirror_hw in step.mirror_hardwares %}
+      - label: "{{ step.agent_pool }}: {{ step.label }}"
+        depends_on: amd-build
+        agents:
+          {% if step.agent_pool %}
+          queue: amd_{{ step.agent_pool }}
+          {% else %}
+          queue: amd_mi300_1
+          {% endif %}
+{{ amd_native_k8s_plugin("        ", step) }}
+{% set cmd_body = (step.command or (step.commands | join("\n"))) | trim %}
+{% set indented_cmd = cmd_body | replace("\n", "\n            ") %}
+        command: bash .buildkite/amd/scripts/run-amd-test.sh
+        env:
+          AMD_CI_RUNTIME: "native"
+          NATIVE_CI: "true"
+          VLLM_CI_DOCKER_DISABLED: "1"
+          VLLM_CI_EXPECTED_GPU_COUNT: "{{ step.agent_pool[-1:] }}"
+          VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE: "1"
+          HF_HOME: "/home/buildkite-agent/huggingface"
+          HF_HUB_DOWNLOAD_TIMEOUT: "300"
+          HF_HUB_ETAG_TIMEOUT: "60"
+          MIOPEN_DEBUG_CONV_DIRECT: "0"
+          MIOPEN_DEBUG_CONV_GEMM: "0"
+          VLLM_ROCM_USE_AITER: "0"
+          PYTORCH_ROCM_ARCH: ""
+{% if step.env %}
+{% for name in step.env %}
+          {{ name }}: "{{ step.env[name] }}"
+{% endfor %}
+{% endif %}
+          TEST_COMMANDS: |-
+            (command rocm-smi || true) && cd {{ (step.working_dir or default_working_dir) | safe }}
+{% if env %}
+{% for name in env %}
+            export {{ name }}="{{ env[name] }}"
+{% endfor %}
+{% endif %}
+{% if step.env %}
+{% for name in step.env %}
+            export {{ name }}="{{ step.env[name] }}"
+{% endfor %}
+{% endif %}
+{% if "mi250" in step.agent_pool %}
+            python3 -m pip uninstall -y amd-aiter
+{% endif %}
+            {{ indented_cmd | safe }}
+        priority: 100
+{% if step.parallelism %}
+        parallelism: {{ step.parallelism }}
+{% endif %}
+{% if step.artifact_paths %}
+        artifact_paths:
+{% for path in step.artifact_paths %}
+          - "{{ path }}"
+{% endfor %}
+{% endif %}
+        {% if step.timeout_in_minutes %}
+        timeout_in_minutes: {{ step.timeout_in_minutes }}
+        {% endif %}
+        {% if step.grade and step.grade == "Blocking" %}
+        soft_fail: false
+        {% else %}
+        soft_fail: true
+        {% endif%}
+{% endif %}
+{% endfor %}
+{% elif step_group.mirror_hardwares and mirror_hw in step_group.mirror_hardwares %}
+{% set step = step_group %}
+  - label: "{{ step.agent_pool }}: {{ step.label }}"
+    depends_on: amd-build
+    agents:
+      {% if step.agent_pool %}
+      queue: amd_{{ step.agent_pool }}
+      {% else %}
+      queue: amd_mi300_1
+      {% endif %}
+{{ amd_native_k8s_plugin("    ", step) }}
+{% set cmd_body = (step.command or (step.commands | join("\n"))) | trim %}
+{% set indented_cmd = cmd_body | replace("\n", "\n        ") %}
+    command: bash .buildkite/amd/scripts/run-amd-test.sh
+    env:
+      AMD_CI_RUNTIME: "native"
+      NATIVE_CI: "true"
+      VLLM_CI_DOCKER_DISABLED: "1"
+      VLLM_CI_EXPECTED_GPU_COUNT: "{{ step.agent_pool[-1:] }}"
+      VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE: "1"
+      HF_HOME: "/home/buildkite-agent/huggingface"
+      HF_HUB_DOWNLOAD_TIMEOUT: "300"
+      HF_HUB_ETAG_TIMEOUT: "60"
+      MIOPEN_DEBUG_CONV_DIRECT: "0"
+      MIOPEN_DEBUG_CONV_GEMM: "0"
+      VLLM_ROCM_USE_AITER: "0"
+      PYTORCH_ROCM_ARCH: ""
+{% if step.env %}
+{% for name in step.env %}
+      {{ name }}: "{{ step.env[name] }}"
+{% endfor %}
+{% endif %}
+      TEST_COMMANDS: |-
+        (command rocm-smi || true) && cd {{ (step.working_dir or default_working_dir) | safe }}
+{% if env %}
+{% for name in env %}
+        export {{ name }}="{{ env[name] }}"
+{% endfor %}
+{% endif %}
+{% if step.env %}
+{% for name in step.env %}
+        export {{ name }}="{{ step.env[name] }}"
+{% endfor %}
+{% endif %}
+{% if "mi250" in step.agent_pool %}
+        python3 -m pip uninstall -y amd-aiter
+{% endif %}
+        {{ indented_cmd | safe }}
+    priority: 100
+{% if step.parallelism %}
+    parallelism: {{ step.parallelism }}
+{% endif %}
+{% if step.artifact_paths %}
+    artifact_paths:
+{% for path in step.artifact_paths %}
+      - "{{ path }}"
+{% endfor %}
+{% endif %}
+    {% if step.timeout_in_minutes %}
+    timeout_in_minutes: {{ step.timeout_in_minutes }}
+    {% endif %}
+    {% if step.grade and step.grade == "Blocking" %}
+    soft_fail: false
+    {% else %}
+    soft_fail: true
+    {% endif%}
+{% endif %}
+{% endfor %}

.buildkite/bootstrap-intel-omni.sh

@@ -1,51 +0,0 @@
-#!/bin/bash
-# vllm-omni Intel bootstrap
-# Uses static pipeline-intel.yaml for Intel XPU tests
-# Based on: bootstrap-amd-omni.sh
-
-set -euo pipefail
-
-if [[ -z "${DOCS_ONLY_DISABLE:-}" ]]; then
-    DOCS_ONLY_DISABLE=0
-fi
-
-upload_pipeline() {
-    echo "--- 🛠 Preparing Intel pipeline"
-    buildkite-agent pipeline upload .buildkite/pipeline-intel.yaml
-}
-
-get_diff() {
-    $(git add .)
-    echo $(git diff --name-only --diff-filter=ACMDR $(git merge-base origin/main HEAD))
-}
-
-get_diff_main() {
-    $(git add .)
-    echo $(git diff --name-only --diff-filter=ACMDR HEAD~1)
-}
-
-file_diff=$(get_diff)
-if [[ $BUILDKITE_BRANCH == "main" ]]; then
-    file_diff=$(get_diff_main)
-fi
-
-# ----------------------------------------------------------------------
-# Early exit: skip pipeline if all changed files are under docs/
-# ----------------------------------------------------------------------
-if [[ "${DOCS_ONLY_DISABLE}" != "1" ]] && [[ -n "${file_diff:-}" ]]; then
-    docs_only=1
-    while IFS= read -r f; do
-        [[ -z "$f" ]] && continue
-        if [[ "$f" != docs/* ]]; then
-            docs_only=0
-            break
-        fi
-    done < <(printf '%s\n' "$file_diff" | tr ' ' '\n' | tr -d '\r')
-
-    if [[ "$docs_only" -eq 1 ]]; then
-        buildkite-agent annotate ":memo: CI skipped — docs only" --style "info"
-        exit 0
-    fi
-fi
-
-upload_pipeline

.buildkite/common/ci_mirror_hardwares.yml

@@ -0,0 +1,246 @@
+# Uploader-only hardware presets for CUDA / NPU test YAML.
+# Referenced as ``mirror_hardwares: l4_1`` on a step; ``upload_pipeline.py`` expands to
+# ``agents`` (+ optional top-level ``image`` for NPU) + ``plugins``, then strips
+# ``mirror_hardwares`` before Buildkite upload.
+#
+# CUDA naming: {chip}_{count}
+#   l4_1, l4_4, h100_1 … h100_4
+# NPU naming: {queue}_{resource_class}
+#   a2b3_npu_1  — ascend-a2b3, resource_class npu-1 (A2/B3 CI image)
+#   a2b3_npu_4  — ascend-a2b3, resource_class npu-4
+#   a3_npu_2    — ascend-a3, resource_class npu-2
+#
+# The ``l4_*`` presets run under the ``docker`` plugin, which does not put
+# ``buildkite-agent`` on PATH inside the container the way the ``kubernetes``
+# presets do, so they set ``mount-buildkite-agent: true``; without it any step
+# calling ``buildkite-agent`` (artifact upload, annotate, meta-data) fails there.
+
+x-npu-k8s-plugins: &npu_k8s_plugins
+  - kubernetes:
+      podSpecPatch:
+        imagePullSecrets:
+          - name: swr-secret
+
+
+mirror_hardwares:
+  l4_1:
+    agents:
+      queue: gpu_1_queue
+    plugins:
+      - docker#v5.2.0:
+          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+          always-pull: true
+          propagate-environment: true
+          mount-buildkite-agent: true
+          shm-size: "8gb"
+          environment:
+            - HF_HOME=/fsx/hf_cache
+            - HF_TOKEN
+          volumes:
+            - /fsx/hf_cache:/fsx/hf_cache
+
+  l4_4:
+    agents:
+      queue: gpu_4_queue
+    plugins:
+      - docker#v5.2.0:
+          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+          always-pull: true
+          propagate-environment: true
+          mount-buildkite-agent: true
+          shm-size: "8gb"
+          environment:
+            - HF_HOME=/fsx/hf_cache
+            - HF_TOKEN
+          volumes:
+            - /fsx/hf_cache:/fsx/hf_cache
+
+  h100_1:
+    agents:
+      queue: mithril-h100-pool
+    plugins:
+      - kubernetes:
+          podSpec:
+            containers:
+              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 1
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: hf-cache
+                    mountPath: /root/.cache/huggingface
+                env:
+                  - name: HF_HOME
+                    value: /root/.cache/huggingface
+                  - name: HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            nodeSelector:
+              node.kubernetes.io/instance-type: gpu-h100-sxm
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+              - name: hf-cache
+                hostPath:
+                  path: /mnt/hf-cache
+                  type: DirectoryOrCreate
+
+  h100_2:
+    agents:
+      queue: mithril-h100-pool
+    plugins:
+      - kubernetes:
+          podSpec:
+            containers:
+              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 2
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: hf-cache
+                    mountPath: /root/.cache/huggingface
+                env:
+                  - name: HF_HOME
+                    value: /root/.cache/huggingface
+                  - name: HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            nodeSelector:
+              node.kubernetes.io/instance-type: gpu-h100-sxm
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+              - name: hf-cache
+                hostPath:
+                  path: /mnt/hf-cache
+                  type: DirectoryOrCreate
+
+  h100_3:
+    agents:
+      queue: mithril-h100-pool
+    plugins:
+      - kubernetes:
+          podSpec:
+            containers:
+              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 3
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: hf-cache
+                    mountPath: /root/.cache/huggingface
+                env:
+                  - name: HF_HOME
+                    value: /root/.cache/huggingface
+                  - name: HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            nodeSelector:
+              node.kubernetes.io/instance-type: gpu-h100-sxm
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+              - name: hf-cache
+                hostPath:
+                  path: /mnt/hf-cache
+                  type: DirectoryOrCreate
+
+  h100_4:
+    agents:
+      queue: mithril-h100-pool
+    plugins:
+      - kubernetes:
+          podSpec:
+            containers:
+              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 4
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: hf-cache
+                    mountPath: /root/.cache/huggingface
+                env:
+                  - name: HF_HOME
+                    value: /root/.cache/huggingface
+                  - name: HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            nodeSelector:
+              node.kubernetes.io/instance-type: gpu-h100-sxm
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+              - name: hf-cache
+                hostPath:
+                  path: /mnt/hf-cache
+                  type: DirectoryOrCreate
+
+  a2b3_npu_1:
+    agents:
+      queue: "ascend-a2b3"
+      resource_class: "npu-1"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}"
+    plugins: *npu_k8s_plugins
+
+  a2b3_npu_4:
+    agents:
+      queue: "ascend-a2b3"
+      resource_class: "npu-4"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}"
+    plugins: *npu_k8s_plugins
+
+
+  a2b3_npu_8:
+    agents:
+      queue: "ascend-a2b3"
+      resource_class: "npu-8"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}"
+    plugins: *npu_k8s_plugins
+
+  a3_npu_2:
+    agents:
+      queue: "ascend-a3"
+      resource_class: "npu-2"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3"
+    plugins: *npu_k8s_plugins
+
+  a3_npu_4:
+    agents:
+      queue: "ascend-a3"
+      resource_class: "npu-4"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3"
+    plugins: *npu_k8s_plugins
+
+  a3_npu_8:
+    agents:
+      queue: "ascend-a3"
+      resource_class: "npu-8"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3"
+    plugins: *npu_k8s_plugins
+
+  a3_npu_16:
+    agents:
+      queue: "ascend-a3"
+      resource_class: "npu-16"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3"
+    plugins: *npu_k8s_plugins

.buildkite/common/scripts/resolve_skip_ci.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Shared Buildkite skip-ci helpers for AMD / Intel bootstrap scripts.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SKIP_CI_PY="${SCRIPT_DIR}/skip_ci.py"
+
+# Resolve git diff once. Exit 0 when this bootstrap should stop uploading.
+# Decision details are logged by skip_ci.py; this helper only annotates skip-all.
+# Usage: gate_bootstrap_ci <platform> <l2|l3>
+gate_bootstrap_ci() {
+    local platform="$1"
+    local level="$2"
+    local kind
+    if kind="$(python3 "${SKIP_CI_PY}" gate "${platform}" "${level}")"; then
+        if [[ "${kind}" == "skip-all" ]]; then
+            buildkite-agent annotate \
+                ":memo: CI skipped — docs or pytest skip-mark changes only" \
+                --style "info" 2>/dev/null || true
+        fi
+        exit 0
+    fi
+}

.buildkite/common/scripts/run_cov_split.sh

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+# Run a model's E2E tests once per entry mode as separate coverage runs and upload
+# the reports, so per-mode attribution survives (see vllm-project/vllm-omni#5332).
+#
+# Usage: run_cov_split.sh --model-id <id> [--offline <paths>] [--online <paths>] \
+#                         -- <pytest args...>
+#
+#   --model-id  Names the artifacts: coverage-<id>-{offline,online}-<step id>.xml.gz.
+#               Use the model's directory name under vllm_omni/*/models/.
+#   --offline   Pytest path(s) for the offline half. Repeatable.
+#   --online    Pytest path(s) for the online half. Repeatable.
+#   --          Everything after this is forwarded verbatim to every pytest run,
+#               quoting preserved: the job's own -m expression, --run-level, and
+#               anything else it needs (e.g. --test-config-file).
+#
+# At least one of --offline/--online is required, so a job with only one mode runs
+# just that half. Total runtime is bounded by the step's timeout_in_minutes.
+#
+# Coverage + artifact upload run only on scheduled weekly E2E sweeps
+# (WEEKLY=1 and BUILDKITE_BRANCH=main — same gate as bootstrap --e2e).
+# All other invocations (PR merge-test, regular main merge, local) run the
+# same combined pytest as the job used before coverage was added.
+#
+# Exits non-zero if any half or the upload failed.
+set -uo pipefail
+
+MODEL_ID=""
+OFFLINE=()
+ONLINE=()
+PYTEST_ARGS=()
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --model-id) MODEL_ID="$2"; shift 2 ;;
+        --offline)  OFFLINE+=("$2"); shift 2 ;;
+        --online)   ONLINE+=("$2"); shift 2 ;;
+        --)         shift; PYTEST_ARGS=("$@"); break ;;
+        *) echo "run_cov_split.sh: unknown argument: $1" >&2; exit 2 ;;
+    esac
+done
+
+if [[ -z "${MODEL_ID}" ]]; then
+    echo "run_cov_split.sh: missing --model-id" >&2
+    exit 2
+fi
+
+if ((${#OFFLINE[@]} == 0 && ${#ONLINE[@]} == 0)); then
+    echo "run_cov_split.sh: need at least one of --offline / --online" >&2
+    exit 2
+fi
+
+_should_run_coverage() {
+    [[ "${WEEKLY:-}" == "1" && "${BUILDKITE_BRANCH:-}" == "main" ]]
+}
+
+if ! _should_run_coverage; then
+    PATHS=("${OFFLINE[@]}" "${ONLINE[@]}")
+    echo "--- pytest (no coverage): ${MODEL_ID}"
+    pytest -s -v "${PATHS[@]}" "${PYTEST_ARGS[@]}"
+    exit $?
+fi
+
+# Two steps can test the same model, and artifacts are scoped to the build (a
+# nightly one carries the ready, merge and nightly tiers at once), so unsuffixed
+# reports would share a path -- a glob download then races to rename onto one file.
+# Not BUILDKITE_JOB_ID: the step id survives retries, so attempts collapse under
+# Buildkite's latest-attempt lookup instead of accumulating. A parallel step would
+# need BUILDKITE_PARALLEL_JOB as well; no pilot is one.
+REPORT_SUFFIX="${BUILDKITE_STEP_ID:-local}"
+OFFLINE_REPORT="coverage-${MODEL_ID}-offline-${REPORT_SUFFIX}.xml"
+ONLINE_REPORT="coverage-${MODEL_ID}-online-${REPORT_SUFFIX}.xml"
+
+run_half() {
+    local mode="$1"
+    shift
+    local report="coverage-${MODEL_ID}-${mode}-${REPORT_SUFFIX}.xml"
+    echo "--- coverage: ${MODEL_ID} ${mode}"
+    pytest -s -v "$@" "${PYTEST_ARGS[@]}" --cov=vllm_omni --cov-report="xml:${report}"
+    local status=$?
+    # Cobertura lists every statement in vllm_omni whether or not it ran, so the
+    # report is ~7MB no matter how little the job covers. The repetition is what
+    # dominates it, and gzip takes that to ~400KB.
+    gzip -9 -f "${report}" || status=1
+    return "${status}"
+}
+
+# Clear both modes, not just the ones being run: the upload glob matches both, so
+# a report left by an earlier run would otherwise be published as if the mode it
+# belongs to had run this time.
+rm -f "${OFFLINE_REPORT}" "${ONLINE_REPORT}" "${OFFLINE_REPORT}.gz" "${ONLINE_REPORT}.gz"
+
+EXIT=0
+((${#OFFLINE[@]})) && { run_half offline "${OFFLINE[@]}" || EXIT=1; }
+((${#ONLINE[@]})) && { run_half online "${ONLINE[@]}" || EXIT=1; }
+
+buildkite-agent artifact upload "coverage-${MODEL_ID}-*-${REPORT_SUFFIX}.xml.gz" || EXIT=1
+
+exit "${EXIT}"

.buildkite/common/scripts/skip_ci.py

@@ -0,0 +1,617 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+"""Shared Buildkite skip-ci and CI-YAML level targeting logic.
+
+``resolve_ci_decision(changed_files, ...)`` buckets explicit file lists (tests, dry-run).
+``resolve_ci_context_from_git()`` resolves git diff once and returns decision + changed files
+for Buildkite bootstrap / upload scripts.
+
+CLI (for bootstrap scripts; exit code is the signal):
+  python3 skip_ci.py gate <platform> <level>
+      Exit 0 when bootstrap should stop (skip-all, or that L2/L3 target is off).
+      Prints ``skip-all`` or ``skip-l23`` on stdout; logs the decision once.
+  python3 skip_ci.py check-skip-all
+      Exit 0 when only docs/skip-mark changes → bootstrap skips entire CI upload.
+  python3 skip_ci.py check-skip-l2-l3 <platform> <level>
+      Exit 0 when that L2/L3 pipeline should be skipped (CI config YAML-only diffs).
+  python3 skip_ci.py print-annotate
+      Print decision message for Buildkite annotation.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import re
+import subprocess
+import sys
+from dataclasses import dataclass, field, replace
+from pathlib import Path
+
+LOG = "[skip-ci]"
+ROOT = Path(__file__).resolve().parent.parent.parent.parent
+
+PLATFORMS = ("cuda", "amd", "intel", "npu")
+L23_LEVELS = frozenset({"l2", "l3"})
+
+L2_YAML_FILES: dict[str, str] = {
+    ".buildkite/cuda/test-ready.yml": "cuda",
+    ".buildkite/npu/test-npu-ready.yml": "npu",
+    ".buildkite/amd/test-amd-ready.yml": "amd",
+    ".buildkite/intel/pipeline-intel.yml": "intel",
+}
+
+L3_YAML_FILES: dict[str, str] = {
+    ".buildkite/cuda/test-merge.yml": "cuda",
+    ".buildkite/amd/test-amd-merge.yml": "amd",
+}
+
+L45_YAML_FILES: dict[str, str] = {
+    ".buildkite/cuda/test-nightly.yml": "cuda",
+    ".buildkite/cuda/test-weekly.yml": "cuda",
+    ".buildkite/npu/test-npu-nightly.yml": "npu",
+}
+
+
+def _empty_platform_buckets() -> dict[str, list[str]]:
+    return {platform: [] for platform in PLATFORMS}
+
+
+@dataclass(frozen=True)
+class CiDevice:
+    """Per-platform / per-level skip flags for yaml-gated bootstrap (``skip_l2_l3``)."""
+
+    skip_cuda_l2: bool = False
+    skip_cuda_l3: bool = False
+    skip_amd_l2: bool = False
+    skip_amd_l3: bool = False
+    skip_intel_l2: bool = False
+    skip_intel_l3: bool = False
+    skip_npu_l2: bool = False
+    skip_npu_l3: bool = False
+
+    def is_skip(self, platform: str, level: str) -> bool:
+        return bool(getattr(self, f"skip_{platform}_{level}", True))
+
+
+@dataclass
+class DiffBuckets:
+    """Changed files grouped by doc / skip-mark / CI yaml / other."""
+
+    docs: list[str] = field(default_factory=list)
+    skip_tests: list[str] = field(default_factory=list)
+    other: list[str] = field(default_factory=list)
+    l2: dict[str, list[str]] = field(default_factory=_empty_platform_buckets)
+    l3: dict[str, list[str]] = field(default_factory=_empty_platform_buckets)
+    l45: dict[str, list[str]] = field(default_factory=_empty_platform_buckets)
+
+    @property
+    def has_doc_changes(self) -> bool:
+        return bool(self.docs)
+
+    @property
+    def has_skip_test_changes(self) -> bool:
+        return bool(self.skip_tests)
+
+    @property
+    def has_other_changes(self) -> bool:
+        """Product code or paths outside skip-ci scope."""
+        return bool(self.other)
+
+    @property
+    def has_l23_yaml_changes(self) -> bool:
+        """At least one whitelisted L2/L3 CI yaml file changed."""
+        return any(self.l2[platform] or self.l3[platform] for platform in PLATFORMS)
+
+    @property
+    def has_l45_yaml_changes(self) -> bool:
+        """At least one whitelisted L4/L5 (nightly/weekly) CI yaml file changed."""
+        return any(self.l45[platform] for platform in PLATFORMS)
+
+    @property
+    def has_skip_ci_scope_changes(self) -> bool:
+        """At least one doc / skip-mark / whitelisted CI yaml change."""
+        return (
+            self.has_doc_changes or self.has_skip_test_changes or self.has_l23_yaml_changes or self.has_l45_yaml_changes
+        )
+
+    def l2_platforms(self) -> frozenset[str]:
+        return frozenset(platform for platform in PLATFORMS if self.l2[platform])
+
+    def l3_platforms(self) -> frozenset[str]:
+        return frozenset(platform for platform in PLATFORMS if self.l3[platform])
+
+
+@dataclass(frozen=True)
+class SkipL2L3Basis:
+    skip_all_l2: bool
+    skip_all_l3: bool
+
+    @classmethod
+    def from_buckets(cls, buckets: DiffBuckets) -> SkipL2L3Basis | None:
+        """L4/L5 yaml → skip both L2 and L3 (L2/L3 buckets may rescue).
+
+        Docs / skip-mark changes do not block this path; they are ignored for
+        L2/L3 targeting when CI YAML is also present.
+        """
+        if buckets.has_other_changes:
+            return None
+        if not buckets.has_l45_yaml_changes:
+            return None
+        return cls(
+            skip_all_l2=not buckets.l2_platforms(),
+            skip_all_l3=not buckets.l3_platforms(),
+        )
+
+
+@dataclass(frozen=True)
+class SkipL3Basis:
+    enable_l2_platforms: frozenset[str]
+
+    @classmethod
+    def from_buckets(cls, buckets: DiffBuckets) -> SkipL3Basis | None:
+        """L2 yaml changed → skip L3; keep L2 on touched platforms."""
+        platforms = buckets.l2_platforms()
+        if not platforms:
+            return None
+        return cls(enable_l2_platforms=platforms)
+
+
+@dataclass(frozen=True)
+class SkipL2Basis:
+    enable_l3_platforms: frozenset[str]
+
+    @classmethod
+    def from_buckets(cls, buckets: DiffBuckets) -> SkipL2Basis | None:
+        """L3 yaml changed → skip L2; keep L3 on touched platforms."""
+        platforms = buckets.l3_platforms()
+        if not platforms:
+            return None
+        return cls(enable_l3_platforms=platforms)
+
+
+@dataclass(frozen=True)
+class CiDecision:
+    """Final CI skip decision for one diff."""
+
+    skip_all: bool = False
+    skip_l2_l3: bool = False
+    device: CiDevice = field(default_factory=CiDevice)
+    message: str = ""
+
+    @classmethod
+    def skip_all_basis(cls, buckets: DiffBuckets, *, diff_range: str | None) -> bool:
+        """Docs / pytest skip-mark only → suppress entire default CI."""
+        if buckets.has_other_changes:
+            return False
+        if (buckets.has_l23_yaml_changes or buckets.has_l45_yaml_changes) and (
+            buckets.has_doc_changes or buckets.has_skip_test_changes
+        ):
+            return False
+        if not buckets.has_doc_changes and not buckets.has_skip_test_changes:
+            return False
+        if diff_range is None and not buckets.has_doc_changes:
+            return False
+        return True
+
+    @classmethod
+    def from_yaml_basis(
+        cls,
+        l2_l3_basis: SkipL2L3Basis | None,
+        l3_basis: SkipL3Basis | None,
+        l2_basis: SkipL2Basis | None,
+    ) -> CiDecision:
+        """Build a yaml-gated decision by merging L2/L3 skip bases.
+
+        Bases are applied as a union of enables: L2 yaml keeps L2 on touched
+        platforms, L3 yaml keeps L3 on touched platforms. Cross-platform mixes
+        (e.g. CUDA ready + AMD merge) must preserve both enables. The resulting
+        per-platform / per-level matrix is stored on ``device`` as ``skip_*``.
+        """
+        run = {platform: {"l2": True, "l3": True} for platform in PLATFORMS}
+
+        if l2_l3_basis is not None:
+            if l2_l3_basis.skip_all_l2:
+                for platform in PLATFORMS:
+                    run[platform]["l2"] = False
+            if l2_l3_basis.skip_all_l3:
+                for platform in PLATFORMS:
+                    run[platform]["l3"] = False
+
+        if l3_basis is not None:
+            for platform in PLATFORMS:
+                run[platform]["l3"] = False
+                run[platform]["l2"] = platform in l3_basis.enable_l2_platforms
+
+        if l2_basis is not None:
+            for platform in PLATFORMS:
+                if platform in l2_basis.enable_l3_platforms:
+                    run[platform]["l3"] = True
+                    if l3_basis is None or platform not in l3_basis.enable_l2_platforms:
+                        run[platform]["l2"] = False
+                elif l3_basis is None:
+                    # L3-yaml-only: disable platforms that did not change L3.
+                    run[platform]["l2"] = False
+                    run[platform]["l3"] = False
+                # else: leave L2 enables from SkipL3Basis; keep this platform's L3 off.
+
+        return cls(
+            skip_l2_l3=True,
+            device=cls._device_from_run(run),
+        )
+
+    @staticmethod
+    def _device_from_run(run: dict[str, dict[str, bool]]) -> CiDevice:
+        return CiDevice(
+            **{f"skip_{platform}_{level}": not run[platform][level] for platform in PLATFORMS for level in ("l2", "l3")}
+        )
+
+    def is_run(self, platform: str, level: str) -> bool:
+        if self.skip_all:
+            return False
+        if not self.skip_l2_l3:
+            return True
+        if level not in L23_LEVELS:
+            return True
+        return not self.device.is_skip(platform, level)
+
+
+@dataclass(frozen=True)
+class CiContext:
+    """Skip-ci decision plus git diff inputs used to compute it."""
+
+    decision: CiDecision
+    changed_files: list[str] | None
+    diff_range: str | None
+
+
+def _log(message: str) -> None:
+    print(f"{LOG}: {message}", file=sys.stderr)
+
+
+def _git(*args: str) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(
+        ["git", *args],
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+
+
+# --- Diff helpers ---
+
+
+def _resolve_diff_range() -> str | None:
+    """Return a git diff range for PR or main builds, or None when unavailable."""
+    is_pr = os.environ.get("BUILDKITE_PULL_REQUEST", "false") != "false" and os.environ.get(
+        "BUILDKITE_PULL_REQUEST", ""
+    )
+    commit = os.environ.get("BUILDKITE_COMMIT", "")
+
+    if is_pr:
+        base_branch = os.environ.get("BUILDKITE_PULL_REQUEST_BASE_BRANCH", "main")
+        base_ref = f"origin/{base_branch}"
+        if _git("rev-parse", "--verify", base_ref).returncode != 0:
+            _log(f"origin/{base_branch} not found locally; trying fetch")
+            _git("fetch", "--depth=200", "origin", base_branch)
+        if _git("rev-parse", "--verify", base_ref).returncode != 0:
+            if _git("rev-parse", "--verify", base_branch).returncode == 0:
+                base_ref = base_branch
+            else:
+                _log(f"cannot resolve PR base {base_branch}; using safe defaults")
+                return None
+        return f"{base_ref}...{commit}"
+
+    if os.environ.get("BUILDKITE_BRANCH", "") == "main":
+        if _git("rev-parse", "--verify", f"{commit}^").returncode != 0:
+            _log("main commit has no parent; using safe defaults")
+            return None
+        return f"{commit}^..{commit}"
+
+    _log("not PR/main build; using safe defaults")
+    return None
+
+
+def _changed_files_for_diff_range(diff_range: str | None) -> list[str] | None:
+    """Return changed file paths for *diff_range*, or None when unavailable."""
+    if diff_range is None:
+        return None
+
+    result = _git("diff", "--name-only", diff_range)
+    if result.returncode != 0:
+        _log(f"git diff failed ({diff_range}); using safe defaults")
+        return None
+
+    files = [line for line in result.stdout.splitlines() if line.strip()]
+    _log(f"{len(files)} changed file(s)")
+    return files
+
+
+def resolve_ci_context_from_git() -> CiContext:
+    """Resolve git diff once and return skip-ci decision plus changed files."""
+    diff_range = _resolve_diff_range()
+    changed_files = _changed_files_for_diff_range(diff_range)
+    decision = resolve_ci_decision(changed_files, diff_range=diff_range)
+    return CiContext(decision=decision, changed_files=changed_files, diff_range=diff_range)
+
+
+# --- Doc / skip-mark helpers ---
+
+
+_SKIP_MARK_RE = re.compile(r"pytest\.mark\.skip(?:if)?\b|pytest\.skip\s*\(")
+_PYTESTMARK_SKIP_RE = re.compile(r"pytest\.mark\.skip\b")
+_PYTEST_MARK_ONLY_RE = re.compile(r"pytest\.mark\.\w+")
+
+
+def _diff_only_contains_skip_mark_changes(diff_text: str) -> bool:
+    """Return True when a unified diff only *adds* pytest skip marks.
+
+    Deleting a skip mark re-enables the test and must not suppress CI. Editing an
+    existing skip (``-`` old / ``+`` new) still qualifies when at least one skip
+    mark is added.
+    """
+
+    def paren_balance(line: str) -> int:
+        return line.count("(") - line.count(")")
+
+    def is_blank_or_comment(line: str) -> bool:
+        stripped = line.strip()
+        return not stripped or stripped.startswith("#")
+
+    def is_skip_mark_line(line: str) -> bool:
+        return _SKIP_MARK_RE.search(line.strip()) is not None
+
+    def is_pytestmark_adjacent_line(line: str) -> bool:
+        stripped = line.strip().rstrip(",")
+        if not stripped or stripped in {"[", "]"} or stripped.startswith("#"):
+            return True
+        if stripped.startswith("pytestmark"):
+            return True
+        return _PYTEST_MARK_ONLY_RE.search(stripped) is not None
+
+    pending_depth = 0
+    saw_change = False
+    has_added_skip_mark = False
+    for raw_line in diff_text.splitlines():
+        if raw_line.startswith("@@"):
+            pending_depth = 0
+            continue
+        if not (raw_line.startswith("+") or raw_line.startswith("-")):
+            continue
+        if raw_line.startswith("+++") or raw_line.startswith("---"):
+            continue
+
+        saw_change = True
+        is_add = raw_line.startswith("+")
+        content = raw_line[1:]
+
+        if pending_depth > 0:
+            pending_depth += paren_balance(content)
+            if pending_depth < 0:
+                pending_depth = 0
+            continue
+
+        if is_blank_or_comment(content):
+            continue
+
+        if is_skip_mark_line(content):
+            if is_add:
+                has_added_skip_mark = True
+            pending_depth = max(0, paren_balance(content))
+            continue
+
+        if is_pytestmark_adjacent_line(content):
+            continue
+
+        return False
+
+    return saw_change and has_added_skip_mark
+
+
+def _is_skip_testcase_change(file_path: str, *, diff_range: str) -> bool:
+    """Return True when a test file diff is skip-mark-only (including new-file cases)."""
+    diff_result = _git("diff", diff_range, "--", file_path)
+    if diff_result.returncode != 0:
+        _log(f"skip-mark-only: git diff failed for {file_path}")
+        return False
+
+    diff_text = diff_result.stdout
+    if not diff_text.strip():
+        _log(f"skip-mark-only: empty diff for {file_path}")
+        return False
+
+    if _diff_only_contains_skip_mark_changes(diff_text):
+        return True
+
+    if "new file mode" not in diff_text and "--- /dev/null" not in diff_text:
+        _log(f"skip-mark-only: non-skip changes in {file_path}")
+        return False
+
+    commit = os.environ.get("BUILDKITE_COMMIT", "")
+    if not commit:
+        return False
+
+    content_result = _git("show", f"{commit}:{file_path}")
+    if content_result.returncode != 0:
+        _log(f"skip-mark-only: cannot read {file_path} at {commit}")
+        return False
+
+    content = content_result.stdout
+    match = re.search(r"^pytestmark\s*=\s*\[(.*?)^\s*\]", content, re.MULTILINE | re.DOTALL)
+    if match is not None and _PYTESTMARK_SKIP_RE.search(match.group(1)) is not None:
+        _log(f"skip-mark-only: new test file with module-level pytest.mark.skip: {file_path}")
+        return True
+    if re.search(r"^pytestmark\s*=\s*pytest\.mark\.skip\b", content, re.MULTILINE):
+        _log(f"skip-mark-only: new test file with module-level pytest.mark.skip: {file_path}")
+        return True
+
+    _log(f"skip-mark-only: new test file without module-level pytest.mark.skip: {file_path}")
+    return False
+
+
+def _classify_changed_files_into_buckets(changed_files: list[str], *, diff_range: str | None) -> DiffBuckets:
+    """Classify each changed file path into a ``DiffBuckets`` category."""
+    buckets = DiffBuckets()
+    for file_path in changed_files:
+        if not file_path:
+            continue
+        if file_path.startswith("docs/") or file_path.endswith(".md") or file_path == "mkdocs.yml":
+            buckets.docs.append(file_path)
+            continue
+        path = Path(file_path)
+        if path.suffix == ".py" and path.parts and path.parts[0] == "tests":
+            if diff_range is not None and _is_skip_testcase_change(file_path, diff_range=diff_range):
+                buckets.skip_tests.append(file_path)
+            else:
+                buckets.other.append(file_path)
+            continue
+        if file_path in L2_YAML_FILES:
+            platform = L2_YAML_FILES[file_path]
+            buckets.l2[platform].append(file_path)
+        elif file_path in L3_YAML_FILES:
+            platform = L3_YAML_FILES[file_path]
+            buckets.l3[platform].append(file_path)
+        elif file_path in L45_YAML_FILES:
+            platform = L45_YAML_FILES[file_path]
+            buckets.l45[platform].append(file_path)
+        else:
+            buckets.other.append(file_path)
+    return buckets
+
+
+# --- Decision ---
+
+
+def _finish(decision: CiDecision, message: str) -> CiDecision:
+    finished = replace(decision, message=message)
+    _log(message)
+    return finished
+
+
+def _format_yaml_gated_message(buckets: DiffBuckets, decision: CiDecision) -> str:
+    """Human-readable summary of which CI YAML changed and which L2/L3 targets run."""
+    changed_parts: list[str] = []
+    for level_name, by_platform in (("L2", buckets.l2), ("L3", buckets.l3), ("L4/L5", buckets.l45)):
+        files = [path for platform in PLATFORMS for path in by_platform[platform]]
+        if files:
+            changed_parts.append(f"{level_name}=[{', '.join(files)}]")
+
+    run_targets = [
+        f"{platform}/{level}" for platform in PLATFORMS for level in ("l2", "l3") if decision.is_run(platform, level)
+    ]
+    skip_targets = [
+        f"{platform}/{level}"
+        for platform in PLATFORMS
+        for level in ("l2", "l3")
+        if not decision.is_run(platform, level)
+    ]
+
+    changed = "; ".join(changed_parts) if changed_parts else "none"
+    run = ", ".join(run_targets) if run_targets else "none"
+    skip = ", ".join(skip_targets) if skip_targets else "none"
+    if buckets.has_doc_changes or buckets.has_skip_test_changes:
+        return (
+            "CI config YAML + docs/skip-mark changed — follow CI YAML gating; "
+            f"changed: {changed}; run: {run}; skip: {skip}"
+        )
+    return f"only CI config YAML changed — changed: {changed}; run: {run}; skip: {skip}"
+
+
+def resolve_ci_decision(
+    changed_files: list[str] | None = None,
+    *,
+    diff_range: str | None = None,
+) -> CiDecision:
+    if changed_files is None:
+        return _finish(CiDecision(), "could not resolve changed files; run normal CI")
+
+    buckets = _classify_changed_files_into_buckets(changed_files, diff_range=diff_range)
+    if buckets.has_other_changes or not buckets.has_skip_ci_scope_changes:
+        return _finish(CiDecision(), "product or non-whitelisted changes present; run normal CI")
+
+    if CiDecision.skip_all_basis(buckets, diff_range=diff_range):
+        return _finish(
+            CiDecision(skip_all=True),
+            "CI skipped — docs or pytest skip-mark changes only",
+        )
+
+    # Docs / skip-mark mixed with whitelisted CI YAML → same yaml-gated path
+    # as CI-YAML-only (docs/skip-mark do not widen to normal CI).
+    if not buckets.has_l23_yaml_changes and not buckets.has_l45_yaml_changes:
+        if buckets.has_skip_test_changes and diff_range is None:
+            message = (
+                "only pytest skip-mark test changes detected, but no git diff range "
+                "is available to verify them (skip-all requires diff_range); run normal CI"
+            )
+        elif buckets.has_skip_test_changes:
+            message = "pytest skip-mark test changes did not qualify for skip-all and no ci yaml changed; run normal CI"
+        else:
+            message = "doc/skip-mark changes did not qualify for skip-all; run normal CI"
+        return _finish(CiDecision(), message)
+
+    decision = CiDecision.from_yaml_basis(
+        SkipL2L3Basis.from_buckets(buckets),
+        SkipL3Basis.from_buckets(buckets),
+        SkipL2Basis.from_buckets(buckets),
+    )
+    return _finish(decision, _format_yaml_gated_message(buckets, decision))
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    gate_parser = subparsers.add_parser(
+        "gate",
+        help="exit 0 when bootstrap should stop (skip-all or PLATFORM/LEVEL disabled); logs once",
+    )
+    gate_parser.add_argument("platform", help="cuda, amd, intel, or npu")
+    gate_parser.add_argument("level", choices=("l2", "l3"), help="test level")
+
+    subparsers.add_parser(
+        "check-skip-all",
+        help="exit 0 when docs/skip-mark-only diff should skip entire CI upload",
+    )
+
+    skip_parser = subparsers.add_parser(
+        "check-skip-l2-l3",
+        help="exit 0 when PLATFORM l2/l3 pipeline should be skipped (CI config YAML-only diffs)",
+    )
+    skip_parser.add_argument("platform", help="cuda, amd, intel, or npu")
+    skip_parser.add_argument("level", choices=("l2", "l3"), help="test level")
+
+    subparsers.add_parser(
+        "print-annotate",
+        help="print human-readable skip reason for Buildkite annotation",
+    )
+
+    args = parser.parse_args()
+    decision = resolve_ci_context_from_git().decision
+
+    if args.command == "gate":
+        if decision.skip_all:
+            print("skip-all")
+            return 0
+        if not decision.is_run(args.platform, args.level):
+            print("skip-l23")
+            return 0
+        return 1
+
+    if args.command == "check-skip-all":
+        return 0 if decision.skip_all else 1
+
+    if args.command == "check-skip-l2-l3":
+        return 0 if not decision.is_run(args.platform, args.level) else 1
+
+    if args.command == "print-annotate":
+        print(decision.message or "CI will run normally")
+        return 0
+
+    return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.buildkite/common/scripts/upload_pipeline.py

@@ -0,0 +1,437 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+"""Render and optionally upload Buildkite pipeline YAML with diff-aware logic.
+
+Bootstrap mode (``bootstrap-upload-steps.yml``):
+  - Hook uploads the entry YAML (``pipeline.yml`` / ``pipeline-npu.yml``) with one step that runs
+    ``upload_pipeline.py --upload <platform>/bootstrap-upload-steps.yml``.
+  - Injects ``if`` by step ``key`` from skip-ci and uploads child steps (image build, L2–L5 upload).
+  - Detect docs-only, pytest skip-mark-only, or combined skip-ci from git diff.
+  - When only CI level YAML changes, enable **L2/L3** upload steps for affected levels only.
+
+Test pipeline mode (e.g. test-merge.yml):
+  - Drop steps whose ``source_file_dependencies`` do not match changed files.
+  - Expand uploader-only ``mirror_hardwares: l4_1`` into ``agents`` (+ optional ``image``
+    for NPU) + ``plugins`` (see ci_mirror_hardwares.yml).
+
+Usage:
+  python3 upload_pipeline.py [--upload] [--all | --e2e] <pipeline.yml>
+
+Requires PyYAML (``pip install pyyaml``); installs it automatically when missing.
+"""
+
+from __future__ import annotations
+
+import argparse
+import copy
+import subprocess
+import sys
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+try:
+    import yaml
+except ModuleNotFoundError:
+    subprocess.run(
+        [sys.executable, "-m", "pip", "install", "-q", "pyyaml"],
+        check=True,
+    )
+    import yaml
+
+from skip_ci import (
+    ROOT,
+    resolve_ci_context_from_git,
+)
+
+# --- Constants ---
+
+LOG = "upload_pipeline"
+BOOTSTRAP_STEPS_FILENAME = "bootstrap-upload-steps.yml"
+BOOTSTRAP_IMAGE_BUILD_KEYS = frozenset({"image-build", "image-build-a2", "image-build-a3"})
+BOOTSTRAP_UPLOAD_IF_KEYS = {
+    "upload-ready-pipeline": "ready",
+    "upload-merge-pipeline": "merge",
+    "upload-nightly-pipeline": "nightly",
+    "upload-weekly-pipeline": "weekly",
+}
+E2E_GROUP_MARKER = "E2E Test"
+CI_MIRROR_HARDWARES_PATH = ROOT / ".buildkite/common/ci_mirror_hardwares.yml"
+
+# Bootstrap Buildkite ``if`` expressions.
+# ``*_MAIN_IF``: main + env schedule. ``*_LABEL_IF``: PR label (and/or composed with MAIN).
+# ``*_UPLOAD_IF``: full gate for uploading that child pipeline.
+NIGHTLY_MAIN_IF = 'build.branch == "main" && build.env("NIGHTLY") == "1"'
+NIGHTLY_LABEL_IF = (
+    f"({NIGHTLY_MAIN_IF}) || "
+    '(build.branch != "main" && ('
+    'build.pull_request.labels includes "nightly-test" || '
+    'build.pull_request.labels includes "omni-test" || '
+    'build.pull_request.labels includes "tts-test" || '
+    'build.pull_request.labels includes "diffusion-x2iat-test" || '
+    'build.pull_request.labels includes "diffusion-x2v-test"'
+    "))"
+)
+WEEKLY_E2E_IF = 'build.branch == "main" && build.env("WEEKLY") == "1"'
+WEEKLY_MAIN_IF = 'build.branch == "main" && (build.env("WEEKLY") == "1" || build.env("NON_CRITICAL") == "1")'
+WEEKLY_LABEL_IF = f'({WEEKLY_MAIN_IF}) || (build.branch != "main" && build.pull_request.labels includes "weekly-test")'
+READY_LABEL_IF = 'build.branch != "main" && build.pull_request.labels includes "ready"'
+MERGE_LABEL_IF = 'build.branch != "main" && build.pull_request.labels includes "merge-test"'
+MERGE_MAIN_IF = (
+    'build.branch == "main" && build.env("NIGHTLY") != "1" && '
+    'build.env("WEEKLY") != "1" && build.env("NON_CRITICAL") != "1"'
+)
+READY_UPLOAD_IF = f"({WEEKLY_E2E_IF}) || ({READY_LABEL_IF})"
+MERGE_UPLOAD_IF = f"({WEEKLY_E2E_IF}) || (({MERGE_MAIN_IF}) || ({MERGE_LABEL_IF}))"
+BOOTSTRAP_DISABLED_IF = "false"
+BOOTSTRAP_ENABLED_IF = "true"
+
+
+# --- Logging ---
+
+
+def _log(message: str) -> None:
+    print(f"{LOG}: {message}", file=sys.stderr)
+
+
+# --- Bootstrap pipeline (bootstrap-upload-steps.yml) ---
+
+
+def _get_bootstrap_platform(path: Path) -> str:
+    parts = path.as_posix().split("/")
+    return "npu" if "npu" in parts else "cuda"
+
+
+def _load_bootstrap_steps(path: Path) -> str:
+    if path.name != BOOTSTRAP_STEPS_FILENAME:
+        raise ValueError(f"expected {BOOTSTRAP_STEPS_FILENAME}, got {path.name}")
+    return path.read_text(encoding="utf-8")
+
+
+def _format_bootstrap_if(expr: str) -> str:
+    """Return a Buildkite ``if`` string. Buildkite rejects YAML bool ``if`` values."""
+    if expr in ("true", "false"):
+        return expr
+    return f"({expr})"
+
+
+def _compute_bootstrap_if_exprs(*, decision, platform: str) -> dict[str, str]:
+    if platform == "npu":
+        ready_upload = READY_LABEL_IF
+        merge_upload = BOOTSTRAP_DISABLED_IF
+        weekly_label_if = BOOTSTRAP_DISABLED_IF
+    else:
+        ready_upload = READY_UPLOAD_IF
+        merge_upload = MERGE_UPLOAD_IF
+        weekly_label_if = WEEKLY_LABEL_IF
+
+    if decision.skip_all:
+        # Docs / skip-mark only: no PR-label escape hatch. Main scheduled
+        # NIGHTLY=1 still runs L4; WEEKLY=1 / NON_CRITICAL=1 still run L5.
+        # main+WEEKLY=1 also uploads L2/L3 (those steps then pass --e2e).
+        image_expr = f"({NIGHTLY_MAIN_IF}) || ({WEEKLY_MAIN_IF})" if platform == "cuda" else NIGHTLY_MAIN_IF
+        ready_expr = WEEKLY_E2E_IF if platform == "cuda" else BOOTSTRAP_DISABLED_IF
+        merge_expr = WEEKLY_E2E_IF if platform == "cuda" else BOOTSTRAP_DISABLED_IF
+        nightly_expr = NIGHTLY_MAIN_IF
+        weekly_expr = WEEKLY_MAIN_IF if platform == "cuda" else BOOTSTRAP_DISABLED_IF
+    elif decision.skip_l2_l3:
+        l2_enabled = decision.is_run("npu", "l2") if platform == "npu" else decision.is_run("cuda", "l2")
+        l3_enabled = platform == "cuda" and decision.is_run("cuda", "l3")
+
+        ready_expr = ready_upload if l2_enabled else BOOTSTRAP_DISABLED_IF
+        merge_expr = merge_upload if l3_enabled else BOOTSTRAP_DISABLED_IF
+        nightly_expr = NIGHTLY_LABEL_IF
+        weekly_expr = weekly_label_if if platform == "cuda" else BOOTSTRAP_DISABLED_IF
+
+        image_parts = [f"({NIGHTLY_LABEL_IF})"]
+        if platform == "cuda":
+            image_parts.append(f"({weekly_label_if})")
+        if l2_enabled:
+            image_parts.insert(0, f"({ready_upload})")
+        if l3_enabled:
+            image_parts.insert(1 if l2_enabled else 0, f"({merge_upload})")
+        image_expr = " || ".join(image_parts)
+    else:
+        image_expr = BOOTSTRAP_ENABLED_IF
+        ready_expr = ready_upload
+        merge_expr = merge_upload if platform == "cuda" else BOOTSTRAP_DISABLED_IF
+        nightly_expr = NIGHTLY_LABEL_IF
+        weekly_expr = weekly_label_if if platform == "cuda" else BOOTSTRAP_DISABLED_IF
+
+    return {
+        "image": _format_bootstrap_if(image_expr),
+        "ready": _format_bootstrap_if(ready_expr),
+        "merge": _format_bootstrap_if(merge_expr),
+        "nightly": _format_bootstrap_if(nightly_expr),
+        "weekly": _format_bootstrap_if(weekly_expr),
+    }
+
+
+def _apply_bootstrap_if(steps: list[Any], if_exprs: dict[str, str]) -> list[Any]:
+    """Inject ``if`` by step key; drop steps that are unconditionally disabled."""
+    kept: list[Any] = []
+    for step in steps:
+        if not isinstance(step, dict):
+            kept.append(step)
+            continue
+        nested = step.get("steps")
+        if isinstance(nested, list):
+            nested_kept = _apply_bootstrap_if(nested, if_exprs)
+            if nested_kept:
+                kept.append({**step, "steps": nested_kept})
+            continue
+        key = step.get("key")
+        if key in BOOTSTRAP_IMAGE_BUILD_KEYS:
+            step["if"] = if_exprs["image"]
+        elif key in BOOTSTRAP_UPLOAD_IF_KEYS:
+            step["if"] = if_exprs[BOOTSTRAP_UPLOAD_IF_KEYS[key]]
+        if_expr = step.get("if")
+        if if_expr == "false":
+            _log(f"omit disabled bootstrap step {key!r}")
+            continue
+        if if_expr == "true":
+            # Unconditional step: omit ``if`` (Buildkite requires string ``if``, not YAML bool).
+            step.pop("if", None)
+        kept.append(step)
+    return kept
+
+
+def _render_bootstrap_pipeline(
+    steps_yaml: str,
+    *,
+    decision,
+    path: Path,
+) -> str:
+    """Load bootstrap steps YAML and inject ``if`` expressions from skip-ci decision."""
+    doc = yaml.safe_load(steps_yaml)
+    if not isinstance(doc, dict):
+        raise ValueError(f"invalid bootstrap steps YAML for {path}")
+    steps = doc.get("steps")
+    if not isinstance(steps, list):
+        raise ValueError(f"bootstrap steps YAML must contain steps: list in {path}")
+
+    platform = _get_bootstrap_platform(path)
+    if_exprs = _compute_bootstrap_if_exprs(decision=decision, platform=platform)
+    doc["steps"] = _apply_bootstrap_if(steps, if_exprs)
+    return yaml.safe_dump(doc, sort_keys=False)
+
+
+# --- Test pipeline (test-ready.yml, test-merge.yml) ---
+
+
+@lru_cache(maxsize=1)
+def _load_mirror_hardwares() -> dict[str, dict[str, Any]]:
+    if not CI_MIRROR_HARDWARES_PATH.is_file():
+        raise FileNotFoundError(f"missing CI mirror_hardwares registry: {CI_MIRROR_HARDWARES_PATH}")
+    doc = yaml.safe_load(CI_MIRROR_HARDWARES_PATH.read_text(encoding="utf-8"))
+    if not isinstance(doc, dict):
+        raise ValueError(f"invalid CI mirror_hardwares registry: {CI_MIRROR_HARDWARES_PATH}")
+    presets = doc.get("mirror_hardwares")
+    if not isinstance(presets, dict):
+        raise ValueError(f"mirror_hardwares must be a mapping in {CI_MIRROR_HARDWARES_PATH}")
+    return presets
+
+
+def _expand_mirror_hardwares(step: dict[str, Any]) -> dict[str, Any]:
+    """Replace uploader-only ``mirror_hardwares`` with preset fields from ci_mirror_hardwares.yml."""
+    hardware = step.get("mirror_hardwares")
+    if hardware is None:
+        return step
+
+    if not isinstance(hardware, str) or not hardware.strip():
+        raise ValueError(
+            f"mirror_hardwares must be a non-empty string in step {_get_step_label(step)!r}",
+        )
+
+    preset = _load_mirror_hardwares().get(hardware)
+    if preset is None:
+        known = ", ".join(sorted(_load_mirror_hardwares()))
+        raise ValueError(
+            f"unknown mirror_hardwares {hardware!r} in step {_get_step_label(step)!r}; known: {known}",
+        )
+
+    if step.get("agents") is not None or step.get("plugins") is not None or step.get("image") is not None:
+        raise ValueError(
+            f"step {_get_step_label(step)!r} sets mirror_hardwares together with agents/plugins/image; "
+            "use mirror_hardwares only",
+        )
+
+    expanded = copy.deepcopy(preset)
+    return {key: value for key, value in step.items() if key != "mirror_hardwares"} | expanded
+
+
+def _match_source_file(changed_files: list[str], prefixes: list[str]) -> bool:
+    for path in changed_files:
+        for prefix in prefixes:
+            normalized = prefix.rstrip("/")
+            if path == normalized or path.startswith(f"{normalized}/"):
+                return True
+    return False
+
+
+def _get_step_label(step: dict[str, Any]) -> str:
+    return str(step.get("group") or step.get("label") or "<step>")
+
+
+def _process_test_steps(
+    steps: list[Any],
+    changed_files: list[str] | None,
+) -> list[Any]:
+    """Drop steps by ``source_file_dependencies`` when *changed_files* is set; always strip that field."""
+    processed: list[Any] = []
+    for step in steps:
+        if not isinstance(step, dict):
+            processed.append(step)
+            continue
+
+        deps = step.get("source_file_dependencies")
+        if deps is not None and not isinstance(deps, list):
+            raise ValueError(
+                f"source_file_dependencies must be a list in step {_get_step_label(step)!r}",
+            )
+        if changed_files is not None and deps is not None and not _match_source_file(changed_files, deps):
+            _log(f"skip {_get_step_label(step)!r} (no changes under {deps})")
+            continue
+
+        nested = step.get("steps")
+        if nested is not None:
+            kept_nested = _process_test_steps(nested, changed_files)
+            if changed_files is not None and not kept_nested:
+                _log(f"omit empty group {_get_step_label(step)!r}")
+                continue
+            new_step = {key: value for key, value in step.items() if key != "source_file_dependencies"}
+            new_step["steps"] = kept_nested
+            processed.append(new_step)
+            continue
+
+        if deps is not None:
+            processed.append(
+                _expand_mirror_hardwares(
+                    {key: value for key, value in step.items() if key != "source_file_dependencies"},
+                ),
+            )
+        else:
+            processed.append(_expand_mirror_hardwares(step))
+
+    return processed
+
+
+def _select_e2e_group_steps(steps: list[Any]) -> list[Any]:
+    """Keep only top-level groups whose name contains ``E2E_GROUP_MARKER``."""
+    selected = [
+        step
+        for step in steps
+        if isinstance(step, dict) and isinstance(step.get("group"), str) and E2E_GROUP_MARKER in step["group"]
+    ]
+    if not selected:
+        _log(f"no group matching {E2E_GROUP_MARKER!r} found")
+    else:
+        _log(f"keep {len(selected)} group(s) matching {E2E_GROUP_MARKER!r}")
+    return selected
+
+
+def _render_test_pipeline(
+    doc: dict[str, Any],
+    changed_files: list[str] | None,
+    *,
+    e2e_only: bool = False,
+) -> dict[str, Any]:
+    """Filter steps by PR diff and strip uploader-only ``source_file_dependencies`` metadata."""
+    steps = doc.get("steps")
+    if not isinstance(steps, list):
+        return doc
+    if e2e_only:
+        steps = _select_e2e_group_steps(steps)
+    steps = _process_test_steps(steps, changed_files)
+    return {**doc, "steps": steps}
+
+
+# --- Entry (read file → bootstrap or test render → YAML string) ---
+
+
+def _render_pipeline(
+    path: Path,
+    *,
+    force_all: bool = False,
+    e2e_only: bool = False,
+) -> str:
+    if path.name == BOOTSTRAP_STEPS_FILENAME:
+        ctx = resolve_ci_context_from_git()
+        continuation = _load_bootstrap_steps(path)
+        return _render_bootstrap_pipeline(
+            continuation,
+            decision=ctx.decision,
+            path=path,
+        )
+
+    text = path.read_text(encoding="utf-8")
+    ctx = resolve_ci_context_from_git()
+    if force_all or e2e_only:
+        changed_files = None
+    else:
+        changed_files = ctx.changed_files
+
+    doc = yaml.safe_load(text)
+    if not isinstance(doc, dict):
+        raise ValueError(f"invalid pipeline YAML: {path}")
+
+    doc = _render_test_pipeline(doc, changed_files, e2e_only=e2e_only)
+    return yaml.safe_dump(doc, sort_keys=False)
+
+
+def _upload_to_buildkite(content: str) -> None:
+    subprocess.run(
+        ["buildkite-agent", "pipeline", "upload"],
+        input=content,
+        text=True,
+        check=True,
+    )
+
+
+# --- CLI ---
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "pipeline",
+        nargs="?",
+        default=".buildkite/cuda/bootstrap-upload-steps.yml",
+        help="Pipeline YAML path (default: .buildkite/cuda/bootstrap-upload-steps.yml)",
+    )
+    parser.add_argument(
+        "--upload",
+        action="store_true",
+        help="Pipe rendered YAML to buildkite-agent pipeline upload",
+    )
+    mode = parser.add_mutually_exclusive_group()
+    mode.add_argument(
+        "--all",
+        action="store_true",
+        help="Keep all steps (disable diff-aware skipping)",
+    )
+    mode.add_argument(
+        "--e2e",
+        action="store_true",
+        help="Keep only the E2E Test group",
+    )
+    args = parser.parse_args()
+
+    path = Path(args.pipeline)
+    if not path.is_absolute():
+        path = ROOT / path
+    if not path.is_file():
+        _log(f"missing pipeline file: {path}")
+        return 1
+
+    rendered = _render_pipeline(path, force_all=args.all, e2e_only=args.e2e)
+    if args.upload:
+        _upload_to_buildkite(rendered)
+    else:
+        sys.stdout.write(rendered)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.buildkite/cuda/bootstrap-upload-steps.yml

@@ -0,0 +1,55 @@
+# Uploaded by upload_pipeline.py --upload (not the Buildkite hook).
+# ``if`` is injected by step ``key`` from skip-ci at upload time.
+steps:
+  - label: ":docker: Build image"
+    key: image-build
+    commands:
+      - "bash .buildkite/cuda/scripts/build-ci-image.sh"
+    agents:
+      queue: "cpu_queue_premerge"
+
+  # L2 Test — PR ready label (diff-aware) or main+WEEKLY=1 (--e2e, full E2E group)
+  - label: "Upload Ready Pipeline"
+    depends_on: image-build
+    key: upload-ready-pipeline
+    commands:
+      - |
+        EXTRA=""
+        if [ "${WEEKLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then
+          EXTRA="--e2e"
+        fi
+        python3 .buildkite/common/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-ready.yml
+    agents:
+      queue: "cpu_queue_premerge"
+
+  # L3 Test — PR merge-test label (diff-aware) or main+WEEKLY=1 (--e2e, full E2E group)
+  - label: "Upload Merge Pipeline"
+    depends_on: image-build
+    key: upload-merge-pipeline
+    commands:
+      - |
+        EXTRA=""
+        if [ "${WEEKLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then
+          EXTRA="--e2e"
+        fi
+        python3 .buildkite/common/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-merge.yml
+    agents:
+      queue: "cpu_queue_premerge"
+
+  # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
+  - label: "Upload Nightly Pipeline"
+    depends_on: image-build
+    key: upload-nightly-pipeline
+    commands:
+      - python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/cuda/test-nightly.yml
+    agents:
+      queue: "cpu_queue_premerge"
+
+  # L5 Test — main+WEEKLY=1 / NON_CRITICAL=1 (scheduled), or PR with weekly-test label
+  - label: "Upload Weekly Pipeline"
+    depends_on: image-build
+    key: upload-weekly-pipeline
+    commands:
+      - python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/cuda/test-weekly.yml
+    agents:
+      queue: "cpu_queue_premerge"

.buildkite/cuda/pipeline.yml

@@ -0,0 +1,9 @@
+# Hook uploads this entry YAML; upload_pipeline.py --upload reads bootstrap-upload-steps.yml
+# and injects ``if`` by step ``key`` from skip-ci.
+steps:
+  - label: ":github: Resolve skip-ci & upload pipeline"
+    key: upload-ci-pipeline
+    commands:
+      - "python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/cuda/bootstrap-upload-steps.yml"
+    agents:
+      queue: "cpu_queue_premerge"

.buildkite/cuda/rebase-pipeline.yml

@@ -2,10 +2,7 @@ steps:
   - label: ":docker: Build image"
     key: image-build
     commands:
-      - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
-      - "docker build --progress=plain --file docker/Dockerfile.ci -t vllm-omni-ci ."
-      - "docker tag vllm-omni-ci public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
-      - "docker push public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
+      - "bash .buildkite/cuda/scripts/build-ci-image.sh"
     agents:
       queue: "cpu_queue_premerge"
 
@@ -14,7 +11,7 @@ steps:
     depends_on: image-build
     key: upload-ready-pipeline
     commands:
-      - python3 .buildkite/scripts/upload_pipeline.py --upload .buildkite/test-ready.yml
+      - python3 .buildkite/common/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-ready.yml
     agents:
       queue: "cpu_queue_premerge"
 
@@ -23,7 +20,7 @@ steps:
     depends_on: image-build
     key: upload-merge-pipeline
     commands:
-      - python3 .buildkite/scripts/upload_pipeline.py --upload .buildkite/test-merge.yml
+      - python3 .buildkite/common/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-merge.yml
     agents:
       queue: "cpu_queue_premerge"
 
@@ -33,6 +30,6 @@ steps:
     key: upload-nightly-pipeline
     if: build.env("NIGHTLY") == "1"
     commands:
-      - buildkite-agent pipeline upload .buildkite/test-nightly.yml
+      - python3 .buildkite/common/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-nightly.yml
     agents:
       queue: "cpu_queue_premerge"

.buildkite/cuda/scripts/build-ci-image.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+set -euo pipefail
+
+# Build and push the CI Docker image with registry-based layer caching.
+#
+# Usage: build-ci-image.sh
+#
+# Expects the following environment variables (set by Buildkite):
+#   BUILDKITE_COMMIT          - commit SHA used as the image tag
+ECR_NAMESPACE="public.ecr.aws/q9t5s3a7"
+REGISTRY="${ECR_NAMESPACE}/vllm-ci-test-repo"
+REGION="us-east-1"
+DOCKERFILE="docker/Dockerfile.ci"
+BUILDER_NAME="vllm-omni-builder"
+
+# Ensure that the env vars are actually set, otherwise exit early
+if [ -z "${BUILDKITE_COMMIT:-}" ]; then
+    echo "ERROR: BUILDKITE_COMMIT is not set"
+    exit 1
+fi
+echo "BUILDKITE_COMMIT: ${BUILDKITE_COMMIT}"
+
+# Authenticate to ECR Public
+aws ecr-public get-login-password --region "$REGION" \
+    | docker login --username AWS --password-stdin "$ECR_NAMESPACE"
+
+# Compute the cache tag for the dependencies image as the hash of the contents.
+# This is because dependencies don't change often, so most PRs will have the same hash.
+#
+# The Dockerfile is also included to avoid churn in the cached image based on the vLLM version.
+DEP_FILES="pyproject.toml setup.py requirements/*.txt docker/Dockerfile.ci"
+CACHE_KEY=$(cat ${DEP_FILES} | sha256sum | cut -c1-16)
+CACHE_TAG="deps-cache-${CACHE_KEY}"
+echo "Cache key: ${CACHE_TAG}"
+
+# Set up buildx with docker-container driver; we need to do this
+# since cache export is not supported for the default docker driver
+# that is running in the CI.
+docker buildx inspect $BUILDER_NAME >/dev/null 2>&1 \
+|| docker buildx create --name $BUILDER_NAME --driver docker-container
+docker buildx use $BUILDER_NAME
+
+echo "Building image tag: ${REGISTRY}:${BUILDKITE_COMMIT}"
+
+# Build + push the image to the registry, using the hash from the dependencies.
+# Note that the caching here is layerwise, so we're just making sure we always
+# can cache hit on everything but the last copy at the moment.
+docker buildx build --push --progress=plain \
+    --cache-from "type=registry,ref=${REGISTRY}:${CACHE_TAG}" \
+    --cache-to "type=registry,ref=${REGISTRY}:${CACHE_TAG},mode=max,compression=zstd" \
+    --file "$DOCKERFILE" \
+    -t "${REGISTRY}:${BUILDKITE_COMMIT}" .

.buildkite/cuda/test-merge.yml

@@ -0,0 +1,231 @@
+env:
+  VLLM_WORKER_MULTIPROC_METHOD: spawn
+  VLLM_USE_FLASHINFER_MOE_FP16: 0
+  HF_HUB_DOWNLOAD_TIMEOUT: 300
+  HF_HUB_ETAG_TIMEOUT: 60
+
+steps:
+  - group: ":card_index_dividers: Simple Test"
+    depends_on: upload-merge-pipeline
+    steps:
+      - label: "Simple · Model Executor Test"
+        timeout_in_minutes: 20
+        commands:
+          - "pytest -sv tests/model_executor -m 'core_model and cpu'"
+        mirror_hardwares: l4_1
+
+      - label: "Simple · Diffusion Test"
+        timeout_in_minutes: 20
+        commands:
+          - "pytest -sv tests/diffusion -m 'core_model and cpu'"
+        mirror_hardwares: l4_1
+
+      - label: "Simple · Engine&Entrypoints Test"
+        timeout_in_minutes: 20
+        commands:
+          - "pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu'"
+        mirror_hardwares: l4_1
+
+      - label: "Simple · Other Test"
+        timeout_in_minutes: 20
+        commands:
+          - "pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine"
+        mirror_hardwares: l4_1
+
+  - group: ":card_index_dividers: Diffusion Test"
+    depends_on: upload-merge-pipeline
+    steps:
+      - label: "Diffusion · LoRA Test"
+        timeout_in_minutes: 30
+        commands:
+          - pytest -sv tests/diffusion/lora/ -m 'advanced_model and cuda' --run-level "advanced_model"
+        mirror_hardwares: l4_1
+
+  - label: "Distributed Test"
+    depends_on: upload-merge-pipeline
+    timeout_in_minutes: 60
+    commands:
+      - |
+        timeout 55m bash -c "
+          set -e
+          export VLLM_IMAGE_FETCH_TIMEOUT=60
+          pytest -s -v tests/distributed -m 'advanced_model and cuda and H100' --run-level 'advanced_model'
+        "
+    mirror_hardwares: h100_1
+
+  - group: ":card_index_dividers: Entrypoints Test"
+    depends_on: upload-merge-pipeline
+    steps:
+      - label: "Entrypoint Test with H100"
+        # TEMPORARY 60 -> 90: engine startup/sleep-wake is ~13-15 min slower per
+        # run on the vLLM 0.26 image (model loading ~2x, init ~2x; see builds
+        # 2915 vs 2916), which pushed this suite past 60 min. Revert once the
+        # 0.26 startup regression is understood and fixed.
+        timeout_in_minutes: 90
+        commands:
+          - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and H100" --run-level "advanced_model"
+        mirror_hardwares: h100_2
+
+      - label: "Entrypoint Test with L4"
+        timeout_in_minutes: 60
+        commands:
+          - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and L4" --run-level "advanced_model"
+        mirror_hardwares: l4_1
+
+  - group: ":card_index_dividers: E2E Test"
+    depends_on: upload-merge-pipeline
+    steps:
+      - label: "TTS · Qwen3-TTS CustomVoice Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen3_tts_customvoice.py
+          - tests/e2e/offline_inference/test_qwen3_tts_customvoice.py
+          - vllm_omni/model_executor/models/qwen3_tts/
+          - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
+          - vllm_omni/model_executor/models/common/snake_activation.py
+          - vllm_omni/model_executor/models/common/alias_free_activation.py
+          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
+          - vllm_omni/deploy/qwen3_tts.yaml
+        commands:
+          - |
+            timeout 40m bash -c "
+              export VLLM_LOGGING_LEVEL=DEBUG
+              export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
+              pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+            "
+        mirror_hardwares: l4_1
+
+      - label: "TTS · Qwen3-TTS Base Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen3_tts_base.py
+          - tests/e2e/offline_inference/test_qwen3_tts_base.py
+          - vllm_omni/model_executor/models/qwen3_tts/
+          - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
+          - vllm_omni/model_executor/models/common/snake_activation.py
+          - vllm_omni/model_executor/models/common/alias_free_activation.py
+          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
+          - vllm_omni/deploy/qwen3_tts.yaml
+          - .buildkite/common/scripts/run_cov_split.sh
+          - pyproject.toml
+        # 40 -> 60: the coverage split runs the model twice.
+        timeout_in_minutes: 60
+        commands:
+          - |
+            export VLLM_LOGGING_LEVEL=DEBUG
+            export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
+            .buildkite/common/scripts/run_cov_split.sh \
+              --model-id qwen3_tts \
+              --offline tests/e2e/offline_inference/test_qwen3_tts_base.py \
+              --online tests/e2e/online_serving/test_qwen3_tts_base.py \
+              -- -m 'advanced_model and cuda' --run-level 'advanced_model'
+        mirror_hardwares: l4_1
+
+      - label: "Omni · Qwen3-Omni Test"
+        source_file_dependencies:
+          - tests/e2e/offline_inference/test_qwen3_omni.py
+          - tests/e2e/online_serving/test_qwen3_omni.py
+          - tests/helpers/runtime.py
+          - vllm_omni/entrypoints/openai/serving_chat.py
+          - vllm_omni/model_executor/models/qwen3_omni/
+          - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
+          - vllm_omni/model_executor/models/common/snake_activation.py
+          - vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py
+          - vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py
+          - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
+          - vllm_omni/deploy/qwen3_omni_moe.yaml
+        timeout_in_minutes: 50
+        commands:
+          - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+        mirror_hardwares: h100_2
+
+      - label: "Omni · MiniCPM-o 4.5 Test"
+        source_file_dependencies:
+          - tests/e2e/offline_inference/test_minicpmo_4_5.py
+          - tests/e2e/online_serving/test_minicpmo_4_5.py
+          - vllm_omni/model_executor/models/minicpmo_4_5/
+          - vllm_omni/model_executor/models/step_audio2/
+          - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
+          - vllm_omni/model_executor/models/cosyvoice3/utils.py
+          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
+          - vllm_omni/deploy/minicpmo_4_5.yaml
+        timeout_in_minutes: 50
+        commands:
+          - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+        # h100_2, not h100_1: advanced_model asserts transcribe every audio
+        # response with Whisper, and tests/helpers/media.py only uses an
+        # accelerator for Whisper when a spare GPU exists. On a single card it
+        # grades on CPU: ~8-10 min per clip vs ~30 s, which alone overruns the
+        # 50-minute job timeout (build 2956).
+        mirror_hardwares: h100_2
+
+      - label: "Omni · MiniCPM-o 4.5 Duplex Test"
+        source_file_dependencies:
+          - tests/assets/minicpmo_4_5/response_required_16k.wav
+          - tests/assets/minicpmo_4_5/soft_interrupt_16k.wav
+          - tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
+          - tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
+          - tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py
+          - tests/e2e/online_serving/helpers/minicpmo_realtime_duplex_scenarios.py
+          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py
+          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py
+          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
+          - vllm_omni/deploy/minicpmo_4_5.yaml
+          - tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py
+          - vllm_omni/experimental/fullduplex/
+          - vllm_omni/model_executor/models/minicpmo_4_5/
+          - vllm_omni/model_executor/models/step_audio2/
+          - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
+          - vllm_omni/model_executor/models/cosyvoice3/utils.py
+          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
+          - vllm_omni/core/sched/
+          - vllm_omni/outputs/multimodal_accumulation.py
+          - vllm_omni/engine/stage_init_utils.py
+          - vllm_omni/engine/async_omni_engine.py
+          - vllm_omni/engine/orchestrator.py
+          - vllm_omni/entrypoints/cli/serve.py
+          - vllm_omni/entrypoints/openai/api_server.py
+          - vllm_omni/entrypoints/openai/duplex_capability.py
+          - vllm_omni/config/stage_config.py
+        timeout_in_minutes: 50
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · Qwen Image Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen_image.py
+          - vllm_omni/diffusion/models/qwen_image/
+          - vllm_omni/diffusion/models/dmd2/
+          - vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_qwenimage.py
+        commands:
+          - |
+            timeout 40m bash -c '
+              export VLLM_IMAGE_FETCH_TIMEOUT=60
+              pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+            '
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · Wan22 Test"
+        source_file_dependencies:
+          - tests/e2e/offline_inference/test_wan22_t2v.py
+          - tests/e2e/online_serving/test_wan22_t2v.py
+          - vllm_omni/diffusion/models/wan2_2/
+          - vllm_omni/diffusion/models/dmd2/
+        commands:
+          - |
+            timeout 40m bash -c "
+              export VLLM_IMAGE_FETCH_TIMEOUT=60
+              pytest -s -v tests/e2e/offline_inference/test_wan22_t2v.py tests/e2e/online_serving/test_wan22_t2v.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
+            "
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · HunyuanVideo-1.5 Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_hunyuan_video_15_expansion.py
+          - vllm_omni/diffusion/models/hunyuan_video/
+        commands:
+          - |
+            timeout 40m bash -c "
+              export VLLM_IMAGE_FETCH_TIMEOUT=60
+              pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
+            "
+        mirror_hardwares: h100_1

.buildkite/cuda/test-nightly.yml

@@ -0,0 +1,431 @@
+env:
+  VLLM_WORKER_MULTIPROC_METHOD: spawn
+  VLLM_USE_FLASHINFER_MOE_FP16: 0
+  HF_HUB_DOWNLOAD_TIMEOUT: 300
+  HF_HUB_ETAG_TIMEOUT: 60
+
+steps:
+  # Group: collapses under one heading in the Buildkite UI; child steps still run in parallel.
+  - group: ":card_index_dividers: Omni Model Test"
+    key: nightly-omni-test-group
+    depends_on: upload-nightly-pipeline
+    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "omni-test"
+    steps:
+
+      - label: ":full_moon: Omni · Function Test with H100"
+        timeout_in_minutes: 120
+        commands:
+          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Omni · MiniCPM-o 4.5 Duplex Test"
+        timeout_in_minutes: 50
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and omni" --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: ":full_moon: Omni · Doc Test with L4"
+        timeout_in_minutes: 90
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - pytest -s -v tests/examples/ -m "full_model and omni and L4" --run-level "full_model"
+        mirror_hardwares: l4_4
+
+      - label: ":full_moon: Omni · Doc Test with H100"
+        timeout_in_minutes: 90
+        commands:
+          - pytest -s -v tests/examples/ -m "full_model and omni and H100" --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Omni · Accuracy Test"
+        timeout_in_minutes: 180
+        commands:
+          - export SEED_TTS_WER_EVAL=1
+          - export SEED_TTS_EVAL_DEVICE=cuda:1
+          - |
+            set +e
+            pytest -s -v tests/e2e/accuracy/qwen3_omni/test_qwen3_omni.py -m "full_model" --run-level full_model
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc/*.json"
+            exit $$EXIT
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Omni · MiniCPM-o 4.5 · Accuracy Test"
+        timeout_in_minutes: 180
+        commands:
+          - export SEED_TTS_WER_EVAL=1
+          - export SEED_TTS_EVAL_DEVICE=cuda:1
+          - |
+            set +e
+            pytest -s -v tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py -m "full_model and cuda" --run-level full_model
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/e2e/accuracy/minicpmo_4_5/results/*.json"
+            exit $$EXIT
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Omni · Perf Test · No Async Chunk"
+        key: nightly-omni-performance-no-async-chunk
+        timeout_in_minutes: 300
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
+            exit $$EXIT
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Omni · Perf Test · Async Chunk"
+        key: nightly-omni-performance-async-chunk
+        timeout_in_minutes: 180
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json -m "H100 and full_model"
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
+            exit $$EXIT
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test"
+        key: nightly-omni-performance-minicpmo-4-5
+        timeout_in_minutes: 180
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
+            exit $$EXIT
+        mirror_hardwares: h100_1
+
+      - label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Seed-TTS Perf Test"
+        key: nightly-omni-performance-minicpmo-4-5-duplex-seed-tts
+        timeout_in_minutes: 180
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
+            exit $$EXIT
+        mirror_hardwares: h100_1
+
+      - label: ":full_moon: Omni · Multi-Replica Startup Test with 4x H100"
+        timeout_in_minutes: 45
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py -m "full_model" --run-level "core_model"
+        mirror_hardwares: h100_4
+
+  - group: ":card_index_dividers: TTS Model Test"
+    key: nightly-tts-test-group
+    depends_on: upload-nightly-pipeline
+    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test"
+    steps:
+
+      - label: ":full_moon: TTS · Function Test with L4"
+        timeout_in_minutes: 120
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - export VLLM_USE_DEEP_GEMM="0"
+          - export VLLM_MOE_USE_DEEP_GEMM="0"
+          - pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy
+        mirror_hardwares: l4_1
+
+      - label: ":full_moon: TTS · Perf Test"
+        key: nightly-tts-performance
+        timeout_in_minutes: 180
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
+            exit $$EXIT
+        mirror_hardwares: h100_2
+
+  - group: ":card_index_dividers: Tiny Model Tests [Multi-GPU]"
+    key: nightly-tiny-model-test-group
+    depends_on: upload-nightly-pipeline
+    if: >-
+      build.env("NIGHTLY") == "1" ||
+      build.pull_request.labels includes "nightly-test"
+    steps:
+
+      - label: ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations)"
+        timeout_in_minutes: 30
+        # NOTE: Single-GPU (core_model) tiny model tests run on the ready label,
+        # so nightly only adds the multi-GPU parallelism configurations.
+        #
+        # This is a bit confusing, but `full_model` is used to mark the multigpu tests, and
+        # --run-level `core_model` is used to indicate that we should use tiny model weights
+        # for the CI, since these tests should pass irrespective of whether or not we use
+        # the full weights or not.
+        commands:
+          - timeout 25m pytest -sv tests/model_tests/diffusion -m 'full_model and cuda' --run-level "core_model"
+        mirror_hardwares: l4_4
+
+  - group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test"
+    key: nightly-diffusion-x2iat-group
+    depends_on: upload-nightly-pipeline
+    if: >-
+      build.env("NIGHTLY") == "1" ||
+      build.pull_request.labels includes "nightly-test" ||
+      build.pull_request.labels includes "diffusion-x2iat-test"
+    steps:
+      # Use explicit file shards so single-card cases do not reserve 2+ GPUs.
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Single-GPU"
+        timeout_in_minutes: 120
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_qwen_image_expansion.py
+            -m "full_model and diffusion and H100 and not distributed_cuda"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Qwen-Image"
+        timeout_in_minutes: 120
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_qwen_image_expansion.py
+            -m "full_model and diffusion and H100 and distributed_cuda"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4"
+        timeout_in_minutes: 120
+        commands:
+          - pytest -sv tests/e2e/ -k "not test_wan and not hunyuan" -m "full_model and diffusion and L4" --run-level "full_model" --ignore=tests/e2e/accuracy
+        mirror_hardwares: l4_4
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · Doc Test"
+        timeout_in_minutes: 60
+        commands:
+          - pytest -s -v tests/examples/*/test_text_to_image.py -m "full_model and example and H100" --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · Accuracy Test"
+        timeout_in_minutes: 180
+        commands:
+          - export VLLM_HTTP_TIMEOUT_KEEP_ALIVE=120
+          - pytest -s -v tests/e2e/accuracy/test_qwen_image.py --run-level full_model
+          - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2i_matches_diffusers' -m full_model --run-level full_model
+        mirror_hardwares: h100_1
+
+      - label: ":full_moon: HunyuanImage3-DIT · Accuracy Test"
+        timeout_in_minutes: 180
+        commands:
+          - export HUNYUAN_IMAGE3_MODEL="tencent/HunyuanImage-3.0-Instruct"
+          - export HUNYUAN_IMAGE3_DEVICES="0,1,2,3"
+          - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m full_model --run-level full_model
+        mirror_hardwares: h100_4
+
+      - label: ":full_moon: HiDream-O1-Image · Accuracy Test"
+        timeout_in_minutes: 180
+        commands:
+          - pytest -s -v tests/e2e/accuracy/test_hidream_o1_image.py -m full_model --run-level full_model
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image"
+        key: nightly-diffusion-x2iat-performance-qwen-image
+        timeout_in_minutes: 180
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
+          - export CACHE_DIT_VERSION=1.5.0
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+        mirror_hardwares: h100_4
+
+  - group: ":card_index_dividers: Diffusion X2V Model Test"
+    key: nightly-diffusion-x2v-group
+    depends_on: upload-nightly-pipeline
+    if: >-
+      build.env("NIGHTLY") == "1" ||
+      build.pull_request.labels includes "nightly-test" ||
+      build.pull_request.labels includes "diffusion-x2v-test"
+    steps:
+
+      - label: ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test"
+        timeout_in_minutes: 90
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda" --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test"
+        timeout_in_minutes: 90
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda" --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Function Test"
+        timeout_in_minutes: 90
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m "full_model and cuda" --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Diffusion X2V · Doc Test"
+        timeout_in_minutes: 60
+        commands:
+          - pytest -s -v tests/examples/offline_inference/test_image_to_video.py -m "full_model and example and H100" --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: ":full_moon: Diffusion X2V · Other Function Test"
+        timeout_in_minutes: 90
+        commands:
+          - pytest -s -v tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py -m "full_model and cuda" --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Diffusion X2V · Wan2.2 Accuracy Test"
+        timeout_in_minutes: 180
+        commands:
+          - pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py -m full_model --run-level full_model
+          - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m full_model --run-level full_model
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Diffusion X2V · MiniMax H3 I2VA/Ref2VA Accuracy Test"
+        timeout_in_minutes: 180
+        commands:
+          - pytest -s -v tests/e2e/accuracy/minimax_h3/test_minimax_h3_i2va_ref2va_similarity.py -m full_model --run-level full_model
+        mirror_hardwares: h100_4
+
+      - label: ":full_moon: Diffusion X2V · Other Accuracy Test"
+        timeout_in_minutes: 180
+        commands:
+          - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py -m full_model --run-level full_model
+          - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m full_model --run-level full_model
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Diffusion X2V · Perf Test"
+        key: nightly-diffusion-x2v-performance
+        timeout_in_minutes: 180
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json
+            EXIT1=$$?
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_cosmos3_vllm_omni.json
+            EXIT2=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            if [ $$EXIT1 -ne 0 ]; then exit $$EXIT1; fi
+            exit $$EXIT2
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Diffusion X2V · Perf Test · HunyuanVideo-1.5"
+        key: nightly-diffusion-x2v-performance-hunyuanvideo15
+        timeout_in_minutes: 180
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json
+            EXIT1=$$?
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json
+            EXIT2=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            if [ $$EXIT1 -ne 0 ]; then exit $$EXIT1; fi
+            exit $$EXIT2
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Diffusion X2V · Perf Test · MiniMax-H3"
+        key: nightly-diffusion-x2v-performance-minimax-h3
+        timeout_in_minutes: 180
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+        mirror_hardwares: h100_4
+
+  - group: ":card_index_dividers: Diffusion Test"
+    key: nightly-diffusion-test-group
+    depends_on: upload-nightly-pipeline
+    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
+    steps:
+      - label: ":full_moon: Diffusion · Distributed Test with L4"
+        timeout_in_minutes: 60
+        commands:
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4' --run-level "full_model"
+        mirror_hardwares: l4_4
+
+      - label: ":full_moon: Diffusion · Distributed Test with H100"
+        timeout_in_minutes: 60
+        commands:
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100' --run-level "full_model"
+        mirror_hardwares: h100_4
+
+      - label: ":full_moon: Diffusion · Offloader Test"
+        timeout_in_minutes: 60
+        commands:
+          - pytest -sv tests/diffusion/offloader -m "full_model and cuda"
+        mirror_hardwares: l4_1
+
+      - label: ":full_moon: Diffusion · Quantization Test with H100 · Single-GPU"
+        timeout_in_minutes: 60
+        commands:
+          - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100' --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: ":full_moon: Diffusion · MiniMax H3 FP8 Quality Test with H100 · 2-GPU"
+        timeout_in_minutes: 60
+        commands:
+          - pytest -sv tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py -m 'full_model and cuda and H100' --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Diffusion · Quantization Test with L4"
+        timeout_in_minutes: 60
+        commands:
+          - pip install "vllm-gguf-plugin==0.0.4"
+          - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4' --run-level "full_model"
+        mirror_hardwares: l4_1
+
+  # NOTE: ":email: Nightly Collection & Email" is deprecated and will be removed soon.
+  # Do not add new depends_on entries or extend this step.
+  - label: ":email: Nightly Collection & Email"
+    key: nightly-perf-distribution
+    depends_on:
+      - nightly-omni-performance-no-async-chunk
+      - nightly-omni-performance-async-chunk
+      - nightly-tts-performance
+      - nightly-diffusion-x2iat-performance-qwen-image
+      - nightly-diffusion-x2v-performance
+    if: build.env("NIGHTLY") == "1" && build.env("EMAIL_DISTRIBUTION") == "1"
+    commands:
+      - pip install openpyxl
+      - export DEFAULT_INPUT_DIR=tests/dfx/perf/results
+      - export DEFAULT_OUTPUT_DIR=tests/dfx/perf/results
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-no-async-chunk
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-async-chunk
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance
+      - python tools/nightly/generate_nightly_perf_excel.py
+      - python tools/nightly/generate_nightly_perf_html.py
+      - python tools/nightly/send_nightly_email.py --report-file "tests/dfx/perf/results/*.xlsx, tests/dfx/perf/results/*.html"
+      - buildkite-agent artifact upload "tests/dfx/perf/results/*.xlsx"
+      - buildkite-agent artifact upload "tests/dfx/perf/results/*.html"
+    agents:
+      queue: "cpu_queue_premerge"

.buildkite/cuda/test-ready.yml

@@ -0,0 +1,352 @@
+env:
+  VLLM_WORKER_MULTIPROC_METHOD: spawn
+  VLLM_USE_FLASHINFER_MOE_FP16: 0
+  HF_HUB_DOWNLOAD_TIMEOUT: 300
+  HF_HUB_ETAG_TIMEOUT: 60
+
+steps:
+  - group: ":card_index_dividers: Simple Test"
+    depends_on: upload-ready-pipeline
+    steps:
+      - label: "Simple · Model Executor Test"
+        timeout_in_minutes: 20
+        commands:
+          - "pytest -sv tests/model_executor -m 'core_model and cpu'"
+        mirror_hardwares: l4_1
+
+      - label: "Simple · Diffusion Test"
+        timeout_in_minutes: 20
+        commands:
+          - "pytest -sv tests/diffusion -m 'core_model and cpu'"
+        mirror_hardwares: l4_1
+
+      - label: "Simple · Engine&Entrypoints Test"
+        timeout_in_minutes: 20
+        commands:
+          - "pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu'"
+        mirror_hardwares: l4_1
+
+      - label: "Simple · Other Test"
+        timeout_in_minutes: 20
+        commands:
+          - "pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine"
+        mirror_hardwares: l4_1
+
+  - group: ":card_index_dividers: Diffusion Test"
+    depends_on: upload-ready-pipeline
+    steps:
+
+      - label: "Diffusion · Offloader Test"
+        commands:
+          - timeout 40m pytest -s -v tests/diffusion/offloader -m "core_model and cuda" --run-level "core_model"
+        mirror_hardwares: l4_1
+
+      - label: "Diffusion · Cache Test"
+        commands:
+          - timeout 15m pytest -sv tests/diffusion/cache -m 'core_model and cuda' --run-level "core_model"
+        mirror_hardwares: l4_1
+
+      - label: "Diffusion · Batch Test"
+        commands:
+          - timeout 15m pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model"
+        mirror_hardwares: l4_1
+
+      - label: "Diffusion · Distributed & Attention Test"
+        source_file_dependencies:
+          - tests/diffusion/distributed/
+          - tests/diffusion/attention/test_attention_sp.py
+          - vllm_omni/diffusion/distributed/
+          - vllm_omni/diffusion/attention/
+          - vllm_omni/diffusion/data.py
+          - vllm_omni/diffusion/models/
+        commands:
+          - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda' --run-level "core_model"
+          - timeout 20m pytest -sv tests/diffusion/attention/test_attention_sp.py -m 'core_model and cuda' --run-level "core_model"
+        mirror_hardwares: l4_4
+
+      - label: "Diffusion · Model Test"
+        commands:
+          - timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model"
+        mirror_hardwares: l4_1
+
+      - label: "Diffusion · Other Test"
+        commands:
+          - timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda' --run-level "core_model"
+        mirror_hardwares: l4_1
+
+  - group: ":card_index_dividers: Tiny Model Tests [Single GPU]"
+    depends_on: upload-ready-pipeline
+    steps:
+
+      - label: "Tiny Model Tests · Diffusion (base)"
+        commands:
+          - timeout 15m pytest -sv -n 4 tests/model_tests/diffusion -m 'core_model and cuda' --run-level "core_model"
+        mirror_hardwares: l4_1
+
+  - label: "Engine&Model Executor Test"
+    depends_on: upload-ready-pipeline
+    commands:
+      - |
+        timeout 15m bash -c "
+                  pytest -sv tests/engine/ tests/model_executor/ -m 'core_model and cuda' --run-level 'core_model'
+        "
+    mirror_hardwares: l4_1
+
+  - label: "Distributed Test"
+    depends_on: upload-ready-pipeline
+    commands:
+      - |
+        timeout 20m bash -c "
+          pytest -sv tests/distributed/ -m 'core_model and cuda and L4' --run-level 'core_model'
+        "
+    mirror_hardwares: l4_1
+
+  - label: "Entrypoints Test"
+    depends_on: upload-ready-pipeline
+    commands:
+      - |
+        timeout 30m bash -c "
+          export VLLM_WORKER_MULTIPROC_METHOD=spawn
+          pytest -sv tests/entrypoints/ -m 'core_model and cuda' --run-level 'core_model'
+        "
+    mirror_hardwares: l4_1
+
+  - group: ":card_index_dividers: E2E Test"
+    depends_on: upload-ready-pipeline
+    steps:
+      - label: "Custom Pipeline Test"
+        source_file_dependencies:
+          - tests/e2e/features/custom_pipeline/
+          - tests/e2e/features/helpers/custom_pipeline.py
+          - vllm_omni/diffusion/
+          - vllm_omni/engine/
+          - vllm_omni/entrypoints/
+          - vllm_omni/inputs/
+          - vllm_omni/outputs/
+        commands:
+          - timeout 20m pytest -s -v tests/e2e/features/custom_pipeline/ -m "core_model"
+        mirror_hardwares: l4_1
+
+
+      - label: "RLHF · VeRL-Omni E2E Test"
+        source_file_dependencies:
+          - tests/e2e/features/rlhf_test/test_verl_omni_e2e.py
+          - tests/e2e/features/helpers/verl_omni_server.py
+          - tests/e2e/features/helpers/custom_pipeline.py
+          - vllm_omni/diffusion/
+          - vllm_omni/engine/
+          - vllm_omni/entrypoints/
+          - vllm_omni/inputs/
+          - vllm_omni/outputs/
+        commands:
+          - |
+            timeout 20m bash -c "
+              pytest -s -v tests/e2e/features/rlhf_test/test_verl_omni_e2e.py
+            "
+        mirror_hardwares: l4_1
+
+      - label: "Omni · Qwen3-Omni Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen3_omni.py
+          - tests/helpers/runtime.py
+          - vllm_omni/entrypoints/openai/serving_chat.py
+          - vllm_omni/model_executor/models/qwen3_omni/
+          - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
+          - vllm_omni/model_executor/models/common/snake_activation.py
+          - vllm_omni/model_executor/models/qwen2_5_omni/qwen2_5_omni_thinker.py
+          - vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py
+          - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
+          - vllm_omni/deploy/qwen3_omni_moe.yaml
+        commands:
+          - |
+            timeout 20m bash -c "
+              pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m 'core_model' --run-level 'core_model'
+            "
+        mirror_hardwares: h100_2
+
+      - label: "Omni · MiniCPM-o 4.5 Offline Test"
+        source_file_dependencies:
+          - tests/e2e/offline_inference/test_minicpmo_4_5.py
+          - vllm_omni/model_executor/models/minicpmo_4_5/
+          - vllm_omni/model_executor/models/step_audio2/
+          - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
+          - vllm_omni/model_executor/models/cosyvoice3/utils.py
+          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
+          - vllm_omni/deploy/minicpmo_4_5.yaml
+        commands:
+          - |
+            timeout 20m bash -c "
+              pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model'
+            "
+        mirror_hardwares: h100_1
+
+      - label: "Omni · MiniCPM-o 4.5 Online Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_minicpmo_4_5.py
+          - vllm_omni/model_executor/models/minicpmo_4_5/
+          - vllm_omni/model_executor/models/step_audio2/
+          - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
+          - vllm_omni/model_executor/models/cosyvoice3/utils.py
+          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
+          - vllm_omni/deploy/minicpmo_4_5.yaml
+        commands:
+          - |
+            timeout 20m bash -c "
+              pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model'
+            "
+        mirror_hardwares: h100_1
+
+      - label: "Omni · MiniCPM-o 4.5 Duplex Test"
+        source_file_dependencies:
+          - tests/assets/minicpmo_4_5/response_required_16k.wav
+          - tests/assets/minicpmo_4_5/soft_interrupt_16k.wav
+          - tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
+          - tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
+          - tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py
+          - tests/e2e/online_serving/helpers/minicpmo_realtime_duplex_scenarios.py
+          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py
+          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py
+          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
+          - vllm_omni/deploy/minicpmo_4_5.yaml
+          - tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py
+          - vllm_omni/experimental/fullduplex/
+          - vllm_omni/model_executor/models/minicpmo_4_5/
+          - vllm_omni/model_executor/models/step_audio2/
+          - vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
+          - vllm_omni/model_executor/models/cosyvoice3/utils.py
+          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
+          - vllm_omni/core/sched/
+          - vllm_omni/outputs/multimodal_accumulation.py
+          - vllm_omni/engine/stage_init_utils.py
+          - vllm_omni/engine/async_omni_engine.py
+          - vllm_omni/engine/orchestrator.py
+          - vllm_omni/entrypoints/cli/serve.py
+          - vllm_omni/entrypoints/openai/api_server.py
+          - vllm_omni/entrypoints/openai/duplex_capability.py
+          - vllm_omni/config/stage_config.py
+        commands:
+          - |
+            timeout 20m bash -c "
+              pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'core_model and cuda' --run-level 'core_model'
+            "
+        mirror_hardwares: h100_1
+
+      - label: "Omni · MiniCPM-o 4.5 Perf Test"
+        timeout_in_minutes: 60
+        source_file_dependencies:
+          - tests/dfx/perf/tests/test_minicpmo_4_5_ready.json
+          - vllm_omni/deploy/minicpmo_4_5.yaml
+          - vllm_omni/model_executor/models/minicpmo_4_5/
+          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and cuda'
+        mirror_hardwares: h100_1
+
+      - label: "TTS · Qwen3-TTS CustomVoice Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen3_tts_customvoice.py
+          - vllm_omni/model_executor/models/qwen3_tts/
+          - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
+          - vllm_omni/model_executor/models/common/snake_activation.py
+          - vllm_omni/model_executor/models/common/alias_free_activation.py
+          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
+          - vllm_omni/deploy/qwen3_tts.yaml
+        commands:
+          - |
+            timeout 20m bash -c "
+              export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
+              pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'
+            "
+        mirror_hardwares: l4_1
+
+      - label: "TTS · Qwen3-TTS Base Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen3_tts_base.py
+          - vllm_omni/model_executor/models/qwen3_tts/
+          - vllm_omni/model_executor/models/common/qwen3_code_predictor.py
+          - vllm_omni/model_executor/models/common/snake_activation.py
+          - vllm_omni/model_executor/models/common/alias_free_activation.py
+          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
+          - vllm_omni/deploy/qwen3_tts.yaml
+        commands:
+          - |
+            timeout 20m bash -c "
+              export VLLM_WORKER_MULTIPROC_METHOD=spawn
+              export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
+              export VLLM_OMNI_USE_V2_RUNNER=1
+              pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py -m 'core_model' --run-level 'core_model'
+            "
+        mirror_hardwares: l4_1
+
+      - label: "Diffusion · Wan22 Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_wan22_t2v.py
+          - vllm_omni/diffusion/models/wan2_2/
+          - vllm_omni/diffusion/models/dmd2/
+        commands:
+          - |
+            timeout 40m bash -c "
+              export VLLM_IMAGE_FETCH_TIMEOUT=60
+              pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model' --run-level 'core_model'
+            "
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · Cosmos3 Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_cosmos3.py
+          - vllm_omni/diffusion/models/cosmos3/
+          - vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_wan.py
+          - vllm_omni/experimental/world_models/
+          - vllm_omni/model_extras/cosmos3.py
+        commands:
+          - |
+            timeout 40m bash -c "
+              export VLLM_IMAGE_FETCH_TIMEOUT=60
+              pytest -s -v tests/e2e/online_serving/test_cosmos3.py -m 'core_model' --run-level 'core_model'
+            "
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · MiniMax-H3 DLO DP2 T2VA Test"
+        timeout_in_minutes: 90
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py
+          - tests/helpers/assertions.py
+          - tests/helpers/runtime.py
+          - vllm_omni/diffusion/models/minimax_h3/
+          - vllm_omni/diffusion/offloader/
+          - vllm_omni/diffusion/distributed/
+          - vllm_omni/entrypoints/openai/
+        commands:
+          - |
+            timeout 80m bash -c "
+              export VLLM_IMAGE_FETCH_TIMEOUT=60
+              pytest -s -v tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py -m 'core_model and cuda and diffusion' --run-level 'core_model'
+            "
+        mirror_hardwares: h100_2
+
+      - label: "Diffusion · Qwen Image Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen_image.py
+          - vllm_omni/diffusion/models/qwen_image/
+          - vllm_omni/diffusion/models/dmd2/
+          - vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_qwenimage.py
+        commands:
+          - |
+            timeout 40m bash -c "
+              export VLLM_IMAGE_FETCH_TIMEOUT=60
+              pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model' --run-level 'core_model'
+            "
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · HunyuanVideo-1.5 Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_hunyuan_video_15_expansion.py
+          - vllm_omni/diffusion/models/hunyuan_video/
+        commands:
+          - |
+            timeout 40m bash -c "
+              export VLLM_IMAGE_FETCH_TIMEOUT=60
+              pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m 'core_model and diffusion' --run-level 'core_model'
+            "
+        mirror_hardwares: h100_1

.buildkite/cuda/test-weekly.yml

@@ -0,0 +1,197 @@
+env:
+  VLLM_WORKER_MULTIPROC_METHOD: spawn
+  VLLM_USE_FLASHINFER_MOE_FP16: 0
+  HF_HUB_DOWNLOAD_TIMEOUT: 300
+  HF_HUB_ETAG_TIMEOUT: 60
+
+steps:
+  - label: "Simple · CPU Coverage Test"
+    key: weekly-simple-cpu-coverage
+    depends_on: upload-weekly-pipeline
+    if: build.env("WEEKLY") == "1"
+    timeout_in_minutes: 60
+    commands:
+      - |
+        set +e
+        REPORT="coverage-core-model-cpu-$${BUILDKITE_STEP_ID:-local}.xml"
+        pytest -sv tests/ -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml:$${REPORT}
+        EXIT=$$?
+        gzip -9 -f "$${REPORT}"
+        buildkite-agent artifact upload "$${REPORT}.gz"
+        exit $$EXIT
+    mirror_hardwares: l4_1
+
+  - group: "Reliability Test"
+    depends_on: upload-weekly-pipeline
+    if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
+    steps:
+
+      - label: "Reliability Test · qwen3-omni"
+        timeout_in_minutes: 180
+        commands:
+          - pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m "slow"
+        mirror_hardwares: h100_2
+
+      - label: "Reliability Test · wan22"
+        timeout_in_minutes: 180
+        commands:
+          - pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m "slow"
+        mirror_hardwares: h100_2
+
+      - label: "Reliability Test · hunyuan-image"
+        timeout_in_minutes: 180
+        commands:
+          - pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m "slow"
+        mirror_hardwares: h100_4
+
+      - label: "Reliability Test · Invalid parameters · H100"
+        timeout_in_minutes: 120
+        commands:
+          - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100"
+        mirror_hardwares: h100_2
+
+      - label: "Reliability Test · Invalid parameters · L4"
+        timeout_in_minutes: 120
+        commands:
+          - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and L4"
+        mirror_hardwares: l4_1
+
+  - group: "Perf Test"
+    depends_on: upload-weekly-pipeline
+    if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
+    steps:
+
+      - label: ":full_moon: Omni · Perf Test · vLLM Text"
+        key: weekly-omni-performance-vllm-text
+        timeout_in_minutes: 180
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
+            exit $$EXIT
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Omni · Perf Test · Async Chunk · Random"
+        key: weekly-omni-performance-async-chunk-random
+        timeout_in_minutes: 180
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json -m "H100 and slow"
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
+            exit $$EXIT
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Omni · Perf Test · Multi-Replica"
+        key: weekly-omni-performance-multi-replicas
+        timeout_in_minutes: 180
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
+            exit $$EXIT
+        mirror_hardwares: h100_3
+
+  - group: ":card_index_dividers: E2E Tests"
+    depends_on: upload-weekly-pipeline
+    if: build.env("NON_CRITICAL") == "1"
+    steps:
+
+      - label: "Omni · L4"
+        timeout_in_minutes: 90
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model"
+          - >-
+            pytest -s -v tests/e2e/
+            --ignore=tests/e2e/online_serving/test_qwen2_5_omni_expansion.py
+            --ignore=tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py
+            -m "slow and L4 and omni"
+            --run-level "full_model"
+        mirror_hardwares: l4_4
+
+      - label: "Omni · H100"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/ -m "slow and H100 and omni"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: "TTS · L4"
+        timeout_in_minutes: 120
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - export VLLM_USE_DEEP_GEMM="0"
+          - export VLLM_MOE_USE_DEEP_GEMM="0"
+          - |
+            failed=0
+            for f in $(pytest --collect-only -q tests/e2e/ -m "slow and L4 and tts" --run-level "full_model" | grep '\.py::' | sed -E 's|::.*||' | sort -u); do
+              pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1
+            done
+            exit $$failed
+        mirror_hardwares: l4_1
+
+      - label: "TTS · H100"
+        timeout_in_minutes: 90
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - export VLLM_USE_DEEP_GEMM="0"
+          - export VLLM_MOE_USE_DEEP_GEMM="0"
+          - >-
+            pytest -sv
+            tests/e2e/ -m "slow and H100 and tts"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      # Own step: the image has no NanoCodec, so this file import-skips out of
+      # the mark-selected TTS steps above. The overrides keep nemo_toolkit[tts]
+      # from downgrading transformers; see
+      # examples/offline_inference/text_to_speech/README.md.
+      - label: "TTS · Gepard-1.0 · L4"
+        timeout_in_minutes: 60
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - printf '%s\n' "transformers>=5.5.3" "huggingface-hub>=1.0" > /tmp/gepard-overrides.txt
+          - uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" "nemo_toolkit[tts]==2.7.3" --overrides /tmp/gepard-overrides.txt
+          - >-
+            pytest -s -v tests/e2e/offline_inference/test_gepard_tts.py
+            -m "tts and cuda"
+            --run-level "full_model"
+        mirror_hardwares: l4_1
+
+      - label: "Diffusion · L4"
+        timeout_in_minutes: 60
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/ -m "slow and diffusion and L4"
+            --run-level "full_model"
+        mirror_hardwares: l4_4
+
+      - label: "Diffusion · H100 · Single-GPU"
+        timeout_in_minutes: 150
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/ -m "slow and diffusion and H100 and not distributed_cuda"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · H100 · Multi-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/ -m "slow and diffusion and H100 and distributed_cuda"
+            --run-level "full_model"
+        mirror_hardwares: h100_4

.buildkite/intel/pipeline-intel.yml

@@ -1,16 +1,16 @@
-# .buildkite/pipeline-intel.yaml
+# .buildkite/intel/pipeline-intel.yml
 steps:
   - group: "INTEL Tests"
     steps:
       - label: "Intel: XPU Test"
-        command: "bash .buildkite/scripts/hardware_ci/run-xpu-test.sh"
+        command: "bash .buildkite/intel/scripts/run-xpu-test.sh"
         agents:
           queue: "intel-gpu-omni"
         env:
           DOCKER_BUILDKIT: "1"
           # Buildkite will automatically replace this with the actual commit hash
           VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
-          VLLM_VERSION: "v0.22.0"
+          VLLM_VERSION: "v0.27.0"
         priority: 100
         timeout_in_minutes: 60
         soft_fail: false

.buildkite/intel/scripts/bootstrap-intel-omni.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+# vllm-omni Intel bootstrap
+# Uses static pipeline-intel.yml for Intel XPU tests
+
+set -euo pipefail
+
+source .buildkite/common/scripts/resolve_skip_ci.sh
+
+upload_pipeline() {
+    echo "--- 🛠 Preparing Intel pipeline"
+    buildkite-agent pipeline upload .buildkite/intel/pipeline-intel.yml
+}
+
+gate_bootstrap_ci intel l2
+
+upload_pipeline

.buildkite/intel/scripts/run-xpu-test.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+
+# This script build the XPU docker image and run the offline inference inside the container.
+set -ex
+
+omni_source_dir=$(git rev-parse --show-toplevel)
+
+base_image_name="xpu/vllm-omni-ci-base:${VLLM_VERSION:?VLLM_VERSION must be set}"
+image_name="xpu/vllm-omni-ci:${BUILDKITE_COMMIT:?BUILDKITE_COMMIT must be set}"
+container_name="xpu_${BUILDKITE_COMMIT}_$(
+    tr -dc A-Za-z0-9 </dev/urandom | head -c 10
+    echo
+)"
+
+cd "${omni_source_dir}"
+
+# The XPU base image is ~37GB; the default gzip layer exporter is single-threaded
+# and takes ~16min to compress it. zstd is multi-threaded (uses all cores) at a
+# similar ratio; set EXPORT_COMPRESSION=uncompressed to skip compression entirely
+# for local-only images. This requires the containerd image store (buildx docker
+# driver), which is the default here.
+EXPORT_COMPRESSION="${EXPORT_COMPRESSION:-zstd}"
+if [ "${EXPORT_COMPRESSION}" = "uncompressed" ]; then
+    export_args=(--output "type=image,name={{IMAGE}},compression=uncompressed")
+else
+    export_args=(--output "type=image,name={{IMAGE}},compression=${EXPORT_COMPRESSION},compression-level=3,force-compression=true")
+fi
+
+docker_build() {
+    # $1 = image name; remaining args passed through to docker build.
+    local image="$1"
+    shift
+    local out=("${export_args[@]/'{{IMAGE}}'/${image}}")
+    docker build "${out[@]}" "$@" -f docker/Dockerfile.xpu .
+}
+
+build_vllm_base() (
+    local vllm_source_dir
+    vllm_source_dir=$(mktemp -d)
+    trap 'rm -rf "${vllm_source_dir}"' EXIT
+
+    git clone --depth 1 --branch "${VLLM_VERSION}" \
+        https://github.com/vllm-project/vllm "${vllm_source_dir}"
+
+    local out=("${export_args[@]/'{{IMAGE}}'/${base_image_name}}")
+    docker build "${out[@]}" \
+        --target vllm-openai \
+        -f "${vllm_source_dir}/docker/Dockerfile.xpu" \
+        "${vllm_source_dir}"
+)
+
+if [ -z "$(docker images -q "${base_image_name}")" ]; then
+    build_vllm_base
+fi
+
+# Try building the docker image
+docker_build "${image_name}" --build-arg "VLLM_BASE=${base_image_name}" --build-arg "VLLM_VERSION=${VLLM_VERSION}"
+
+# Setup cleanup
+remove_docker_container() {
+    docker rm -f "${container_name}" || true
+    docker image rm -f "${image_name}" || true
+    docker system prune -f || true
+}
+trap remove_docker_container EXIT
+
+HF_CACHE="${HF_CACHE:-$(realpath ~)/.cache/huggingface}"
+mkdir -p "${HF_CACHE}"
+HF_MOUNT="/root/.cache/huggingface"
+
+time timeout -k 30 30m docker run \
+    --device /dev/dri:/dev/dri \
+    --net=host \
+    --ipc=host \
+    -v /dev/dri/by-path:/dev/dri/by-path \
+    -v "${HF_CACHE}:${HF_MOUNT}" \
+    --security-opt seccomp=unconfined \
+    --entrypoint="" \
+    -e VLLM_LOGGING_LEVEL \
+    -e VLLM_OMNI_LOGGING_LEVEL \
+    -e HF_TOKEN \
+    -e ZE_AFFINITY_MASK \
+    --name "${container_name}" \
+    "${image_name}" \
+    bash -c '
+    set -e
+    echo $ZE_AFFINITY_MASK
+    pip install tblib==3.1.0
+    cd /workspace/vllm-omni
+    XPU_TEST_PATHS="tests/diffusion tests/dfx tests/e2e"
+    pytest -v -s $XPU_TEST_PATHS -m "core_model and xpu and B60"
+    pytest -v -s tests/diffusion/quantization/test_mxfp8_config.py
+    pytest -v -s $XPU_TEST_PATHS -m "advanced_model and xpu and B60"
+    pytest -v -s $XPU_TEST_PATHS -m "omni and xpu and B60"
+'

.buildkite/npu/bootstrap-upload-steps.yml

@@ -0,0 +1,181 @@
+# Uploaded by upload_pipeline.py --upload (not the Buildkite hook).
+# ``if`` is injected by step ``key`` from skip-ci at upload time.
+steps:
+  - label: ":buildkit: Build and Push NPU Test Image - A2"
+    key: image-build-a2
+    agents:
+      queue: "ascend-a2b3"
+      resource_class: "npu-1"
+    plugins:
+      - kubernetes:
+          metadata:
+            annotations:
+              vault.hashicorp.com/agent-init-first: "true"
+              vault.hashicorp.com/agent-inject: "true"
+              vault.hashicorp.com/agent-inject-perms-ca.pem: "0400"
+              vault.hashicorp.com/agent-inject-perms-cert.pem: "0400"
+              vault.hashicorp.com/agent-inject-perms-config.json: "0400"
+              vault.hashicorp.com/agent-inject-perms-key.pem: "0400"
+              vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd
+              vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd
+              vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd
+              vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd
+              vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}"
+              vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}"
+              vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}"
+              vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}"
+              vault.hashicorp.com/agent-pre-populate-only: "true"
+              vault.hashicorp.com/agent-run-as-group: "1000"
+              vault.hashicorp.com/agent-run-as-user: "1000"
+              vault.hashicorp.com/agent-service-account-token-volume-name: token-vol
+              vault.hashicorp.com/role: ascend-gha-runners
+              vault.hashicorp.com/secret-volume-path: /home/user/.docker/
+              vault.hashicorp.com/tls-skip-verify: "true"
+          podSpecPatch:
+            volumes:
+            - name: token-vol
+              projected:
+                defaultMode: 420
+                sources:
+                - serviceAccountToken:
+                    audience: api
+                    expirationSeconds: 600
+                    path: token
+    env:
+      VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
+      IMAGE_NAME: "vllm-omni-ci-npu"
+      IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
+      BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
+    command: |
+      set -ex
+
+      echo "--- Building and pushing NPU Test Image"
+      echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
+      echo "buildkitd address: $${BUILDKITD_ADDR}"
+
+      if ! command -v buildctl &> /dev/null; then
+        echo "Installing buildctl..."
+        mkdir -p /tmp/buildkit
+        BUILDKIT_VERSION="v0.29.0"
+        wget -q "https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz
+        tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
+        cp /tmp/buildkit/bin/buildctl /usr/local/bin/
+      fi
+
+      export DOCKER_CONFIG=/home/user/.docker
+      buildctl \
+        --addr="$${BUILDKITD_ADDR}" \
+        --tlscacert=/home/user/.docker/ca.pem \
+        --tlscert=/home/user/.docker/cert.pem \
+        --tlskey=/home/user/.docker/key.pem \
+        build \
+        --frontend dockerfile.v0 \
+        --local context=. \
+        --local dockerfile=./docker \
+        --opt filename=Dockerfile.npu.ci \
+        --secret id=dockerconfig,src=/home/user/.docker/config.json \
+        --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \
+        --progress=plain
+
+      echo "--- Image pushed successfully"
+      echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
+
+  - label: ":buildkit: Build and Push NPU Test Image - A3"
+    key: image-build-a3
+    agents:
+      queue: "ascend-a2b3"
+      resource_class: "npu-1"
+    plugins:
+      - kubernetes:
+          metadata:
+            annotations:
+              vault.hashicorp.com/agent-init-first: "true"
+              vault.hashicorp.com/agent-inject: "true"
+              vault.hashicorp.com/agent-inject-perms-ca.pem: "0400"
+              vault.hashicorp.com/agent-inject-perms-cert.pem: "0400"
+              vault.hashicorp.com/agent-inject-perms-config.json: "0400"
+              vault.hashicorp.com/agent-inject-perms-key.pem: "0400"
+              vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd
+              vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd
+              vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd
+              vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd
+              vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}"
+              vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}"
+              vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}"
+              vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}"
+              vault.hashicorp.com/agent-pre-populate-only: "true"
+              vault.hashicorp.com/agent-run-as-group: "1000"
+              vault.hashicorp.com/agent-run-as-user: "1000"
+              vault.hashicorp.com/agent-service-account-token-volume-name: token-vol
+              vault.hashicorp.com/role: ascend-gha-runners
+              vault.hashicorp.com/secret-volume-path: /home/user/.docker/
+              vault.hashicorp.com/tls-skip-verify: "true"
+          podSpecPatch:
+            volumes:
+            - name: token-vol
+              projected:
+                defaultMode: 420
+                sources:
+                - serviceAccountToken:
+                    audience: api
+                    expirationSeconds: 600
+                    path: token
+    env:
+      VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}-a3"
+      IMAGE_NAME: "vllm-omni-ci-npu"
+      IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
+      BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
+    command: |
+      set -ex
+
+      echo "--- Building and pushing NPU Test Image"
+      echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
+      echo "buildkitd address: $${BUILDKITD_ADDR}"
+
+      if ! command -v buildctl &> /dev/null; then
+        echo "Installing buildctl..."
+        mkdir -p /tmp/buildkit
+        BUILDKIT_VERSION="v0.29.0"
+        wget -q "https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz
+        tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
+        cp /tmp/buildkit/bin/buildctl /usr/local/bin/
+      fi
+
+      export DOCKER_CONFIG=/home/user/.docker
+      buildctl \
+        --addr="$${BUILDKITD_ADDR}" \
+        --tlscacert=/home/user/.docker/ca.pem \
+        --tlscert=/home/user/.docker/cert.pem \
+        --tlskey=/home/user/.docker/key.pem \
+        build \
+        --frontend dockerfile.v0 \
+        --local context=. \
+        --local dockerfile=./docker \
+        --opt filename=Dockerfile.npu.ci.a3 \
+        --secret id=dockerconfig,src=/home/user/.docker/config.json \
+        --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \
+        --progress=plain
+
+      echo "--- Image pushed successfully"
+      echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
+
+  - label: "Upload Ready Pipeline"
+    depends_on:
+      - image-build-a2
+      - image-build-a3
+    key: upload-ready-pipeline
+    agents:
+      queue: "cpu_queue_premerge"
+    command: |
+      python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/npu/test-npu-ready.yml
+
+  # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
+  - label: "Upload Nightly Pipeline"
+    depends_on:
+      - image-build-a2
+      - image-build-a3
+    key: upload-nightly-pipeline
+    commands:
+      - python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/npu/test-npu-nightly.yml
+    agents:
+      queue: "cpu_queue_premerge"

.buildkite/npu/pipeline-npu-a3.yml

@@ -88,7 +88,7 @@
 #   #     queue: "ascend-a3"
 #   #     resource_class: "npu-2"
 #   #   command: |
-#   #     buildkite-agent pipeline upload .buildkite/test-npu-ready.yaml
+#   #     buildkite-agent pipeline upload .buildkite/npu/test-npu-ready.yml
 
   # # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
   # - label: "Upload Nightly Pipeline"
@@ -104,7 +104,7 @@
   #       build.pull_request.labels includes "diffusion-x2v-test"
   #     ))
   #   commands:
-  #     - buildkite-agent pipeline upload .buildkite/test-npu-nightly.yaml
+  #     - buildkite-agent pipeline upload .buildkite/npu/test-npu-nightly.yml
   #   agents:
   #     queue: "ascend-a3"
   #     resource_class: "npu-2"

.buildkite/npu/pipeline-npu.yml

@@ -0,0 +1,9 @@
+# Hook uploads this entry YAML; upload_pipeline.py --upload reads bootstrap-upload-steps.yml
+# and injects ``if`` by step ``key`` from skip-ci.
+steps:
+  - label: ":github: Resolve skip-ci & upload pipeline"
+    key: upload-ci-pipeline
+    commands:
+      - "python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/npu/bootstrap-upload-steps.yml"
+    agents:
+      queue: "cpu_queue_premerge"

.buildkite/npu/test-npu-nightly.yml

@@ -0,0 +1,261 @@
+steps:
+  - group: ":card_index_dividers: Omni Model Test"
+    key: nightly-omni-test-group
+    depends_on: upload-nightly-pipeline
+    if: >-
+      build.env("NIGHTLY") == "1" ||
+      build.pull_request.labels includes "nightly-test" ||
+      build.pull_request.labels includes "omni-test"
+    steps:
+      - label: ":full_moon: Omni · Perf Test · No Async Chunk"
+        key: nightly-omni-performance-no-async-chunk
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_4
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
+            exit $$EXIT
+      - label: ":full_moon: Omni · Perf Test · Async Chunk"
+        key: nightly-omni-performance-async-chunk
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_4
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
+            exit $$EXIT
+      - label: ":full_moon: Omni · MiniCPM-o 4.5 Test"
+        mirror_hardwares: a3_npu_2
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+        commands:
+          - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5_expansion.py -m 'full_model' --run-level 'full_model'
+      - label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Test"
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_2
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+        commands:
+          - >-
+            pytest -s -v
+            tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
+            tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
+            -m "npu and A3 and omni"
+            --run-level full_model
+      - label: ":full_moon: Omni · MiniCPM-o 4.5 · Accuracy Test"
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_2
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+        commands:
+          # Pin the Seed-TTS scorer to the second card: _get_eval_device() otherwise
+          # auto-picks npu:0, which the omni server under test already fills with its
+          # KV cache, so Whisper OOMs on `.to(device)` and every clip reports
+          # "ASR / WER failed".
+          - export SEED_TTS_WER_EVAL=1
+          - export SEED_TTS_EVAL_DEVICE=npu:1
+          - |
+            set +e
+            pytest -s -v tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py -m "full_model and npu and A3" --run-level full_model
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/e2e/accuracy/minicpmo_4_5/results/*.json"
+            exit $$EXIT
+      - label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test"
+        key: nightly-omni-performance-minicpmo-4-5-npu
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_2
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m "npu and A3" --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
+            exit $$EXIT
+      - label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test (A2)"
+        key: nightly-omni-performance-minicpmo-4-5-npu-a2
+        timeout_in_minutes: 180
+        mirror_hardwares: a2b3_npu_1
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+          HF_HUB_OFFLINE: "1"
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m "npu and A2" --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
+            exit $$EXIT
+      - label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Seed-TTS Perf Test"
+        key: nightly-omni-performance-minicpmo-4-5-duplex-seed-tts-npu
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_2
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m "npu and A3" --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
+            exit $$EXIT
+  - group: ":card_index_dividers: TTS Model Test"
+    key: nightly-tts-test-group
+    depends_on: upload-nightly-pipeline
+    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test"
+    steps:
+      - label: ":full_moon: TTS · Perf Test"
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_2
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_tts.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
+            exit $$EXIT
+  - group: ":card_index_dividers: Diffusion I2I Model Test"
+    key: nightly-HunyuanImage3-x2i-test-group
+    depends_on: upload-nightly-pipeline
+    if: >-
+      build.env("NIGHTLY") == "1" ||
+      build.pull_request.labels includes "nightly-test" ||
+      build.pull_request.labels includes "diffusion-x2iat-test"
+    steps:
+      - label: ":full_moon: HunyuanImage3-DIT · Accuracy Test"
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_4
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          HUNYUAN_IMAGE3_MODEL: "tencent/HunyuanImage-3.0-Instruct"
+          HUNYUAN_IMAGE3_DEVICES: "0,1,2,3"
+          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
+        commands:
+          - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m "full_model and A3" --run-level full_model
+  - group: ":card_index_dividers: Diffusion Benchmark Perf Test"
+    key: nightly-diffusion-benchmark-group
+    depends_on: upload-nightly-pipeline
+    if: >-
+      build.env("NIGHTLY") == "1" ||
+      build.pull_request.labels includes "nightly-test" ||
+      build.pull_request.labels includes "diffusion-x2iat-test"
+    steps:
+      - label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=2 CFGP=2"
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_4
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          HF_TOKEN: "${HF_TOKEN}"
+          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m A3 --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+      - label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=2 SP=2"
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_4
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          HF_TOKEN: "${HF_TOKEN}"
+          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m A3 --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+      - label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=4"
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_4
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          HF_TOKEN: "${HF_TOKEN}"
+          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m A3 --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp4.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+  - group: ":card_index_dividers: Diffusion X2V Model Test"
+    key: nightly-diffusion-x2v-group
+    depends_on: upload-nightly-pipeline
+    if: >-
+      build.env("NIGHTLY") == "1" ||
+      build.pull_request.labels includes "nightly-test" ||
+      build.pull_request.labels includes "diffusion-x2v-test"
+    steps:
+      - label: ":full_moon: Diffusion X2V · Function Test"
+        mirror_hardwares: a2b3_npu_4
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k npu_i2v --run-level "full_model"
+      - label: ":full_moon: Diffusion X2V · Perf Test"
+        key: nightly-diffusion-x2v-performance
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_4
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT

.buildkite/npu/test-npu-ready.yml

@@ -0,0 +1,52 @@
+steps:
+  - label: "Platforms Test"
+    key: ready-platforms-test
+    depends_on: upload-ready-pipeline
+    mirror_hardwares: a3_npu_2
+    commands:
+      - >-
+        pytest -s -v
+        tests/platforms/npu/test_minicpmo_code2wav_npugraph.py
+        -m 'core_model and npu and A3 and omni'
+        --run-level 'core_model'
+
+  - group: ":card_index_dividers: Omni Model Test"
+    key: ready-omni-group
+    depends_on: upload-ready-pipeline
+    steps:
+      - label: "Omni · MiniCPM-o 4.5 Test"
+        mirror_hardwares: a3_npu_2
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+        commands:
+          - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model and npu and A3 and omni' --run-level 'core_model'
+      - label: "Omni · MiniCPM-o 4.5 Duplex Test"
+        mirror_hardwares: a3_npu_2
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'core_model and npu and A3 and omni' --run-level 'core_model'
+      - label: "Omni · MiniCPM-o 4.5 Perf Test"
+        timeout_in_minutes: 60
+        mirror_hardwares: a3_npu_2
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and npu and A3 and omni'
+
+  - group: ":card_index_dividers: TTS Model Test"
+    key: ready-tts-group
+    depends_on: upload-ready-pipeline
+    steps:
+      - label: ":sound: TTS · Qwen3-TTS CustomVoice Test"
+        mirror_hardwares: a3_npu_2
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model and npu' --run-level 'core_model'

.buildkite/pipeline-npu.yaml

@@ -1,112 +0,0 @@
-steps:
-  - label: ":buildkit: Build and Push NPU Test Image - A2"
-    key: image-build
-    agents:
-      queue: "ascend-a2b3"
-      resource_class: "npu-1"
-    plugins:
-      - kubernetes:
-          metadata:
-            annotations:
-              vault.hashicorp.com/agent-init-first: "true"
-              vault.hashicorp.com/agent-inject: "true"
-              vault.hashicorp.com/agent-inject-perms-ca.pem: "0400"
-              vault.hashicorp.com/agent-inject-perms-cert.pem: "0400"
-              vault.hashicorp.com/agent-inject-perms-config.json: "0400"
-              vault.hashicorp.com/agent-inject-perms-key.pem: "0400"
-              vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}"
-              vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}"
-              vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}"
-              vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}"
-              vault.hashicorp.com/agent-pre-populate-only: "true"
-              vault.hashicorp.com/agent-run-as-group: "1000"
-              vault.hashicorp.com/agent-run-as-user: "1000"
-              vault.hashicorp.com/agent-service-account-token-volume-name: token-vol
-              vault.hashicorp.com/role: ascend-gha-runners
-              vault.hashicorp.com/secret-volume-path: /home/user/.docker/
-              vault.hashicorp.com/tls-skip-verify: "true"
-          podSpecPatch:
-            volumes:
-            - name: token-vol
-              projected:
-                defaultMode: 420
-                sources:
-                - serviceAccountToken:
-                    audience: api
-                    expirationSeconds: 600
-                    path: token
-    env:
-      VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
-      IMAGE_NAME: "vllm-omni-ci-npu"
-      IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
-      BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
-    command: |
-      set -ex
-
-      echo "--- Building and pushing NPU Test Image"
-      echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
-      echo "buildkitd address: $${BUILDKITD_ADDR}"
-
-      if ! command -v buildctl &> /dev/null; then
-        echo "Installing buildctl..."
-        mkdir -p /tmp/buildkit
-        BUILDKIT_VERSION="v0.29.0"
-        wget -q "https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz
-        tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
-        cp /tmp/buildkit/bin/buildctl /usr/local/bin/
-      fi
-
-      export DOCKER_CONFIG=/home/user/.docker
-      buildctl \
-        --addr="$${BUILDKITD_ADDR}" \
-        --tlscacert=/home/user/.docker/ca.pem \
-        --tlscert=/home/user/.docker/cert.pem \
-        --tlskey=/home/user/.docker/key.pem \
-        build \
-        --frontend dockerfile.v0 \
-        --local context=. \
-        --local dockerfile=./docker \
-        --opt filename=Dockerfile.npu.ci \
-        --secret id=dockerconfig,src=/home/user/.docker/config.json \
-        --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \
-        --progress=plain
-
-      echo "--- Image pushed successfully"
-      echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
-
-  - label: "Upload Ready Pipeline"
-    depends_on: image-build
-    key: upload-ready-pipeline
-    if: >-
-      build.branch != "main" && (
-        build.pull_request.labels includes "npu-test" ||
-        build.pull_request.labels includes "ready"
-      )
-    agents:
-      queue: "ascend-a2b3"
-      resource_class: "npu-1"
-    command: |
-      buildkite-agent pipeline upload .buildkite/test-npu-ready.yaml
-
-  # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
-  - label: "Upload Nightly Pipeline"
-    depends_on: image-build
-    key: upload-nightly-pipeline
-    if: >-
-      (build.branch == "main" && build.env("NIGHTLY") == "1") ||
-      (build.branch != "main" && (
-        build.pull_request.labels includes "nightly-test" ||
-        build.pull_request.labels includes "omni-test" ||
-        build.pull_request.labels includes "tts-test" ||
-        build.pull_request.labels includes "diffusion-x2iat-test" ||
-        build.pull_request.labels includes "diffusion-x2v-test"
-      ))
-    commands:
-      - buildkite-agent pipeline upload .buildkite/test-npu-nightly.yaml
-    agents:
-      queue: "ascend-a2b3"
-      resource_class: "npu-1"

.buildkite/pipeline.yml

@@ -1,76 +0,0 @@
-# Document 1: Buildkite loads only this block on first parse. The next step resolves docs-only skip-ci
-# from git diff, then uploads document 2. When docs-only skip applies, image-build still runs for the nightly
-# exception in upload_pipeline.py (nightly-test label or main NIGHTLY=1); otherwise upload-nightly
-# can be skipped together with test-ready/test-merge. Weekly uploads also depend on image-build.
-
-# Document 2: appended after `---`; same file, read by upload_pipeline.py (not evaluated as a second pipeline by Buildkite). Child uploads include test-ready.yml, test-merge.yml, test-nightly.yml, and test-weekly.yml.
-steps:
-  - label: ":github: Resolve skip-ci & upload pipeline"
-    key: upload-ci-pipeline
-    commands:
-      - "python3 .buildkite/scripts/upload_pipeline.py --upload .buildkite/pipeline.yml"
-    agents:
-      queue: "cpu_queue_premerge"
-
----
-steps:
-  - label: ":docker: Build image"
-    key: image-build
-    if: __IMAGE_BUILD_IF__
-    commands:
-      - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
-      - "docker build --progress=plain --file docker/Dockerfile.ci -t vllm-omni-ci ."
-      - "docker tag vllm-omni-ci public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
-      - "docker push public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
-    agents:
-      queue: "cpu_queue_premerge"
-
-  # L2 Test
-  - label: "Upload Ready Pipeline"
-    depends_on: image-build
-    key: upload-ready-pipeline
-    if: __UPLOAD_READY_IF__
-    commands:
-      - python3 .buildkite/scripts/upload_pipeline.py --upload .buildkite/test-ready.yml
-    agents:
-      queue: "cpu_queue_premerge"
-
-  # L3 Test
-  - label: "Upload Merge Pipeline"
-    depends_on: image-build
-    key: upload-merge-pipeline
-    if: __UPLOAD_MERGE_IF__
-    commands:
-      - python3 .buildkite/scripts/upload_pipeline.py --upload .buildkite/test-merge.yml
-    agents:
-      queue: "cpu_queue_premerge"
-
-  # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
-  - label: "Upload Nightly Pipeline"
-    depends_on: image-build
-    key: upload-nightly-pipeline
-    if: >-
-      (build.branch == "main" && build.env("NIGHTLY") == "1") ||
-      (build.branch != "main" && (
-        build.pull_request.labels includes "nightly-test" ||
-        build.pull_request.labels includes "omni-test" ||
-        build.pull_request.labels includes "tts-test" ||
-        build.pull_request.labels includes "diffusion-x2iat-test" ||
-        build.pull_request.labels includes "diffusion-x2v-test"
-      ))
-    commands:
-      - buildkite-agent pipeline upload .buildkite/test-nightly.yml
-    agents:
-      queue: "cpu_queue_premerge"
-
-  # L5 Test — main+WEEKLY=1 (scheduled), or PR with weekly-test label
-  - label: "Upload Weekly Pipeline"
-    depends_on: image-build
-    key: upload-weekly-pipeline
-    if: >-
-      (build.branch == "main" && build.env("WEEKLY") == "1") ||
-      (build.branch != "main" && build.pull_request.labels includes "weekly-test")
-    commands:
-      - buildkite-agent pipeline upload .buildkite/test-weekly.yml
-    agents:
-      queue: "cpu_queue_premerge"

.buildkite/release/release-pipeline.yml

@@ -16,7 +16,7 @@ steps:
       - "uv venv --python=3.12 && source .venv/bin/activate"
       - "uv pip install --upgrade build"
       - "python3 -m build"
-      - "bash .buildkite/scripts/upload-nightly-wheels.sh"
+      - "bash .buildkite/release/scripts/upload-nightly-wheels.sh"
 
   - label: "Generate and upload wheel indices"
     depends_on: "build-wheel"
@@ -24,7 +24,7 @@ steps:
     agents:
       queue: small_cpu_queue_release
     commands:
-      - "bash .buildkite/scripts/generate-and-upload-nightly-index.sh"
+      - "bash .buildkite/release/scripts/generate-and-upload-nightly-index.sh"
 
   - block: "Upload release wheels to PyPI"
     depends_on: "build-wheel"
@@ -38,11 +38,13 @@ steps:
     agents:
       queue: small_cpu_queue_release
     commands:
-      - "bash .buildkite/scripts/publish-release-wheels-pypi.sh"
+      - "bash .buildkite/release/scripts/publish-release-wheels-pypi.sh"
 
+  # Auto-skip when NIGHTLY=1 (scheduled); otherwise require manual unblock.
   - block: "Unblock to build release Docker images"
     depends_on: ~
     key: block-build-release-images
+    if: build.env("NIGHTLY") != "1"
 
   - group: "Build release Docker images"
     key: "build-release-images"
@@ -63,7 +65,7 @@ steps:
               --progress plain \
               -f docker/Dockerfile.cuda .
           - "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m)"
-          # re-tag to default image tag and push, just in case arm64 build fails
+          # Fallback default tag (no arch suffix) if arm64 build fails.
           - "docker tag public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m) public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT"
           - "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT"
         env:
@@ -86,6 +88,26 @@ steps:
         env:
           DOCKER_BUILDKIT: "1"
 
+  # NIGHTLY=1 only: promote ECR images to DockerHub nightly tags, then prune.
+  - label: "Publish nightly multi-arch image to DockerHub"
+    depends_on:
+      - build-release-image-x86
+      - build-release-image-arm64
+    if: build.env("NIGHTLY") == "1"
+    key: publish-nightly-images-dockerhub
+    agents:
+      queue: small_cpu_queue_release
+    commands:
+      - "bash .buildkite/release/scripts/publish-release-images.sh nightly"
+      - "bash .buildkite/release/scripts/cleanup-nightly-builds.sh"
+    plugins:
+      - docker-login#v3.0.0:
+          username: vllmbot
+          password-env: DOCKERHUB_TOKEN
+    env:
+      DOCKER_BUILDKIT: "1"
+      DOCKERHUB_USERNAME: "vllmbot"
+
   - block: "Publish release images to DockerHub"
     key: block-publish-release-images
     depends_on:
@@ -100,7 +122,7 @@ steps:
     agents:
       queue: small_cpu_queue_release
     commands:
-      - "bash .buildkite/scripts/publish-release-images.sh"
+      - "bash .buildkite/release/scripts/publish-release-images.sh"
     plugins:
       - docker-login#v3.0.0:
           username: vllmbot

.buildkite/release/scripts/cleanup-nightly-builds.sh

@@ -0,0 +1,147 @@
+#!/bin/bash
+# SPDX-License-Identifier: Apache-2.0
+#
+# Clean up old nightly builds from DockerHub (vllm/vllm-omni), keeping the
+# newest 14 tags with the nightly- prefix.
+
+set -ex
+
+TAG_PREFIX="nightly-"
+REPO="vllm/vllm-omni"
+REPO_API_URL="https://hub.docker.com/v2/repositories/${REPO}/tags"
+
+echo "Cleaning up tags with prefix: $TAG_PREFIX in repository: $REPO"
+
+if [ -z "$DOCKERHUB_TOKEN" ]; then
+  echo "Error: DOCKERHUB_TOKEN environment variable is not set"
+  exit 1
+fi
+
+if [ -z "$DOCKERHUB_USERNAME" ]; then
+  echo "Error: DOCKERHUB_USERNAME environment variable is not set"
+  exit 1
+fi
+
+echo "Getting DockerHub bearer token..."
+set +x
+BEARER_TOKEN=$(curl -s -X POST \
+  -H "Content-Type: application/json" \
+  -d "{\"username\": \"$DOCKERHUB_USERNAME\", \"password\": \"$DOCKERHUB_TOKEN\"}" \
+  "https://hub.docker.com/v2/users/login" | jq -r '.token')
+set -x
+
+if [ -z "$BEARER_TOKEN" ] || [ "$BEARER_TOKEN" = "null" ]; then
+  echo "Error: Failed to get DockerHub bearer token"
+  exit 1
+fi
+
+get_all_tags() {
+  local page=1
+  local all_tags=""
+  # Hub `name` is a partial match (not a strict prefix). Use it to narrow the
+  # list, then enforce startswith() client-side. Exhaust API pages; do not
+  # stop merely because a page has no matching tags after filtering.
+  local name_query
+  name_query=$(printf '%s' "$TAG_PREFIX" | jq -sRr @uri)
+
+  while true; do
+    set +x
+    local response
+    response=$(curl -s -H "Authorization: Bearer $BEARER_TOKEN" \
+      "$REPO_API_URL?page=$page&page_size=100&name=${name_query}")
+    set -x
+
+    local page_count
+    page_count=$(echo "$response" | jq -r '.results | length')
+    if [ -z "$page_count" ] || [ "$page_count" = "0" ] || [ "$page_count" = "null" ]; then
+      break
+    fi
+
+    local tags
+    tags=$(echo "$response" | jq -r --arg prefix "$TAG_PREFIX" \
+      '.results[] | select(.name | startswith($prefix)) | "\(.last_updated)|\(.name)"')
+
+    if [ -n "$tags" ]; then
+      all_tags="$all_tags$tags"$'\n'
+    fi
+    page=$((page + 1))
+  done
+
+  echo "$all_tags" | sort -r | cut -d'|' -f2
+}
+
+delete_tag() {
+  local tag_name="$1"
+  echo "Deleting tag: $tag_name"
+
+  local delete_url="https://hub.docker.com/v2/repositories/${REPO}/tags/$tag_name"
+  set +x
+  local response http_code body
+  response=$(curl -s -w "\n%{http_code}" -X DELETE \
+    -H "Authorization: Bearer $BEARER_TOKEN" "$delete_url")
+  set -x
+  http_code=$(printf '%s\n' "$response" | tail -n1)
+  body=$(printf '%s\n' "$response" | sed '$d')
+
+  if [ "$http_code" -ge 200 ] && [ "$http_code" -lt 300 ]; then
+    echo "Successfully deleted tag: $tag_name (HTTP ${http_code})"
+    return 0
+  fi
+
+  local detail=""
+  if echo "$body" | jq -e '.detail' > /dev/null 2>&1; then
+    detail=$(echo "$body" | jq -r '.detail')
+  elif [ -n "$body" ]; then
+    detail="$body"
+  else
+    detail="(empty response body)"
+  fi
+  echo "Error: Failed to delete tag $tag_name (HTTP ${http_code}): ${detail}"
+  return 1
+}
+
+echo "Fetching all tags from DockerHub..."
+all_tags=$(get_all_tags)
+
+if [ -z "$all_tags" ]; then
+  echo "No tags found to clean up"
+  exit 0
+fi
+
+total_tags=$(echo "$all_tags" | wc -l)
+echo "Found $total_tags tags"
+
+tags_to_keep=14
+tags_to_delete=$((total_tags - tags_to_keep))
+
+if [ $tags_to_delete -le 0 ]; then
+  echo "No tags need to be deleted (only $total_tags tags found, keeping $tags_to_keep)"
+  exit 0
+fi
+
+echo "Will delete $tags_to_delete old tags, keeping the newest $tags_to_keep"
+
+tags_to_delete_list=$(echo "$all_tags" | tail -n +$((tags_to_keep + 1)))
+
+if [ -z "$tags_to_delete_list" ]; then
+  echo "No tags to delete"
+  exit 0
+fi
+
+echo "Deleting old tags..."
+delete_failures=0
+while IFS= read -r tag; do
+  if [ -n "$tag" ]; then
+    if ! delete_tag "$tag"; then
+      delete_failures=$((delete_failures + 1))
+    fi
+    sleep 1
+  fi
+done <<< "$tags_to_delete_list"
+
+if [ "$delete_failures" -gt 0 ]; then
+  echo "Cleanup finished with ${delete_failures} deletion failure(s)"
+  exit 1
+fi
+
+echo "Cleanup completed successfully"

.buildkite/release/scripts/generate-and-upload-nightly-index.sh

@@ -36,12 +36,12 @@ mkdir -p "$INDICES_OUTPUT_DIR"
 
 # HACK: we do not need regex module here, but it is required by pre-commit hook
 # To avoid any external dependency, we simply replace it back to the stdlib re module
-sed -i.bak 's/import regex as re/import re/g' .buildkite/scripts/generate-nightly-index.py && rm -f .buildkite/scripts/generate-nightly-index.py.bak
+sed -i.bak 's/import regex as re/import re/g' .buildkite/release/scripts/generate-nightly-index.py && rm -f .buildkite/release/scripts/generate-nightly-index.py.bak
 
 # Generate indices -- the version is just the commit hash (not omni/{commit})
 # because relative paths are computed between the index and wheel directories,
 # both of which live under the omni/ prefix in S3.
-$PYTHON .buildkite/scripts/generate-nightly-index.py \
+$PYTHON .buildkite/release/scripts/generate-nightly-index.py \
     --version "$BUILDKITE_COMMIT" \
     --current-objects "$obj_json" \
     --output-dir "$INDICES_OUTPUT_DIR" \
@@ -78,7 +78,7 @@ if [[ "$version" != *"dev"* ]]; then
     rm -rf "${INDICES_OUTPUT_DIR:?}"
     mkdir -p "$INDICES_OUTPUT_DIR"
     # wheel-dir is overridden to be the commit directory, so that the indices point to the correct wheel path
-    $PYTHON .buildkite/scripts/generate-nightly-index.py \
+    $PYTHON .buildkite/release/scripts/generate-nightly-index.py \
         --version "$s3_version" \
         --wheel-dir "$BUILDKITE_COMMIT" \
         --current-objects "$obj_json" \

.buildkite/release/scripts/publish-release-images.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+# SPDX-License-Identifier: Apache-2.0
+#
+# Publish Docker images from ECR staging to DockerHub.
+#
+# Usage:
+#   publish-release-images.sh            # release mode (default)
+#   publish-release-images.sh release
+#   publish-release-images.sh nightly
+#
+# Release tags: latest, v<version>
+# Nightly tags: nightly, nightly-<commit>
+
+set -euo pipefail
+
+MODE="${1:-release}"
+
+DOCKERHUB_REPO="vllm/vllm-omni"
+ECR_REPO="public.ecr.aws/q9t5s3a7/vllm-omni-release-repo"
+COMMIT="$BUILDKITE_COMMIT"
+PRIMARY_TAGS=()
+EXTRA_MANIFEST_TAGS=()
+
+case "${MODE}" in
+  release)
+    RELEASE_VERSION=$(buildkite-agent meta-data get release-version --default "" | sed 's/^v//')
+    if [ -z "${RELEASE_VERSION}" ]; then
+      echo "ERROR: release-version metadata not set"
+      exit 1
+    fi
+    PRIMARY_TAGS=("latest" "v${RELEASE_VERSION}")
+    ;;
+  nightly)
+    # Commit-pinned manifest reuses nightly-{x86_64,aarch64} arch images.
+    PRIMARY_TAGS=("nightly")
+    EXTRA_MANIFEST_TAGS=("nightly-${COMMIT}")
+    ;;
+  *)
+    echo "Usage: $0 {release|nightly}"
+    exit 2
+    ;;
+esac
+
+echo "========================================"
+echo "Publishing ${MODE} images to ${DOCKERHUB_REPO}"
+echo "  Commit: ${COMMIT}"
+echo "  Primary tags: ${PRIMARY_TAGS[*]}"
+if [ ${#EXTRA_MANIFEST_TAGS[@]} -gt 0 ]; then
+  echo "  Extra manifests: ${EXTRA_MANIFEST_TAGS[*]}"
+fi
+echo "========================================"
+
+aws ecr-public get-login-password --region us-east-1 | \
+  docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
+
+SRC_X86="${ECR_REPO}:${COMMIT}-x86_64"
+SRC_ARM="${ECR_REPO}:${COMMIT}-aarch64"
+
+docker pull "${SRC_X86}"
+docker pull "${SRC_ARM}"
+
+publish_manifest() {
+  local tag="$1"
+  local arch_tag_base="$2"
+  docker manifest rm "${DOCKERHUB_REPO}:${tag}" || true
+  docker manifest create \
+    "${DOCKERHUB_REPO}:${tag}" \
+    "${DOCKERHUB_REPO}:${arch_tag_base}-x86_64" \
+    "${DOCKERHUB_REPO}:${arch_tag_base}-aarch64"
+  docker manifest push "${DOCKERHUB_REPO}:${tag}"
+}
+
+for tag in "${PRIMARY_TAGS[@]}"; do
+  docker tag "${SRC_X86}" "${DOCKERHUB_REPO}:${tag}-x86_64"
+  docker tag "${SRC_ARM}" "${DOCKERHUB_REPO}:${tag}-aarch64"
+  docker push "${DOCKERHUB_REPO}:${tag}-x86_64"
+  docker push "${DOCKERHUB_REPO}:${tag}-aarch64"
+  publish_manifest "${tag}" "${tag}"
+done
+
+ARCH_BASE="${PRIMARY_TAGS[0]}"
+for tag in "${EXTRA_MANIFEST_TAGS[@]}"; do
+  publish_manifest "${tag}" "${ARCH_BASE}"
+done
+
+echo ""
+echo "Successfully published ${MODE} images"

.buildkite/scripts/docker_login_ecr_public.sh

@@ -1,73 +0,0 @@
-#!/bin/bash
-# Helper function to safely login to ECR Public with per-job config isolation
-# Uses DOCKER_CONFIG environment variable to prevent race conditions
-#
-# This script prevents the "device or resource busy" error by giving each
-# Buildkite job its own isolated Docker config directory.
-#
-# Usage:
-#   source docker_login_ecr_public.sh && safe_docker_login_ecr_public
-
-set -euo pipefail
-
-# Configuration
-ECR_REGISTRY="public.ecr.aws"
-
-setup_isolated_docker_config() {
-    # Use BUILDKITE_JOB_ID for job-specific isolation
-    # Fallback to PID if running outside Buildkite
-    local job_id="${BUILDKITE_JOB_ID:-$$}"
-
-    # Set Docker config to job-specific directory
-    export DOCKER_CONFIG="/tmp/docker-config-${job_id}"
-
-    # Create directory if it doesn't exist
-    mkdir -p "$DOCKER_CONFIG"
-
-    echo "[docker-config] Using isolated Docker config: $DOCKER_CONFIG"
-}
-
-check_docker_auth() {
-    # Check if already authenticated to the given registry
-    # Returns 0 if authenticated, 1 if not
-    local registry="$1"
-
-    # Check if credentials exist in the isolated config
-    if [[ -f "$DOCKER_CONFIG/config.json" ]]; then
-        # Check if registry is present in config
-        if grep -q "$registry" "$DOCKER_CONFIG/config.json" 2>/dev/null; then
-            return 0
-        fi
-    fi
-
-    return 1
-}
-
-safe_docker_login_ecr_public() {
-    # Setup isolated config first
-    setup_isolated_docker_config
-
-    local registry="$ECR_REGISTRY"
-
-    # Check if already authenticated (within this job)
-    if check_docker_auth "$registry"; then
-        echo "[docker-login] Already authenticated to $registry in this job"
-        return 0
-    fi
-
-    # Perform login to isolated config directory
-    echo "[docker-login] Logging in to $ECR_REGISTRY (isolated config)..."
-    if aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$ECR_REGISTRY"; then
-        echo "[docker-login] Login successful (config: $DOCKER_CONFIG)"
-        return 0
-    else
-        local exit_code=$?
-        echo "[docker-login] ERROR: Login failed with exit code $exit_code" >&2
-        return $exit_code
-    fi
-}
-
-# Execute if run as script (not sourced)
-if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
-    safe_docker_login_ecr_public
-fi

.buildkite/scripts/hardware_ci/run-amd-test.sh

@@ -1,154 +0,0 @@
-#!/bin/bash
-# vllm-omni customized version
-# Based on: vllm/.buildkite/scripts/hardware_ci/run-amd-test.sh
-# Last synced: 2025-12-15
-# Modifications: docker image name for vllm-omni
-
-# This script runs test inside the corresponding ROCm docker container.
-set -o pipefail
-
-# Export Python path
-export PYTHONPATH=".."
-
-# Print ROCm version
-echo "--- ROCm info"
-rocminfo
-
-# cleanup older docker images
-cleanup_docker() {
-  # Get Docker's root directory
-  docker_root=$(docker info -f '{{.DockerRootDir}}')
-  if [ -z "$docker_root" ]; then
-    echo "Failed to determine Docker root directory."
-    exit 1
-  fi
-  echo "Docker root directory: $docker_root"
-  # Check disk usage of the filesystem where Docker's root directory is located
-  disk_usage=$(df "$docker_root" | tail -1 | awk '{print $5}' | sed 's/%//')
-  # Define the threshold
-  threshold=70
-  if [ "$disk_usage" -gt "$threshold" ]; then
-    echo "Disk usage is above $threshold%. Cleaning up Docker images and volumes..."
-    # Remove dangling images (those that are not tagged and not used by any container)
-    docker image prune -f
-    # Remove unused volumes / force the system prune for old images as well.
-    docker volume prune -f && docker system prune --force --filter "until=72h" --all
-    echo "Docker images and volumes cleanup completed."
-  else
-    echo "Disk usage is below $threshold%. No cleanup needed."
-  fi
-}
-
-# Call the cleanup docker function
-cleanup_docker
-
-echo "--- Pulling container"
-## Temporary change to use AMD Docker Hub to store the vllm-omni image
-# to bypass the rate limit issue with ECR Public Gallery.
-# Images are now stored in a separate repository for vllm-omni, instead of vllm-ci.
-# TODO: @tjtanaa point back to ECR Public Gallery
-# once the amd agents are configured to use ECR Public Gallery.
-# image_name="public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:${BUILDKITE_COMMIT}-rocm-omni"
-image_name="rocm/vllm-omni:${BUILDKITE_COMMIT}"
-container_name="rocm_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)"
-
-# TODO: @tjtanaa uncomment this once the amd agents are configured to use ECR Public Gallery.
-# # Install AWS CLI to authenticate to ECR Public Gallery to get higher rate limit for pulling images
-# sudo apt-get update && sudo apt-get install -y awscli
-##  Use safe docker login helper to prevent race conditions
-# source "$(dirname "${BASH_SOURCE[0]}")/../docker_login_ecr_public.sh"
-# safe_docker_login_ecr_public
-
-## Pull the container from AMD Docker Hub
-
-docker pull "${image_name}"
-
-remove_docker_container() {
-   docker rm -f "${container_name}" || docker image rm -f "${image_name}" || true
-}
-trap remove_docker_container EXIT
-
-echo "--- Running container"
-
-HF_CACHE="$(realpath ~)/huggingface"
-mkdir -p "${HF_CACHE}"
-HF_MOUNT="/root/.cache/huggingface"
-
-if [[ -n "${TEST_COMMAND:-}" ]]; then
-    commands="$TEST_COMMAND"
-else
-    commands="$@"
-fi
-echo "Commands:$commands"
-
-PARALLEL_JOB_COUNT=8
-MYPYTHONPATH=".."
-
-# Test that we're launching on the machine that has
-# proper access to GPUs
-render_gid=$(getent group render | cut -d: -f3)
-if [[ -z "$render_gid" ]]; then
-  echo "Error: 'render' group not found. This is required for GPU access." >&2
-  exit 1
-fi
-
-# check if the command contains shard flag, we will run all shards in parallel because the host have 8 GPUs.
-# TODO: @tjtanaa reenable to run VLLM_ROCM_USE_AITER=1 when AITER is shipped with prebuilt kernels.
-if [[ $commands == *"--shard-id="* ]]; then
-  # assign job count as the number of shards used
-  commands=$(echo "$commands" | sed -E "s/--num-shards[[:blank:]]*=[[:blank:]]*[0-9]*/--num-shards=${PARALLEL_JOB_COUNT} /g" | sed 's/ \\ / /g')
-  for GPU in $(seq 0 $(($PARALLEL_JOB_COUNT-1))); do
-    # assign shard-id for each shard
-    commands_gpu=$(echo "$commands" | sed -E "s/--shard-id[[:blank:]]*=[[:blank:]]*[0-9]*/--shard-id=${GPU} /g" | sed 's/ \\ / /g')
-    echo "Shard ${GPU} commands:$commands_gpu"
-    echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES"
-    docker run \
-        --device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \
-        --network=host \
-        --shm-size=16gb \
-        --group-add "$render_gid" \
-        --rm \
-        -e MIOPEN_DEBUG_CONV_DIRECT=0 \
-        -e MIOPEN_DEBUG_CONV_GEMM=0 \
-        -e VLLM_ROCM_USE_AITER=0 \
-        -e HIP_VISIBLE_DEVICES="${GPU}" \
-        -e HF_TOKEN \
-        -v "${HF_CACHE}:${HF_MOUNT}" \
-        -e "HF_HOME=${HF_MOUNT}" \
-        -e "PYTHONPATH=${MYPYTHONPATH}" \
-        --name "${container_name}_${GPU}" \
-        "${image_name}" \
-        /bin/bash -c "${commands_gpu}" \
-        |& while read -r line; do echo ">>Shard $GPU: $line"; done &
-    PIDS+=($!)
-  done
-  #wait for all processes to finish and collect exit codes
-  for pid in "${PIDS[@]}"; do
-    wait "${pid}"
-    STATUS+=($?)
-  done
-  for st in "${STATUS[@]}"; do
-    if [[ ${st} -ne 0 ]]; then
-      echo "One of the processes failed with $st"
-      exit "${st}"
-    fi
-  done
-else
-  echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES"
-  docker run \
-          --device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \
-          --network=host \
-          --shm-size=16gb \
-          --group-add "$render_gid" \
-          --rm \
-          -e MIOPEN_DEBUG_CONV_DIRECT=0 \
-          -e MIOPEN_DEBUG_CONV_GEMM=0 \
-          -e VLLM_ROCM_USE_AITER=0 \
-          -e HF_TOKEN \
-          -v "${HF_CACHE}:${HF_MOUNT}" \
-          -e "HF_HOME=${HF_MOUNT}" \
-          -e "PYTHONPATH=${MYPYTHONPATH}" \
-          --name "${container_name}" \
-          "${image_name}" \
-          /bin/bash -c "${commands}"
-fi

.buildkite/scripts/hardware_ci/run-xpu-test.sh

@@ -1,58 +0,0 @@
-#!/bin/bash
-
-# This script build the XPU docker image and run the offline inference inside the container.
-set -ex
-
-omni_source_dir=$(git rev-parse --show-toplevel)
-
-base_image_name="xpu/vllm-omni-ci-base:${VLLM_VERSION:?VLLM_VERSION must be set}"
-image_name="xpu/vllm-omni-ci:${BUILDKITE_COMMIT:?BUILDKITE_COMMIT must be set}"
-container_name="xpu_${BUILDKITE_COMMIT}_$(
-    tr -dc A-Za-z0-9 </dev/urandom | head -c 10
-    echo
-)"
-
-cd "${omni_source_dir}"
-if [ -z "$(docker images -q "${base_image_name}")" ]; then
-    docker build --target vllm-base -t "${base_image_name}" --build-arg "VLLM_VERSION=${VLLM_VERSION}" -f docker/Dockerfile.xpu .
-fi
-
-# Try building the docker image
-docker build --build-arg "VLLM_BASE=${base_image_name}" --build-arg "VLLM_VERSION=${VLLM_VERSION}" -t "${image_name}" -f docker/Dockerfile.xpu .
-
-# Setup cleanup
-remove_docker_container() {
-    docker rm -f "${container_name}" || true
-    docker image rm -f "${image_name}" || true
-    docker system prune -f || true
-}
-trap remove_docker_container EXIT
-
-HF_CACHE="${HF_CACHE:-$(realpath ~)/.cache/huggingface}"
-mkdir -p "${HF_CACHE}"
-HF_MOUNT="/root/.cache/huggingface"
-
-time timeout -k 30 30m docker run \
-    --device /dev/dri:/dev/dri \
-    --net=host \
-    --ipc=host \
-    -v /dev/dri/by-path:/dev/dri/by-path \
-    -v "${HF_CACHE}:${HF_MOUNT}" \
-    --security-opt seccomp=unconfined \
-    --entrypoint="" \
-    -e VLLM_LOGGING_LEVEL \
-    -e VLLM_OMNI_LOGGING_LEVEL \
-    -e HF_TOKEN \
-    -e ZE_AFFINITY_MASK \
-    --name "${container_name}" \
-    "${image_name}" \
-    bash -c '
-    set -e
-    echo $ZE_AFFINITY_MASK
-    pip install tblib==3.1.0
-    cd /workspace/vllm-omni
-    pytest -v -s -m "core_model and xpu and B60"
-    pytest -v -s tests/diffusion/quantization/test_mxfp8_config.py
-    export VLLM_XPU_USE_SAMPLER_KERNEL=0    # NOTE: Remove this after vLLM v0.21.1 is merged. Fixes Qwen2-5 omni-expansion tests.
-    pytest -v -s -m "advanced_model and xpu and B60"
-'

.buildkite/scripts/hardware_ci/run_npu_test.sh

@@ -1,145 +0,0 @@
-#!/bin/bash
-
-# This script build the Ascend NPU docker image and run the offline inference inside the container.
-# It serves a sanity check for compilation and basic model usage.
-set -ex
-
-# Base ubuntu image with basic ascend development libraries and python installed
-VLLM_OMNI_REPO="https://github.com/vllm-project/vllm-omni.git"
-BASE_IMAGE_NAME="quay.nju.edu.cn/ascend/vllm-ascend:v0.11.0rc2"
-image_name="npu/vllm-omni-ci:${BUILDKITE_COMMIT}_${EPOCHSECONDS}"
-# image_name="npu/vllm-ci:${BUILDKITE_COMMIT}_${EPOCHSECONDS}"
-container_name="npu_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)"
-
-# BUILDKITE_AGENT_NAME format is {hostname}-{agent_idx}-{npu_card_num}cards
-agent_idx=$(echo "${BUILDKITE_AGENT_NAME}" | awk -F'-' '{print $(NF-1)}')
-echo "agent_idx: ${agent_idx}"
-builder_name="cachebuilder${agent_idx}"
-builder_cache_dir="/mnt/docker-cache${agent_idx}"
-mkdir -p ${builder_cache_dir}
-
-# Try building the docker image
-cat <<EOF | DOCKER_BUILDKIT=1 docker build \
-    --add-host pypi-cache:${PYPI_CACHE_HOST} \
-    --builder ${builder_name} --cache-from type=local,src=${builder_cache_dir} \
-                           --cache-to type=local,dest=${builder_cache_dir},mode=max \
-    --build-arg BUILDKITE_PULL_REQUEST="${BUILDKITE_PULL_REQUEST}" \
-    --build-arg BUILDKITE_PULL_REQUEST_REPO="${BUILDKITE_PULL_REQUEST_REPO}" \
-    --progress=plain --load -t ${image_name} -f - .
-FROM ${BASE_IMAGE_NAME}
-
-# Define environments
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN pip config set global.index-url http://pypi-cache:${PYPI_CACHE_PORT}/pypi/simple && \
-    pip config set global.trusted-host pypi-cache && \
-    apt-get update -y && \
-    apt-get install -y python3-pip git vim wget net-tools gcc g++ cmake libnuma-dev && \
-    rm -rf /var/cache/apt/* && \
-    rm -rf /var/lib/apt/lists/*
-
-# Install for pytest to make the docker build cache layer always valid
-RUN --mount=type=cache,target=/root/.cache/pip \
-    pip install pytest>=6.0  pytest-cov modelscope
-
-COPY . .
-
-# Install vllm-omni
-WORKDIR /workspace
-ARG VLLM_OMNI_REPO=https://github.com/vllm-project/vllm-omni.git
-ARG VLLM_OMNI_TAG=main
-ARG BUILDKITE_PULL_REQUEST
-ARG BUILDKITE_PULL_REQUEST_REPO
-RUN git config --global url."https://gh-proxy.test.osinfra.cn/https://github.com/".insteadOf "https://github.com/" && \
-    if [ "\$BUILDKITE_PULL_REQUEST" != "false" ] && [ -n "\$BUILDKITE_PULL_REQUEST" ]; then \
-        echo "Cloning and checking out PR #\$BUILDKITE_PULL_REQUEST..." && \
-        git clone \$VLLM_OMNI_REPO /workspace/vllm-omni && \
-        cd /workspace/vllm-omni && \
-        git fetch origin pull/\$BUILDKITE_PULL_REQUEST/head:pr-\$BUILDKITE_PULL_REQUEST && \
-        git checkout pr-\$BUILDKITE_PULL_REQUEST; \
-    else \
-        echo "Not a PR build, using main branch" && \
-        git clone --depth 1 \$VLLM_OMNI_REPO /workspace/vllm-omni; \
-    fi
-
-RUN --mount=type=cache,target=/root/.cache/pip \
-    export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi && \
-    source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
-    source /usr/local/Ascend/nnal/atb/set_env.sh && \
-    export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
-    python3 -m pip install -v -e /workspace/vllm-omni/
-
-ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-WORKDIR /workspace/vllm-omni
-CMD ["/bin/bash"]
-
-EOF
-
-# Setup cleanup
-remove_docker_container() {
-  docker rm -f "${container_name}" || true;
-  docker image rm -f "${image_name}" || true;
-  docker system prune -f || true;
-}
-trap remove_docker_container EXIT
-
-# Generate corresponding --device args based on BUILDKITE_AGENT_NAME
-# Ascend NPU BUILDKITE_AGENT_NAME format is {hostname}-{agent_idx}-{npu_card_num}cards, and agent_idx starts from 1.
-#   e.g. atlas-a2-001-1-2cards means this is the 1-th agent on atlas-a2-001 host, and it has 2 NPU cards.
-#   returns --device /dev/davinci0 --device /dev/davinci1
-parse_and_gen_devices() {
-    local input="$1"
-    local index cards_num
-    if [[ "$input" =~ ([0-9]+)-([0-9]+)cards$ ]]; then
-        index="${BASH_REMATCH[1]}"
-        cards_num="${BASH_REMATCH[2]}"
-    else
-        echo "parse error" >&2
-        return 1
-    fi
-
-    local devices=""
-    local i=0
-    while (( i < cards_num )); do
-        local dev_idx=$(((index - 1)*cards_num + i ))
-        devices="$devices --device /dev/davinci${dev_idx}"
-        ((i++))
-    done
-
-    # trim leading space
-    devices="${devices#"${devices%%[![:space:]]*}"}"
-    # Output devices: assigned to the caller variable
-    printf '%s' "$devices"
-}
-
-devices=$(parse_and_gen_devices "${BUILDKITE_AGENT_NAME}") || exit 1
-
-# Run the image and execute the Out-Of-Tree (OOT) platform interface test case on Ascend NPU hardware.
-# This test checks whether the OOT platform interface is functioning properly in conjunction with
-# the hardware plugin vllm-ascend.
-hf_model_cache_dir=/mnt/hf_cache${agent_idx}
-ms_model_cache_dir=/mnt/modelscope${agent_idx}
-mkdir -p ${hf_model_cache_dir}
-mkdir -p ${ms_model_cache_dir}
-docker run \
-    --init \
-    ${devices} \
-    --device /dev/davinci_manager \
-    --device /dev/devmm_svm \
-    --device /dev/hisi_hdc \
-    -v /usr/local/dcmi:/usr/local/dcmi \
-    -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
-    -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
-    -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
-    -v /etc/ascend_install.info:/etc/ascend_install.info \
-    -v ${hf_model_cache_dir}:/root/.cache/huggingface \
-    -v ${ms_model_cache_dir}:/root/.cache/modelscope \
-    --network host \
-    --entrypoint="" \
-    --name "${container_name}" \
-    "${image_name}" \
-    bash -c '
-    set -e
-    VLLM_USE_MODELSCOPE=True pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py
-'

.buildkite/scripts/publish-release-images.sh

@@ -1,50 +0,0 @@
-#!/bin/bash
-# SPDX-License-Identifier: Apache-2.0
-#
-# Publish release Docker images from ECR to DockerHub.
-# Pulls per-arch images, tags with `latest` and versioned tags, pushes them,
-# then creates and pushes a multi-arch manifest.
-
-set -euo pipefail
-
-RELEASE_VERSION=$(buildkite-agent meta-data get release-version --default "" | sed 's/^v//')
-if [ -z "${RELEASE_VERSION}" ]; then
-  echo "ERROR: release-version metadata not set"
-  exit 1
-fi
-
-COMMIT="$BUILDKITE_COMMIT"
-DOCKERHUB_REPO="vllm/vllm-omni"
-ECR_REPO="public.ecr.aws/q9t5s3a7/vllm-omni-release-repo"
-
-echo "========================================"
-echo "Publishing release images v${RELEASE_VERSION}"
-echo "  Commit: ${COMMIT}"
-echo "========================================"
-
-# Login to ECR to pull staging images
-aws ecr-public get-login-password --region us-east-1 | \
-  docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
-
-docker pull "${ECR_REPO}:${COMMIT}-x86_64"
-docker pull "${ECR_REPO}:${COMMIT}-aarch64"
-
-docker tag "${ECR_REPO}:${COMMIT}-x86_64" "${DOCKERHUB_REPO}:latest-x86_64"
-docker tag "${ECR_REPO}:${COMMIT}-x86_64" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-x86_64"
-docker push "${DOCKERHUB_REPO}:latest-x86_64"
-docker push "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-x86_64"
-
-docker tag "${ECR_REPO}:${COMMIT}-aarch64" "${DOCKERHUB_REPO}:latest-aarch64"
-docker tag "${ECR_REPO}:${COMMIT}-aarch64" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-aarch64"
-docker push "${DOCKERHUB_REPO}:latest-aarch64"
-docker push "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-aarch64"
-
-docker manifest rm "${DOCKERHUB_REPO}:latest" || true
-docker manifest rm "${DOCKERHUB_REPO}:v${RELEASE_VERSION}" || true
-docker manifest create "${DOCKERHUB_REPO}:latest" "${DOCKERHUB_REPO}:latest-x86_64" "${DOCKERHUB_REPO}:latest-aarch64"
-docker manifest create "${DOCKERHUB_REPO}:v${RELEASE_VERSION}" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-x86_64" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-aarch64"
-docker manifest push "${DOCKERHUB_REPO}:latest"
-docker manifest push "${DOCKERHUB_REPO}:v${RELEASE_VERSION}"
-
-echo ""
-echo "Successfully published release images for v${RELEASE_VERSION}"

.buildkite/scripts/upload_pipeline.py

@@ -1,308 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: Apache-2.0
-"""Render and optionally upload Buildkite pipeline YAML with diff-aware logic.
-
-Bootstrap mode (pipeline.yml with __IMAGE_BUILD_IF__ placeholders):
-  - Detect docs-only PR/main changes and substitute skip-ci ``if`` expressions.
-
-Test pipeline mode (e.g. test-merge.yml):
-  - Drop steps or groups whose ``source_file_dependencies`` do not match changed files.
-  - ``source_file_dependencies`` may be set on a leaf step or on a group step.
-  - Always strip ``source_file_dependencies`` from uploaded YAML (Buildkite does not
-    recognize this uploader-only key).
-
-Usage:
-  python3 upload_pipeline.py [--upload] <pipeline.yml>
-
-  # Bootstrap (replaces upload_pipeline_with_skip_ci.sh):
-  python3 upload_pipeline.py --upload .buildkite/pipeline.yml
-
-  # Test pipeline (replaces upload_test_pipeline_with_diff_skip.py):
-  python3 upload_pipeline.py --upload .buildkite/test-merge.yml
-"""
-
-from __future__ import annotations
-
-import argparse
-import os
-import subprocess
-import sys
-from pathlib import Path
-from typing import Any
-
-try:
-    import yaml
-except ImportError:
-    subprocess.check_call(
-        [sys.executable, "-m", "pip", "install", "-q", "pyyaml"],
-    )
-    import yaml
-
-LOG = "upload_pipeline"
-ROOT = Path(__file__).resolve().parent.parent.parent
-DOC_SEP = "\n---\n"
-BOOTSTRAP_MARKER = "__IMAGE_BUILD_IF__"
-
-
-def _log(message: str) -> None:
-    print(f"{LOG}: {message}", file=sys.stderr)
-
-
-def _git(*args: str) -> subprocess.CompletedProcess[str]:
-    return subprocess.run(
-        ["git", *args],
-        check=False,
-        capture_output=True,
-        text=True,
-    )
-
-
-def resolve_changed_files() -> list[str] | None:
-    """Return changed file paths, or None when diff cannot be resolved."""
-    is_pr = os.environ.get("BUILDKITE_PULL_REQUEST", "false") != "false" and os.environ.get(
-        "BUILDKITE_PULL_REQUEST", ""
-    )
-    commit = os.environ.get("BUILDKITE_COMMIT", "")
-
-    if is_pr:
-        base_branch = os.environ.get("BUILDKITE_PULL_REQUEST_BASE_BRANCH", "main")
-        base_ref = f"origin/{base_branch}"
-        if _git("rev-parse", "--verify", base_ref).returncode != 0:
-            _log(f"origin/{base_branch} not found locally; trying fetch")
-            _git("fetch", "--depth=200", "origin", base_branch)
-        if _git("rev-parse", "--verify", base_ref).returncode != 0:
-            if _git("rev-parse", "--verify", base_branch).returncode == 0:
-                base_ref = base_branch
-            else:
-                _log(f"cannot resolve PR base {base_branch}; using safe defaults")
-                return None
-        diff_range = f"{base_ref}...{commit}"
-    elif os.environ.get("BUILDKITE_BRANCH", "") == "main":
-        if _git("rev-parse", "--verify", f"{commit}^").returncode != 0:
-            _log("main commit has no parent; using safe defaults")
-            return None
-        diff_range = f"{commit}^..{commit}"
-    else:
-        _log("not PR/main build; using safe defaults")
-        return None
-
-    result = _git("diff", "--name-only", diff_range)
-    if result.returncode != 0:
-        _log(f"git diff failed ({diff_range}); using safe defaults")
-        return None
-
-    files = [line for line in result.stdout.splitlines() if line.strip()]
-    _log(f"{len(files)} changed file(s)")
-    return files
-
-
-def is_docs_only_change(changed_files: list[str]) -> bool:
-    has_any = False
-    for file_path in changed_files:
-        if not file_path:
-            continue
-        has_any = True
-        if file_path.startswith("docs/"):
-            continue
-        if file_path.endswith(".md"):
-            continue
-        if file_path == "mkdocs.yml":
-            continue
-        return False
-    return has_any
-
-
-def resolve_skip_ci(changed_files: list[str] | None) -> bool:
-    if changed_files is None:
-        _log("skip-ci=0 (could not resolve changed files)")
-        return False
-    if is_docs_only_change(changed_files):
-        _log("docs-only change detected; skip-ci=1")
-        return True
-    _log("non-doc changes detected; skip-ci=0")
-    return False
-
-
-def render_bootstrap_pipeline(text: str, *, skip_ci: bool) -> str:
-    if DOC_SEP in text:
-        _, continuation = text.split(DOC_SEP, 1)
-    else:
-        continuation = text
-
-    nightly_only = (
-        '(build.pull_request.labels includes "nightly-test") || (build.branch == "main" && build.env("NIGHTLY") == "1")'
-    )
-    if skip_ci:
-        image_if = f"'{nightly_only}'"
-        ready_if = "'false'"
-        merge_if = "'false'"
-    else:
-        image_if = "'true'"
-        ready_if = '\'build.branch != "main" && build.pull_request.labels includes "ready"\''
-        merge_if = (
-            '\'(build.branch == "main" && build.env("NIGHTLY") != "1" '
-            '&& build.env("WEEKLY") != "1") || '
-            '(build.branch != "main" && build.pull_request.labels includes "merge-test")\''
-        )
-
-    return (
-        continuation.replace("__IMAGE_BUILD_IF__", image_if)
-        .replace("__UPLOAD_READY_IF__", ready_if)
-        .replace("__UPLOAD_MERGE_IF__", merge_if)
-    )
-
-
-def _matches_dependencies(changed_files: list[str], prefixes: list[str]) -> bool:
-    for path in changed_files:
-        for prefix in prefixes:
-            normalized = prefix.rstrip("/")
-            if path == normalized or path.startswith(f"{normalized}/"):
-                return True
-    return False
-
-
-def _strip_source_file_dependencies(step: dict[str, Any]) -> dict[str, Any]:
-    return {key: value for key, value in step.items() if key != "source_file_dependencies"}
-
-
-def _step_label(step: dict[str, Any]) -> str:
-    return str(step.get("group") or step.get("label") or "<step>")
-
-
-def _filter_steps(steps: list[Any], changed_files: list[str]) -> list[Any]:
-    filtered: list[Any] = []
-    for step in steps:
-        if not isinstance(step, dict):
-            filtered.append(step)
-            continue
-
-        deps = step.get("source_file_dependencies")
-        if deps is not None and not isinstance(deps, list):
-            raise ValueError(
-                f"source_file_dependencies must be a list in step {_step_label(step)!r}",
-            )
-        if deps is not None and not _matches_dependencies(changed_files, deps):
-            _log(
-                f"skip {_step_label(step)!r} (no changes under {deps})",
-            )
-            continue
-
-        nested = step.get("steps")
-        if nested is not None:
-            kept_nested = _filter_steps(nested, changed_files)
-            if not kept_nested:
-                _log(f"omit empty group {_step_label(step)!r}")
-                continue
-            new_step = _strip_source_file_dependencies(step)
-            new_step["steps"] = kept_nested
-            filtered.append(new_step)
-            continue
-
-        if deps is not None:
-            filtered.append(_strip_source_file_dependencies(step))
-        else:
-            filtered.append(step)
-
-    return filtered
-
-
-def _strip_uploader_metadata_from_steps(steps: list[Any]) -> list[Any]:
-    """Remove uploader-only keys while keeping all steps (no diff filtering)."""
-    stripped: list[Any] = []
-    for step in steps:
-        if not isinstance(step, dict):
-            stripped.append(step)
-            continue
-
-        deps = step.get("source_file_dependencies")
-        if deps is not None and not isinstance(deps, list):
-            raise ValueError(
-                f"source_file_dependencies must be a list in step {_step_label(step)!r}",
-            )
-
-        nested = step.get("steps")
-        new_step = _strip_source_file_dependencies(step)
-        if nested is not None:
-            new_step["steps"] = _strip_uploader_metadata_from_steps(nested)
-        stripped.append(new_step)
-
-    return stripped
-
-
-def render_test_pipeline(
-    doc: dict[str, Any],
-    changed_files: list[str] | None,
-) -> dict[str, Any]:
-    steps = doc.get("steps")
-    if not isinstance(steps, list):
-        return doc
-    if changed_files is not None:
-        steps = _filter_steps(steps, changed_files)
-    else:
-        steps = _strip_uploader_metadata_from_steps(steps)
-    return {**doc, "steps": steps}
-
-
-def resolve_pipeline_path(arg: str) -> Path:
-    path = Path(arg)
-    if path.is_absolute():
-        return path
-    return ROOT / path
-
-
-def render_pipeline(path: Path) -> str:
-    text = path.read_text(encoding="utf-8")
-    changed_files = resolve_changed_files()
-
-    if BOOTSTRAP_MARKER in text:
-        rendered = render_bootstrap_pipeline(text, skip_ci=resolve_skip_ci(changed_files))
-        return rendered
-
-    doc = yaml.safe_load(text)
-    if not isinstance(doc, dict):
-        raise ValueError(f"invalid pipeline YAML: {path}")
-
-    doc = render_test_pipeline(doc, changed_files)
-
-    return yaml.safe_dump(doc, sort_keys=False)
-
-
-def upload_to_buildkite(content: str) -> None:
-    subprocess.run(
-        ["buildkite-agent", "pipeline", "upload"],
-        input=content,
-        text=True,
-        check=True,
-    )
-
-
-def main() -> int:
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument(
-        "pipeline",
-        nargs="?",
-        default=".buildkite/pipeline.yml",
-        help="Pipeline YAML path (default: .buildkite/pipeline.yml)",
-    )
-    parser.add_argument(
-        "--upload",
-        action="store_true",
-        help="Pipe rendered YAML to buildkite-agent pipeline upload",
-    )
-    args = parser.parse_args()
-
-    path = resolve_pipeline_path(args.pipeline)
-    if not path.is_file():
-        _log(f"missing pipeline file: {path}")
-        return 1
-
-    rendered = render_pipeline(path)
-    if args.upload:
-        upload_to_buildkite(rendered)
-    else:
-        sys.stdout.write(rendered)
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

.buildkite/test-amd.yaml

@@ -1,118 +0,0 @@
-steps:
-
-- label: "Diffusion Model Test"
-  timeout_in_minutes: 30
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - pytest -s -v tests/e2e/offline_inference/test_z_image.py
-
-- label: "Diffusion Images API LoRA E2E"
-  timeout_in_minutes: 30
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/diffusion/lora/test_images_generations_lora.py
-
-- label: "Diffusion Model CPU offloading Test"
-  timeout_in_minutes: 20
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
-    - pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
-
-- label: "Diffusion Cache Backend Test"
-  timeout_in_minutes: 15
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v -m 'core_model and cache and diffusion and not distributed_rocm and MI325'
-
-- label: "Diffusion Sequence Parallelism Test"
-  timeout_in_minutes: 20
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py -m core_model
-
-- label: "Diffusion Tensor Parallelism Test"
-  timeout_in_minutes: 20
-  agent_pool: mi325_2
-  depends_on: amd-build
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/diffusion/distributed/test_zimage_parallelism.py
-
-- label: "Diffusion GPU Worker Test"
-  timeout_in_minutes: 20
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - pytest -s -v tests/diffusion/test_diffusion_worker.py
-
-- label: "Omni Model Test Qwen2-5-Omni"
-  timeout_in_minutes: 15
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py
-    - pytest -s -v tests/engine/test_async_omni_engine_abort.py
-
-- label: "Omni Model Test Qwen3-Omni"
-  timeout_in_minutes: 15
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - export VLLM_ROCM_USE_AITER=0
-    - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py
-    - pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py
-
-
-- label: "Omni Sleep Mode Test"
-  timeout_in_minutes: 40
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/entrypoints/test_omni_sleep_mode.py -m "advanced_model and omni and MI325" --run-level "advanced_model"

.buildkite/test-merge.yml

@@ -1,884 +0,0 @@
-env:
-  VLLM_WORKER_MULTIPROC_METHOD: spawn
-  VLLM_USE_FLASHINFER_MOE_FP16: 0
-  HF_HUB_DOWNLOAD_TIMEOUT: 300
-  HF_HUB_ETAG_TIMEOUT: 60
-
-steps:
-  - group: ":card_index_dividers: Simple Test"
-    depends_on: upload-merge-pipeline
-    steps:
-      - label: "Simple · Diffusion & Model Executor Test"
-        commands:
-          - "timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "Simple · Other Test"
-        commands:
-          - "timeout 20m pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - group: ":card_index_dividers: Diffusion Test"
-    depends_on: upload-merge-pipeline
-    steps:
-      - label: "Diffusion · LoRA Test"
-        timeout_in_minutes: 30
-        commands:
-          - pytest -sv tests/diffusion/lora/ -m 'advanced_model and cuda' --run-level "advanced_model"
-        agents:
-          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Distributed Test"
-    timeout_in_minutes: 60
-    commands:
-      - |
-        timeout 55m bash -c "
-          set -e
-          export VLLM_IMAGE_FETCH_TIMEOUT=60
-          pytest -s -v tests/distributed -m 'advanced_model and cuda' --run-level 'advanced_model'
-        "
-    agents:
-      queue: "mithril-h100-pool"
-    plugins:
-      - kubernetes:
-          podSpec:
-            containers:
-              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                resources:
-                  limits:
-                    nvidia.com/gpu: 1
-                volumeMounts:
-                  - name: devshm
-                    mountPath: /dev/shm
-                  - name: hf-cache
-                    mountPath: /root/.cache/huggingface
-                env:
-                  - name: HF_HOME
-                    value: /root/.cache/huggingface
-                  - name: HF_TOKEN
-                    valueFrom:
-                      secretKeyRef:
-                        name: hf-token-secret
-                        key: token
-            nodeSelector:
-              node.kubernetes.io/instance-type: gpu-h100-sxm
-            volumes:
-              - name: devshm
-                emptyDir:
-                  medium: Memory
-              - name: hf-cache
-                hostPath:
-                  path: /mnt/hf-cache
-                  type: DirectoryOrCreate
-
-  - group: ":card_index_dividers: Entrypoints Test"
-    depends_on: upload-merge-pipeline
-    steps:
-      - label: "Entrypoint Test with H100"
-        timeout_in_minutes: 60
-        commands:
-          - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and H100" --run-level "advanced_model"
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 2
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: "Entrypoint Test with L4"
-        timeout_in_minutes: 30
-        commands:
-          - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and L4" --run-level "advanced_model"
-        agents:
-          queue: "gpu_4_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - group: ":card_index_dividers: E2E Test"
-    depends_on: upload-merge-pipeline
-    steps:
-      - label: "TTS · Qwen3-TTS CustomVoice Test"
-        commands:
-          - |
-            timeout 20m bash -c "
-              export VLLM_LOGGING_LEVEL=DEBUG
-              export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
-              pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-            "
-        agents:
-          queue: "gpu_4_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "TTS · Qwen3-TTS Base Test"
-        commands:
-          - |
-            timeout 20m bash -c "
-              export VLLM_LOGGING_LEVEL=DEBUG
-              export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
-              pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-            "
-        agents:
-          queue: "gpu_4_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "TTS · Ming-Omni-TTS Test"
-        timeout_in_minutes: 20
-        commands:
-          - |
-            timeout 20m bash -c "
-              export VLLM_LOGGING_LEVEL=DEBUG
-              pytest -s -v tests/e2e/online_serving/test_ming_tts.py tests/e2e/offline_inference/test_ming_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-            "
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "TTS · Voxtral-TTS Test"
-        source_file_dependencies:
-          - tests/e2e/offline_inference/test_voxtral_tts.py
-          - tests/e2e/online_serving/test_voxtral_tts.py
-          - vllm_omni/model_executor/models/voxtral_tts/
-          - vllm_omni/model_executor/stage_input_processors/voxtral_tts.py
-          - vllm_omni/deploy/voxtral_tts.yaml
-        timeout_in_minutes: 20
-        commands:
-          - |
-            timeout 20m bash -c "
-              export VLLM_LOGGING_LEVEL=DEBUG
-              pytest -s -v tests/e2e/offline_inference/test_voxtral_tts.py tests/e2e/online_serving/test_voxtral_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-            "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: "TTS · GLM-TTS Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_glm_tts.py
-          - tests/e2e/offline_inference/test_glm_tts.py
-          - vllm_omni/model_executor/models/glm_tts/
-          - vllm_omni/model_executor/stage_input_processors/glm_tts.py
-          - vllm_omni/deploy/glm_tts.yaml
-        timeout_in_minutes: 20
-        commands:
-          - |
-            timeout 20m bash -c "
-              export VLLM_LOGGING_LEVEL=DEBUG
-              pytest -s -v tests/e2e/online_serving/test_glm_tts.py tests/e2e/offline_inference/test_glm_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-            "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: "Omni · Qwen3-Omni Test"
-        source_file_dependencies:
-          - tests/e2e/offline_inference/test_qwen3_omni.py
-          - tests/e2e/online_serving/test_qwen3_omni.py
-          - vllm_omni/model_executor/models/qwen3_omni/
-          - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
-          - vllm_omni/deploy/qwen3_omni_moe.yaml
-        timeout_in_minutes: 30
-        commands:
-          - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 2
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: "Omni · Mimo Audio Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_mimo_audio.py
-          - vllm_omni/model_executor/models/mimo_audio/
-          - vllm_omni/model_executor/stage_input_processors/mimo_audio.py
-          - vllm_omni/deploy/mimo_audio.yaml
-        timeout_in_minutes: 30
-        commands:
-          - pytest -s -v tests/e2e/online_serving/test_mimo_audio.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 2
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: "TTS · VoxCPM2 Test"
-        timeout_in_minutes: 30
-        commands:
-          - |
-            timeout 30m bash -c "
-              export VLLM_LOGGING_LEVEL=DEBUG
-              export VLLM_WORKER_MULTIPROC_METHOD=spawn
-              pytest -s -v tests/e2e/offline_inference/test_voxcpm2_tts.py tests/e2e/online_serving/test_voxcpm2_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-            "
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "Omni · Higgs-Audio V2 Test"
-        timeout_in_minutes: 30
-        commands:
-          - |
-            timeout 30m bash -c "
-              export VLLM_LOGGING_LEVEL=DEBUG
-              export VLLM_WORKER_MULTIPROC_METHOD=spawn
-              export VLLM_USE_DEEP_GEMM=0
-              export VLLM_MOE_USE_DEEP_GEMM=0
-              pytest -s -v tests/e2e/online_serving/test_higgs_audio_v2.py tests/e2e/offline_inference/test_higgs_audio_v2.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-            "
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "Diffusion · Qwen Image Edit Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_qwen_image_edit.py
-          - vllm_omni/diffusion/models/qwen_image/
-        timeout_in_minutes: 20
-        commands:
-          - pytest -s -v tests/e2e/online_serving/test_qwen_image_edit.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: "Diffusion · Longcat Image Edit Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_longcat_image_edit.py
-          - vllm_omni/diffusion/models/longcat_image/
-        timeout_in_minutes: 20
-        commands:
-          - pytest -s -v tests/e2e/online_serving/test_longcat_image_edit.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: "Diffusion · Longcat Image Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_longcat_image.py
-          - vllm_omni/diffusion/models/longcat_image/
-        timeout_in_minutes: 20
-        commands:
-          - pytest -s -v tests/e2e/online_serving/test_longcat_image.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: "Diffusion · Bagel Test"
-        source_file_dependencies:
-          - tests/e2e/offline_inference/test_bagel.py
-          - tests/e2e/online_serving/test_bagel.py
-          - vllm_omni/model_executor/models/bagel/
-          - vllm_omni/diffusion/models/bagel/
-          - vllm_omni/model_executor/stage_input_processors/bagel.py
-          - vllm_omni/deploy/bagel.yaml
-        timeout_in_minutes: 20
-        commands:
-          - pytest -s -v tests/e2e/offline_inference/test_bagel.py tests/e2e/online_serving/test_bagel.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-
-      - label: "Diffusion · Qwen Image Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_qwen_image.py
-          - vllm_omni/diffusion/models/qwen_image/
-        commands:
-          - |
-            timeout 40m bash -c '
-              export VLLM_IMAGE_FETCH_TIMEOUT=60
-              pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-            '
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: "Diffusion · Qwen Image Layered Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_qwen_image_layered.py
-          - vllm_omni/diffusion/models/qwen_image/
-        commands:
-          - |
-            timeout 40m bash -c "
-              export VLLM_IMAGE_FETCH_TIMEOUT=60
-              pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-            "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: "Diffusion · Wan22 Test"
-        source_file_dependencies:
-          - tests/e2e/offline_inference/test_wan22_t2v.py
-          - tests/e2e/online_serving/test_wan22_t2v.py
-          - vllm_omni/diffusion/models/wan2_2/
-        commands:
-          - |
-            timeout 40m bash -c "
-              export VLLM_IMAGE_FETCH_TIMEOUT=60
-              pytest -s -v tests/e2e/offline_inference/test_wan22_t2v.py tests/e2e/online_serving/test_wan22_t2v.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
-            "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: "Diffusion · HunyuanVideo 1.5 Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_hunyuan_video_15.py
-          - vllm_omni/diffusion/models/hunyuan_video/
-        timeout_in_minutes: 40
-        commands:
-          - |
-            timeout 40m bash -c "
-              pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
-            "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: "Diffusion · Z Image Test"
-        timeout_in_minutes: 30
-        commands:
-          - pytest -s -v tests/e2e/offline_inference/test_z_image.py tests/e2e/online_serving/test_z_image.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
-        agents:
-          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "Diffusion · FLUX.2 Klein Test"
-        timeout_in_minutes: 30
-        commands:
-          - pytest -s -v tests/e2e/offline_inference/test_flux2_klein.py tests/e2e/online_serving/test_flux2_klein.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
-        agents:
-          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "Diffusion · SD3 Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_sd3.py
-          - vllm_omni/diffusion/models/sd3/
-        timeout_in_minutes: 45
-        commands:
-          - |
-            timeout 45m bash -c "
-              export VLLM_IMAGE_FETCH_TIMEOUT=60
-              pytest -s -v tests/e2e/online_serving/test_sd3.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
-            "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 4
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate

.buildkite/test-npu-nightly.yaml

@@ -1,24 +0,0 @@
-steps:
-  - group: ":card_index_dividers: Diffusion X2V Model Test"
-    key: nightly-diffusion-x2v-group
-    depends_on: upload-nightly-pipeline
-    if: >-
-      build.env("NIGHTLY") == "1" ||
-      build.pull_request.labels includes "nightly-test" ||
-      build.pull_request.labels includes "diffusion-x2v-test"
-    steps:
-      - label: ":full_moon: Diffusion X2V · Function Test"
-        agents:
-          queue: "ascend-a2b3"
-          resource_class: "npu-4"
-        image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}"
-        plugins:
-          - kubernetes:
-              podSpecPatch:
-                imagePullSecrets:
-                  - name: swr-secret
-        env:
-          VLLM_WORKER_MULTIPROC_METHOD: spawn
-          HF_TOKEN: "${HF_TOKEN}"
-        commands:
-          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k npu_i2v --run-level "full_model"

.buildkite/test-npu-ready.yaml

@@ -1,21 +0,0 @@
-steps:
-  - group: ":card_index_dividers: TTS Model Test"
-    key: ready-tts-group
-    depends_on: upload-ready-pipeline
-    steps:
-      - label: ":sound: TTS · Qwen3-TTS CustomVoice Test"
-        agents:
-          queue: "ascend-a2b3"
-          resource_class: "npu-1"
-        image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}"
-        plugins:
-          - kubernetes:
-              podSpecPatch:
-                imagePullSecrets:
-                  - name: swr-secret
-        env:
-          VLLM_WORKER_MULTIPROC_METHOD: spawn
-          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
-          HF_TOKEN: "${HF_TOKEN}"
-        commands:
-          - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'

.buildkite/test-ready.yml

@@ -1,624 +0,0 @@
-env:
-  VLLM_WORKER_MULTIPROC_METHOD: spawn
-  VLLM_USE_FLASHINFER_MOE_FP16: 0
-  HF_HUB_DOWNLOAD_TIMEOUT: 300
-  HF_HUB_ETAG_TIMEOUT: 60
-
-steps:
-  - group: ":card_index_dividers: Simple Test"
-    depends_on: upload-ready-pipeline
-    steps:
-      - label: "Simple · Diffusion & Model Executor Test"
-        commands:
-          - "timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "Simple · Other Test"
-        commands:
-          - "timeout 20m pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Custom Pipeline Test"
-    depends_on: upload-ready-pipeline
-    commands:
-      - timeout 20m pytest -s -v tests/e2e/offline_inference/custom_pipeline/ -m "core_model"
-    agents:
-      queue: "gpu_1_queue"
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - group: ":card_index_dividers: Diffusion Test"
-    depends_on: upload-ready-pipeline
-    steps:
-      - label: "Diffusion · Offloader Test"
-        commands:
-          - timeout 10m pytest -s -v tests/diffusion/offloader -m "core_model and cuda" --run-level "core_model"
-        agents:
-          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "Diffusion · Cache Test"
-        commands:
-          - timeout 15m pytest -sv tests/diffusion/cache -m 'core_model and cuda' --run-level "core_model"
-        agents:
-          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "Diffusion · Batch Test"
-        commands:
-          - timeout 15m pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model"
-        agents:
-          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "Diffusion · Distributed Test"
-        commands:
-          - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda' --run-level "core_model"
-        agents:
-          queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "Diffusion · Model Test"
-        commands:
-          - timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model"
-        agents:
-          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "Diffusion · Other Test"
-        commands:
-          - timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda' --run-level "core_model"
-        agents:
-          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-
-  - label: "Engine&Model Executor Test"
-    depends_on: upload-ready-pipeline
-    commands:
-      - |
-        timeout 15m bash -c "
-                  pytest -sv tests/engine/ tests/model_executor/ -m 'core_model and cuda' --run-level 'core_model'
-        "
-    agents:
-      queue: "gpu_1_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          shm-size: "8gb"
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-            - "HF_TOKEN"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-
-  - label: "Distributed Test"
-    depends_on: upload-ready-pipeline
-    commands:
-      - |
-        timeout 20m bash -c "
-                  pytest -sv tests/distributed/ -m 'core_model and cuda' --run-level 'core_model'
-        "
-    agents:
-      queue: "mithril-h100-pool"
-    plugins:
-      - kubernetes:
-          podSpec:
-            containers:
-              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                resources:
-                  limits:
-                    nvidia.com/gpu: 1
-                volumeMounts:
-                  - name: devshm
-                    mountPath: /dev/shm
-                  - name: hf-cache
-                    mountPath: /root/.cache/huggingface
-                env:
-                  - name: HF_HOME
-                    value: /root/.cache/huggingface
-            nodeSelector:
-              node.kubernetes.io/instance-type: gpu-h100-sxm
-            volumes:
-              - name: devshm
-                emptyDir:
-                  medium: Memory
-              - name: hf-cache
-                hostPath:
-                  path: /mnt/hf-cache
-                  type: DirectoryOrCreate
-
-  - label: "Entrypoints Test"
-    depends_on: upload-ready-pipeline
-    commands:
-      - |
-        timeout 30m bash -c "
-          export VLLM_WORKER_MULTIPROC_METHOD=spawn
-          pytest -sv tests/entrypoints/ -m 'core_model and cuda' --run-level 'core_model'
-        "
-    agents:
-      queue: "mithril-h100-pool"
-    plugins:
-      - kubernetes:
-          podSpec:
-            containers:
-              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                resources:
-                  limits:
-                    nvidia.com/gpu: 2
-                volumeMounts:
-                  - name: devshm
-                    mountPath: /dev/shm
-                  - name: hf-cache
-                    mountPath: /root/.cache/huggingface
-                env:
-                  - name: HF_HOME
-                    value: /root/.cache/huggingface
-            nodeSelector:
-              node.kubernetes.io/instance-type: gpu-h100-sxm
-            volumes:
-              - name: devshm
-                emptyDir:
-                  medium: Memory
-              - name: hf-cache
-                hostPath:
-                  path: /mnt/hf-cache
-                  type: DirectoryOrCreate
-
-  - group: ":card_index_dividers: E2E Test"
-    depends_on: upload-ready-pipeline
-    steps:
-      - label: "RLHF · VeRL-Omni E2E Test"
-        commands:
-          - |
-            timeout 20m bash -c "
-              pytest -s -v tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py
-            "
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "Omni · Qwen3-Omni Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_qwen3_omni.py
-          - vllm_omni/model_executor/models/qwen3_omni/
-          - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
-          - vllm_omni/deploy/qwen3_omni_moe.yaml
-        commands:
-          - |
-            timeout 20m bash -c "
-              pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m 'core_model' --run-level 'core_model'
-            "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 2
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: "TTS · VoxCPM2 Test"
-        timeout_in_minutes: 30
-        commands:
-          - |
-            timeout 30m bash -c "
-              export VLLM_LOGGING_LEVEL=DEBUG
-              export VLLM_WORKER_MULTIPROC_METHOD=spawn
-              pytest -s -v tests/e2e/offline_inference/test_voxcpm2_tts.py tests/e2e/online_serving/test_voxcpm2_tts.py -m 'core_model' --run-level 'core_model'
-            "
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "TTS · Qwen3-TTS CustomVoice Test"
-        commands:
-          - |
-            timeout 20m bash -c "
-              export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
-              pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'
-            "
-        agents:
-          queue: "gpu_4_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "TTS · Qwen3-TTS Base Test"
-        commands:
-          - |
-            timeout 20m bash -c "
-              export VLLM_WORKER_MULTIPROC_METHOD=spawn
-              export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
-              export VLLM_OMNI_USE_V2_RUNNER=1
-              pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py -m 'core_model' --run-level 'core_model'
-            "
-        agents:
-          queue: "gpu_4_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "TTS · Higgs-Audio-V3 Test"
-        commands:
-          - |
-            timeout 20m bash -c "
-              export VLLM_WORKER_MULTIPROC_METHOD=spawn
-              export VLLM_USE_DEEP_GEMM=0
-              export VLLM_MOE_USE_DEEP_GEMM=0
-              pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py -m 'core_model' --run-level 'core_model'
-            "
-        agents:
-          queue: "gpu_4_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "Diffusion · Qwen Image Edit Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_qwen_image_edit.py
-          - vllm_omni/diffusion/models/qwen_image/
-        commands:
-          - |
-            timeout 20m bash -c "
-              pytest -s -v tests/e2e/online_serving/test_qwen_image_edit.py -m 'core_model' --run-level 'core_model'
-            "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: "Diffusion · Bagel Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_bagel.py
-          - vllm_omni/model_executor/models/bagel/
-          - vllm_omni/diffusion/models/bagel/
-          - vllm_omni/model_executor/stage_input_processors/bagel.py
-          - vllm_omni/deploy/bagel.yaml
-        commands:
-          - |
-            timeout 40m bash -c "
-              export VLLM_IMAGE_FETCH_TIMEOUT=60
-              pytest -s -v tests/e2e/online_serving/test_bagel.py -m 'core_model' --run-level 'core_model'
-            "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: "Diffusion · Wan22 Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_wan22_t2v.py
-          - vllm_omni/diffusion/models/wan2_2/
-        commands:
-          - |
-            timeout 40m bash -c "
-              export VLLM_IMAGE_FETCH_TIMEOUT=60
-              pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model' --run-level 'core_model'
-            "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-
-      - label: "Diffusion · Qwen Image Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_qwen_image.py
-          - vllm_omni/diffusion/models/qwen_image/
-        commands:
-          - |
-            timeout 40m bash -c "
-              export VLLM_IMAGE_FETCH_TIMEOUT=60
-              pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model' --run-level 'core_model'
-            "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: "Diffusion · Qwen Image Layered Test"
-        source_file_dependencies:
-          - tests/e2e/online_serving/test_qwen_image_layered.py
-          - vllm_omni/diffusion/models/qwen_image/
-        commands:
-          - |
-            timeout 40m bash -c "
-              export VLLM_IMAGE_FETCH_TIMEOUT=60
-              pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'core_model' --run-level 'core_model'
-            "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate

.buildkite/test-template-amd-omni.j2

@@ -1,141 +0,0 @@
-{# vllm-omni customized version
-   Based on: https://github.com/vllm-project/ci-infra/blob/main/buildkite/test-template-amd.j2
-   Last synced: 2025-12-15
-   Modifications: Removed unused CUDA/NVIDIA logic, keeping only AMD tests
-#}
-{% set docker_image_amd = "rocm/vllm-omni:$BUILDKITE_COMMIT" %}
-{% set default_working_dir = "/app/vllm-omni" %}
-
-{% if env %}
-env:
-{% for name in env %}
-  {{ name }}: "{{ env[name] }}"
-{% endfor %}
-{% endif %}
-steps:
-  - group: "AMD Tests"
-    depends_on: ~
-    steps:
-      - label: "AMD: :docker: build image"
-        depends_on: ~
-        soft_fail: false
-        commands:
-          - "docker build -f docker/Dockerfile.rocm -t {{ docker_image_amd }} --target test --build-arg ARG_PYTORCH_ROCM_ARCH=gfx942 --progress plain ."
-          - "docker push {{ docker_image_amd }}"
-        key: "amd-build"
-        env:
-          DOCKER_BUILDKIT: "1"
-        retry:
-          automatic:
-            - exit_status: -1  # Agent was lost
-              limit: 2
-            - exit_status: -10  # Agent was lost
-              limit: 2
-            - exit_status: 128  # Git connectivity issues
-              limit: 2
-            - exit_status: 1  # Machine occasionally fail
-              limit: 1
-        agents:
-          queue: amd-cpu
-
-{% for step_group in steps %}
-{% if step_group.group %}
-  - group: "{{ step_group.group }}"
-{% if step_group.depends_on %}
-    depends_on: {{ step_group.depends_on }}
-{% endif %}
-    steps:
-{% for step in step_group.steps %}
-{% if step.mirror_hardwares and mirror_hw in step.mirror_hardwares %}
-      - label: "{{ step.agent_pool }}: {{ step.label }}"
-        depends_on: amd-build
-        agents:
-          {% if step.agent_pool %}
-          queue: amd_{{ step.agent_pool }}
-          {% else %}
-          queue: amd_mi325_1
-          {% endif %}
-{% set cmd_body = (step.command or (step.commands | join("\n"))) | trim %}
-{% set indented_cmd = cmd_body | replace("\n", "\n            ") %}
-        command: bash .buildkite/scripts/hardware_ci/run-amd-test.sh
-        env:
-          DOCKER_BUILDKIT: "1"
-{% if step.env %}
-{% for name in step.env %}
-          {{ name }}: "{{ step.env[name] }}"
-{% endfor %}
-{% endif %}
-          TEST_COMMAND: |-
-            (command rocm-smi || true) && cd {{ (step.working_dir or default_working_dir) | safe }}
-{% if env %}
-{% for name in env %}
-            export {{ name }}="{{ env[name] }}"
-{% endfor %}
-{% endif %}
-{% if step.env %}
-{% for name in step.env %}
-            export {{ name }}="{{ step.env[name] }}"
-{% endfor %}
-{% endif %}
-{% if "mi250" in step.agent_pool %}
-            python3 -m pip uninstall -y amd-aiter
-{% endif %}
-            {{ indented_cmd | safe }}
-        priority: 100
-        {% if step.timeout_in_minutes %}
-        timeout_in_minutes: {{ step.timeout_in_minutes }}
-        {% endif %}
-        {% if step.grade and step.grade == "Blocking" %}
-        soft_fail: false
-        {% else %}
-        soft_fail: true
-        {% endif%}
-{% endif %}
-{% endfor %}
-{% elif step_group.mirror_hardwares and mirror_hw in step_group.mirror_hardwares %}
-{% set step = step_group %}
-  - label: "{{ step.agent_pool }}: {{ step.label }}"
-    depends_on: amd-build
-    agents:
-      {% if step.agent_pool %}
-      queue: amd_{{ step.agent_pool }}
-      {% else %}
-      queue: amd_mi325_1
-      {% endif %}
-{% set cmd_body = (step.command or (step.commands | join("\n"))) | trim %}
-{% set indented_cmd = cmd_body | replace("\n", "\n        ") %}
-    command: bash .buildkite/scripts/hardware_ci/run-amd-test.sh
-    env:
-      DOCKER_BUILDKIT: "1"
-{% if step.env %}
-{% for name in step.env %}
-      {{ name }}: "{{ step.env[name] }}"
-{% endfor %}
-{% endif %}
-      TEST_COMMAND: |-
-        (command rocm-smi || true) && cd {{ (step.working_dir or default_working_dir) | safe }}
-{% if env %}
-{% for name in env %}
-        export {{ name }}="{{ env[name] }}"
-{% endfor %}
-{% endif %}
-{% if step.env %}
-{% for name in step.env %}
-        export {{ name }}="{{ step.env[name] }}"
-{% endfor %}
-{% endif %}
-{% if "mi250" in step.agent_pool %}
-        python3 -m pip uninstall -y amd-aiter
-{% endif %}
-        {{ indented_cmd | safe }}
-    priority: 100
-    {% if step.timeout_in_minutes %}
-    timeout_in_minutes: {{ step.timeout_in_minutes }}
-    {% endif %}
-    {% if step.grade and step.grade == "Blocking" %}
-    soft_fail: false
-    {% else %}
-    soft_fail: true
-    {% endif%}
-{% endif %}
-{% endfor %}

.buildkite/test-weekly.yml

@@ -1,148 +0,0 @@
-env:
-  VLLM_WORKER_MULTIPROC_METHOD: spawn
-  VLLM_USE_FLASHINFER_MOE_FP16: 0
-  HF_HUB_DOWNLOAD_TIMEOUT: 300
-  HF_HUB_ETAG_TIMEOUT: 60
-
-steps:
-  - label: "Reliability Test - qwen3-omni"
-    timeout_in_minutes: 180
-    depends_on: upload-weekly-pipeline
-    if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
-    commands:
-      - pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m "slow"
-    agents:
-      queue: "mithril-h100-pool"
-    plugins:
-      - kubernetes:
-          podSpec:
-            containers:
-              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                resources:
-                  limits:
-                    nvidia.com/gpu: 2
-                volumeMounts:
-                  - name: devshm
-                    mountPath: /dev/shm
-                  - name: hf-cache
-                    mountPath: /root/.cache/huggingface
-                env:
-                  - name: HF_HOME
-                    value: /root/.cache/huggingface
-                  - name: HF_TOKEN
-                    valueFrom:
-                      secretKeyRef:
-                        name: hf-token-secret
-                        key: token
-            nodeSelector:
-              node.kubernetes.io/instance-type: gpu-h100-sxm
-            volumes:
-              - name: devshm
-                emptyDir:
-                  medium: Memory
-              - name: hf-cache
-                hostPath:
-                  path: /mnt/hf-cache
-                  type: DirectoryOrCreate
-
-  - label: "Reliability Test - wan22"
-    timeout_in_minutes: 180
-    depends_on: upload-weekly-pipeline
-    if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
-    commands:
-      - pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m "slow"
-    agents:
-      queue: "mithril-h100-pool"
-    plugins:
-      - kubernetes:
-          podSpec:
-            containers:
-              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                resources:
-                  limits:
-                    nvidia.com/gpu: 2
-                volumeMounts:
-                  - name: devshm
-                    mountPath: /dev/shm
-                  - name: hf-cache
-                    mountPath: /root/.cache/huggingface
-                env:
-                  - name: HF_HOME
-                    value: /root/.cache/huggingface
-                  - name: HF_TOKEN
-                    valueFrom:
-                      secretKeyRef:
-                        name: hf-token-secret
-                        key: token
-            nodeSelector:
-              node.kubernetes.io/instance-type: gpu-h100-sxm
-            volumes:
-              - name: devshm
-                emptyDir:
-                  medium: Memory
-              - name: hf-cache
-                hostPath:
-                  path: /mnt/hf-cache
-                  type: DirectoryOrCreate
-
-  # Invalid HTTP / WS payloads vs live Omni servers (Issue #3649 tracks known skips).
-  - group: "Reliability Test - Invalid parameters Test"
-    depends_on: upload-weekly-pipeline
-    if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
-    steps:
-      - label: "Invalid parameters Test · H100"
-        timeout_in_minutes: 120
-        commands:
-          - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100"
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 2
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      # Stable Audio Open, Qwen3-TTS speech/batch/voices paths marked ``L4``.
-      - label: "Invalid parameters Test · L4"
-        timeout_in_minutes: 120
-        commands:
-          - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and L4"
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"

.claude/skills/add-diffusion-model/SKILL.md

@@ -24,7 +24,7 @@ Check the model's HF repo for `model_index.json`. This determines your path:
 
 | Scenario | How to identify | Migration path |
 |----------|----------------|----------------|
-| **Already supported** | `_class_name` in `model_index.json` matches a key in `_DIFFUSION_MODELS` in `registry.py` | Skip to Step 5 (test) and Step 7 (docs) |
+| **Already supported** | `_class_name` in `model_index.json` matches a key in `_DIFFUSION_MODELS` in `registry.py` | Skip implementation, then validate model-specific examples, tests, and docs as needed |
 | **Diffusers-based** | Has standard `model_index.json` with `_diffusers_version`, subfolders for `transformer/`, `vae/`, etc. | Follow **Path A** below |
 | **Custom/private repo** | No diffusers `model_index.json`, weights in non-standard format, custom model code in a separate git repo | Follow **Path B** below |
 | **Hybrid** | Has some diffusers components (VAE) but custom transformer/fusion | Mix of Path A and Path B |
@@ -52,7 +52,7 @@ vllm_omni/diffusion/models/your_model_name/
 2. Replace attention with `vllm_omni.diffusion.attention.layer.Attention` (QKV shape: `[B, seq, heads, head_dim]`).
 3. Add `od_config: OmniDiffusionConfig | None = None` to `__init__`.
 4. Add `load_weights()` method mapping diffusers weight names to vllm-omni names.
-5. Add class attributes: `_repeated_blocks`, `_layerwise_offload_blocks_attr`.
+5. Add class attributes for acceleration features such as `_repeated_blocks` and `_layerwise_offload_blocks_attrs` (see [references/transformer-adaptation.md](references/transformer-adaptation.md) for examples).
 
 ### A4. Adapt pipeline
 
@@ -78,13 +78,34 @@ class YourPipeline(nn.Module):
 
 Add post/pre-process functions in the same pipeline file. Register them in `registry.py`.
 
+### A4.1 Add progress bar support (recommended)
+
+For pipelines with a standard denoising loop, prefer the existing progress bar pattern instead of hand-rolled logging.
+
+```python
+from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin
+
+class YourPipeline(nn.Module, ProgressBarMixin):
+    def forward(self, req: OmniDiffusionRequest) -> DiffusionOutput:
+        # ... prepare timesteps / latents ...
+        with self.progress_bar(total=len(timesteps)) as progress_bar:
+            for i, t in enumerate(timesteps):
+                # predict noise / scheduler step
+                latents = ...
+                progress_bar.update()
+
+        return DiffusionOutput(output=output)
+```
+
+For custom loop structures, follow `vllm_omni/diffusion/models/progress_bar.py` and existing pipelines using `ProgressBarMixin`.
+
 ### A5. Register, test, docs → continue at Step 4 below.
 
 ---
 
 ## Path B: Custom/Private Repo Model
 
-For models without a diffusers pipeline — weights in custom format, model code in a private repo. Real examples: DreamID-Omni, BAGEL, HunyuanImage3.
+For models without a diffusers pipeline — weights in custom format, model code in a private repo. Real examples: BAGEL, HunyuanImage3. See [references/custom-model-patterns.md](references/custom-model-patterns.md) for concrete custom-model integration patterns.
 
 ### B1. Understand the reference repo
 
@@ -105,9 +126,9 @@ This is the key design decision for custom models. Follow these placement rules:
 | **Pipeline orchestration** (init, forward, denoise loop) | `vllm_omni/diffusion/models/<name>/pipeline_<name>.py` | Always required |
 | **Custom transformer/backbone** (ported and adapted to vllm-omni) | `vllm_omni/diffusion/models/<name>/<name>_transformer.py` or similar | `wan2_2.py`, `fusion.py`, `bagel_transformer.py` |
 | **Custom sub-models** (VAE, fusion, autoencoder) | `vllm_omni/diffusion/models/<name>/` as separate files | `autoencoder.py`, `fusion.py` |
-| **External dependency code** (original repo utilities) | **External repo**, installed via download script or pip | `dreamid_omni` package via git clone |
+| **External dependency code** (original repo utilities) | **External repo**, installed via download script or pip | original-repo package via git clone |
 | **Hardcoded model configs** | Module-level dicts in pipeline file | `VIDEO_CONFIG`, `AUDIO_CONFIG` dicts |
-| **Download/setup script** | `examples/offline_inference/<name>/download_<name>.py` | `download_dreamid_omni.py` |
+| **Download/setup script** | `examples/offline_inference/<name>/download_<name>.py` | `download_<name>.py` |
 | **Custom `model_index.json`** | Generated by download script, placed at model root | Minimal: `{"_class_name": "YourPipeline", ...}` |
 
 ### B3. Handle external dependencies
@@ -134,9 +155,9 @@ Copy the essential model files into `vllm_omni/diffusion/models/<name>/` and ada
 
 ### B4. Handle custom weight loading
 
-Custom models have two patterns for weight loading:
+Custom models have two common patterns for weight loading:
 
-**Pattern 1: Bypass standard loader** (DreamID-Omni style)
+**Pattern 1: Bypass standard loader** (eager custom init)
 
 When the original model has complex custom init functions that load weights in `__init__`:
 
@@ -149,8 +170,10 @@ class CustomPipeline(nn.Module):
         self.vae = custom_init_vae(model, device=self.device)
         self.text_encoder = custom_init_text_encoder(model, device=self.device)
         self.transformer = CustomFusionModel(CONFIG)
-        load_custom_checkpoint(self.transformer,
-            checkpoint_path=os.path.join(model, "model.safetensors"))
+        load_custom_checkpoint(
+            self.transformer,
+            checkpoint_path=os.path.join(model, "model.safetensors"),
+        )
         # NO weights_sources defined — bypasses standard loader
 
     def load_weights(self, weights):
@@ -162,6 +185,8 @@ class CustomPipeline(nn.Module):
 When weights are in safetensors format but need name remapping:
 
 ```python
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
+
 class CustomPipeline(nn.Module):
     def __init__(self, *, od_config, prefix=""):
         super().__init__()
@@ -187,7 +212,9 @@ class CustomPipeline(nn.Module):
             # Remap original weight names to vllm-omni module names
             name = self._remap_weight_name(name)
             if name in params:
-                default_weight_loader(params[name], tensor)
+                param = params[name]
+                weight_loader = getattr(param, "weight_loader", default_weight_loader)
+                weight_loader(param, tensor)
                 loaded.add(name)
         return loaded
 ```
@@ -281,16 +308,43 @@ For Omni or custom models, create:
 ### Step 7: Update Documentation
 
 Required updates:
-1. `docs/user_guide/diffusion/parallelism_acceleration.md` — parallelism support table
-2. `docs/user_guide/diffusion/cpu_offload_diffusion.md` — if CPU offload supported (add to supported models table)
-3. `docs/user_guide/diffusion/teacache.md` — if TeaCache supported
-4. `docs/user_guide/diffusion/cache_dit_acceleration.md` — if Cache-DiT supported
-5. `examples/offline_inference/xxx/README.md` — offline example docs
-6. `examples/online_serve/xxx/README.md` — online serve docs
+1. `docs/user_guide/diffusion/parallelism/overview.md` — parallelism support overview/table
+2. `docs/user_guide/diffusion/cpu_offload.md` — if CPU offload supported (add to supported models table)
+3. `docs/user_guide/diffusion/cache_acceleration/teacache.md` — if TeaCache supported
+4. `docs/user_guide/diffusion/cache_acceleration/cache_dit.md` — if Cache-DiT supported
+5. Offline example docs under `examples/offline_inference/<name>/` (`README.md` or category-specific `.md`)
+6. `examples/online_serving/<name>/README.md` — online serving docs
 
-### Step 8: Add E2E Tests (Recommended)
+### Step 8: Add E2E Tests
 
-Create `tests/e2e/online_serving/test_your_model_expansion.py`.
+**Follow the [vllm-omni-test skill](../vllm-omni-test/SKILL.md)** for markers, file naming, Buildkite wiring, and run commands. Also read [l4_functionality_tests.inc.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/test_examples/l4_functionality_tests.inc.md), [test_system_overview.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/test_system_overview.md), and [test_writing_guide.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/test_writing_guide.md).
+
+Classify the model's **CI priority** first:
+
+| Priority | Required test levels | Files & markers |
+|----------|---------------------|-----------------|
+| **High** (listed in [#1832](https://github.com/vllm-project/vllm-omni/issues/1832) or on the diffusion hot path) | **L1** · **L2** online · **L3** online + offline · **L4** feature + performance | See table below |
+| **Medium** (*normal priority* in L4 docs) | **L3** online + offline · **L4** feature only | Fewer L4 parametrized rows |
+| **Low** | **L4** feature only | One or two `*_expansion.py` cases |
+
+**Per-level deliverables (diffusion / `pytest.mark.diffusion`):**
+
+| Level | Location | Marker | CI pipeline | Notes |
+|-------|----------|--------|-------------|-------|
+| **L1** | `tests/diffusion/models/{slug}/`, `tests/diffusion/cache/`, transformer unit tests | `core_model` + `cpu` | `test-ready.yml` | Weight remap, `_sp_plan`, cache enabler registration, shape contracts |
+| **L2** | `tests/e2e/online_serving/test_{slug}.py` (and offline if the category is offline-first) | **`core_model` + `advanced_model`** (both on baseline smoke) + `diffusion` + `@hardware_test` / `hardware_marks` | `test-ready.yml` | Default deploy smoke — minimal `num_inference_steps`, single prompt |
+| **L3** | `tests/e2e/online_serving/test_{slug}.py` **and** `tests/e2e/offline_inference/test_{slug}.py` when offline matters | Baseline smoke: **`core_model` + `advanced_model`**; heavier cases: `advanced_model` only (+ `diffusion`) | `test-merge.yml` **or** merged into nightly diffusion function job | Real weights, streaming/API paths, LoRA/offload smoke |
+| **L4** | `tests/e2e/online_serving/test_{slug}_expansion.py` (+ offline expansion if needed) | `full_model` + `diffusion` | `test-nightly.yml` (X2I / X2V / X2A function groups) | Feature combos per [#1832](https://github.com/vllm-project/vllm-omni/issues/1832); perf → `tests/dfx/perf/tests/test_{model}_vllm_omni.json` with per-case **`mark`** (`hardware_marks` + `full_model` + `diffusion`) |
+
+**L2 & L3 online — same file, dual marks on the baseline smoke:** The **first / simplest** case in `test_{slug}.py` (default deploy, minimal steps, single prompt) should carry **both** `@pytest.mark.core_model` and `@pytest.mark.advanced_model` on the **same** function so L2 (`test-ready.yml`) and L3 (`test-merge.yml`) share one smoke test. Heavier deploy variants or API paths in the same file use **`advanced_model` only**. When L3 moves to nightly, migrate those heavier cases into `test_{slug}_expansion.py` with `full_model` and remove the dedicated `test-merge.yml` job (see `test_longcat_image_expansion.py`, `test_qwen_image_expansion.py`).
+
+**L4 design (high priority):** Combine multiple supported features (Cache-DiT, TP, USP, CFG, HSDP, CPU offload, quantization) into **few parametrized** `OmniServerParams` rows so each feature appears in at least one case without exploding GPU jobs. Shard single-GPU vs multi-GPU cases across the nightly X2I/X2V function steps (1-GPU vs `distributed_cuda`).
+
+**L4 design (medium / low):** One or two parametrized rows covering the best quality/perf trade-off; skip perf JSON unless the model is high priority.
+
+**Reference implementations:** `tests/e2e/online_serving/test_qwen_image_edit_expansion.py`, `tests/e2e/online_serving/test_longcat_image_expansion.py`, `tests/e2e/online_serving/test_hunyuan_video_15_expansion.py`.
+
+**Keep model-specific code inside test modules — not `tests/helpers/{slug}.py`:** deploy constants, prompts, sampling dicts, and inline `request_config` / `form_data` belong in each `test_{slug}.py` and `test_{slug}_expansion.py`. Do not add per-model files under `tests/helpers/`; reuse only repo-wide harness (`mark`, `media`, `runtime`, `stage_config`, `assertions`). **L2+ online/offline e2e:** reuse or add `send_*_request` in `tests/helpers/runtime.py` — tests call the handler, not raw `omni.generate` / HTTP. See [vllm-omni-test skill](../vllm-omni-test/SKILL.md) § **Runtime send helpers**.
 
 ### Step 9: Add Cache-DiT Acceleration
 
@@ -364,7 +418,8 @@ def enable_cache_for_your_model(pipeline, cache_config):
 
 #### 9d. Register the custom enabler
 
-Add your enabler to `CUSTOM_DIT_ENABLERS` in `vllm_omni/diffusion/cache/cache_dit_backend.py`:
+Add your enabler to `CUSTOM_DIT_ENABLERS` in
+`vllm_omni/diffusion/cache/cachedit/model_specific.py`:
 
 ```python
 CUSTOM_DIT_ENABLERS = {
@@ -510,7 +565,7 @@ class YourTransformer(nn.Module):
 #### 10e. Update parallelism documentation
 
 After adding parallelism support, update:
-1. `docs/user_guide/diffusion/parallelism_acceleration.md` — add your model to the support table
+1. `docs/user_guide/diffusion/parallelism/overview.md` — add your model to the support overview/table
 2. Record which parallelism methods are supported (USP, Ring, CFG, TP, HSDP, VAE-Patch)
 
 ### Step 11: Add CPU Offload Support
@@ -555,6 +610,24 @@ See the [Profiling Single-Stage Diffusion](../../../docs/contributing/profiling.
 
 ---
 
+## Pre-commit conventions
+
+New library files must pass the local gates in
+[docs/contributing/README.md](../../../docs/contributing/README.md#linting).
+That page is the full hook list (SPDX, forbidden imports including Hugging Face
+Hub / Triton / pickle, `torch.cuda`, mypy, test marks, markdownlint, Buildkite,
+shellcheck). In particular:
+
+- SPDX copyright is `vLLM-Omni project` (stale `vLLM project` is rewritten).
+- Use `import regex as re` and `pybase64` in `vllm_omni/`; do not import stdlib
+  `re` or `base64`. Hugging Face Hub downloads go through
+  `vllm.transformers_utils.repo_utils`.
+- Do not add `torch.cuda.*` call sites; use `current_omni_platform`.
+- New `tests/**/test_*.py` files need a CI level mark and a hardware mark.
+- Do not expand `CHECK_IMPORTS[*].allowed_files` or `ALLOWED_FILES` without review.
+- GitHub Actions skips SPDX/shellcheck/mypy-3.10/test-marks/markdownlint; run
+  `pre-commit` locally.
+
 ## Iterative Development Tips
 
 1. **Start minimal**: Basic generation first, no parallelism/caching
@@ -568,6 +641,7 @@ See the [Profiling Single-Stage Diffusion](../../../docs/contributing/profiling.
 
 ## Reference Files
 
+- [vllm-omni-test skill](../vllm-omni-test/SKILL.md) — L1–L4 markers, naming, Buildkite wiring, run commands
 - [Transformer Adaptation](references/transformer-adaptation.md) — porting transformers from diffusers
 - [Custom Model Patterns](references/custom-model-patterns.md) — patterns for non-diffusers models
 - [Parallelism Patterns](references/parallelism-patterns.md) — TP, SP/USP, CFG parallel, HSDP implementation details

.claude/skills/add-diffusion-model/references/cache-dit-patterns.md

@@ -29,8 +29,13 @@ vLLM-Omni integrates cache-dit through `CacheDiTBackend`:
 | `cache_dit.refresh_context()` | Updates cache context when `num_inference_steps` changes |
 
 **Source files**:
-- `vllm_omni/diffusion/cache/cache_dit_backend.py` — `CacheDiTBackend`, enablers, `CUSTOM_DIT_ENABLERS`
-- `vllm_omni/diffusion/cache/` — cache backend implementations
+- `vllm_omni/diffusion/cache/cachedit/__init__.py` — public package API
+- `vllm_omni/diffusion/cache/cachedit/backend.py` — generic backend lifecycle and integration
+- `vllm_omni/diffusion/cache/cachedit/config.py` — Cache-DiT configuration types
+- `vllm_omni/diffusion/cache/cachedit/model_specific.py` — model-specific adapters, enablers, and registry
+
+Outside `vllm_omni/diffusion/cache`, import Cache-DiT symbols only from
+`vllm_omni.diffusion.cache.cachedit`.
 
 ## Standard Models: Automatic Support
 
@@ -175,7 +180,8 @@ Inspect your block's `forward()` return type and residual connection pattern to
 
 ## Registering Custom Enablers
 
-Add your enabler to `CUSTOM_DIT_ENABLERS` in `vllm_omni/diffusion/cache/cache_dit_backend.py`:
+Add your enabler to `CUSTOM_DIT_ENABLERS` in
+`vllm_omni/diffusion/cache/cachedit/model_specific.py`:
 
 ```python
 CUSTOM_DIT_ENABLERS = {
@@ -248,7 +254,7 @@ Models listed in `_NO_CACHE_ACCELERATION` in `vllm_omni/diffusion/registry.py` d
 
 | Model | Path | Notes |
 |-------|------|-------|
-| Standard DiT | `cache_dit_backend.py::enable_cache_for_dit` | Default enabler, automatic |
-| Wan2.2 | `cache_dit_backend.py::enable_cache_for_wan22` | Dual-transformer, auto-detects mode |
-| LongCat | `cache_dit_backend.py::enable_cache_for_longcat_image` | Multi-block-list |
-| BAGEL | `cache_dit_backend.py::enable_cache_for_bagel` | Complex omni model |
+| Standard DiT | `cachedit.backend::enable_cache_for_dit` | Default enabler, automatic |
+| Wan2.2 | `cachedit.model_specific::enable_cache_for_wan22` | Dual-transformer, auto-detects mode |
+| LongCat | `cachedit.config::CacheDiTAdapterConfig` | Declarative multi-block-list adapter |
+| BAGEL | `cachedit.model_specific::BagelCachedAdapter` | Custom cached adapter |

.claude/skills/add-diffusion-model/references/custom-model-patterns.md

@@ -16,20 +16,19 @@ vllm_omni/diffusion/models/wan2_2/
 
 The transformer is loaded separately via `weights_sources` + `load_weights()`. Non-transformer components (VAE, text encoder) are loaded in `__init__` via `from_pretrained()`.
 
-### Custom model with external deps (e.g., DreamID-Omni)
+### Custom model with external deps
 
 ```
-vllm_omni/diffusion/models/dreamid_omni/
+vllm_omni/diffusion/models/<name>/
 ├── __init__.py                    # Exports pipeline only
-├── pipeline_dreamid_omni.py       # Pipeline: loads ALL weights in __init__ via custom helpers
-├── fusion.py                      # Custom fusion architecture (video + audio cross-attention)
-└── wan2_2.py                      # Re-implemented Wan backbone with split API
+├── pipeline_<name>.py             # Pipeline: loads ALL weights in __init__ via custom helpers
+└── <backbone>.py                  # Custom transformer / fusion architecture
 
-examples/offline_inference/x_to_video_audio/
-└── download_dreamid_omni.py       # Downloads weights from 3 HF repos + clones code repo
+examples/offline_inference/<task>/
+└── download_<name>.py             # Downloads weights and/or clones code repo
 ```
 
-All weights loaded eagerly in `__init__`. `load_weights()` is a no-op. External dependency (`dreamid_omni` package) imported with try/except.
+All weights loaded eagerly in `__init__`. `load_weights()` is a no-op. External dependency imported with try/except.
 
 ### Custom model with ported code (e.g., BAGEL)
 
@@ -76,7 +75,7 @@ self.weights_sources = [
 ]
 ```
 
-### Pattern 3: Fully custom loading (DreamID-Omni)
+### Pattern 3: Fully custom loading
 
 ```
 init → load ALL weights eagerly via custom helpers → load_weights() = no-op
@@ -108,8 +107,8 @@ Standard diffusers `model_index.json`:
 Custom model `model_index.json` (minimal):
 ```json
 {
-    "_class_name": "DreamIDOmniPipeline",
-    "fusion": "DreamID-Omni/dreamid_omni.safetensors"
+    "_class_name": "YourCustomPipeline",
+    "fusion": "weights/model.safetensors"
 }
 ```
 
@@ -117,7 +116,7 @@ The only **required** field is `_class_name` — it must match a key in `_DIFFUS
 
 ## External Dependency Management
 
-### Git clone + .pth injection (DreamID-Omni pattern)
+### Git clone + .pth injection
 
 ```python
 def download_dependency():
@@ -163,7 +162,7 @@ The engine checks `isinstance(pipeline, SupportImageInput)` at startup to config
 
 Diffusers models use `config.json` in each subfolder. Custom models often use:
 
-**Module-level config dicts** (DreamID-Omni):
+**Module-level config dicts**:
 ```python
 VIDEO_CONFIG = {
     "patch_size": [1, 2, 2], "model_type": "ti2v",
@@ -181,7 +180,7 @@ vae_cfg = bagel_cfg.get("vae_config", {})
 
 ## Custom Architecture Patterns
 
-### Split forward API (DreamID-Omni)
+### Split forward API
 
 When a fusion model needs to interleave blocks from two backbones:
 

.claude/skills/add-diffusion-model/references/parallelism-patterns.md

@@ -569,4 +569,4 @@ Check which parallelism methods a model supports:
 | **TP** | Uses `ColumnParallelLinear` / `QKVParallelLinear`. Search: `grep -r 'ParallelLinear\|QKVParallel' vllm_omni/diffusion/models/<model>/` |
 | **HSDP** | Transformer defines `_hsdp_shard_conditions`. Search: `grep -r '_hsdp_shard_conditions' vllm_omni/diffusion/models/` |
 
-The canonical per-model support table is in `docs/user_guide/diffusion/parallelism_acceleration.md`.
+The canonical per-model support overview/table is in `docs/user_guide/diffusion/parallelism/overview.md`.

.claude/skills/add-diffusion-model/references/transformer-adaptation.md

@@ -219,13 +219,14 @@ def load_weights(self, weights):
 
 ### Class-Level Attributes for Features
 
+
 ```python
 class MyTransformer(nn.Module):
     # torch.compile: list block class names that repeat and can be compiled
     _repeated_blocks = ["MyTransformerBlock"]
 
     # CPU offload: attribute name of the nn.ModuleList containing blocks
-    _layerwise_offload_blocks_attr = "blocks"
+    _layerwise_offload_blocks_attrs = ["blocks"]
 
     # LoRA: mapping of fused param names to original param names
     packed_modules_mapping = {"to_qkv": ["to_q", "to_k", "to_v"]}

.claude/skills/add-tts-model/SKILL.md

@@ -138,9 +138,12 @@ A shared buffer silently corrupts audio across concurrent requests — the sympt
    - Load codec weights (may need lazy loading from separate checkpoint)
    - Implement `forward()`: codec codes -> audio waveform
    - Return `OmniOutput` with `multimodal_outputs`
-5. **Create stage config YAML** defining both stages, memory allocation, and model paths
-6. **Create stage input processor** for prompt building
-7. **Write end2end.py** test script
+5. **Define and register the pipeline topology** in
+   `vllm_omni/model_executor/models/<model>/pipeline.py`
+6. **Create the deploy YAML** under `vllm_omni/deploy/` for placement,
+   memory sizing, connectors, and runtime overrides
+7. **Create stage input processor** for prompt building and inter-stage handoff
+8. **Write end2end.py** test script
 
 ### Critical Parameters to Get Right
 
@@ -174,7 +177,7 @@ These bugs appear in almost every new TTS PR. Check all before the first push. S
 - **All return paths must emit `model_outputs`** — if any early-return branch skips setting `model_outputs`, the serving layer silently drops that step's audio (fish speech: `fix: ensure ALL return paths emit model_outputs`)
 - **Per-request state isolation** — for batched concurrent requests, key all state by request ID; a shared buffer corrupts audio across requests (fish speech: `fix: per-request vocode + delta emission`)
 - **Codec tensor device** — move codec codes to the codec decoder's device before calling decode; mismatches cause silent CPU fallback or crashes (fish speech: `fix: use model device for CUDA stream`)
-- **AR stage `max_num_seqs`** — set to at least 4 in production configs; for single-stage models this is the only stage. For two-stage models, Stage 0 (AR) needs `max_num_seqs ≥ 4` to pipeline concurrent requests; Stage 1 (codec decoder) typically uses `max_num_seqs: 1` intentionally. Default of 1 everywhere causes audio gaps under concurrency because the codec window round-robins across requests (RFC #2568)
+- **AR stage `max_num_seqs`** — set to at least 4 in production deploy configs; for single-stage models this is the only stage. For two-stage models, Stage 0 (AR) needs `max_num_seqs ≥ 4` to pipeline concurrent requests; Stage 1 (codec decoder) is model-specific and may intentionally use `max_num_seqs: 1`. Defaulting the AR stage to 1 causes audio gaps under concurrency because the codec window round-robins across requests (RFC #2568)
 
 ### Optional Dependency Handling
 
@@ -190,18 +193,18 @@ pattern, signature constraints, and MOSS-TTS-Nano reference.
 When the upstream model cannot be cleanly split into an AR stage and a
 separate decoder, run the full pipeline inside a single AR worker and
 stream audio through a per-request `inference_stream()` generator keyed by
-`_omni_req_id`. Stage config must set `worker_type: ar`,
-`engine_output_type: audio`, `final_output: true`, `is_comprehension: true`,
-and `async_chunk: false` at the top level. Only extract params from
+`_omni_req_id`. Define one `StagePipelineConfig` with
+`execution_type=StageExecutionType.LLM_AR`, `engine_output_type="audio"`,
+`final_output=True`, and `owns_tokenizer=True`. Set `async_chunk: false` in
+the deploy YAML. Only extract params from
 `additional_information` that you actually forward, or pre-commit fails
 `ruff F841`.
 
 Full walkthrough with the complete `forward()` / `_create_stream_gen()`
-skeleton and stage-config fields:
+skeleton plus pipeline/deploy definitions:
 [references/single-stage-ar.md](references/single-stage-ar.md). For an
 in-tree reference, look for any single-stage AR model under
-`vllm_omni/model_executor/models/` — e.g. the MOSS-TTS-Nano integration when
-it lands.
+`vllm_omni/model_executor/models/`, such as MOSS-TTS-Nano.
 
 **VoxCPM2 is a different pattern** and should not reuse this skeleton — it
 runs the base LM under vLLM PagedAttention with external side-computation.
@@ -210,7 +213,8 @@ See `plan/voxcpm2_native_ar_design.md`.
 ### Deliverables
 
 - Model files in `vllm_omni/model_executor/models/<model_name>/`
-- Stage config YAML
+- Registered `pipeline.py` topology
+- Deploy YAML under `vllm_omni/deploy/`
 - Working `end2end.py` at `examples/offline_inference/text_to_speech/<model>/end2end.py`
 - New section in `examples/offline_inference/text_to_speech/README.md` (table row + per-model section). Do **not** create a top-level `examples/offline_inference/<model>/` dir or a per-model `README.md` inside `text_to_speech/<model>/` — the hub README is the documented surface and the mkdocs `generate_examples` hook only descends one level into `examples/<category>/`.
 
@@ -220,65 +224,70 @@ See `plan/voxcpm2_native_ar_design.md`.
 
 ### Steps
 
-1. **Register in `serving_speech.py`** — add all 5 points in a **single commit**;
-   partial integration causes hard-to-debug failures. This file is modified by every
-   model PR and is the most common source of rebase conflicts — see conflict note below.
+1. **Write one adapter** under `vllm_omni/entrypoints/openai/tts_adapters/`.
+   `serving_speech.py` should not need an edit — detection, stage discovery and
+   dispatch are all derived from what the adapter declares.
 
-   **Point 1** — stage constant (near the top, alongside the other `_*_TTS_MODEL_STAGES` sets):
+   Create `vllm_omni/entrypoints/openai/tts_adapters/your_model.py`:
    ```python
-   _YOUR_MODEL_TTS_MODEL_STAGES = {"your_stage_key"}
+   @register_tts_adapter
+   class YourModelAdapter(ARTTSAdapter):
+       name = "your_model"                              # registry key + log label
+       stage_keys = frozenset({"your_stage_key"})       # the deploy yaml's model_stage
+
+       def validate(self, request) -> str | None:
+           if not request.input or not request.input.strip():
+               return "Input text cannot be empty"
+           return None
+
+       async def build(self, request, sampling_params_list, has_inline_ref_audio):
+           params = {"text": [request.input]}
+           if request.voice is not None:
+               params["voice"] = [request.voice]
+           return PreparedRequest(
+               prompt={"prompt": request.input},
+               tts_params=params,
+               model_type=self.name,
+           )
    ```
 
-   **Point 2** — union into `_TTS_MODEL_STAGES`:
-   ```python
-   _TTS_MODEL_STAGES: set[str] = (
-       ...
-       | _YOUR_MODEL_TTS_MODEL_STAGES
-   )
-   ```
+   Then add the module to the import block at the bottom of
+   `tts_adapters/__init__.py` so it registers. That import line is the only shared
+   file a new model touches — which is also why the old rebase-conflict hotspot is
+   gone.
 
-   **Point 3** — model type detection in `_detect_tts_model_type()`:
-   ```python
-   if model_stage in _YOUR_MODEL_TTS_MODEL_STAGES:
-       return "your_model"
-   ```
+   > **Pure-diffusion TTS does not go through adapters yet.** Under
+   > `for_diffusion()`, `create_speech()` routes straight to
+   > `_create_diffusion_speech()` and never calls `validate()`/`build()`.
+   > `DiffusionTTSAdapter` is scaffolding with no production subclass, so logic
+   > placed in one would silently never run. Diffusion-engine models follow the
+   > existing diffusion path; wiring it through adapters is open work (#4855).
 
-   **Point 4** — validation dispatch in `_validate_tts_request()`:
-   ```python
-   if self._tts_model_type == "your_model":
-       return self._validate_your_model_request(request)
-   ```
+   **If a stage key alone cannot identify the model**, declare
+   `model_archs = frozenset({"YourModelForConditionalGeneration"})`; add
+   `arch_identifies_entry_stage = True` when the model owns no stage key at all
+   (Ming dense). For a rule that is not set membership, override `matches()` (see
+   `covo_audio.py`). For a genuine overlap with another adapter, give one an
+   explicit `detect_priority` — `test_tts_detection.py` fails on an unordered
+   overlap. Models that only serve speech in some topologies override
+   `stage_serves_speech()` (see `audex.py`).
 
-   **Point 5** — validation + parameter-builder methods:
-   ```python
-   def _validate_your_model_request(self, request) -> str | None:
-       if not request.input or not request.input.strip():
-           return "Input text cannot be empty"
-       return None
+   **Reuse shared helpers via `self.ctx.server`** rather than reimplementing them:
+   `_resolve_ref_audio`, `_apply_uploaded_speaker`, `_validate_ref_audio_format`,
+   `_max_instructions_length`. Read a comparable adapter first — `fish_speech.py`
+   (voice cloning), `higgs_audio_v3.py` (parameter-heavy), `moss_tts.py` (family
+   sharing a base class).
 
-   def _build_your_model_params(self, request) -> dict:
-       params = {"text": [request.input]}
-       if request.voice is not None:
-           params["voice"] = [request.voice]
-       return params
-   ```
-   Wire `_build_your_model_params` into `_create_tts_request()` alongside the other
-   model-specific param builders.
+   > **Do not add `self._tts_model_type == ...` branches to `serving_speech.py`.**
+   > Older models predate the adapter framework and still have them; they are being
+   > migrated out (RFC #4327, #4855). `tools/pre_commit/check_tts_adapter.py` is a
+   > ratchet on the remaining count and fails the commit if it grows. Behaviour that
+   > no adapter hook can express is a missing hook — propose it on the RFC.
 
-   > **Two dispatch patterns coexist**: Fish Speech uses a `self._is_fish_speech` boolean
-   > instance attribute checked before `elif self._is_tts`, while all newer models
-   > (CosyVoice3, MOSS-TTS-Nano) use the `_tts_model_type` string returned by
-   > `_detect_tts_model_type()`. For new models, always use the `_tts_model_type` string
-   > pattern — do not add new `_is_*` flags.
-
-   > **Unused variable rule**: only extract fields in `_build_your_model_params` that
-   > are actually forwarded to the model. Unused extractions fail `ruff F841`.
-   > For voice-cloning fields (`ref_audio` → `prompt_audio_path`, `ref_text` →
-   > `prompt_text`), add them to the param builder and verify they reach the model call.
-
-   **Rebase conflict note**: when rebasing onto `main` after another model was merged,
-   `serving_speech.py` will conflict. Resolution: always keep *both* the upstream
-   model's additions and your own — never discard either side.
+   > **Unused variable rule**: only extract fields in `build()` that are actually
+   > forwarded to the model. Unused extractions fail `ruff F841`. For voice-cloning
+   > fields (`ref_audio` -> `prompt_audio_path`, `ref_text` -> `prompt_text`), add
+   > them to the params and verify they reach the model call.
 
 2. **Handle model-specific parameters**:
    - Voice cloning: `ref_audio` encoding and prompt injection
@@ -295,7 +304,7 @@ import base64
 from pathlib import Path
 
 def build_voice_clone_prompt(ref_audio_path: str, text: str, codec) -> list:
-    """Build prompt with reference audio for voice cloning in serving_speech.py."""
+    """Build prompt with reference audio for voice cloning, called from the adapter."""
     audio_bytes = Path(ref_audio_path).read_bytes()
     codes = codec.encode(audio_bytes)  # Encode on CPU using model's codec (e.g., DAC)
     token_ids = [code + codec.vocab_offset for code in codes.flatten().tolist()]
@@ -305,13 +314,72 @@ def build_voice_clone_prompt(ref_audio_path: str, text: str, codec) -> list:
     ]
 ```
 
+### Test Case Writing (CI Levels)
+
+**Follow the [vllm-omni-test skill](../vllm-omni-test/SKILL.md)** for markers, file naming (`test_{slug}.py` / `test_{slug}_expansion.py`), Buildkite wiring, and copy-paste run commands. Also read [test_system_overview.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/test_system_overview.md) and [test_writing_guide.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/test_writing_guide.md).
+
+Classify the model's **CI priority** first (high / medium / low). High-priority TTS models are typically those on the integration hot path or listed in tracking issues such as [#1832](https://github.com/vllm-project/vllm-omni/issues/1832); medium and low tiers cover the long tail. When unsure, ask the reviewer which tier applies.
+
+| Priority | Required test levels | Files & markers |
+|----------|---------------------|-----------------|
+| **High** | **L1** unit/logic · **L2** online smoke · **L3** online + offline integration · **L4** feature + performance | See table below |
+| **Medium** | **L3** online + offline · **L4** feature only | Skip dedicated L1/L2 unless fixing a logic bug |
+| **Low** | **L4** feature only | One or two `*_expansion.py` parametrized cases |
+
+**Per-level deliverables (TTS / `pytest.mark.tts`):**
+
+| Level | Location | Marker | CI pipeline | Notes |
+|-------|----------|--------|-------------|-------|
+| **L1** | `tests/model_executor/…`, `tests/entrypoints/openai_api/…`, stage-processor tests | `core_model` + `cpu` | `test-ready.yml` | Prompt assembly, async_chunk helpers, adapter validation — no GPU |
+| **L2** | `tests/e2e/online_serving/test_{slug}.py` | **`core_model` + `advanced_model`** (both on baseline smoke) + `tts` + `@hardware_test(...)` | `test-ready.yml` (`ready` label) | Default deploy smoke: single `/v1/audio/speech` or offline `OmniRunner` path |
+| **L3** | `tests/e2e/online_serving/test_{slug}.py` **and** `tests/e2e/offline_inference/test_{slug}.py` | Baseline smoke: **`core_model` + `advanced_model`**; heavier cases: `advanced_model` only (+ `tts`) | `test-merge.yml` **or** merged into nightly TTS function job | Streaming, voice clone, batch/queue, async_chunk |
+| **L4** | `tests/e2e/online_serving/test_{slug}_expansion.py`, optional offline expansion | `full_model` + `tts` | `test-nightly.yml` (`:full_moon: TTS · Function Test with L4`) | Feature matrix; perf → `tests/dfx/perf/tests/test_tts.json` |
+
+**L2 & L3 online — same file, dual marks on the baseline smoke:** The **first / simplest** case in `test_{slug}.py` (default deploy, single non-streaming `/v1/audio/speech` or equivalent offline path) should carry **both** `@pytest.mark.core_model` **and** `@pytest.mark.advanced_model` on the **same** function so it runs in L2 (`test-ready.yml`, `--run-level core_model`, basic validation) and L3 (`test-merge.yml`, `--run-level advanced_model`, deeper validation) without duplicating the test. In-tree examples: `test_voxcpm2_tts.py::test_text_to_audio_001`, `test_qwen3_tts_customvoice.py::test_text_to_audio_001`.
+
+Heavier scenarios in the same file use **`advanced_model` only** (streaming, extra languages, concurrency, async_chunk, batch). Example: `test_voice_clone_en_streaming_001` → `advanced_model` only. When migrating L3 to nightly, move those heavier cases into `test_{slug}_expansion.py` with `full_model` and drop the dedicated merge job (see `test_ming_tts_expansion.py`, `test_glm_tts_expansion.py`).
+
+```python
+@pytest.mark.core_model
+@pytest.mark.advanced_model
+@pytest.mark.tts
+@hardware_test(res={"cuda": "L4"}, num_cards=1)
+@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
+def test_voice_clone_en_non_streaming_001(omni_server, online_client) -> None:
+    online_client.send_audio_speech_request({...})
+```
+
+**L4 consolidation:** Prefer parametrized `OmniServerParams` rows (`default`, `async_chunk`, feature flags) in one expansion module rather than many merge-only files ([#1832](https://github.com/vllm-project/vllm-omni/issues/1832)).
+
+**L4 performance (high-priority models):** Add latency / throughput / stress rows in `tests/dfx/perf/tests/test_tts.json`, or a dedicated `tests/dfx/perf/tests/test_{slug}.json` when the model must not join the shared nightly server matrix before integration lands (see VoxCPM2 / Coqui XTTS pattern). Register the model in `benchmarks/tts/model_configs.yaml` for local `bench_tts.py`. Wire a **separate** `test-nightly.yml` Perf Test step when the JSON is not merged into `test_tts.json` yet.
+
+**Keep model-specific code inside test modules — not `tests/helpers/{slug}.py`:**
+
+- Put `MODEL`, deploy path, vendored `REF_AUDIO_URL`, `get_prompt()`, and inline `request_config` dicts **in each** `test_{slug}.py`, `test_{slug}_expansion.py`, offline `test_{slug}.py`, and L1 `test_{slug}_*.py` as needed.
+- **Do not** add `tests/helpers/{slug}.py` (or `tests/helpers/{model_name}.py`) to deduplicate constants or request builders across those files. A little duplication is intentional; follow in-tree references such as `tests/e2e/online_serving/test_glm_tts.py` and `tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py`.
+- `tests/helpers/` is for **repo-wide** harness code only (`mark.py`, `media.py`, `runtime.py`, `stage_config.py`, `assertions.py`, `fixtures/`). Import those; do not extend the tree with per-model modules.
+
+**Runtime send helpers (`tests/helpers/runtime.py`) — online and offline e2e:**
+
+| Path | Fixture | Call |
+|------|---------|------|
+| Online `/v1/*` | `online_client` | `online_client.send_*_request(request_config)` |
+| Offline inference | `offline_client` | `offline_client.send_*_request(request_config)` |
+
+1. **Grep `runtime.py` first** — reuse `send_omni_request`, `send_diffusion_request`, `send_audio_speech_request` (online + offline Qwen-style TTS), `send_single_stage_tts_request` (Coqui XTTS / MOSS-TTS-Nano offline), etc.
+2. **No matching helper** → add `send_<feature>_request` (or `send_<route>_http_request` for negative/dfx) in **`runtime.py`** with general `assert_*` bundled inside, **then** call it from the test.
+3. **Test file** holds `request_config` dicts only — not `omni.generate`, not `_collect_audio()`, not raw HTTP/SDK.
+4. Wire `tests/helpers/runtime.py` into Buildkite `source_file_dependencies` when you add helpers.
+
+See [vllm-omni-test skill](../vllm-omni-test/SKILL.md) § **Runtime send helpers** for full tables and exceptions.
+
 ### Deliverables
 
-- Updated `serving_speech.py` with all 5 integration points (single commit)
+- One adapter file under `tts_adapters/` plus its line in the package import block
 - Client scripts and server launcher under `examples/online_serving/text_to_speech/<model>/`
 - Gradio demo with streaming and voice cloning UI in the same dir
-- E2E online serving test (`tests/e2e/online_serving/test_<model>.py`)
-- Buildkite CI entry in `.buildkite/test-merge.yml`
+- E2E tests per **Test Case Writing (CI Levels)** above (priority tier determines L1–L4 scope)
+- Buildkite wired per level: `test-ready.yml` (L1/L2), `test-merge.yml` or nightly function job (L3), `test-nightly.yml` (L4) — see [vllm-omni-test skill](../vllm-omni-test/SKILL.md)
 - New section in `examples/online_serving/text_to_speech/README.md` (table row + per-model section). Do **not** create a top-level `examples/online_serving/<model>/` dir or a per-model `README.md` inside `text_to_speech/<model>/`.
 
 ### E2E test pitfalls to avoid
@@ -326,9 +394,9 @@ def build_voice_clone_prompt(ref_audio_path: str, text: str, codec) -> list:
 - **Use the harness readiness gate.** The fixture waits for HTTP 200 on
   `/health`; don't add `time.sleep` in tests. If warmup is incomplete, make
   `/health` return non-200 until you're actually ready.
-- **Mark with `@pytest.mark.core_model` + `hardware_test(res={"cuda": "H100"})`**
-  so the test lands in `test-ready.yml` (triggered by the `ready` label) rather
-  than only nightly.
+- **Mark tests per the CI Levels table** — baseline smoke: `core_model` + `advanced_model`; heavier cases: `advanced_model` only; L4 expansion: `full_model`
+- **No per-model helper modules** — do not create `tests/helpers/{slug}.py`; keep constants and `request_config` payloads in the test file
+- **Online and offline e2e go through `runtime.py`** — `online_client.send_audio_speech_request` (online); `offline_client.send_audio_speech_request` (Qwen-style offline) or `send_single_stage_tts_request` (single-stage offline). Add a new `send_*_request` in `runtime.py` when none fits; do not embed `omni.generate` or HTTP in tests
 
 ## Phase 4: Async Chunk (Streaming)
 
@@ -336,21 +404,29 @@ def build_voice_clone_prompt(ref_audio_path: str, text: str, codec) -> list:
 
 ### Steps
 
-1. **Update stage config YAML**:
+1. **Update the pipeline topology** so the producing stage declares its async
+   handoff processor with `async_chunk_process_next_stage_input_func`.
+2. **Update the deploy YAML** to enable async chunk and configure the connector:
    ```yaml
    async_chunk: true
-   codec_chunk_frames: 25      # frames per chunk
-   codec_left_context_frames: 25  # overlap for smooth boundaries
+
+   connectors:
+     connector_of_shared_memory:
+       name: SharedMemoryConnector
+       extra:
+         codec_streaming: true
+         codec_chunk_frames: 25
+         codec_left_context_frames: 25
    ```
-2. **Implement chunk handling in Stage 1**:
+3. **Implement chunk handling in Stage 1**:
    - Accept partial input (chunk of codec codes)
    - Handle left context for smooth audio boundaries
    - Return partial audio in `OmniOutput`
-3. **Test streaming**:
+4. **Test streaming**:
    - Verify audio quality matches non-streaming output
    - Check for artifacts at chunk boundaries
    - Measure TTFA (time to first audio)
-4. **Update online serving** to support `stream=true` with PCM output
+5. **Update online serving** to support `stream=true` with PCM output
 
 ### Streaming Architecture
 
@@ -371,7 +447,8 @@ Stage 0 (AR)                    Stage 1 (Decoder)
 
 ### Deliverables
 
-- Updated stage config with async_chunk enabled
+- Updated pipeline async handoff processor and deploy config with
+  `async_chunk: true`
 - Smooth streaming audio without boundary artifacts
 - TTFA metrics
 
@@ -443,31 +520,33 @@ Use this checklist when integrating a new TTS model:
 - [ ] Config classes created with `model_type` registration
 - [ ] Stage 0 (AR) implemented and generates correct tokens
 - [ ] Stage 1 (Decoder) produces correct audio from tokens — dtype float32 for codec decoder
-- [ ] Stage 1 `max_num_seqs` ≥ 4 in production config (default 1 causes gaps under concurrency)
+- [ ] AR stage `max_num_seqs` ≥ 4 in the production deploy config unless the model has a tested lower limit
 - [ ] Optional dependency fallbacks handled at `load_weights()` time (torchaudio/soundfile/etc.)
 - [ ] Streaming: codec codes accumulated across AR steps (not reset per step)
 - [ ] Streaming: delta audio emitted per chunk, not full re-decoded waveform
 - [ ] Streaming: all `forward()` return paths emit `model_outputs`
 - [ ] Streaming: per-request state keyed by request ID (not shared across requests)
 - [ ] Streaming: codec tensors moved to codec decoder device before decode
-- [ ] Stage config YAML created
+- [ ] Pipeline topology defined in `pipeline.py` and registered in `OMNI_PIPELINES`
+- [ ] Deploy YAML created under `vllm_omni/deploy/`
 - [ ] `end2end.py` produces audio matching reference quality
 - [ ] README.md written
 
 ### Phase 3: Online Serving
-- [ ] All 5 `serving_speech.py` integration points added in one commit
-- [ ] Only extract params in `_build_*_params` that are forwarded to the model call (ruff F841)
+- [ ] Adapter written under `tts_adapters/` and registered in the import block
+- [ ] Only extract params in `build()` that are forwarded to the model call (ruff F841)
 - [ ] Prompt builder handles text input correctly
 - [ ] Voice cloning works (if supported)
 - [ ] All response formats work (wav, mp3, flac, pcm)
 - [ ] Client scripts and server launcher created
-- [ ] E2E online serving test written (`tests/e2e/online_serving/test_<model>.py`)
-- [ ] Buildkite CI entry added to `.buildkite/test-merge.yml`
+- [ ] E2E tests added per model priority tier (see **Test Case Writing (CI Levels)**)
+- [ ] Buildkite entries match level: `test-ready.yml` / `test-merge.yml` or nightly TTS job / `test-nightly.yml`
 - [ ] Gradio demo working
 - [ ] Documentation added (offline + online docs, nav, supported models)
 
 ### Phase 4: Async Chunk
-- [ ] Stage config updated with `async_chunk: true`
+- [ ] Pipeline declares the async handoff processor
+- [ ] Deploy config sets `async_chunk: true` and connector chunk parameters
 - [ ] Stage 1 handles partial chunks correctly
 - [ ] No audio artifacts at chunk boundaries
 - [ ] Streaming via API (`stream=true`) works
@@ -484,7 +563,7 @@ Use this checklist when integrating a new TTS model:
 - [ ] `pre-commit run --files <changed>` passes before every push
 - [ ] Every commit has `Signed-off-by` matching the author email (`git commit -s`)
 - [ ] `git config user.email` matches the email registered on your GitHub account
-- [ ] Details and failure-recovery commands: [references/precommit-dco.md](references/precommit-dco.md)
+- [ ] Hook list, failure table, and DCO recovery: [references/precommit-dco.md](references/precommit-dco.md)
 
 ## References
 
@@ -500,5 +579,6 @@ Project docs and adjacent skills:
 - [TTS audio skill](../vllm-omni-audio-tts/SKILL.md) — supported models and usage
 - [Fish Speech integration](../vllm-omni-audio-tts/references/fish-speech.md) — complete example of Phases 1–3
 - [Qwen3-TTS reference](../vllm-omni-audio-tts/references/qwen-tts.md) — complete example of all 5 phases
+- [vllm-omni-test skill](../vllm-omni-test/SKILL.md) — L1–L4 markers, naming, Buildkite wiring, run commands
 - [Adding a TTS model (developer guide)](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/model/adding_tts_model.md)
 - `plan/voxcpm2_native_ar_design.md` — VoxCPM2's vLLM-native AR + side-computation pattern (distinct from the generator-based single-stage described above)

.claude/skills/add-tts-model/references/precommit-dco.md

@@ -30,6 +30,18 @@ but the changes are correct — stage the modified files and re-commit.
 | `ruff F841` | Variable extracted but never forwarded to model call | Remove the extraction or wire it through |
 | `ruff E402` | Import added below function definitions | Move to top-level import block |
 | `ruff format` | Line length, spacing, quote style | Accept auto-fix, stage, re-commit |
+| `check-spdx-header` | Missing header, or copyright still says `vLLM project` | Two-line Omni header (`vLLM-Omni project`); restage the rewrite |
+| `check-forbidden-imports` | Stdlib `re`/`base64`, pickle, Hugging Face Hub API, or direct Triton/TileLang | `import regex as re` and `pybase64`; use `vllm.transformers_utils.repo_utils`. Do **not** add the file to `allowed_files` without review |
+| `check-torch-cuda-call` | New `torch.cuda.*` call site | Use `current_omni_platform` / `OmniPlatform`; do not grow `ALLOWED_FILES` without review |
+| `check-tts-adapter-migration` | New `self._tts_model_type` branch in `serving_speech.py` | Put per-model logic in `tts_adapters/`. Lower `MAX_MODEL_TYPE_BRANCHES` when removing branches; do not raise it without review |
+| `check-test-ci-coverage` | New `tests/**/test_*.py` missing level or hardware mark | Add `core_model`/`advanced_model`/… plus `cpu`/`cuda`/`hardware_test(` |
+| `mypy-3.10` | Type error on changed `vllm_omni/` files | Fix the types; model trees are excluded. Extra versions: `pre-commit run --hook-stage manual mypy-3.12` |
+| `markdownlint-cli2` | Docs/README markdown lint | Accept auto-fix under `docs/` / `recipes/` / root README |
+| `check-buildkite` | Invalid `.buildkite/*.yml` | Fix the YAML; do not grow `SKIP_FILES` without review |
+| `shellcheck` | No `shellcheck` on PATH, or script warning | `apt-get`/`dnf`/`brew install shellcheck`, or `shellcheck.exe` on Windows PATH |
+
+Canonical policy, CI `SKIP` list, and allowlist locations:
+[docs/contributing/README.md](../../../docs/contributing/README.md#linting).
 
 ## DCO sign-off
 

.claude/skills/add-tts-model/references/single-stage-ar.md

@@ -86,12 +86,51 @@ class YourModelForCausalLM(nn.Module):
 - Handle both `"audio"` (incremental) and `"result"` (final combined) event
   types from upstream models.
 
-## Stage config
+## Pipeline and deploy config
 
-Single stage with `worker_type: ar`, `engine_output_type: audio`,
-`final_output: true`, `is_comprehension: true`, and `async_chunk: false` at
-the top level. Omitting any of these causes silent misclassification in the
-serving layer.
+Define the fixed single-stage topology in `pipeline.py`:
+
+```python
+from vllm_omni.config.stage_config import (
+    PipelineConfig,
+    StageExecutionType,
+    StagePipelineConfig,
+)
+
+YOUR_TTS_PIPELINE = PipelineConfig(
+    model_type="your_model_name",
+    default_deploy_config_name="your_model_name.yaml",
+    model_arch="YourModelForCausalLM",
+    stages=(
+        StagePipelineConfig(
+            stage_id=0,
+            model_stage="your_model_name",
+            execution_type=StageExecutionType.LLM_AR,
+            input_sources=(),
+            owns_tokenizer=True,
+            engine_output_type="audio",
+            final_output=True,
+            final_output_type="audio",
+        ),
+    ),
+)
+```
+
+Register it in `vllm_omni/config/pipeline_registry.py`. Keep placement and
+runtime sizing in `vllm_omni/deploy/your_model_name.yaml`:
+
+```yaml
+async_chunk: false
+
+stages:
+  - stage_id: 0
+    devices: "0"
+    max_num_seqs: 4
+    gpu_memory_utilization: 0.8
+```
+
+Do not move `execution_type`, output ownership, tokenizer ownership, or model
+architecture into the deploy YAML; those are pipeline topology.
 
 ## Lint discipline
 
@@ -101,8 +140,5 @@ pre-commit.
 
 ## Reference implementation
 
-Look for any single-stage AR model under
-`vllm_omni/model_executor/models/` — e.g. `moss_tts_nano/` when its
-integration lands. If none is in tree yet, follow the skeleton above and
-cross-check against the `is_comprehension: true` / `async_chunk: false`
-dispatch in `vllm_omni/entrypoints/openai/serving_speech.py`.
+Use `vllm_omni/model_executor/models/moss_tts_nano/pipeline.py` and
+`vllm_omni/deploy/moss_tts_nano.yaml` as the in-tree reference.

.claude/skills/find-simplifications/SKILL.md

@@ -0,0 +1,135 @@
+---
+name: find-simplifications
+description: Find evidence-backed simplification candidates in vLLM-Omni and, when requested, turn them into focused proposals or code changes. Use for audits of dead, duplicated, speculative, over-generalized, unnecessarily defensive, or hand-rolled code. Use review-pr for ordinary correctness review and diffusion-perf-opt for performance-first optimization.
+---
+
+# Find vLLM-Omni Simplifications
+
+Find a small number of well-proven ways to reduce code, state, APIs, or maintenance cost without weakening supported behavior. Treat this as an architecture and ownership audit, not a dead-code search or style cleanup.
+
+The default deliverable is a read-only report. Do not add TODOs, design documents, issues, or code unless the user asks for those changes.
+
+## Establish the Contract
+
+Before judging a candidate:
+
+1. Read the repository instructions supplied by the host.
+2. Read [`docs/design/architecture_overview.md`](../../../docs/design/architecture_overview.md).
+3. Select only the relevant active module and feature documents from [`docs/design/index.md`](../../../docs/design/index.md). Archived design pages are historical context, not current contracts.
+4. Identify the comparison base with `git merge-base HEAD origin/main`; do not assume the current branch represents the intended baseline.
+5. If tests or CI coverage may change, use [`vllm-omni-test`](../vllm-omni-test/SKILL.md) for test placement, markers, and commands.
+
+Separate an implementation accident from an intentional boundary. In particular:
+
+- vLLM-Omni extends and sometimes overrides upstream vLLM. A similar upstream implementation is evidence for reuse only after comparing API, lifecycle, and multi-stage semantics.
+- Registries, import strings, model configuration, deployment YAML, plugins, and optional dependencies create dynamic consumers that a static reference count can miss.
+- Hardware backends, model families, execution modes, and connector implementations may be intentionally parallel. Do not propose collapsing one only because it is inactive in the local environment.
+- Stage, replica, request, cache, and transport state can look redundant while representing different owners. Prove that two fields or barriers encode the same fact before merging them.
+- A shorter hot path is not a simplification if it adds synchronization, tensor copies, host materialization, recompilation, or a measurable quality/performance regression.
+
+## Strong Candidates
+
+A strong candidate removes a meaningful surface and has a clear replacement or deletion boundary. Examples include:
+
+- A public method, configuration option, registry entry, route, message field, or event has no production consumer and no compatibility obligation.
+- Tests or documentation are the only consumers of behavior that is not part of a supported contract.
+- Multiple layers mirror the same request, lifecycle, cache-validity, readiness, or topology state.
+- AR, diffusion, platform, or model-specific implementations duplicate a stable common operation and can share it without hiding performance-critical differences.
+- An inherited upstream vLLM primitive is reimplemented locally without an Omni-specific semantic difference.
+- Compatibility or fallback branches protect environments the project no longer supports.
+- Serialization, IPC, output formatting, or connector code materializes the same payload more than once or converts between equivalent representations.
+- A helper, abstraction, package split, or extension point has only one real implementation and adds more indirection than policy.
+- Hand-written infrastructure can be replaced by an existing project dependency, Python standard-library feature, PyTorch primitive, or upstream vLLM facility with net deletion and equal observability.
+- Tests maintain large mocks, snapshots, or expected data solely for an unused API or representation.
+
+Do not elevate typo fixes, formatting, isolated renames, or vague complexity complaints into simplification candidates. Bundle small cleanups only when they support one substantive removal.
+
+## Survey by Ownership Boundary
+
+For broad audits, cover the largest or riskiest production deltas first. Useful domains are:
+
+- `vllm_omni/engine/`: orchestration, stage pools, request state, output convergence, cancellation, and replica lifecycle.
+- `vllm_omni/worker/` and AR model execution: upstream vLLM overlays, scheduler/cache state, collective RPC, adapters, and weight loading.
+- `vllm_omni/diffusion/`: request/step execution, batching, output materialization, IPC, offload, parallelism, and model pipelines.
+- `vllm_omni/entrypoints/`: duplicated validation/defaulting, offline/online parity, streaming, and route compatibility.
+- Connectors and distributed code: ownership, copies, barriers, handles, cleanup, and failure propagation across process/device/node boundaries.
+- Platform backends: genuine vendor differences versus copied control flow that has drifted.
+- Configuration, registries, recipes, and examples: duplicate sources of truth and options that never reach a runtime consumer.
+- Tests and CI: redundant fixtures, mocks of obsolete contracts, and expensive coverage that can move to a cheaper level without losing the guarded behavior.
+
+When the user explicitly requests a repository-wide or many-candidate audit and parallel agent work is available and authorized, divide these domains among agents and require the same evidence format from each. Otherwise survey them sequentially.
+
+## Prove or Reject Each Candidate
+
+Use `rg` first, then read every relevant caller and implementation. Search exact Python symbols plus serialized names, route paths, configuration keys, registry strings, YAML values, and CLI spellings.
+
+Classify evidence:
+
+- **Production:** `vllm_omni/`, runtime configuration, registries, serving entrypoints, connectors, recipes used for supported deployments, and executable loader paths.
+- **Validation/support:** `tests/`, benchmarks, examples, docs, CI, and developer tooling.
+- **External/dynamic:** upstream vLLM contracts, third-party plugins, model repositories, optional backends, and string-loaded objects. State what was checked and what remains unknown.
+
+For each candidate, answer:
+
+1. What exact surface would disappear or merge?
+2. Who owns the state or behavior today?
+3. Which production consumers exist, including dynamic consumers?
+4. What supported behavior, compatibility, performance, memory, or observability could change?
+5. What is the smallest validation that would catch a bad simplification?
+6. Is the result net deletion after tests, adapters, migration glue, and documentation are counted?
+
+Reject or downgrade the candidate when a live consumer exists and removal is actually a feature decision; when an active design contract explains the separation; when the change only moves complexity; or when hardware/model evidence is unavailable for a performance-sensitive claim.
+
+## Lifecycle and Data-Movement Audits
+
+For asynchronous or distributed code, map the ownership graph before recommending deletion:
+
+```text
+request admission -> orchestrator -> stage/replica -> worker/device
+  -> connector or IPC boundary -> output aggregation -> terminal cleanup
+```
+
+Associate every lock, event, sentinel, queue, readiness flag, timeout, abort path, and cache marker with an owner and transition. Coalesce mechanisms only when they represent the same state and have the same failure boundary. Preserve separate mechanisms when they protect publication versus rollback, local versus remote ownership, first-terminal-outcome arbitration, or cleanup after partial failure.
+
+For tensors and media, also trace device, process, and ownership transitions. Count GPU-to-CPU copies, shared-memory/object-store handles, serialization, decode/encode steps, and unlink/release responsibility. Prefer eliminating a transfer or representation over merely hiding it behind a helper.
+
+## Dependency Replacements
+
+Prefer facilities already available in the repository before adding a dependency. For a proposed replacement:
+
+- Name the exact local behavior it covers and any semantics that remain in glue code.
+- Check maintenance, license, optional-dependency boundaries, platform support, and transitive footprint.
+- Compare net deletion, test burden, import/startup cost, and runtime behavior.
+- Benchmark when the replaced code is on a model, scheduler, connector, serialization, or serving hot path.
+
+A wrapper that retains the same state machine or conversion logic is not a simplification.
+
+## Deliver the Result
+
+Report only candidates supported by concrete evidence. For each one include:
+
+- **Surface:** files and symbols involved.
+- **Evidence:** production consumers found, dynamic paths checked, and relevant design contract.
+- **Simplification:** exactly what to remove, merge, inline, or reuse.
+- **Benefit:** deleted API/state/code, reduced data movement, or reduced maintenance/test burden.
+- **Risk and validation:** behavior that could change and the narrowest useful tests or measurements.
+- **Confidence:** high or medium, with the unresolved evidence named for medium-confidence items.
+
+Also list representative ideas that were rejected when that prevents repeated investigation. Prefer three high-confidence candidates over a long speculative inventory.
+
+If the user asks for a durable design proposal, place it under `docs/design/feature/` and register it in `docs/design/index.md`; use current design-document conventions and keep implementation details proportional to the decision. If the user asks for implementation, make the smallest coherent change, update affected tests/docs, and preserve public compatibility unless the user explicitly accepts a breaking change.
+
+## Folding Work from Another Branch
+
+Diff the source branch against its own merge base with `origin/main`, not against the current feature branch. Port only independently justified candidates. Consolidate overlaps into the candidate with stronger evidence, and do not preserve a candidate count by carrying weaker duplicates.
+
+## Validation
+
+Match validation to the outgoing diff:
+
+- Documentation or skill changes: run the repository skill validator when available, check links/paths, search for stale source-specific terminology, and run `git diff --check`.
+- Python changes: run the narrowest relevant pytest coverage through the repository environment, then lint/format the changed files.
+- Distributed, model, hardware, performance, or accuracy changes: run available CPU/static checks and state the exact GPU/NPU/model gap; never present simulated evidence as device validation.
+- Before submission, use [`precheck-pr`](../precheck-pr/SKILL.md) for the branch-level self-review.
+
+When summarizing, distinguish checks that passed from checks that were not runnable, and state the snapshot or branch against which the audit was performed.

.claude/skills/precheck-pr/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: precheck-pr
-description: Self-check your branch before creating a PR — catch dead code, verify accuracy/perf claims, validate PR title format, and confirm merge readiness. Use when the user says "precheck", "self review", "pre-submit check", or "check my PR before I open it." Never posts to GitHub.
+description: Self-check your branch before creating a PR — catch dead code, prevent new model-specific Python examples, verify accuracy/perf claims, validate PR title format, and confirm merge readiness. Use when the user says "precheck", "self review", "pre-submit check", or "check my PR before I open it." Never posts to GitHub.
 ---
 
 # PR Pre-Check
@@ -63,6 +63,14 @@ If multiple rows apply (e.g., a diffusion model is also a new model), union the
 
 Ask: "Quick mode or full mode?" Then walk the checklist for the detected PR type from [references/checklists.md](references/checklists.md). Each item produces ✓, ✗, or ⚠.
 
+**Also run the Code-Quality sweep on every PR**, regardless of type or mode: the five diff-scoped checks in [references/code-quality.md](references/code-quality.md) — kwargs fragility, broad-except swallow, `Any`/wrong type hints, hot-path `.clone()`/`deepcopy`, and event-loop blocking — plus the advisory conventions (log level, structured logging, synchronization, cleanup, dependencies, naming) in [checklists.md](references/checklists.md). These count only lines the PR *adds* — the pre-existing backlog across the repo is out of scope.
+
+**Also run the Examples-Policy check on every PR** using [references/examples-policy.md](references/examples-policy.md). Inspect only Python paths introduced by the diff. Treat a new model-, checkpoint-, vendor-, or family-specific Python example as blocking; do not report pre-existing example debt.
+
+**Also run the Simplification pass on every PR** using [find-simplifications](../find-simplifications/SKILL.md). Keep it diff-scoped: inspect added or modified public APIs, state, abstractions, fallback and compatibility paths, data movement, and only the adjacent ownership needed to prove a candidate. Do not expand the pass into a repository-wide audit. Zero candidates is a valid pass; report only evidence-backed opportunities. Mark a candidate as blocking only when it proves a correctness or merge-readiness problem, such as newly unreachable code. Otherwise report it as a warning.
+
+**Also confirm local pre-commit gates** from [docs/contributing/README.md](../../../docs/contributing/README.md#linting). GitHub Actions `SKIP` does not prove they passed. Full new-hook list: Omni SPDX (`vLLM-Omni project`); forbidden imports (stdlib `re`/`base64`, pickle, Hugging Face Hub API, Triton/TileLang); no new `torch.cuda` call sites; shellcheck (macOS/Windows must install the binary); mypy-3.10; test files have CI level + hardware marks; TTS `_tts_model_type` branches in `serving_speech.py` must not increase; markdownlint on `docs/`; Buildkite YAML schema. ✗ for growing `allowed_files` / `ALLOWED_FILES` or raising `MAX_MODEL_TYPE_BRANCHES` without review.
+
 ### Step 5: Print Report
 
 ```
@@ -74,6 +82,9 @@ Pre-check report for <branch>
   Dimension          Result
   ─────────────────  ──────
   PR title format    ✓
+  Code quality       ⚠ 1 broad except, 2 Any hints
+  Examples policy    ✗ new model-specific Python example
+  Simplification     ⚠ helper duplicates an existing owner
   PR desc integrity  ✓
   Registry/config    ✓
   Dead code          ⚠ 2 warnings

.claude/skills/precheck-pr/references/checklists.md

@@ -35,6 +35,44 @@ PR title must follow the project convention documented in `docs/contributing/REA
 
 ---
 
+## All PRs: Code Quality (run on every PR, quick & full)
+
+Diff-scoped sweep for five fragility patterns that are pervasive in the existing codebase — **only lines the PR adds count**, not the pre-existing backlog. Detection commands, codebase examples, and full severity rules live in [code-quality.md](code-quality.md).
+
+- [ ] **No new `**kwargs` string-lookup plumbing:** the diff adds no `**kwargs` + `kwargs["..."]` / `kwargs.get("...")` / `"..." in kwargs`. ⚠ any new instance; ✗ if a new string key is duplicated across ≥2 files without a shared constant, or unknown keys are silently dropped on a fail-fast path.
+- [ ] **No new broad exception swallow:** the diff adds no `except Exception:` / bare `except:`. ⚠ any new instance; ✗ for `except: pass` / `except Exception: return None` / `continue` on a fail-fast path (init, config validation, weight loading, request handling).
+- [ ] **No new `Any` / wrong type hints:** the diff adds no `: Any` / `-> Any` / untyped production signature, and no new `SimpleNamespace` in tests faking an object that already has a real typed stub. ⚠ any new instance; ✗ for a *wrong* (actively misleading) annotation or a `SimpleNamespace` test fake of a typed object.
+- [ ] **No new hot-path copy:** the diff adds no `.clone()` / `.copy_()` / `copy.deepcopy(...)` inside a per-step loop or per-request path. ⚠ any new instance; ✗ for `.clone()` of a full latent/activations tensor inside an AR or diffusion step loop without a comment, or `deepcopy` of a scheduler / sampling-config on the request path.
+- [ ] **No new event-loop blocking:** the diff adds no `time.sleep` / blocking HTTP (`requests`, `urllib`) / `await` inside a held lock in `async def` code. ⚠ any new instance; ✗ for blocking sleep or HTTP on the serving path (`engine/`, `entrypoints/`, async `connectors/`), or a lock held across an `await`. Blocking inside a dedicated worker thread is fine.
+
+Roll the ⚠/✗ counts into the report as a single **Code quality** dimension row.
+
+---
+
+## All PRs: Examples Policy (run on every PR, quick & full)
+
+Inspect Python files introduced under `examples/` using
+[examples-policy.md](examples-policy.md). This is a diff-scoped ratchet: existing
+model-specific examples may be edited or removed without triggering the check.
+
+- [ ] **No new model-specific Python examples:** the diff adds, copies, or renames no Python file whose path or behavior is specific to one model, checkpoint, vendor, or model family. ✗ for any violation. Route inference through a shared task/protocol entrypoint, model request behavior through `vllm_omni/model_extras`, and commands through task docs or `recipes/`.
+- [ ] **New generic example code is justified:** any new task-neutral Python file under `examples/` is an actual user-facing entrypoint rather than reusable production logic, a benchmark, an app, a setup tool, or a reproducer. ⚠ when the placement is questionable.
+
+Roll the result into the report as a separate **Examples policy** dimension.
+
+### Conventions (eyeball — no grep)
+
+Judgment calls that don't grep cleanly — apply them while reading the diff, not as a mechanical check:
+
+- **Right log level** — `info` is for end-user-visible events; routine internal events are `debug`; hot paths are `trace` or removed. Logging is not free.
+- **Structured fields over f-strings** — pass dynamic values as structured fields (`logger.info("msg", extra={...})`), not interpolated strings, so they stay greppable and parseable.
+- **No reflexive synchronization** — don't add `threading.Lock` / `asyncio.Lock` / `deepcopy` / re-wrap shared state unless there is real concurrent mutation; the GIL covers most Python code. Owners decide their own synchronization — don't pre-lock in a constructor.
+- **Context managers for cleanup** — use `with` / `contextlib` for files, locks, CUDA streams, and profilers; not raw `open()` + manual `.close()` or ad-hoc `finally`.
+- **No new dependencies without justification** — prefer the stdlib and existing vllm-omni deps; a new top-level dep needs a reason (check `pyproject.toml` / `requirements*.txt`).
+- **Naming / interface discipline** — don't rename or change interfaces just for taste in this PR, but do fix names that actively mislead (e.g. `serve` implying a long-running server, `Instance` where multiple instance kinds coexist).
+
+---
+
 ## Bug Fix PRs
 
 ### Quick
@@ -49,7 +87,7 @@ PR title must follow the project convention documented in `docs/contributing/REA
 - [ ] **Regression test exists:** `git diff --name-only` includes at least one `tests/` file
 - [ ] **Regression test reproduces the original error:** the test would fail on main, pass on this branch
 - [ ] **Fix matches root cause exactly:** no "fix the symptom + something else" — if the root cause is dtype, the fix is dtype, not dtype + reformatting
-- [ ] **No silent failure risk:** no bare `except: pass`, no `try: ... except: return None`, no empty fallback added
+- [ ] **No silent failure risk:** no bare `except: pass`, no `try: ... except: return None`, no empty fallback added. See the broad-except pattern in [code-quality.md](code-quality.md#2-broad-exception-swallow) for the full severity rule (⚠ any new broad catch; ✗ for a swallow on a fail-fast path).
 - [ ] **Upstream pattern match:** the fix follows the same pattern as existing code for similar cases (grep for analogous `from_pretrained` calls, etc.)
 - [ ] **Environment documented:** torch/transformers/vllm versions listed if the bug is version-dependent
 
@@ -126,6 +164,7 @@ Diffusion models live under `vllm_omni/diffusion/`. In addition to the New Model
 - [ ] **PR body matches diff:** description doesn't claim changes not in the diff
 - [ ] **Branch is rebased:** `git merge-base HEAD origin/main` is recent, no merge conflicts
 - [ ] **CI gates passing:** DCO, pre-commit, build — check `gh pr view --json statusCheckRollup` if a PR already exists
+- [ ] **Local pre-commit gates:** full list in [Linting](../../../../docs/contributing/README.md#linting). Blocking when the diff introduces: stale `vLLM project` SPDX; stdlib `re`/`base64` or pickle in library code; new `torch.cuda` call sites; `tests/**/test_*.py` without level+hardware marks; new `_tts_model_type` branches in `serving_speech.py`; invalid `.buildkite` YAML. macOS/Windows need a native `shellcheck`. ✗ for growing `CHECK_IMPORTS[*].allowed_files`, `ALLOWED_FILES`, or `MAX_MODEL_TYPE_BRANCHES` without review.
 
 ### Full (adds)
 

.claude/skills/precheck-pr/references/code-quality.md

@@ -0,0 +1,179 @@
+# Code-Quality Patterns
+
+Five fragility patterns to sweep on **every** PR, in both quick and full mode. These are pervasive in the existing codebase (hundreds of pre-existing sites) — so the checks are **diff-scoped**: only lines the PR *adds* count. Whole-repo greps light up every PR with the backlog and are useless.
+
+All commands assume:
+```bash
+BASE="$(git merge-base HEAD origin/main)"
+```
+
+**Severity:** ⚠ for any *new* instance introduced by the diff; ✗ for the acute sub-cases defined per pattern. Acute ✗ findings belong in the report's blocking count.
+
+---
+
+## 1. `**kwargs` string-lookup passthrough
+
+### Anti-pattern
+```python
+def forward(self, **kwargs):
+    # BLOCKER-prone: raw dict + string keys duplicated across files
+    if "runtime_additional_information" in kwargs and \
+       "model_intermediate_buffer" not in kwargs:
+        ...
+```
+Real instances of this exact guard are copy-pasted across `cosyvoice3.py`, `indextts2_talker.py` (×2), `indextts2_s2mel_decoder.py`, `qwen3_tts_talker.py`, `qwen3_omni.py`. A typo in any string key silently no-ops; there is no type check, no IDE completion, and unknown keys are silently dropped.
+
+### Why dangerous
+Raw `dict` + `**kwargs` + lists of param-name strings is **fragile by construction**: the compiler can't see a missing or misspelled key, so failures are silent. Each duplicated copy is a new place for the strings to drift.
+
+### Severity
+- ⚠ any *new* `**kwargs` signature whose body does `kwargs["..."]` / `kwargs.get("...")` / `"..." in kwargs`.
+- ✗ when the diff introduces a **new** string key that is duplicated across ≥2 files without a shared module constant, or when `**kwargs` plumbing **silently drops unknown keys** on a fail-fast path (init, config validation, weight loading).
+
+### Detect (diff-scoped)
+```bash
+git diff ${BASE}...HEAD -- "*.py" | grep "^+[^+]" \
+  | grep -E '\*\*kwargs|kwargs(\.get|\[)|"[a-z_]+"\s+in\s+kwargs'
+```
+
+### Fix
+Prefer explicit typed params. The repo already has the right primitive: `msgspec.Struct` with `forbid_unknown_fields=True` rejects unknown keys instead of silently dropping them — see `_StructBase` in `vllm_omni/data_entry_keys.py:105` and `MoriPullRequest` in `vllm_omni/distributed/omni_connectors/connectors/mori_transfer_engine_connector.py:81`. A `@dataclass` or `TypedDict` works too.
+
+If `**kwargs` is unavoidable (vLLM base-class compat), centralize the key strings as module-level constants behind **one** typed accessor, instead of repeating the literal across files.
+
+---
+
+## 2. Broad exception swallow
+
+### Anti-pattern
+```python
+# BLOCKER-prone: catches everything, returns None on any failure
+try:
+    evt = build_event(...)
+    return evt
+except Exception:
+    return None
+```
+Real clusters: `vllm_omni/metrics/stats.py:240,274,367,450` and `vllm_omni/metrics/{modality,utils}.py` — all `except Exception: return None`.
+
+### Why dangerous
+A generic `except Exception:` is **too flexible**. If we catch an exception type we don't expect, it is almost always an unhandled edge case that needs to be *fixed* — not swallowed into `None`/`pass`/`continue`. Broad catches turn fixable bugs into silent wrong behavior and make debugging impossible.
+
+The repo backs this with ruff `BLE001` (broad-exception catch); it is currently suppressed via `# noqa: BLE001` in `vllm_omni/patch.py:413`. Prefer fixing over suppressing.
+
+### Severity
+- ⚠ any *new* `except Exception:` / `except BaseException:` / bare `except:`.
+- ✗ for `except:` / `except Exception: pass|return None|continue` on a **fail-fast path** (init, config validation, weight loading, request handling, connector setup). This extends the Bug-Fix "No silent failure risk" ✗ in [checklists.md](checklists.md).
+
+### Detect (diff-scoped)
+```bash
+# Any new broad catch (⚠ baseline):
+git diff ${BASE}...HEAD -- "*.py" | grep "^+[^+]" \
+  | grep -E 'except\s*(Exception|BaseException)?\s*:'
+# Note: this matches only the immediate-colon swallow forms (`except :`,
+# `except Exception:`) — it deliberately excludes specific catches like
+# `except ValueError:` and the `except Exception as e:` variant. Eyeball
+# the latter; `as e` usually means it is logged, but still verify.
+```
+
+### Fix
+Catch the **specific** types you actually expect (`ValueError`, `KeyError`, `AttributeError`, `OSError`, `torch.cuda.OutOfMemoryError`, `msgspec.ValidationError`, …). At a genuine top-level / best-effort boundary (metrics, signal handlers), at minimum **log** the exception; never swallow into a bare `pass`/`return None` on a path that should fail loudly.
+
+---
+
+## 3. `Any` / wrong type annotations
+
+### Anti-pattern
+```python
+# ⚠ Any param + Any return — no contract, no narrowing
+def _extract_mm_output(engine_outputs: Any) -> dict[str, Any]:
+    ...
+```
+
+### Why dangerous (root cause)
+`Any` disables the type checker for that position, so typos and wrong shapes pass silently. The repo has 636 `: Any` params, 244 `-> Any` returns, and **1,173 `SimpleNamespace` in `tests/`**. The `Any` leak is largely test-driven: vibe-coded unit tests fake objects with `SimpleNamespace`, which forces the production code consuming them to be typed `Any`, and the `Any` then propagates outward into public signatures.
+
+### Severity
+- ⚠ any *new* `: Any` / `-> Any` / fully-untyped signature in production code.
+- ✗ for a **wrong** annotation (actively misleading — e.g. `-> bool` that returns `Optional[bool]`), or a *new* `SimpleNamespace` in a test that mimics an object which already has a real typed stub / `@dataclass` / `TypedDict` / `Protocol`.
+
+### Detect (diff-scoped)
+```bash
+# Any leaking into new signatures:
+git diff ${BASE}...HEAD -- "*.py" | grep "^+[^+]" \
+  | grep -E ':\s*Any\b|->\s*Any\b'
+# SimpleNamespace newly added in tests (the Any-leak source):
+git diff ${BASE}...HEAD -- "tests/" | grep "^+[^+]" | grep "SimpleNamespace"
+```
+
+### Fix
+Replace `Any` with the concrete type, a `Protocol`, or a `Union`. If a type is genuinely dynamic, prefer `object` + `isinstance` narrowing over `Any` — `object` keeps the checker engaged. In tests, replace `SimpleNamespace` with the real class, a small `@dataclass`/`TypedDict`, or a `Protocol` + fake, so the system under test stays fully typed.
+
+---
+
+## 4. Unnecessary copy — `.clone()` / `copy.deepcopy` in hot paths
+
+### Anti-pattern
+```python
+# Per-step in a diffusion/AR loop: clones the full latent every iteration
+for t in timesteps:
+    trajectory_latents.append(x_t.clone())   # O(N) GPU alloc + memcpy each step
+```
+Real clusters: `vllm_omni/diffusion/models/bagel/bagel_transformer.py` calls `x_t.clone()` ~9× across a trajectory loop (lines 1820–2186); the cache hooks `vllm_omni/diffusion/cache/teacache/hook.py:155` and `magcache/hook.py:182` clone `hidden_states` per step. Separately, `copy.deepcopy(self.scheduler)` / `copy.deepcopy(req.sampling_params)` is copy-pasted across diffusion pipelines — `pipeline_qwen_image.py:786`, `pipeline_hunyuan_image3.py:1904`, `pipeline_ltx2*.py` (×3), `pipeline_dreamzero.py:1085`, `diffusion_model_runner.py:398` — a full object-graph deepcopy on the request path. (Contrast `dreamzero/causal_wan_model.py:203`, which carries a `# No .clone() needed.` comment — the aware pattern.)
+
+### Why dangerous
+A `.clone()` of a GPU tensor or a `copy.deepcopy` of a config object inside a per-step / per-request path is a hidden allocator: it reads as one Python line but triggers a GPU memcpy or a deep object-graph walk every iteration. Across 50 diffusion steps or hundreds of AR steps it dominates latency and VRAM bandwidth. `deepcopy` additionally ignores any custom device / `__copy__` logic and can drag tensors through CPU.
+
+### Severity
+- ⚠ any *new* `.clone()` / `.copy_()` / `copy.deepcopy(...)` inside a per-step loop or per-request path.
+- ✗ for `.clone()` of a full latent/activations tensor inside an AR or diffusion step loop without an explanatory comment, or `copy.deepcopy` of a scheduler / sampling-config on the request hot path.
+
+### Detect (diff-scoped)
+```bash
+git diff ${BASE}...HEAD -- "*.py" | grep "^+[^+]" \
+  | grep -E '\.clone\(\)|\.copy_\(|copy\.deepcopy|deepcopy\('
+```
+
+### Fix
+Tensors: prefer a view (`x[..., :k]`, `torch.narrow`, `as_strided`) or write into a pre-allocated buffer with `dst.copy_(src)` instead of `.clone()`; if the caller no longer needs the original, `move`/`del` it rather than copy. Schedulers / configs: build a fresh lightweight instance per request, or split out the mutable per-request bits so the shared immutable part isn't deepcopied. If a copy is genuinely required (e.g. the source is mutated in place later), keep it and add a one-line comment saying why.
+
+---
+
+## 5. Blocking the asyncio event loop / lock held across `await`
+
+### Anti-pattern
+```python
+# BLOCKER-prone: blocking sleep / blocking HTTP / lock held across an await,
+# all inside an async (event-loop) function
+async def handle(self, req):
+    with self._lock:
+        resp = await self._client.call(req)   # lock held across await -> serializes all requests
+    time.sleep(0.1)                           # stalls the single loop for every concurrent request
+```
+
+### Why dangerous
+vLLM-Omni serves over a single asyncio event loop. Any synchronous blocking work on an `async def` path — `time.sleep`, blocking HTTP (`requests`/`urllib`), heavy CPU work, or a lock held across an `await` — stalls the loop for *every* concurrent request, not just the blocking one. A lock held across `await` silently serializes the whole pipeline; blocking I/O tanks throughput and inflates tail latency.
+
+### Severity
+- ⚠ any *new* `time.sleep(`, blocking HTTP call (`requests.get/post`, `urllib.request`, `urlopen`), or `await` inside a `with Lock` / `async with` critical section, in `async def` code.
+- ✗ for `time.sleep` or blocking HTTP on the serving path (`engine/`, `entrypoints/`, async `connectors/`), or a lock acquired and held across an `await` on the request path.
+
+### Detect (diff-scoped)
+```bash
+# New blocking primitives introduced by the diff — then eyeball whether each
+# sits in an async function on the loop vs a dedicated worker thread:
+git diff ${BASE}...HEAD -- "*.py" | grep "^+[^+]" \
+  | grep -E 'time\.sleep|requests\.(get|post|put|delete)|urllib\.request|urlopen'
+# Lock acquired with an await potentially inside the critical section (manual eyeball):
+git diff ${BASE}...HEAD -- "*.py" | grep "^+[^+]" | grep -E 'async with|\.acquire\(\)'
+```
+Note: blocking inside a **dedicated worker thread is correct** — the repo already does this (`diffusion/model_loader/hub_prefetch.py`, `distributed/omni_connectors/kv_transfer_manager.py`, the `mooncake`/`mori` connectors poll with `time.sleep` on their own threads). The check targets the *event-loop* path only.
+
+### Fix
+Move blocking work off the loop with `asyncio.to_thread(...)` / `run_in_executor`, or use a native async client (`httpx.AsyncClient` / `aiohttp`). Never `await` while holding a lock — drop the lock before the `await` and re-acquire after, or restructure so the critical section contains no `await`.
+
+---
+
+## Running the sweep
+
+Run all five detections against `${BASE}...HEAD`, then for each hit decide ⚠ vs ✗ using the severity rules above. Roll the results into the report as a single **Code quality** dimension row (count of ⚠ and ✗), alongside the type-specific checklist from [checklists.md](checklists.md).

.claude/skills/precheck-pr/references/examples-policy.md

@@ -0,0 +1,68 @@
+# Examples Python Policy
+
+Prevent new model-specific Python examples without forcing cleanup of the
+existing backlog. Run this check on every PR in quick and full modes.
+
+## Scope
+
+Set the merge base as in the main workflow, then list Python paths introduced
+by the PR, including additions, copies, and renames:
+
+```bash
+BASE="$(git merge-base HEAD origin/main)"
+git diff --name-status --diff-filter=ACR --find-renames --find-copies \
+  "${BASE}"...HEAD -- 'examples/**/*.py'
+```
+
+Inspect destination paths for copies and renames. Existing model-specific files
+that are only modified or deleted are grandfathered and must not be reported.
+
+## Blocking classification
+
+Mark ✗ when a newly introduced Python file does any of the following:
+
+- Names or scopes a path to one model, checkpoint, vendor, or model family.
+- Hard-codes a model identifier, model-specific prompt, request shape, output
+  conversion, or launch configuration behind an otherwise generic filename.
+- Duplicates an existing task/protocol runner to demonstrate one model.
+- Adds a per-model helper below a generic task hub such as `text_to_speech/`.
+
+Do not infer safety from a generic filename alone. For example, a file named
+`text_to_audio.py` is still model-specific if it only implements one model's
+contract.
+
+## Acceptable additions
+
+A new Python example may pass when it is a genuinely model-neutral,
+user-facing task or protocol entrypoint. It must accept model selection through
+configuration and keep model-specific behavior out of the script.
+
+Use ⚠ when the file is model-neutral but reusable logic appears misplaced under
+`examples/`. Route code by purpose:
+
+| Code purpose | Preferred location |
+|---|---|
+| Model prompt, defaults, request/output adaptation | `vllm_omni/model_extras/` or production model modules |
+| Runnable model command and validation evidence | Task documentation or `recipes/` |
+| UI or interactive application | `apps/` |
+| Benchmark or evaluator | `benchmarks/` |
+| Regression reproducer | `tests/` |
+| Download, conversion, or setup utility | `tools/` |
+
+If a model cannot use an existing shared runner, report the missing generic
+capability. Do not solve the gap by adding a model-specific Python example.
+
+## Report format
+
+Add one row to the pre-check report:
+
+```text
+Examples policy    ✓ no new Python example paths
+Examples policy    ⚠ generic helper belongs in tools/
+Examples policy    ✗ examples/offline_inference/new_model/end2end.py is model-specific
+Examples policy    ✗ examples/offline_inference/x_to_y_model_name.py is model-specific
+```
+
+For a blocker, name the path, the model-specific behavior, and the appropriate
+shared runner or destination. Do not propose deleting unrelated existing
+examples as part of the PR.

.claude/skills/readme.md

@@ -1,9 +1,9 @@
-# Claude Skills for vLLM-Omni
+# Repository Skills for vLLM-Omni
 
-This directory contains Claude Code skills maintained for the `vllm-omni`
-repository. These skills capture repeatable workflows for common contributor
-tasks such as model integration, pull request review, and release note
-generation.
+This directory contains repository-scale skills maintained for `vllm-omni`.
+They capture repeatable workflows for common contributor and maintainer tasks
+such as model integration, CI-aligned testing, performance optimization, and
+pull request review.
 
 ## Directory Structure
 
@@ -14,25 +14,44 @@ include:
 - `references/`: focused reference material used by the skill
 - `scripts/`: small helper scripts used by the skill
 
+## Using the Skills
+
+Coding agents that discover repository skills can invoke the relevant skill by
+name. For other agents, point them to the skill's `SKILL.md` and ask them to
+read it before changing code. Combine a domain skill with `vllm-omni-test` when
+tests or CI coverage change, then use `precheck-pr` for the contributor's final
+self-check.
+
 ## Available Skills
 
-- `add-diffusion-model`: guides integration of a new diffusion model into
-  `vllm-omni`
-- `diffusion-perf-opt`: guides diffusion model performance optimization,
-  including profiling traces, parallel strategies, stage timing analysis, and
-  benchmark-driven tuning
-- `quantization`: guides quantization method selection, model integration,
-  checkpoint loading, and quality/performance validation for vLLM-Omni
-- `add-omni-model`: covers addition of new omni-modality model support
-- `add-tts-model`: covers integration of new TTS models and related serving
-  workflows
-- `generate-release-note`: helps prepare release notes for repository changes
-- `precheck-pr`: self-check a branch before creating a PR — validates PR title
-  format, catches dead code, verifies accuracy/perf claims, and confirms merge
-  readiness
-- `review-pr`: provides a structured workflow for reviewing pull requests
-- `vllm-omni-npu-model-runner-upgrade`: upgrades NPU model runners to align with the
-  latest vllm-ascend NPUModelRunner
+- [`add-diffusion-model`](add-diffusion-model/SKILL.md): guides integration of
+  a new diffusion model into `vllm-omni`
+- [`add-tts-model`](add-tts-model/SKILL.md): covers integration of new TTS
+  models and related serving workflows
+- [`diffusion-perf-opt`](diffusion-perf-opt/SKILL.md): guides diffusion model
+  performance optimization, including profiling traces, parallel strategies,
+  stage timing analysis, and benchmark-driven tuning
+- [`find-simplifications`](find-simplifications/SKILL.md): finds evidence-backed
+  opportunities to remove or merge dead, duplicated, speculative,
+  over-generalized, or unnecessarily defensive vLLM-Omni code
+- [`precheck-pr`](precheck-pr/SKILL.md): self-checks a branch before creating a
+  PR by validating title format, dead code, simplification opportunities,
+  accuracy and performance claims, and merge readiness
+- [`quantization`](quantization/SKILL.md): guides quantization method selection,
+  model integration, checkpoint loading, and quality/performance validation
+  for vLLM-Omni
+- [`review-pr`](review-pr/SKILL.md): provides a frozen-snapshot, contract-aware
+  workflow for maintainers and reviewers; contributors should use
+  `precheck-pr` for self-review
+- [`vllm-omni-npu-model-runner-upgrade`](vllm-omni-npu-upgrade/SKILL.md):
+  upgrades NPU model runners to align with the latest vllm-ascend
+  `NPUModelRunner`
+- [`vllm-omni-test`](vllm-omni-test/SKILL.md): guides generation and execution
+  of CI-aligned tests (L1–L4), pytest marker selection (`core_model` /
+  `advanced_model` / `full_model`, `omni` / `tts` / `diffusion`), Buildkite
+  wiring (`test-ready.yml`, `test-merge.yml`, `test-nightly.yml`,
+  `test-weekly.yml`), and copy-paste local plus CI-like `pytest` commands; see
+  `references/test-routing.md` for level-to-command mapping
 
 ## Maintenance Guidelines
 

.claude/skills/review-pr/SKILL.md

@@ -0,0 +1,247 @@
+---
+name: review-pr
+description: Review pull requests and local branches for vllm-project/vllm-omni with a frozen snapshot, module-design ownership, feature-design overlays, targeted validation, and concise evidence-backed findings. Use for default, detailed, or repeat maintainer reviews; checking correctness, compatibility, tests, benchmarks, model additions, distributed changes, or breaking behavior; and identifying or explicitly requesting the most relevant code-owner reviewers. Use precheck-pr instead for an author's pre-submit self-check.
+---
+
+# Review vLLM-Omni Pull Requests
+
+Review like a maintainer: direct, selective, and focused on issues that CI does
+not prove. Prefer a few high-confidence findings over exhaustive commentary.
+Zero findings is a valid result.
+
+## Quality contract
+
+Make every finding:
+
+- **Correct:** prove a reachable failure, not a suspicion.
+- **Prioritized:** lead with merge blockers and high-impact defects.
+- **Actionable:** identify the smallest safe fix direction.
+- **Evidence-based:** cite code, tests, docs, CI, or measurements.
+- **Concise:** avoid review templates and repeated summaries.
+- **Calibrated:** match severity to user and maintainer impact.
+
+Do not report unrelated backlog, style already enforced by pre-commit, or a
+missing test that would not protect changed behavior. Do report new allowlist
+or budget entries in `check_forbidden_imports.py` / `check_torch_cuda.py` /
+`check_tts_adapter.py` / `check_buildkite.py` unless the PR justifies them:
+those are policy changes, not lint noise.
+
+## Select the input and depth
+
+Use `vllm-project/vllm-omni` as the base repository. Accept its forks and local
+checkouts; use another skill for unrelated repositories.
+
+| Input | Review surface |
+| --- | --- |
+| PR number or URL | Frozen PR metadata, full diff, and relevant threads. |
+| Local branch/worktree | Frozen target-base SHA through committed, staged, unstaged, and in-scope untracked changes. |
+| Pre-filled context | Reuse supplied metadata; fetch only missing facts and the full diff. |
+
+Default to maintainer brevity. A detailed or audit request expands coverage and
+lists `path:line` findings, but keeps the same confidence and severity bar.
+
+## Reference guide
+
+Load references in review order: process, one primary module contract, matching
+feature designs, evidence checks, then delivery. Every file is linked directly
+below; do not load unrelated module references.
+
+Each concise reference links to the maintained
+[vLLM-Omni documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/).
+For branch-specific behavior, inspect the matching `docs/` file in the reviewed
+checkout first; use the published latest docs for current guidance and discovery.
+If docs and live code disagree, verify the code/tests and report the drift.
+
+### Review process
+
+| Reference | Read when |
+| --- | --- |
+| [review-execution.md](references/process/review-execution.md) | Every review; freeze inputs, inspect safely, and deliver against the same snapshot. |
+| [general-checks.md](references/process/general-checks.md) | Every review; apply repository-wide correctness and evidence rules. |
+| [design-contracts.md](references/process/design-contracts.md) | Every production review; resolve branch-local module and feature design status. |
+| [review-routing.md](references/process/review-routing.md) | After the diff census; select one primary module and conditional overlays. |
+
+### Primary module contract
+
+| Reference | Read when |
+| --- | --- |
+| [entrypoints.md](references/modules/entrypoints.md) | Offline/CLI/API ingress, validation, rendering, streaming, or sessions change. |
+| [configuration.md](references/modules/configuration.md) | Config construction, deploy/stage schema, defaults, registry, or topology changes. |
+| [input-output-modality.md](references/modules/input-output-modality.md) | Requests, messages, serialization, output types, accumulation, or completion change. |
+| [error-contracts.md](references/modules/error-contracts.md) | Error classification, fatality, propagation, sanitization, or rendering changes. |
+| [engine-orchestration.md](references/modules/engine-orchestration.md) | Cross-stage routing, request state, output ordering, RPC correlation, or terminal convergence changes. |
+| [stage-runtime.md](references/modules/stage-runtime.md) | Placement, startup, readiness, replica identity, affinity, membership, or shutdown changes. |
+| [omni-connector.md](references/modules/omni-connector.md) | Cross-stage/process/device/node transport or synchronization changes. |
+| [model-integration.md](references/modules/model-integration.md) | Registration, preprocessing, loading, runners, or model-specific execution changes. |
+| [ar-runtime.md](references/modules/ar-runtime.md) | AR scheduling, request/cache state, adapters, workers, or upstream vLLM semantics change. |
+| [diffusion.md](references/modules/diffusion.md) | Diffusion runtime, models, batching, parallelism, or offload changes. |
+| [execution-platforms.md](references/modules/execution-platforms.md) | Hardware selection, capabilities, vendor workers, kernels, or patches change. |
+| [cache-management.md](references/modules/cache-management.md) | Cache identity, reuse, validity, reset, eviction, or teardown changes. |
+| [quantization.md](references/modules/quantization.md) | Quantization selection, checkpoint metadata, layer mapping, precision, or constraints change. |
+| [observability.md](references/modules/observability.md) | Metrics, logs, units, labels, correlation, or lifecycle changes. |
+| [profiling.md](references/modules/profiling.md) | Profiling instrumentation, traces, start/stop lifecycle, or overhead changes. |
+| [benchmarking.md](references/modules/benchmarking.md) | Benchmark workload, metric calculation, CLI, or result metadata changes. |
+
+### Feature-design overlays
+
+| Reference | Read when |
+| --- | --- |
+| [runtime-stage-execution.md](references/features/runtime-stage-execution.md) | Disaggregated inference, async chunk/output/materialization, or prefix caching changes. |
+| [communication.md](references/features/communication.md) | A concrete OmniConnector backend or its deployment contract changes. |
+| [diffusion-acceleration.md](references/features/diffusion-acceleration.md) | Diffusion parallelism, attention, quantization, cache, batching, or offload changes. |
+| [infrastructure-performance.md](references/features/infrastructure-performance.md) | Metrics infrastructure or documented speech optimization stacks change. |
+
+### Evidence and quality checks
+
+| Reference | Read when |
+| --- | --- |
+| [model-addition-checklist.md](references/checks/model-addition-checklist.md) | A model, architecture, loader, processor, registry, pipeline config, or deploy config is added. |
+| [perf-verification.md](references/checks/perf-verification.md) | The PR makes a latency, throughput, memory, or quality claim. |
+| [test-quality-evaluation.md](references/checks/test-quality-evaluation.md) | Tests change, are absent for risky code, or may not exercise production behavior. |
+| [tests-docs-checklist.md](references/checks/tests-docs-checklist.md) | Coverage, CI markers, examples, user docs, or PR evidence need review. |
+| [verification.md](references/checks/verification.md) | Hardware, a server, or a runnable affected path is available for active verification. |
+| [examples-policy.md](../precheck-pr/references/examples-policy.md) | The PR adds, copies, or renames Python under `examples/`; apply the canonical policy shared with `precheck-pr`. |
+| [find-simplifications](../find-simplifications/SKILL.md) | Every review; run a diff-scoped subtraction and simplification pass after correctness blockers. |
+
+### Delivery and reviewer coordination
+
+| Reference | Read when |
+| --- | --- |
+| [maintainer-style-study.md](references/delivery/maintainer-style-study.md) | Findings are ready for concise maintainer-style delivery. |
+| [review-requests.md](references/delivery/review-requests.md) | The user asks to identify, suggest, request, or ping code-owner reviewers. |
+
+## Workflow
+
+### 1. Freeze and report the snapshot
+
+Pin the base and head before reading source or running validation. Within 60 seconds,
+report the pinned head, CI, mergeability, and preliminary findings in the host
+conversation. Do not wait for CI or post this update to GitHub.
+
+If the target changes while fetching, discard the evidence and retry once. If
+it changes again, report the churn and wait for a stable target.
+
+For a trusted PR head, materialize the pinned head in an isolated detached
+worktree. A worktree freezes identity but is not a security sandbox. Treat fork
+heads as untrusted unless the user and environment policy explicitly establish
+otherwise: execute them only in a disposable, secret-free sandbox with restricted
+filesystem, network, and resources; without one, use static SHA-addressed reads
+and CI evidence only. For a local review, freeze the committed, index, worktree,
+and NUL-safe in-scope untracked contents. Follow
+[review-execution.md](references/process/review-execution.md) for trust gates,
+state fingerprints, and byte-for-byte staleness checks.
+
+### 2. Build the diff census
+
+Group files into production code, tests, docs, configuration, build/CI, and
+generated artifacts. Map each changed production file and test group to the PR
+goal. Compare the title/body claims with the actual diff; use linked issues only
+when they define the contract or reproduction.
+
+Mark unrelated scope and unexplained generated artifacts. Do not infer behavior
+from the PR description without tracing the live code.
+
+### 3. Route from the live behavior
+
+Trace each claimed behavior through the changed producer to its live consumer,
+then use [design-contracts.md](references/process/design-contracts.md) and
+[review-routing.md](references/process/review-routing.md) to select one primary
+module contract, a second only for a real documented cross-boundary call path,
+and every matching feature-design and evidence overlay. Treat titles and paths
+as hints; live behavior and the frozen head's current design metadata are
+authoritative. For docs-, tests-, or CI-only changes, route to the production
+contract they protect or use only the applicable evidence checks.
+
+### 4. Run the blocker scan
+
+Apply every category in [general-checks.md](references/process/general-checks.md) before
+lower-priority comments.
+
+If the diff census contains an added, copied, or renamed Python path under
+`examples/`, read and apply the canonical
+[examples policy](../precheck-pr/references/examples-policy.md). Treat a new
+model-specific Python example as blocking. Do not flag model-specific example
+debt that the PR only modifies or removes, and do not run the rest of the
+author-oriented `precheck-pr` workflow.
+
+For each changed value or behavior, trace:
+
+```text
+public ingress -> validation/defaulting -> producer -> transformations
+  -> stage/worker/connector boundary -> final consumer -> terminal cleanup
+```
+
+Cover every applicable offline/online, streaming/non-streaming, sync/async,
+feature-on/off, topology, and compatibility path. Search bounded callers and
+sibling implementations rather than assuming the changed hunk is the only path.
+
+### 5. Apply module and feature contracts
+
+Apply the reference set selected in step 3 and any matching repo-local skill.
+Read the exact module and feature pages in the frozen head, including status,
+ownership boundary, dependencies, candidate invariants, safe-change guide, and
+promotion gate. Candidate or draft rules are questions, not blockers, unless
+current code, tests, or policy enforce them. Inspect both sides of any config,
+registry, serialization, connector, cache, or stage boundary.
+
+Read and apply [find-simplifications](../find-simplifications/SKILL.md) on every
+review. Constrain it to the diff and the adjacent ownership, callers, or
+consumers needed to prove a candidate. Check whether added or expanded helpers,
+classes, state, fallback and compatibility branches, data movement, or public
+behavior can be deleted, merged, moved, or inlined. Zero candidates is a valid
+result. Do not widen the review into repository backlog or report speculative
+style preferences as simplification findings.
+
+### 6. Verify the changed path
+
+Before each validation group, verify the frozen SHA plus the tracked, index,
+untracked, and ignored-file fingerprint, or recreate a pristine snapshot. On a
+trusted head or inside the required sandbox, run an import/version preflight,
+then the narrowest relevant tests and low-cost static checks. Bind every result
+to the head SHA, snapshot fingerprint, and environment fingerprint. Never run
+imports, tests, builds, hooks, or repo-configurable tooling from an untrusted
+head on the reviewer host.
+
+- Treat CI as status evidence; inspect only the first overlapping failure.
+- For docs-only changes, use diff hygiene, links/build checks, and bounded live
+  contract verification instead of dependency setup or pytest.
+- For hardware-dependent paths, run available static/CPU checks and name the
+  exact GPU/NPU gap. Never simulate device evidence.
+- For performance or accuracy claims, require comparable base/head runs with
+  the same environment, workload, warmup, repetitions, and quality criteria.
+
+Stop when each changed semantic path has a supported finding or an explicit
+no-issue conclusion. Do not search further only to increase confidence.
+
+### 7. Consolidate and deliver
+
+Verify each finding against the current diff, deduplicate by root cause, and
+order by severity.
+
+Re-read the remote head and reverify or recreate the pristine validation
+snapshot immediately before delivery. If either changed, mark the review stale
+and restart from the new snapshot.
+
+Return findings first. Use
+[maintainer-style-study.md](references/delivery/maintainer-style-study.md) to keep them
+direct and brief. Each finding must include an exact `path:line`, trigger or
+call path, current behavior, impact, and smallest fix direction. If there are
+no findings, say so briefly and name material validation gaps.
+
+Keep the review read-only unless the user explicitly authorizes posting. Do not
+submit `APPROVE`, `COMMENT`, or `REQUEST_CHANGES`, add labels, edit code, or push
+commits as an implied part of review.
+
+### 8. Optionally request focused owner reviews
+
+Only when the user asks to identify or request reviewers, read
+[review-requests.md](references/delivery/review-requests.md). Rank path-matched
+CODEOWNERS with the frozen module page's owners or required reviewers and
+documented governance expertise; propose one to three focused reviewers with an
+explicit contract rationale.
+
+Identifying or suggesting reviewers is read-only. Requesting reviewers or
+posting `@mention` comments changes external state and requires explicit user
+authorization. When authorized, recheck the head, deduplicate existing
+requests, and post at most one consolidated comment. Do not infer this
+permission from a request to review the code.

.claude/skills/review-pr/agents/openai.yaml

@@ -0,0 +1,4 @@
+interface:
+  display_name: "vLLM-Omni PR Review"
+  short_description: "Review vLLM-Omni PRs and route owner requests"
+  default_prompt: "Use $review-pr to review this vLLM-Omni pull request with owner-specific rules and evidence-backed findings."

.claude/skills/review-pr/references/checks/model-addition-checklist.md

@@ -0,0 +1,121 @@
+# Model Addition Checklist
+
+Read this reference when a PR adds or expands a model architecture, pipeline,
+loader, processor, registry entry, pipeline config, or deploy config.
+
+Official docs: [model contribution guides](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/model/),
+[adding an omni model](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/model/adding_omni_model/),
+[adding a diffusion model](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/model/adding_diffusion_model/),
+[adding a TTS model](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/model/adding_tts_model/),
+[supported models](https://docs.vllm.ai/projects/vllm-omni/en/latest/models/supported_models/),
+[diffusion features](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion_features/),
+and [feature compatibility](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/feature_compatibility/).
+
+## Match claims to the diff
+
+- Reconcile every claimed architecture, modality, variant, feature, and example
+  with changed files and registry entries.
+- Resolve config model types, architecture names, callable paths, exports, and
+  dependency extras against live symbols.
+- Check sibling generations sharing an architecture name are disambiguated by
+  a stable config predicate rather than import order or an accidental default.
+
+## Close the integration chain
+
+Trace:
+
+```text
+public model id -> config/pipeline selection -> registry -> loader/processor
+  -> stage inputs -> model execution -> stage/public output
+```
+
+Verify optional dependencies fail with an actionable message, weight names and
+dtypes map correctly, and each advertised serving mode reaches the production
+dispatcher. Confirm outputs are non-empty and valid for their modality, shape,
+sample rate, or response schema.
+
+## Keep support documentation truthful
+
+- Build an evidence-backed inventory of the new model's exact architecture and
+  checkpoint identifiers, modalities and tasks, hardware backends, serving
+  modes, and supported acceleration, parallelism, cache, offload, and
+  quantization features. Treat untested support as unknown, not supported.
+- Update `docs/models/supported_models.md` with the architecture, model family,
+  example checkpoints, and only the backends validated by the PR.
+- For a diffusion model, add it to the applicable ImageGen, VideoGen, or
+  AudioGen table in `docs/user_guide/diffusion_features.md`. Fill every feature
+  column according to its legend and document partial support or constraints.
+- Update `docs/user_guide/feature_compatibility.md` when the model changes valid
+  feature combinations, configuration constraints, examples, or limitations,
+  and update any other branch-local support table affected by the claims.
+- Cross-check every documented support mark against the registry, configuration,
+  production-path tests, and PR evidence; do not copy a sibling model's support
+  claims without validation.
+
+## Ship the model recipe
+
+- Require every new model PR to add or update one model-family recipe under
+  `recipes/<vendor>/` and its row in the `recipes/README.md` Available Recipes
+  table. Follow [`recipes/TEMPLATE.md`](../../../../../recipes/TEMPLATE.md) and
+  the layout rules in
+  [`recipes/README.md`](../../../../../recipes/README.md); the
+  external recipes repository is a structural reference, not a substitute.
+- Include only tested task, serving-mode, and hardware configurations. Keep the
+  exact model identifiers, flags, commands, verification, feature support,
+  constraints, and limitations consistent with code, tests, examples, and the
+  support documentation.
+- Cover each validated platform in its own hardware section. Use one recipe per
+  model family by default; justify a split when configurations cannot be kept
+  clear in one file, and do not add placeholder sections for untested platforms.
+
+## Compare the reference implementation
+
+Require every new model PR to include a head-versus-reference accuracy or
+quality and performance comparison:
+
+- Name the canonical implementation and pin its repository revision, package
+  version, configuration, and checkpoint. If it cannot run, explain why and use
+  the closest runnable reference; do not silently omit the comparison.
+- Match inputs or dataset, seed, output settings, precision or quantization,
+  hardware, software, warmup, repetitions, synchronization, and timing and
+  memory scope. Include exact commands and environment details for both sides.
+- Compare accuracy or output quality with a task-appropriate metric and explicit
+  tolerance. When no reliable metric exists, provide paired fixed-input outputs,
+  artifacts, and evaluation criteria, and explain expected numerical or
+  qualitative differences.
+- Report end-to-end latency, task-appropriate throughput, peak device memory,
+  run-to-run variability, and head/reference ratios at equivalent output quality.
+- Prefer a timing split for preprocessing or encoding, each model stage,
+  inter-stage transfer or communication, decoding or postprocessing, and output
+  assembly. Define the boundaries, account for overlap, reconcile the split with
+  end-to-end time, and explain any unavailable stage measurement.
+
+Use [perf-verification.md](perf-verification.md) for the comparison protocol and
+repository benchmark and profiling links.
+
+## Remove accidental surface
+
+Search bounded call sites for:
+
+- inference-dead training `forward()` paths, unused factories/wrappers, never-set
+  branch keys, and immediately discarded parameters;
+- duplicate payload strings, validation, or shape coercion across stages;
+- compatibility aliases in brand-new code without an existing caller;
+- private symbols re-exported as public API or module-level side effects.
+
+Keep code only when it has a distinct live caller, invariant, or compatibility
+contract. Prefer one typed producer-consumer schema to repeated string keys.
+
+## Require proportionate evidence
+
+- Run a representative production-path inference and assert output content, not
+  only process survival.
+- Require profiling or A/B tables when the PR makes performance, memory,
+  precision, or quality claims, or when a suspected hot-path/device bug needs
+  that evidence. Do not impose fixed utilization or regression thresholds that
+  the repository does not define.
+- Require focused registry/config/loading tests, a representative E2E path, and
+  user-facing model/usage documentation.
+
+Use [diffusion.md](../modules/diffusion.md) for diffusion models and
+[perf-verification.md](perf-verification.md) for quantitative claims.

.claude/skills/review-pr/references/checks/perf-verification.md

@@ -0,0 +1,51 @@
+# Performance and Accuracy Verification
+
+Activate this reference when the PR adds a model or claims or intentionally
+changes latency, throughput, memory, scaling, precision, or output quality.
+
+Official docs: [profiling](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/profiling/),
+[serving benchmarks](https://docs.vllm.ai/projects/vllm-omni/en/latest/cli/bench/serve/),
+and [metrics](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/metrics/).
+
+## Comparable A/B contract
+
+Compare frozen base and head for an existing path. For a new model absent from
+base, compare head with a pinned canonical reference implementation. Use the same:
+
+- hardware, driver, software versions, model/checkpoint, and dependencies;
+- input or dataset, seed, precision, batch/concurrency, topology, and feature
+  flags;
+- warmup, measured repetitions, synchronization, timing scope, and memory
+  collection method.
+
+Keep exact commands and report variability rather than a cherry-picked run.
+Use the repository's benchmark when it covers the claim; otherwise use the
+smallest reproducible workload that reaches the changed production path.
+
+## Evidence table
+
+| Dimension | Typical evidence |
+| --- | --- |
+| Latency | End-to-end latency and, when available, TTFT plus a defined per-stage and transfer-time split. |
+| Throughput/scaling | Requests, tokens, frames, or audio duration per second across the claimed concurrency/topology. |
+| Memory | Peak allocated/reserved device memory and OOM boundary if relevant. |
+| Quality/accuracy | Repository metric or known-good output comparison with an explicit tolerance; paired samples when no metric exists. |
+
+Every optimization needs correctness or quality evidence. A faster result on a
+different workload, precision, seed, or topology is not a valid comparison.
+
+## Reviewer verification ladder
+
+1. Run base/head A/B on suitable hardware when affordable.
+2. If only one side can run, verify the script and request a comparable pair.
+3. If hardware or assets are unavailable, audit methodology, code path, and
+   contributor evidence; name the exact unverified claim.
+
+Classify discrepancies before reporting: implementation regression, benchmark
+bug, environmental drift, noise, or unsupported claim. Explain material
+regressions against the PR's stated goal or repository contract; do not invent
+universal percentage thresholds.
+
+Report claimed and measured values together, bind them to the frozen SHAs and
+environment, and keep unavailable hardware as a validation gap rather than a
+fabricated pass.

.claude/skills/review-pr/references/checks/test-quality-evaluation.md

@@ -0,0 +1,39 @@
+# Test Quality Evaluation
+
+Read this reference when tests change, a risky production path has no test, or
+a test-only PR may not prove the intended behavior.
+
+Official docs: [test system overview](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/test_system_overview/)
+and [test writing guide](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/test_writing_guide/).
+
+## Static proof check
+
+For each changed semantic path, ask whether the test would fail if that behavior
+were reverted or broken. Check:
+
+- assertions pin the contract, not only non-null output, logs, or “no crash”;
+- the production dispatcher, registry, connector, or scheduler is reached
+  instead of the changed behavior being mocked away;
+- fakes preserve relevant types, MRO, shapes, devices, async behavior, and
+  lifecycle transitions;
+- seeds, ordering, timing, synchronization, external services, and numeric
+  tolerances are deterministic or explicitly controlled;
+- normal, invalid, boundary, feature-off, failure/cancellation, and regression
+  paths are covered where the diff changes them;
+- run-level and domain markers place hardware/model tests in the intended CI
+  lane without silently skipping the contract.
+
+Map source symbols to tests with bounded `rg` searches; do not assume the test
+tree mirrors production paths.
+
+## Runtime check
+
+Run the narrowest affected tests supported by the environment. Record skipped
+hardware/model cases as gaps. Classify failures as code, test, infrastructure,
+or flaky before turning them into findings; one passing rerun does not erase a
+flakiness signal.
+
+Keep grades or matrices internal. Report only concrete code bugs or the one or
+two test defects that leave changed behavior materially unprotected. Use
+[tests-docs-checklist.md](tests-docs-checklist.md) for CI and documentation
+coverage.

.claude/skills/review-pr/references/checks/tests-docs-checklist.md

@@ -0,0 +1,48 @@
+# Tests and Documentation Checklist
+
+Use this reference to connect changed behavior to coverage, CI selection,
+examples, documentation, and reproducible PR evidence.
+
+Official docs: [test writing guide](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/test_writing_guide/),
+[test execution guide](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/test_execution_guide/),
+and [documentation guide](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/DOCS_GUIDE/).
+
+## Coverage packet
+
+For each high-risk change, record a compact internal packet:
+
+```text
+change and failure risk -> existing unit/E2E coverage -> uncovered boundary
+  -> smallest test and stable assertion that closes the gap
+```
+
+Prefer a regression test that fails on the frozen base for bug fixes. Check
+numeric tolerances, deterministic inputs, realistic mocks, hardware/domain
+markers, and Buildkite/CI wiring using the documentation present on the reviewed
+branch. A skipped test is not a pass.
+
+## Documentation sync
+
+Require user-facing updates when the diff changes a model, feature, CLI/API,
+config key, default, compatibility behavior, or supported platform. Verify:
+
+- the actual docs navigation and support tables used by the branch;
+- exact identifiers, defaults, constraints, known limitations, and migration;
+- a minimal runnable example for newly exposed behavior;
+- examples and docs agree on commands, parameters, output schema, and supported
+  modes;
+- links and the bounded live contract build or resolve correctly.
+
+Do not request docs for internal behavior with no user-visible contract.
+
+## PR evidence
+
+For contributor-run tests or benchmarks, require enough provenance to evaluate
+the claim: hardware, relevant software versions, exact commands, inputs/model,
+result, and approximate runtime. Performance or accuracy evidence must compare
+base and head under equivalent conditions; use
+[perf-verification.md](perf-verification.md).
+
+Turn a missing test or document into a finding only when it protects or explains
+changed behavior. Name the smallest useful addition rather than requesting a
+broad coverage expansion.

.claude/skills/review-pr/references/checks/verification.md

@@ -0,0 +1,36 @@
+# Active Verification
+
+Read this reference when a runnable affected path and suitable local or remote
+hardware are available. Never claim device evidence from static analysis or a
+simulated backend.
+
+Official docs: [installation](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/)
+and [test execution guide](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/test_execution_guide/).
+
+## Choose the narrowest level
+
+| Available environment | Verification |
+| --- | --- |
+| Matching accelerator, model/assets, and server | Targeted unit/integration test plus one representative E2E request. |
+| Matching accelerator without serving setup | Targeted model/runtime test and output or metric inspection. |
+| CPU/static environment only | Import/version preflight, focused CPU tests, config/registry resolution, and static contract checks. |
+| Missing dependencies or inaccessible hardware | Record the exact gap and evaluate contributor/CI evidence. |
+
+## Execute safely
+
+1. Bind the run to the frozen head SHA and record Python, platform, dependency,
+   model, device, and relevant environment fingerprints.
+2. Run the smallest affected unit test before an E2E workload.
+3. Exercise the changed public or production-dispatch path with bounded inputs.
+4. Compare actual output, metrics, cleanup, and exit behavior with the PR claim.
+5. For base/head claims, use the equivalent A/B protocol in
+   [perf-verification.md](perf-verification.md).
+
+Avoid disturbing shared servers, ports, models, or accelerators without user
+authorization. Do not install large dependencies, download gated assets, or
+start remote workloads merely because hardware might exist.
+
+Report the command, environment, result, and any skipped path. Classify failures
+as change-induced, pre-existing, infrastructure, environment, or flaky before
+making a finding. A hardware gap lowers confidence; it is not automatic proof
+of a defect.

.claude/skills/review-pr/references/delivery/maintainer-style-study.md

@@ -0,0 +1,105 @@
+# Maintainer Review Style
+
+Use this reference only when turning proved findings into comments. The baseline
+maintainer samples favor short, direct comments and reserve long explanations
+for architecture or non-obvious failure paths.
+
+Official docs: [contributing and review process](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/).
+
+## Evidence basis
+
+This calibration used 928 PRs merged from 2026-05-10 through 2026-08-08.
+Candidates were ranked by substantive human inline-review threads, then by
+distinct reviewers and review rounds. Author self-comments, bots and AI
+reviewers, and bodyless approvals were excluded. Counts selected the sample;
+the practices below came from reading root comments and follow-up replies.
+
+The main set contains the six leaders after filtering; three other high-ranked
+PRs add core-runtime and benchmark coverage. All were merged.
+
+| PR | Review boundary |
+| --- | --- |
+| [#2906](https://github.com/vllm-project/vllm-omni/pull/2906) | New TTS model, shared utilities, recipes, and request state. |
+| [#3907](https://github.com/vllm-project/vllm-omni/pull/3907) | Full-duplex API, session lifecycle, concurrency, and realtime evidence. |
+| [#2162](https://github.com/vllm-project/vllm-omni/pull/2162) | Diffusion world model, OpenPI serving, tests, examples, and docs. |
+| [#3454](https://github.com/vllm-project/vllm-omni/pull/3454) | Diffusion model, generic CLI surface, loaders, and configuration scope. |
+| [#4425](https://github.com/vllm-project/vllm-omni/pull/4425) | Structured configuration, typing, defaults, and compatibility. |
+| [#5397](https://github.com/vllm-project/vllm-omni/pull/5397) | Distributed offload, collectives, failure handling, and load semantics. |
+| [#4281](https://github.com/vllm-project/vllm-omni/pull/4281) | Composable parallelism, configuration contracts, and examples. |
+| [#3628](https://github.com/vllm-project/vllm-omni/pull/3628) | Stage metrics, benchmark definitions, opt-in API behavior, and cleanup. |
+| [#3855](https://github.com/vllm-project/vllm-omni/pull/3855) | Stage runtime, thread safety, startup, and compatibility. |
+
+Historical comments calibrate delivery; they do not create merge policy. Live
+code, tests, repository docs, and the current `CODEOWNERS` file remain
+authoritative. When refreshing this study, promote a practice only when it
+recurs across at least three PRs or is supported by current repository guidance.
+
+## Recurring review behavior
+
+- Start from a changed contract, not a style preference. State the exact
+  request, configuration, topology, or failure path that triggers the issue.
+- Protect repository boundaries. Keep model-specific behavior in its domain,
+  move genuinely reusable logic to a shared module, and reuse existing loaders,
+  protocols, definitions, and config sources instead of bypassing them.
+- Challenge unrelated scope and abstractions. Ask why a new flag, config, class,
+  helper, fallback, or file is needed when an existing path appears sufficient.
+- Trace concurrency and terminal paths: overlapping requests, disconnect,
+  cancellation, partial failure, stale replies, collective schedule mismatch,
+  cleanup, and retry. Prefer an explicit startup error for unsupported
+  combinations over silently running with incomplete state or wrong weights.
+- Preserve user-visible behavior across variants. An accepted CLI/API/config
+  field must be applied, explicitly rejected, or visibly warned about; compare
+  normal and headless, offline and online, and streaming and non-streaming paths.
+- Ask for evidence at the affected boundary: a regression test using the real
+  tensor or protocol shape, correct pytest markers so CI runs it, comparable A/B
+  performance results, or an SLO-oriented latency/quality measurement.
+- Keep interfaces explicit: avoid `Any` and opaque strings where a precise
+  union, enum, or typed envelope exists; remove dead code and duplicated
+  constants; validate requests before expensive execution.
+- Document module scope and non-obvious invariants. Update examples, recipes,
+  supported-model/feature docs, and hardware assumptions when users rely on
+  them.
+- When expertise crosses an ownership boundary, ask a named owner to inspect a
+  specific file or contract and explain why. Do not send a generic `PTAL` list.
+- Keep non-blocking cleanup in a linked follow-up only after the current PR is
+  safe. If a revision fixes only part of a failure chain, continue the existing
+  thread with the remaining trigger instead of opening a duplicate finding.
+
+## Write the finding
+
+- Lead with the issue or decisive question; omit a review preamble.
+- Name the concrete trigger and impact, then the smallest fix direction.
+- Keep obvious fixes to one sentence. Use more detail only when the call path or
+  ownership boundary is otherwise unclear.
+- Prefer one root-cause comment over repeated symptom comments.
+- Hedge only when evidence is genuinely incomplete; mark that item as a
+  question or validation gap rather than overstating severity.
+- Say briefly that no findings were found when the review is clean; zero
+  comments is a valid outcome when posting is authorized.
+
+Useful forms:
+
+```text
+This leaves <state> allocated when <failure/cancellation path>. Release it in
+the shared terminal cleanup path.
+```
+
+```text
+Does this producer also update <consumer>? It still reads the old field here.
+```
+
+```text
+With <configuration>, this bypasses <existing loader/validator>, so <impact>.
+Reuse that path or reject this combination before startup.
+```
+
+```text
+@<owner>, could you check <specific boundary>? This changes <contract> used by
+<consumer>.
+```
+
+Use a priority label only for a proved high-impact defect. Avoid generic praise,
+routine “Nit:” prefixes, decorative severity labels, audit templates, rule IDs,
+comment-count narration, and “I left a few comments.” Keep approvals, review
+events, reviewer requests, mentions, and GitHub replies subject to the user's
+explicit authorization.

.claude/skills/review-pr/references/delivery/review-requests.md

@@ -0,0 +1,99 @@
+# Focused Code-Owner Review Requests
+
+Read only when the user asks to identify, suggest, request, add, or ping
+reviewers. Finding candidates is read-only; GitHub review requests and
+`@mention` comments are external writes.
+
+Owner sources, in order:
+
+1. the reviewed head's
+   [CODEOWNERS](https://github.com/vllm-project/vllm-omni/blob/main/.github/CODEOWNERS)
+   for path ownership;
+2. the active module design page's `owners` and `required_reviewers` metadata
+   for contract ownership and mandatory design review; and
+3. documented
+   [governance expertise](https://docs.vllm.ai/projects/vllm-omni/en/latest/community/governance/)
+   for specialization and tie-breaking.
+
+Resolve all three from the reviewed head when available because ownership
+changes over time. Use published docs only to discover a page absent from an
+older checkout, never to override the frozen head, and do not copy a static
+owner list into review output. CODEOWNERS remains the path authority; module
+metadata identifies contract experts and required design reviewers.
+
+## Rank candidates
+
+1. Freeze the PR head and collect changed files, author, requested reviewers,
+   submitted reviews, and existing comments.
+2. Resolve the **last matching CODEOWNERS rule** for each changed file. Do not
+   union an earlier broad rule with a later specific rule.
+3. Resolve the primary module and feature overlays through
+   [design-contracts.md](../process/design-contracts.md). Read the active module
+   page's status, `owners`, and `required_reviewers`; do not treat owners on an
+   archived, superseded, or unrelated page as current authority.
+4. Group files by the selected primary owner and the exact module or feature
+   contract needing review. Weight production owners and live consumers above
+   tests/docs/CI owners and the repository-wide fallback. Include an applicable
+   `required_reviewers` entry even when it is not the path owner, and label why.
+5. Use documented governance specialties only to choose among matching owners:
+   for example serving versus configuration, diffusion model versus cache or
+   parallelism, TTS versus generic model execution, or CI versus tests.
+6. Exclude the PR author, bots, users who already completed the needed review,
+   and duplicate pending requests. Do not infer availability or invent
+   expertise from a username.
+7. Select one reviewer for a single-owner change and at most three for a real
+   cross-module change. If more are plausible, rank them but do not ping a
+   broad group; ask the user before exceeding three.
+
+If every specific owner is excluded, consider one repository-wide fallback
+owner and label that choice as a fallback rather than a path match. Never fan
+out to the full fallback list. Prefer a qualified owner who covers multiple
+critical path groups over separate reviewers for each file. If CODEOWNERS is
+missing or invalid, state that gap and use governance only as a labeled fallback.
+
+Recent file history may break a tie only when CODEOWNERS and governance remain
+ambiguous. Never use commit count alone to override an explicit current owner.
+
+## Preview before writing
+
+Show the user the exact candidate set and rationale:
+
+```text
+@owner — <owned paths or module role>; please verify <specific module or feature contract>.
+```
+
+Draft one consolidated comment tied to the frozen head:
+
+```text
+Requesting focused review for `<short-head-sha>`:
+
+- @owner1: could you check the <module contract> in `<path/group>`?
+- @owner2: could you check the <feature or cross-boundary contract> in `<path/group>`?
+
+Relevant validation: <one concise result or exact remaining gap>.
+```
+
+Mention only owners being requested. Keep the reason specific enough that each
+person knows what decision or invariant needs attention; do not post a generic
+“please review” or a generated ownership audit.
+
+## External-write gate
+
+- “Who should review?”, “suggest reviewers”, or “draft a request” authorizes
+  only the preview.
+- “Request/add these reviewers” authorizes the GitHub review-request mutation.
+- “Ping/comment/tag the owners” authorizes the consolidated comment.
+- When the user explicitly asks to request reviews **with an owner comment**,
+  both writes are authorized. Obey narrower wording when only one action is
+  requested.
+
+Immediately before an authorized write, verify the PR is still open, the head
+SHA is unchanged, and the candidate is not the author or already satisfied.
+Report draft/WIP state and avoid notifications unless the user explicitly wants
+reviewers requested before ready-for-review.
+
+Use the GitHub connector for reviewer requests and the single top-level comment;
+use `gh` only when connector coverage is unavailable. Do not submit an approval,
+request-changes event, label, assignment, or multiple comments as part of this
+action. After writing, reread the PR and report the reviewers requested and the
+comment URL. If the head changed, discard the draft and rerank before posting.

.claude/skills/review-pr/references/features/communication.md

@@ -0,0 +1,24 @@
+# Communication Features
+
+Use with the OmniConnector, stage-runtime, configuration, and I/O module
+contracts. Load the concrete backend design from the reviewed head:
+
+| Backend | Design |
+| --- | --- |
+| Mooncake store | [MooncakeStoreConnector](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/omni_connectors/mooncake_store_connector/) |
+| Mooncake transfer engine | [MooncakeTransferEngineConnector](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/omni_connectors/mooncake_transfer_engine_connector/) |
+| Mori transfer engine | [MoriTransferEngineConnector](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/omni_connectors/mori_transfer_engine_connector/) |
+| Shared memory | [SharedMemoryConnector](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/omni_connectors/shared_memory_connector/) |
+| Yuanrong store | [YuanrongConnector](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/omni_connectors/yuanrong_connector/) |
+| Yuanrong transfer engine | [YuanrongTransferEngineConnector](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/omni_connectors/yuanrong_transfer_engine_connector/) |
+
+## Feature checks
+
+- Verify dependency, service, device/network, port, memory-pool, and environment
+  prerequisites with actionable startup errors.
+- Match key/metadata construction, sender and receiver roles, allocation,
+  ownership, completion, and supported fast/serialized paths.
+- Exercise timeout, retry, stale sender/buffer reclamation, partial transfer,
+  cancellation, and connector close semantics.
+- Validate the smallest supported same-host or cross-node topology and document
+  backend limitations and operational diagnostics.

.claude/skills/review-pr/references/features/diffusion-acceleration.md

@@ -0,0 +1,26 @@
+# Diffusion Acceleration Features
+
+Use with the exact diffusion submodule plus platform, cache, quantization, or
+benchmarking contracts selected by the live path.
+
+| Feature family | Designs |
+| --- | --- |
+| Parallelism | [CFG](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/cfg_parallel/), [Expert](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/expert_parallel/), [HSDP](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/hsdp/), [Pipeline](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/pipeline_parallel/), [Sequence](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/sequence_parallel/), [Tensor](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/tensor_parallel/), [VAE patch](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/vae_parallel/) |
+| Attention | [Skip-Softmax](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/skip_softmax/) |
+| Quantization | [Quantization](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/quantization/) |
+| Cache | [Cache-DiT](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/cache_dit/), [TeaCache](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/teacache/) |
+| Batching | [Diffusion Continuous Batching](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/diffusion_continuous_batching/) |
+| Offload | [Distributed Layerwise Offload](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/distributed_layerwise_offload/) |
+
+## Feature checks
+
+- Verify feature selection, supported models/platforms/topologies, defaults,
+  dependencies, and every touched feature combination.
+- Preserve feature-off and single-rank correctness; reject unsupported
+  combinations before collectives, loading, or execution begin.
+- Trace shapes, shards, ranks, collectives, precision, cache validity, residency,
+  and per-request state through the selected runtime hooks.
+- Compare output quality or accuracy against the disabled/reference path before
+  accepting latency or memory improvements.
+- Require exact commands, topology, environment, warmup, repetitions, and
+  measurements for performance claims; update feature compatibility docs.

.claude/skills/review-pr/references/features/infrastructure-performance.md

@@ -0,0 +1,22 @@
+# Infrastructure and Performance Features
+
+Load the matching design from the reviewed head:
+
+- [Prometheus Metrics](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/metrics/)
+  for logger wrapping, gating, lifecycle, stage transfer, and metric definitions.
+- [Speech Generation Performance Optimizations](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/qwen3_omni_tts_performance_optimization/)
+  for stage batching, CUDA graphs, async chunk, streaming, re-prefill, compile,
+  and deployment evidence.
+
+## Feature checks
+
+- Route instrumentation changes through observability or profiling and result
+  methodology through benchmarking and the performance evidence check.
+- Preserve opt-in/default behavior, units, labels, aggregation, timing scope,
+  lifecycle resets, and bounded overhead.
+- For stacked optimizations, measure each increment against the same baseline
+  and explain interactions, quality guards, regressions, and limitations.
+- Require exact model, checkpoint, hardware, software, workload, topology,
+  concurrency, warmup, repetitions, and production commands.
+- Update the design page when metric meaning, supported optimization stack, or
+  deployment guidance changes.

.claude/skills/review-pr/references/features/runtime-stage-execution.md

@@ -0,0 +1,24 @@
+# Runtime and Stage Execution Features
+
+Load the matching design from the reviewed head:
+
+| Feature | Design | Primary module intersections |
+| --- | --- | --- |
+| Disaggregated inference | [Design](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/disaggregated_inference/) | Configuration, stage runtime, OmniConnector, I/O contracts |
+| Async chunk | [Design](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/async_chunk/) | Model integration, orchestration, stage runtime, connector, I/O |
+| Async diffusion output | [Design](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/async_diffusion_output/) | Diffusion runtime, I/O contracts, entrypoints |
+| Async Omni output materialization | [Design](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/omni_async_output_materialization/) | AR runtime, model integration, I/O contracts, orchestration |
+| Automatic prefix caching | [Design](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/prefix_caching/) | AR runtime, cache management, I/O contracts |
+
+## Feature checks
+
+- Confirm the feature's supported models, entrypoints, stage types, topology,
+  configuration, default, and compatibility limitations.
+- Trace enablement from public/config ingress through every participating module
+  and verify the advertised production dispatch actually selects it.
+- Preserve a correct disabled or fallback path and make unsupported combinations
+  fail or warn as documented.
+- Test overlap, ordering, identity, backpressure, cancellation, failure, final
+  partial output, and cleanup at each asynchronous boundary.
+- Update the feature page, support matrices, examples, and recipes when behavior
+  or coverage changes.

.claude/skills/review-pr/references/modules/ar-runtime.md

@@ -0,0 +1,22 @@
+# Autoregressive Runtime
+
+Primary design: [Autoregressive Runtime](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/ar_runtime/).
+
+Use for AR request queues, scheduling state, token and memory budgets, KV/input
+readiness, workers, runners, and upstream vLLM request/cache transitions.
+
+## Contract checks
+
+- Preserve upstream request-state, scheduling, cache, and preemption semantics
+  unless an Omni-specific difference is documented and tested.
+- Convert modality-specific data through explicit input or output adapters; do
+  not inject it through unrelated scheduler state.
+- Keep workers and model runners focused on assigned execution rather than
+  admission or cross-stage routing.
+- Synchronize token, sequence, batch, and memory budgets with worker-visible
+  state and prevent starvation, double counting, or stale readiness.
+- Preserve legal transitions and one terminal cleanup path across completion,
+  abort, timeout, preemption, worker failure, and shutdown.
+
+Test request lifecycle, scheduling decisions, abort, cache state, adapters, and
+every affected worker execution mode against the pinned upstream contract.

.claude/skills/review-pr/references/modules/benchmarking.md

@@ -0,0 +1,24 @@
+# Benchmarking
+
+Primary design: [Benchmarking](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/benchmarking/).
+
+Use when benchmark code, workloads, metric calculations, CLI benchmark
+entrypoints, or result metadata change. Use the performance check separately
+when a PR uses benchmark results as evidence.
+
+## Contract checks
+
+- Record model and revision, code SHA, hardware, software, configuration,
+  workload, precision, topology, concurrency, warmup, repetitions, and
+  measurement window.
+- Validate successful and semantically acceptable outputs before accepting
+  timing, memory, or scaling results.
+- Define every metric's unit, population, aggregation, synchronization, and
+  timing scope.
+- Keep workload parsing and metric computation deterministic and reusable
+  across base and head.
+- Reject comparisons that change material workload or runtime settings without
+  isolating and explaining that difference.
+
+Test workload parsing and metric calculations with deterministic fixtures and
+publish exact commands plus variability for quantitative claims.

.claude/skills/review-pr/references/modules/cache-management.md

@@ -0,0 +1,22 @@
+# Cache Management
+
+Primary design: [Cache Management](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/cache_management/).
+
+Use for AR or diffusion cache identity, reusable state, validity, lifecycle,
+reset, eviction, and teardown.
+
+## Contract checks
+
+- Include every model, request, input, timestep, precision, device, topology,
+  and feature property that can change a reused value in cache identity.
+- Preserve a correct disabled and miss path; never reuse after validity inputs
+  change.
+- Isolate request-scoped state across concurrent requests and define ownership
+  for model-scoped state.
+- Make reset, invalidation, eviction, capacity, and teardown explicit on
+  completion, cancellation, failure, model reload, and shutdown.
+- Keep cache optimizations behind the owning runtime's hooks rather than adding
+  a second request lifecycle.
+
+Test hit, miss, invalidation, disabled behavior, concurrency, capacity, reset,
+and teardown, with quality comparison when reuse is approximate.

.claude/skills/review-pr/references/modules/configuration.md

@@ -0,0 +1,24 @@
+# vLLM-Omni Configuration
+
+Primary design: [vLLM-Omni Configuration](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/vllm_omni_config/).
+
+Use for config construction, deploy YAML, registries, schema, defaults,
+aliases, CLI projection, pipeline config, and topology. Inspect the reviewed
+page's status: while its contract remains deferred or draft, current code and
+tests—not proposed precedence rules—are authoritative.
+
+## Contract checks
+
+- Resolve each supported structured, legacy, CLI, environment, default, alias,
+  and stage-override producer through its actual runtime consumer.
+- Reject unknown or owner-mismatched fields explicitly; keep process-local
+  runtime objects out of transportable configuration.
+- Verify default, explicit, and feature-off values reach every live factory and
+  consumer without silent reinterpretation.
+- Treat stage count, placement, connector, parallelism, device, and memory
+  fields as one topology contract and reject impossible combinations early.
+- Preserve supported construction-path parity until a documented migration
+  removes a path.
+
+Require normalization and schema tests, a live-consumer assertion, and docs for
+public keys, defaults, constraints, precedence, or migration behavior.

.claude/skills/review-pr/references/modules/diffusion.md

@@ -0,0 +1,33 @@
+# Diffusion Module Family
+
+Primary design: [Diffusion Modules](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/diffusion/).
+
+Use for diffusion-stage behavior, then load the exact submodule page from the
+reviewed head:
+
+| Submodule | Signals | Design |
+| --- | --- | --- |
+| Runtime | Admission, scheduling, execution, progress, outputs, cancellation, hooks, cleanup | [Diffusion Runtime](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/diffusion/diffusion_runtime/) |
+| Model Integration | Pipelines, registry, loader, adapters, shared layers, model-specific processing | [Diffusion Model Integration](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/diffusion/diffusion_model_integration/) |
+| Continuous Batching | Compatibility, request/step batches, per-request progress, capacity | [Continuous Batching](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/diffusion/continuous_batching/) |
+| Parallelism | Rank topology, process groups, sharding, collectives, distributed execution | [Parallelism](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/diffusion/parallelism/) |
+| Offloader | Residency, transfers, memory accounting, prefetch, teardown | [Offloader](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/diffusion/offloader/) |
+
+## Family-wide checks
+
+- Keep one scheduler-owned lifecycle from admission through completion,
+  cancellation, failure, and cleanup; optional features must use defined hooks.
+- Select pipelines through registry and loader contracts instead of scattered
+  model-name conditionals; keep genuine model differences in model directories.
+- Trace latent, conditioning, timestep, generator, shape, layout, dtype, device,
+  batch expansion, and output conversion through the actual consumer.
+- Make batch compatibility and per-request isolation explicit; never associate
+  output by unstable batch position.
+- Derive topology from validated configuration, define every sharding boundary,
+  keep collectives symmetric, and preserve a supported single-rank path.
+- Track one residency owner per offloaded component, wait for transfer readiness,
+  preserve model state, bound retained copies, and clean up deterministically.
+
+Load matching feature designs for cache, quantization, parallelism, continuous
+batching, or offload. Require feature-off correctness, representative inference,
+terminal-path tests, and quality evidence for optimizations.

.claude/skills/review-pr/references/modules/engine-orchestration.md

@@ -0,0 +1,23 @@
+# Engine Orchestration
+
+Primary design: [Engine Orchestration](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/engine_orchestration/).
+
+Use for request-state creation, cross-stage routing, output ordering, companion
+tracking, RPC correlation, cancellation propagation, and terminal convergence.
+Orchestration does not own replica placement, stage-process startup, public
+rendering, connector transport, or payload and error schemas.
+
+## Contract checks
+
+- Keep one owner for cross-stage routing; entrypoints, models, and stage clients
+  must not independently select the next logical stage.
+- Preserve request identity and output order across concurrent stages and
+  correlate every control or RPC result to the originating request and wave.
+- Make terminal state monotonic: no new forwarding after success, abort,
+  request failure, or fatal engine failure.
+- Propagate cancellation and failure to every active stage and release
+  orchestrator-owned tasks, queues, companions, and request state.
+- Preserve shutdown ordering across the orchestrator and stage runtime.
+
+Test representative multi-stage routing, ordering, correlation, cancellation,
+failure, and repeated shutdown.

.claude/skills/review-pr/references/modules/entrypoints.md

@@ -0,0 +1,25 @@
+# Entrypoints and Serving Boundaries
+
+Primary design: [Entrypoints and Serving Boundaries](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/entrypoints/).
+
+Use for offline APIs, CLI and server composition, OpenAI-compatible routes,
+request validation, response conversion, streaming, sessions, and engine
+handoff. Entrypoints adapt public contracts; they do not own cross-stage
+routing, stage lifecycle, configuration precedence, payload schemas, or error
+classification.
+
+## Contract checks
+
+- Normalize and validate public values once before engine submission.
+- Preserve request identity, modality, ordering, and exactly one terminal event
+  across streaming, disconnect, cancellation, and failure.
+- Translate internal outputs and errors explicitly for each advertised offline,
+  HTTP, SSE, WebSocket, or asynchronous-job path.
+- Keep model-specific behavior behind a common adapter or processor instead of
+  adding model-name policy to generic entrypoints.
+- Require API or CLI changes to explain necessity, alternatives, compatibility,
+  migration, and the supported route or transport matrix.
+
+Test the affected public protocol, normalization, engine handoff, rendering,
+and terminal outcomes. Route orchestration, I/O schema, and semantic error
+changes to their module contracts.

.claude/skills/review-pr/references/modules/error-contracts.md

@@ -0,0 +1,24 @@
+# Error Classification, Propagation, and Rendering
+
+Primary design: [Error Classification, Propagation, and Rendering](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/error_contracts/).
+
+Use when a change classifies failures, crosses process boundaries, decides
+fatal versus request-scoped impact, sanitizes context, or renders offline,
+HTTP, SSE, WebSocket, or asynchronous-job errors. The I/O module owns the wire
+schema; this module owns the meaning of its error fields.
+
+## Contract checks
+
+- Keep request-scoped failure isolated unless the underlying engine is no
+  longer usable.
+- Preserve stable classification and request context across process and
+  protocol boundaries without exposing prompts, outputs, secrets, or internal
+  stack details.
+- Keep cancellation distinguishable from internal failure.
+- Terminate each public transport exactly once with consistent status, body,
+  event, finish reason, and metrics classification.
+- Maintain compatibility across mixed or older producers only when the
+  reviewed contract defines that behavior.
+
+Test table-driven classification, wire round trips, fatal/request-scoped
+propagation, cancellation, sanitization, and every affected public renderer.

.claude/skills/review-pr/references/modules/execution-platforms.md

@@ -0,0 +1,23 @@
+# Execution Platforms
+
+Primary design: [Execution Platforms](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/execution_platforms/).
+
+Use for hardware capability detection, worker selection, patches, kernels,
+attention backends, and vendor-specific configuration or connectors.
+
+## Contract checks
+
+- Guard hardware behavior through platform selection or capability detection
+  and fail clearly when unsupported.
+- Keep portable modules free of direct vendor imports when a common selector
+  exists; verify portable imports without the target accelerator.
+- Override only behavior that differs from common or upstream implementations,
+  and compare sibling worker hooks when inheritance does not provide parity.
+- Preserve dtype, layout, rank, stream, synchronization, workspace,
+  graph-capture, and resource-lifetime contracts.
+- Do not let simplified CPU doubles hide platform MRO, initialization, or
+  dispatch failures.
+
+Run portable checks first, then the smallest matching-device test in a fresh
+environment. Report unavailable hardware as a validation gap, never simulated
+device evidence.

.claude/skills/review-pr/references/modules/input-output-modality.md

@@ -0,0 +1,23 @@
+# Input, Output, and Modality Contracts
+
+Primary design: [Input, Output, and Modality Contracts](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/input_output_modality_contracts/).
+
+Use for requests, prompt and modality types, message schemas, serialization,
+output types, accumulation, completion, and compatibility shims crossing
+entrypoint, orchestration, stage, connector, or model boundaries.
+
+## Contract checks
+
+- Give every boundary value an explicit modality and validated typed schema.
+- Preserve request and stage identity across conversion, serialization,
+  streaming updates, batching, cancellation, and errors.
+- Distinguish partial, final, empty, and failed output states for every modality.
+- Verify producer and consumer agreement on shape, layout, dtype, device,
+  ownership, metadata, ordering, and optional or unknown fields.
+- Keep internal objects out of public protocols; translate them at the owning
+  entrypoint.
+- Preserve wire compatibility deliberately and give deprecated shims a bounded
+  migration path.
+
+Require construction, round-trip, accumulation, completion, rejection, and
+compatibility tests at each changed producer-consumer boundary.

.claude/skills/review-pr/references/modules/model-integration.md

@@ -0,0 +1,25 @@
+# Model Integration
+
+Primary design: [Model Integration](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/model_integration/).
+
+Use for model registration, configuration adapters, preprocessing, checkpoint
+loading, stage inputs, model-specific hooks, shared layers, runners, and model
+execution. Use the diffusion family reference for diffusion models.
+
+## Contract checks
+
+- Declare explicitly how the model class, loader, input processor, stage
+  configuration, and runtime capabilities are selected.
+- Preserve weight-name mapping, dtype/device placement, optional-dependency
+  errors, and checkpoint-specific loading semantics.
+- Trace each typed input and output through preprocessing, runner state, model
+  call, and its actual consumer.
+- Reuse upstream vLLM behavior when sufficient; document and test the precise
+  difference for every override or patch.
+- Keep model code out of admission, cross-stage routing, public policy, and
+  terminal lifecycle ownership.
+- Place reusable behavior in shared modules and test it with more than one
+  integration.
+
+Require focused registry, loader, processor, runner, and representative model
+execution tests. Apply the model-addition check for new or expanded support.

.claude/skills/review-pr/references/modules/observability.md

@@ -0,0 +1,21 @@
+# Observability
+
+Primary design: [Observability](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/observability/).
+
+Use for metrics, logs, correlation fields, and cross-stage request visibility.
+
+## Contract checks
+
+- Preserve stable request and stage identity across entrypoints, orchestration,
+  stages, workers, errors, and terminal cleanup.
+- Treat metric names, units, labels, aggregation, and lifecycle as public
+  contracts; document any semantic change.
+- Keep labels bounded: never use request IDs, prompts, outputs, paths, or other
+  unbounded user values.
+- Register and reset metrics exactly once across repeated startup and shutdown,
+  and keep disabled instrumentation inexpensive.
+- Align emitted values with their actual timing and lifecycle owner rather than
+  reconstructing them from ambiguous client strings.
+
+Test registration, label sets, units, resets, error classification, and
+representative multi-stage emission.

.claude/skills/review-pr/references/modules/omni-connector.md

@@ -0,0 +1,24 @@
+# OmniConnector
+
+Primary design: [OmniConnector](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/omni_connector/).
+
+Use for connector contracts and model-agnostic data transfer across stages,
+processes, devices, or nodes. Connector implementations transport and
+synchronize data; they do not choose the next stage or implement model policy.
+
+## Contract checks
+
+- Prove producer and consumer agreement on request/stage identity, shape,
+  layout, dtype, placement, ownership, ordering, and completion.
+- Keep backend selection and metadata/key construction explicit; do not silently
+  reinterpret or drop payloads across SHM, store, network, or vendor paths.
+- Bound connect, put/get, retry, synchronization, and readiness waits with
+  backpressure, cancellation, and actionable errors.
+- Prevent route, endpoint, port, key, and buffer collisions across repeated
+  startup, concurrent requests, and replica replacement.
+- Release connections, handles, buffers, listener work, and stale sender state
+  on success, cancellation, failure, and shutdown.
+
+Load the matching communication feature design for a concrete backend. Require
+focused producer-consumer tests and the smallest runnable multi-process or
+multi-node path.

.claude/skills/review-pr/references/modules/profiling.md

@@ -0,0 +1,21 @@
+# Profiling
+
+Primary design: [Profiling](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/profiling/).
+
+Use for opt-in instrumentation and trace collection across supported profiler
+backends.
+
+## Contract checks
+
+- Keep expensive collection disabled by default and bounded by explicit start,
+  stop, timeout, and cleanup behavior.
+- Preserve scheduling, generated outputs, resource ownership, and concurrency
+  semantics apart from documented synchronization overhead.
+- Include stage, rank, worker, operation, and request correlation where useful
+  without embedding user payloads or secrets.
+- Make repeated start/stop and partial failure safe, including multi-rank trace
+  naming and artifact ownership.
+- Measure disabled and enabled overhead before making performance claims.
+
+Test the disabled path, repeated lifecycle, failure cleanup, trace context, and
+each affected backend.

.claude/skills/review-pr/references/modules/quantization.md

@@ -0,0 +1,22 @@
+# Quantization
+
+Primary design: [Quantization](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/quantization/).
+
+Use for method registration, checkpoint interpretation, layer replacement,
+reduced precision, platform capability, and runtime constraints.
+
+## Contract checks
+
+- Validate checkpoint metadata, selected method, supported layers, device
+  capability, required dependencies, and parallel layout before loading.
+- Preserve loader and weight-name mapping semantics, dtype, scale/granularity,
+  zero point, padding, and buffer handling through the real kernel consumer.
+- Fail clearly for unsupported combinations; do not silently choose a different
+  method or precision.
+- Keep online and pre-quantized checkpoint paths explicit and ensure patches or
+  fallbacks do not bypass model-owned loaders.
+- Require numerical comparison against a defined reference with explicit
+  tolerance before treating performance results as valid.
+
+Test registration, loading, representative layers, metadata, platform guards,
+parallel combinations, and numerical correctness separately from performance.

.claude/skills/review-pr/references/modules/stage-runtime.md

@@ -0,0 +1,23 @@
+# Stage Runtime and Replica Lifecycle
+
+Primary design: [Stage Runtime and Replica Lifecycle](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/stage_runtime/).
+
+Use for local or distributed placement, startup, readiness, stage pools,
+replica identity and membership, selection, affinity, liveness, draining, and
+stage-client or process lifecycle. Cross-stage request policy belongs to engine
+orchestration; model scheduling belongs to the owning runtime.
+
+## Contract checks
+
+- Acquire, replace, and retire replicas through one stage-runtime lifecycle.
+- Preserve stable replica identity and request affinity until termination or a
+  declared replica loss.
+- Make readiness explicit before admission; surface partial-startup and
+  membership failures without leaving half-initialized capacity.
+- Keep local and distributed semantics aligned for supported modes.
+- Make drain, shutdown, and cleanup idempotent across repeated signals and
+  process failure.
+
+Test startup, readiness, selection, affinity, membership changes, abort,
+process loss, draining, and repeated shutdown on affected local/distributed
+paths.

.claude/skills/review-pr/references/process/design-contracts.md

@@ -0,0 +1,34 @@
+# Resolve Design Contracts
+
+Use after freezing the review head and before selecting module or feature
+references. The maintained [design index](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/)
+defines two routing axes:
+
+- **Module designs** own code boundaries and invariants. Select one primary
+  module and a second only when the live producer-consumer path crosses it.
+- **Feature designs** describe behavior spanning modules. Load every feature
+  page whose implementation, default, support, or contract changes.
+
+## Resolution order
+
+1. Open `docs/design/index.md` in the reviewed head, not the reviewer's checkout
+   or published latest tree.
+2. Match changed behavior against each module page's `primary_code_paths`,
+   `primary_path_exceptions`, ownership boundary, and `depends_on` metadata.
+3. Read `status`, `architecture_state`, `last_verified_commit`, and any linked
+   in-flight work before treating prose as current behavior.
+4. Use current code and tests when a page is draft, deferred, stale, absent, or
+   conflicts with the frozen implementation. Report material documentation
+   drift separately.
+5. Treat candidate invariants and promotion gates as review questions until
+   repository policy, current code/tests, or owner-approved normative text
+   makes them enforceable. Do not turn a draft identifier into an automatic
+   blocker.
+6. Use published latest docs only for discovery when the reviewed head lacks a
+   page; never import a newer contract into an older branch silently.
+
+When behavior changes an active module or feature contract, require the same PR
+to update that page. Require a new page only when repository policy, an accepted
+RFC/issue, or explicit maintainer acceptance criteria do so; otherwise treat it
+as a non-blocking design suggestion. Do not demand a second page when an existing
+design already owns the exact contract.

.claude/skills/review-pr/references/process/general-checks.md

@@ -0,0 +1,129 @@
+# General Review Checks
+
+Apply these checks to every PR after freezing the review surface. A pattern is
+not a finding until the diff introduces or exposes a reachable trigger, impact,
+and smallest safe fix. Do not report unrelated backlog.
+
+Official docs: [contributing guide](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/)
+[feature compatibility](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/feature_compatibility/),
+and [design documents](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/).
+
+## Contract and scope
+
+- Match the title/body claims to every changed production path, test, config,
+  dependency, generated artifact, and user-visible document.
+- Trace each changed value through public ingress, validation/defaulting,
+  producer, transformations, process or device boundaries, final consumer, and
+  terminal cleanup.
+- Check every applicable sync/async, offline/online, streaming/non-streaming,
+  feature-off, topology, and compatibility path without demanding unsupported
+  modes merely for symmetry.
+
+## Module, API, and class design
+
+Apply these rules to new or materially changed production code; do not require
+unrelated cleanup from untouched files.
+
+1. Start each source module with a concise module docstring that defines its
+   responsibility and boundaries and identifies its primary classes and key
+   functions. Place it after any required license header and before imports.
+2. Give every added or materially changed class and function a clear docstring
+   covering its responsibility, parameters, return value, raised exceptions,
+   and non-obvious invariants. Use explicit typed parameters at owned API
+   boundaries; avoid opaque `**kwargs` and `Any`. Allow them only when an
+   upstream or compatibility interface requires them, and document and validate
+   the accepted keys and concrete types.
+3. Move stable helpers shared by multiple features into an appropriately scoped
+   utility module. Do not let feature modules accumulate large collections of
+   unrelated private utility functions; keep a private helper local only when
+   it is single-purpose and tightly coupled to that feature.
+4. Keep every method on a key class within that class's stated responsibility,
+   and document non-obvious design decisions and invariants. Before adding a
+   method, prefer reuse, consolidation, or extension of an existing abstraction;
+   split the class when new behavior creates a separate responsibility or makes
+   the class unnecessarily long.
+5. Require changes to user-facing behavior to update `docs/`. When an existing
+   active module or feature design page owns the changed contract, update that
+   page. Require a new design page only when the reviewed branch's repository
+   policy, an accepted RFC/issue, or explicit maintainer acceptance criteria do
+   so; otherwise suggest it as non-blocking. For a major architectural change,
+   apply the branch's contributing policy for a design RFC or issue; the current
+   policy uses more than 500 changed production-code lines, excluding kernel,
+   data, config, and test changes. Resolve page status and authority through
+   [design-contracts.md](design-contracts.md), and do not demand a duplicate page
+   when an existing design owns the exact contract. When required, cover scope,
+   data flow, public contracts, failure behavior, alternatives, validation,
+   limitations, and rollout.
+6. Require every performance PR to provide reproducible base-versus-head A/B
+   results under the same hardware, software, model, workload, precision,
+   topology, warmup, repetitions, and measurement scope. Include exact commands,
+   variability, and a correctness or quality guard; one-sided or non-comparable
+   numbers are incomplete evidence. For a new model absent from the frozen base,
+   compare head with its pinned canonical reference implementation under rule 12
+   instead. Follow
+   [perf-verification.md](../checks/perf-verification.md) for the full protocol.
+7. Require every addition, removal, default change, or semantic change to a
+   user-facing inference API or CLI to explain why it is necessary, which
+   alternatives were considered, and why the chosen interface is preferable.
+   Record compatibility, deprecation or migration behavior, and the rationale
+   in the PR. Update the corresponding module and feature design documents when
+   they define the changed interface or workflow, or when policy requires one.
+8. Require every bug-fix PR to add an automated regression test that reproduces
+   the defect on the frozen base and passes on the fixed head. Exercise the
+   production path or nearest stable boundary and assert the corrected behavior,
+   not only process survival. Record the exact base/head command and result, and
+   place hardware- or model-specific coverage in the correct CI lane. A manual
+   reproduction supplements but does not replace the regression test.
+9. Require every new or materially changed recipe to follow
+   [`recipes/TEMPLATE.md`](../../../../../recipes/TEMPLATE.md) and the naming and
+   layout conventions in
+   [`recipes/README.md`](../../../../../recipes/README.md).
+   Complete every applicable summary, use-case, reference, hardware,
+   environment, exact-command, verification, and notes section with tested,
+   reproducible details; explain any intentional deviation in the PR.
+10. Require every new model to verify its supported modalities, platforms,
+    execution modes, and features with production-path evidence. Update
+    `docs/models/supported_models.md` and every applicable feature support or
+    compatibility table; do not claim support that the PR does not validate.
+    Follow
+    [model-addition-checklist.md](../checks/model-addition-checklist.md) for the
+    model-specific documentation checks.
+11. Require every new model PR to add or update its in-repository model-family
+    recipe under `recipes/` and its `recipes/README.md` index entry. Apply rule
+    9, and keep the recipe's model identifiers, tasks, serving modes, hardware,
+    commands, supported features, and limitations aligned with validated PR
+    evidence. An external recipe link does not replace the in-repository recipe.
+12. Require every new model to compare accuracy or output quality and performance
+    against a pinned baseline or canonical reference implementation. Match the
+    checkpoint, inputs, precision, hardware, workload, and measurement method;
+    provide exact commands, results, and explanations for material differences.
+    Prefer a detailed timing split for each stage and explain when it is
+    unavailable. Follow
+    [model-addition-checklist.md](../checks/model-addition-checklist.md) and
+    [perf-verification.md](../checks/perf-verification.md) for the evidence
+    contract.
+
+## Blocking risk scan
+
+| Risk | Prove before reporting |
+| --- | --- |
+| Correctness | A live input reaches a wrong output, exception, silent drop, shape/dtype/device mismatch, or partially initialized state. |
+| Compatibility | A supported API, config key, default, serialization schema, model identifier, or caller breaks without validation, migration, or deprecation. |
+| Lifecycle | Allocation or startup lacks cleanup on partial failure, timeout, cancellation, shutdown, or repeated requests. |
+| Concurrency/distribution | Ordering, identity, rank, world size, completion, timeout, backpressure, or collective assumptions fail on a supported path. |
+| Cache/shared state | Identity omits a correctness-affecting input, or isolation, invalidation, disabled behavior, eviction, or cleanup is incomplete. |
+| Async behavior | Blocking I/O, sleep, a lock across `await`, or unnecessary serialization harms a reachable async path. |
+| Security/data handling | Secrets, user payloads, unsafe deserialization/eval/shell, unvalidated paths, or unbounded metric labels are reachable. |
+| Validation evidence | Tests bypass the production dispatcher, mock away the change, use unrealistic types/MRO, or cannot fail when the defect returns. |
+| User contract | User-visible model, feature, API, CLI, config, default, or compatibility behavior changes without accurate docs or examples. |
+
+Search bounded callers and sibling implementations before reporting dead code,
+duplication, or a missed consumer. Keep new abstractions only when they have a
+distinct live caller, invariant, or compatibility need.
+
+## Finding bar
+
+Anchor each finding to a changed `path:line`; name the trigger or call path,
+current behavior, impact, and smallest fix direction. Treat pending CI, missing
+hardware, or unsupported measurements as validation gaps unless the repository
+contract makes that evidence a merge requirement.

.claude/skills/review-pr/references/process/review-execution.md

@@ -0,0 +1,202 @@
+# Review Execution and Delivery
+
+Use this reference for every review. It defines snapshot collection, gates,
+validation records, delivery permissions, maintainer tone, and re-review.
+
+Official docs: [contributing guide](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/)
+and [CI failure triage](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/failures/).
+
+## Contents
+
+- [Freeze the review surface](#freeze-the-review-surface)
+- [Report status before analysis](#report-status-before-analysis)
+- [Apply review gates](#apply-review-gates)
+- [Run bounded validation](#run-bounded-validation)
+- [Deliver maintainer-style findings](#deliver-maintainer-style-findings)
+- [Re-review safely](#re-review-safely)
+
+## Freeze the review surface
+
+For a GitHub PR, read the base/head SHA before and after fetching metadata and
+the diff:
+
+```bash
+gh api "repos/vllm-project/vllm-omni/pulls/<PR>" \
+  --jq '{base_sha: .base.sha, head_sha: .head.sha}'
+REVIEW_FIELDS="number,url,title,body,isDraft,baseRefName,headRefName,mergeable,mergeStateStatus,statusCheckRollup,files"
+gh pr view <PR> --repo vllm-project/vllm-omni \
+  --json "${REVIEW_FIELDS}"
+gh pr diff <PR> --repo vllm-project/vllm-omni
+gh api "repos/vllm-project/vllm-omni/pulls/<PR>" \
+  --jq '{base_sha: .base.sha, head_sha: .head.sha}'
+```
+
+Discard the snapshot if either SHA changed. Do not mix comments or validation
+from different heads.
+
+After the SHAs stabilize, fetch and materialize a trusted `head_sha` in an
+isolated detached worktree. Run every source read, `rg` search, import, and test
+from that snapshot, not the caller's checkout. A detached worktree provides
+snapshot isolation, not a security boundary.
+
+Treat a fork head as untrusted unless the user and environment policy explicitly
+establish trust. Never run its imports, tests, builds, hooks, package setup, or
+repo-configurable linters/plugins on the reviewer host. Execute them only in a
+disposable sandbox or VM with no credentials or inherited secrets, no host agent
+or service sockets, minimal read-only host mounts, disabled or explicitly
+allowlisted network, resource/time limits, and destruction after the run. If
+that boundary is unavailable, limit the review to the remote diff, SHA-addressed
+reads such as `git show <head_sha>:<path>`, and existing CI evidence; report all
+executable validation as a gap.
+
+Before the first source read, record a pristine snapshot fingerprint outside
+the reviewed worktree. Include `HEAD`, the NUL-delimited index entries and blob
+IDs, and a NUL-safe manifest of every worktree entry except Git metadata,
+including tracked, untracked, and ignored paths with file type, mode, symlink
+target, and content hash. Before each validation group and delivery, recompute
+and byte-compare the fingerprint as well as asserting `HEAD == head_sha`. If a
+tool changes or creates any entry, discard affected evidence and recreate the
+snapshot before another group; do not rely on `HEAD` alone or clean an unknown
+worktree in place. If an exact snapshot cannot be created, use SHA-addressed
+reads and report filesystem-dependent validation as a gap.
+
+For a local branch/worktree, determine the target ref from the user, current PR,
+or configured upstream; never infer it from the branch name. Resolve the target
+once and include all task-owned worktree state:
+
+```bash
+git status --porcelain=v2 -z
+git rev-parse HEAD
+git rev-parse <target-ref>
+git merge-base <target-base-sha> HEAD
+git diff --stat <comparison-commit>
+git diff --name-status <comparison-commit>
+git diff --binary <comparison-commit>
+git diff --cached --binary <comparison-commit>
+git diff --binary
+git ls-files --others --exclude-standard -z
+```
+
+Freeze the exact bytes of every in-scope untracked file from the NUL-delimited
+list, plus a NUL-safe path, file-type, mode, and content-hash manifest. Also
+fingerprint `HEAD`, the target and merge base, porcelain status, and both binary
+index and worktree patches. Recording only untracked names is insufficient.
+Keep these snapshots in the evidence packet and do not mutate the reviewed
+checkout during a read-only review.
+
+For local changes without PR metadata, mark the title/body as synthetic. Base
+them on the user request and commit messages without inventing validation claims.
+
+If no PR, branch, or usable worktree is identifiable, ask for a PR URL/number or
+explicit base and head rather than guessing.
+
+## Report status before analysis
+
+Within 60 seconds of starting, and before source searches or tests, send this
+host update:
+
+```text
+Pinned head: <SHA>
+Base/comparison: <ref and SHA>
+CI: <pass/fail/pending/not applicable>
+Mergeability: <state/not applicable>
+Preliminary findings: <brief finding or none yet>
+```
+
+Mark early findings as preliminary and continue. This is not a GitHub comment.
+
+## Apply review gates
+
+- Report draft/WIP state. Continue a local review when explicitly requested,
+  but do not publish review events without separate authorization.
+- Record DCO, pre-commit, required CI, and mergeability. Pending or unknown gates
+  do not block source review.
+- GitHub Actions `SKIP` omits SPDX, shellcheck, markdownlint, mypy-3.10, and
+  test-mark coverage. A green GHA pre-commit job does not prove those local
+  gates passed. New files still need the full Linting list: Omni SPDX
+  (`vLLM-Omni project`), no stdlib `re`/`base64` in `vllm_omni/`, no new
+  pickle / Hugging Face Hub API / `torch.cuda` call sites, test marks, TTS
+  adapter ratchet, Buildkite schema, and native shellcheck on macOS/Windows.
+  See [Linting](https://vllm-omni.readthedocs.io/en/latest/contributing/#linting).
+- Expanding `CHECK_IMPORTS[*].allowed_files`, `ALLOWED_FILES`,
+  `MAX_MODEL_TYPE_BRANCHES`, or Buildkite `SKIP_FILES` is a policy change.
+  Do not rubber-stamp allowlist/budget growth; require justification and
+  prefer fixing the call site.
+- Treat a failed gate as its own evidence. Do not restate its formatting/lint
+  output as a new code-review finding.
+- Open CI logs only when the first failing step overlaps the frozen diff or
+  blocks the verdict. Start with the first error, not the last cascade.
+- Treat the PR body as navigation. Commands, benchmark tables, and claimed tests
+  become evidence only after their provenance and relevance are checked.
+
+## Run bounded validation
+
+Keep one evidence packet for files read, bounded searches, callers, tests, CI,
+hardware, routes, and findings. Reuse it rather than refetching facts.
+
+Before pytest, run a short import/version compatibility check. Record each
+validation result with:
+
+```text
+repo, head SHA, command, result, Python/platform, dependency or lock fingerprint
+```
+
+After preflight, run targeted tests and low-cost static checks alongside source
+inspection when possible. Imports, tests, builds, and linters may create caches
+or rewrite files, so isolate each potentially mutating group in a disposable
+snapshot or recreate the pinned snapshot before continuing. Map source symbols
+to tests with bounded `rg` searches rather than assuming the test directory
+mirrors production paths.
+
+Classify failures as code, test, infrastructure, or flaky before reporting.
+Skipped hardware tests are gaps, not passes. For available hardware, verify the
+smallest representative unit/E2E path and compare actual output with PR claims.
+
+## Deliver maintainer-style findings
+
+Default to roughly 1-5 short comments, but treat that as calibration rather than
+a quota: report every real blocker and report no finding when there is no issue.
+
+Write each finding as:
+
+```text
+[P1] Short imperative title — path/to/file.py:<line>
+<Trigger or call path>. <Current behavior and impact>. <Smallest fix direction>.
+```
+
+Use [maintainer-style-study.md](../delivery/maintainer-style-study.md) for
+comment tone.
+Keep rule IDs, grades, and audit matrices internal unless the user asks for the
+complete audit.
+
+Verify every inline `path:line` against the frozen diff before delivery. For a
+PR, re-read the remote head and byte-compare the detached snapshot fingerprint,
+or recreate it from `head_sha`, before asserting current lines. For a local
+review, recompute and byte-compare the frozen `HEAD`, target, merge base, status,
+index patch, worktree patch, and untracked-content manifest. Any mismatch makes
+the review stale; discard the affected evidence and restart. Prefer one
+root-cause comment to several symptom comments.
+
+Local presentation is the default. Only explicit authorization permits GitHub
+posting. When authorized, publish one consolidated final review after
+validation; do not post preliminary or incremental comments. Submit a review
+event (`APPROVE`, `COMMENT`, or `REQUEST_CHANGES`) only when the user explicitly
+chooses that event.
+
+Reviewer requests and owner `@mention` comments are separate external writes;
+follow [review-requests.md](../delivery/review-requests.md) when the user asks
+for them.
+
+## Re-review safely
+
+Freeze the new head and compare it with the previously reviewed SHA. Then:
+
+1. inspect the delta and every file affected by conflict resolution or rebase;
+2. revalidate earlier findings against current lines and behavior;
+3. read unresolved, non-outdated threads and the author's evidence;
+4. rerun only checks invalidated by the delta;
+5. look for new regressions introduced by the fix;
+6. do not repeat resolved or outdated comments.
+
+If the target head changes during re-review, discard stale validation and start
+again from the new snapshot.

.claude/skills/review-pr/references/process/review-routing.md

@@ -0,0 +1,76 @@
+# Module and Feature Review Routing
+
+Read [design-contracts.md](design-contracts.md) first. Route from the frozen
+head's live producer-consumer behavior, then confirm the result against its
+`docs/design/` metadata. Titles and paths are evidence, not the deciding rule.
+
+## Primary module contract
+
+Choose one primary module and a second only when the live call path crosses a
+documented dependency or primary-path exception.
+
+| Module | Live behavior signals | Read |
+| --- | --- | --- |
+| Entrypoints | Offline/CLI/API ingress, validation, rendering, streaming, sessions | [entrypoints.md](../modules/entrypoints.md) |
+| Configuration | Schema, defaults, deploy/stage construction, registry, topology | [configuration.md](../modules/configuration.md) |
+| I/O and Modality | Requests, messages, serialization, output types, accumulation, completion | [input-output-modality.md](../modules/input-output-modality.md) |
+| Error Contracts | Classification, fatality, propagation, sanitization, public rendering | [error-contracts.md](../modules/error-contracts.md) |
+| Engine Orchestration | Cross-stage routing, request state, ordering, RPC correlation, terminal convergence | [engine-orchestration.md](../modules/engine-orchestration.md) |
+| Stage Runtime | Placement, startup, readiness, replicas, affinity, membership, shutdown | [stage-runtime.md](../modules/stage-runtime.md) |
+| OmniConnector | Cross-stage/process/device/node transport and synchronization | [omni-connector.md](../modules/omni-connector.md) |
+| Model Integration | Registration, preprocessing, loading, runners, model-specific execution | [model-integration.md](../modules/model-integration.md) |
+| AR Runtime | Scheduling, request/cache state, adapters, workers, upstream vLLM semantics | [ar-runtime.md](../modules/ar-runtime.md) |
+| Diffusion Family | Diffusion runtime, models, batching, parallelism, or offload | [diffusion.md](../modules/diffusion.md) |
+| Execution Platforms | Hardware selection, capabilities, vendor workers, kernels, patches | [execution-platforms.md](../modules/execution-platforms.md) |
+| Cache Management | Cache identity, validity, reuse, eviction, reset, teardown | [cache-management.md](../modules/cache-management.md) |
+| Quantization | Method/metadata selection, layer mapping, reduced precision, constraints | [quantization.md](../modules/quantization.md) |
+| Observability | Metrics, logs, labels, units, correlation, lifecycle | [observability.md](../modules/observability.md) |
+| Profiling | Opt-in instrumentation, traces, start/stop lifecycle, overhead | [profiling.md](../modules/profiling.md) |
+| Benchmarking | Workloads, metric computation, benchmark CLI, result metadata | [benchmarking.md](../modules/benchmarking.md) |
+
+For tests-, docs-, CI-, or recipe-only changes, route to the production module
+whose contract they protect. When none exists, use the applicable evidence
+check without inventing a production owner.
+
+## Feature-design overlays
+
+Load every category whose documented behavior, enablement, default, support, or
+compatibility changes.
+
+| Feature-design section | Read |
+| --- | --- |
+| Runtime and stage execution | [runtime-stage-execution.md](../features/runtime-stage-execution.md) |
+| Communication and concrete connectors | [communication.md](../features/communication.md) |
+| Diffusion acceleration | [diffusion-acceleration.md](../features/diffusion-acceleration.md) |
+| Infrastructure and performance | [infrastructure-performance.md](../features/infrastructure-performance.md) |
+
+## Evidence and change overlays
+
+| Signal | Read | Optional repo-local skill |
+| --- | --- | --- |
+| New or expanded model, loader, processor, registry, pipeline config, or deploy config | [model-addition-checklist.md](../checks/model-addition-checklist.md) | [`add-tts-model`](../../../add-tts-model/SKILL.md) or [`add-diffusion-model`](../../../add-diffusion-model/SKILL.md) |
+| Latency, throughput, memory, scaling, precision, or quality claim | [perf-verification.md](../checks/perf-verification.md) | [`diffusion-perf-opt`](../../../diffusion-perf-opt/SKILL.md) for diffusion |
+| Tests changed, absent for risky behavior, or test-only | [test-quality-evaluation.md](../checks/test-quality-evaluation.md) | [`vllm-omni-test`](../../../vllm-omni-test/SKILL.md) |
+| CI, examples, docs, public behavior, or contributor evidence | [tests-docs-checklist.md](../checks/tests-docs-checklist.md) | None |
+| Suitable hardware/server and runnable affected path | [verification.md](../checks/verification.md) | None |
+| User asks who should review or requests owner notification | [review-requests.md](../delivery/review-requests.md) | None |
+
+Use [`quantization`](../../../quantization/SKILL.md) for a quantization module
+change and [`$vllm-omni-npu-model-runner-upgrade`](../../../vllm-omni-npu-upgrade/SKILL.md)
+for an NPU execution-platform change when those skills are available.
+
+For a bug fix, require a reachable reproduction and regression test. For a
+refactor, prove parity and remove obsolete paths. For a feature, verify the
+public/config contract, module integration, compatibility/default behavior,
+production dispatch, feature design, and docs.
+
+## Calibrate findings
+
+- **P0:** security exposure, data corruption, or broad project unusability.
+- **P1:** reachable runtime failure, wrong output, compatibility break, or
+  unsafe lifecycle in the changed behavior.
+- **P2:** real non-blocking defect with a concrete future failure mode.
+
+Treat draft candidate invariants, missing hardware, and unsupported claims as
+questions or validation gaps unless current code, tests, or policy makes them a
+merge requirement.

.claude/skills/vllm-omni-test/SKILL.md

@@ -0,0 +1,895 @@
+---
+name: vllm-omni-test
+description: Generate and run tests for vllm-project/vllm-omni with CI-aligned levels and markers; wire new tests into Buildkite (test-ready.yml for L1/L2, test-merge.yml for L3, test-nightly.yml for L4). On completion, always provide copy-paste local and CI-like pytest commands plus prerequisites. Use when creating regression tests, adding L1-L4 coverage, selecting pytest markers, or validating fixes from issues/PRs.
+---
+
+# vLLM-Omni Test Generator & Runner
+
+## Purpose
+
+Use this skill to generate minimal, stable test cases and run them with the correct marker/level strategy for [vllm-project/vllm-omni](https://github.com/vllm-project/vllm-omni).
+
+**Link convention:** Paths such as `.buildkite/` and `docs/contributing/` live at the **vllm-omni repo root**. Markdown links use repo-relative paths from this skill file (e.g. `../../../.buildkite/cuda/test-ready.yml`, `../../../docs/contributing/ci/test_system_overview.md`).
+
+Default priorities:
+
+1. Reproducible regression coverage for bug fixes
+2. Correct test level and marker selection
+3. Low flake, low dependency tests first
+4. CI-compatible run commands
+5. **Actionable run commands for the human**: whenever you add or change tests, always finish with **copy-paste-ready** `pytest` lines (local: single file and/or single test; CI-like: markers + `--run-level`), plus short **prerequisites** (GPU tier, HF cache, optional `model_prefix`). Do not assume the reader will infer commands from `test-routing.md` alone.
+
+## Inputs
+
+- Issue/PR link and summary
+- Changed files or suspected code path
+- Whether the user wants local quick validation or CI-equivalent validation
+- Hardware constraints (CPU only / CUDA / ROCm / NPU)
+
+## Workflow
+
+### Step 1: Classify Test Goal
+
+- **Bugfix regression**: start from a minimal failing scenario and add assertions that prevent recurrence. Before writing tests, output **`required` / `recommended` / `not_needed`**: **`required`** — stable logic/contract bug that should have been caught; **`recommended`** — environment-sensitive but a small regression still helps; **`not_needed`** — one-off external/config failure or existing tests already cover the path. Prefer the narrowest stable L1 (CPU) case; escalate to L2/L3 only when the bug needs real weights or serving.
+- **Feature coverage**: verify new behavior and one negative/boundary case.
+- **Perf/benchmark claim**: require benchmark-oriented tests and explicit metrics.
+
+### Step 2: Select Test Level
+
+- **L1**: unit/logic, deterministic, CPU-friendly, fastest feedback.
+- **L2**: basic e2e/integration and platform-dependent checks.
+- **L3/L4**: advanced model/integration/perf validation.
+
+Use [references/test-routing.md](references/test-routing.md) for level-to-marker and command mapping.
+
+### Step 3: Pick Markers
+
+Always attach markers deliberately:
+
+- **Level**: `core_model` (L1/L2) and/or `advanced_model` (L3) and/or `full_model` (L4 nightly)
+- **Model type** (required on model-centric e2e — pick exactly one):
+  - `omni` — end-to-end multimodal LLM pipelines (thinker/talker/stages; Qwen-Omni family)
+  - `tts` — speech synthesis / TTS-only models (`/v1/audio/speech`, voice clone, etc.)
+  - `diffusion` — generative diffusion models (image / audio / text / video from noise)
+- **Cross-cutting area** (when relevant): `parallel`, `cache`, `example`, `benchmark`
+- **Hardware**: `cpu`, `gpu`, `cuda`, `rocm`, `npu`, `L4`, `H100`, `distributed_cuda`, …
+- Optional: `slow`, distributed markers when multi-card is required
+
+**Baseline smoke (L2 + L3):** The simplest e2e case per model — default deploy, minimal request — should usually carry **both** `@pytest.mark.core_model` and `@pytest.mark.advanced_model` on the **same** test function so `test-ready.yml` and `test-merge.yml` share one test. `send_*_request` picks validation depth from `--run-level`. References: `test_voxcpm2_tts.py::test_text_to_audio_001`, `test_qwen3_tts_customvoice.py::test_text_to_audio_001`. Heavier scenarios use **`advanced_model` only**; L4 expansion uses **`full_model`**.
+
+For hardware-aware tests, prefer `@hardware_test(...)` or `hardware_marks(...)` in `tests/helpers/mark.py`.
+
+**Diffusion nightly split** (`test-nightly.yml`): all diffusion tests use `pytest.mark.diffusion`, but CI groups them by **output modality**, not by separate pytest markers:
+
+| Nightly group | Scope | Typical models / paths |
+|---------------|--------|-------------------------|
+| **Diffusion X2I(&A&T) Model Test** | x2**i** (image), x2**a** (audio), x2**t** (text) and other **non-video** diffusion | Qwen-Image*, BAGEL, FLUX, SD3, Z-Image, LongCat, DreamZero, … — `test_*_expansion.py` under X2I shards; L4 sweep uses `-k "not test_wan and not test_bagel_expansion and not hunyuan"` for L4 |
+| **Diffusion X2V Model Test** | x2**v** (video) only | Wan2.2, HunyuanVideo 1.5, Wan VACE, … — e.g. `test_wan22_expansion.py`, `test_hunyuan_video_15_expansion.py` |
+
+Wire new **video** diffusion expansion tests into the **X2V** group; wire **image/audio/text** diffusion expansion tests into **X2I(&A&T)**. Do not place x2v modules in X2I shards (see comment in `test-nightly.yml` above the X2I group).
+
+### Naming: generated test module files (L2–L4, model-centric e2e)
+
+When adding **new** pytest modules whose primary scope is a **specific model** (typical under `tests/e2e/offline_inference/` or `tests/e2e/online_serving/`), use this filename pattern:
+
+| Level | Filename pattern | Example (model `Qwen/Qwen2.5-Omni-7B`) |
+|-------|------------------|----------------------------------------|
+| **L2**, **L3** | `test_{lowercase_model_slug}.py` | `test_qwen2_5_omni.py` |
+| **L4** | `test_{lowercase_model_slug}_expansion.py` | `test_qwen2_5_omni_expansion.py` |
+
+**Slug rules for `{lowercase_model_slug}`:**
+
+1. Start from the HuggingFace-style id (e.g. `Qwen/Qwen2.5-Omni-7B`), but **do not** put the org into the filename: use the **repo segment only** (`Qwen2.5-Omni-7B`), not `Qwen_...` / `qwen_qwen2_5_...`.
+2. **Lowercase**; replace `.`, `-`, and whitespace with a single `_` (e.g. `Qwen2.5-Omni-7B` → `qwen2_5_omni`). **Omit** trailing size tokens such as `7b` / `30b` in the basename when a single file covers that model line in the directory (matches `test_qwen2_5_omni.py` in-tree).
+3. If two checkpoints in the same folder need separate modules, add a **minimal** disambiguator (e.g. `_7b` vs `_3b`) only then.
+4. **L1** unit tests are **not** bound to this pattern; use `tests/<area>/test_<feature>.py` as today.
+
+Routing tables and commands: [references/test-routing.md](references/test-routing.md) § *Model-centric e2e filename convention*.
+
+Existing references: `tests/e2e/offline_inference/test_qwen2_5_omni.py` (L2-style omni), `tests/e2e/offline_inference/test_qwen3_5_9b.py` (L2-style omni, single-stage VL), `tests/e2e/online_serving/test_qwen3_omni_expansion.py` (L4-style omni), `tests/e2e/online_serving/test_qwen_image_edit_expansion.py` / `test_qwen_image_expansion.py` (L4-style diffusion).
+
+### Step 4: Generate Test Case Skeleton
+
+**1. Pick the functional scenario** (then choose directory, fixtures, and markers):
+
+| Scenario | Typical location | Fixtures / runner pattern | Baseline markers & level |
+|----------|------------------|---------------------------|---------------------------|
+| **Offline inference e2e** | `tests/e2e/offline_inference/` | **Module (default):** `omni_runner` + `offline_client`. **Function (isolation only):** `omni_runner_function` + `offline_client_function`. Diffusion/TTS may use `Omni(...).generate` directly | L2: `core_model` + **one of** `omni` / `tts` / `diffusion`; `@hardware_test(...)` when GPU/NPU is required |
+| **Online serving e2e** | `tests/e2e/online_serving/` | **Module (default):** `omni_server` + `online_client`. **Function (isolation only):** `omni_server_function` + `online_client_function`. Clients: `send_omni_request` (omni), `send_audio_speech_request` (tts), `send_diffusion_request` / `send_video_diffusion_request` / `send_images_generations_request` (diffusion) | Baseline smoke: **`core_model` + `advanced_model`**; heavier paths: `advanced_model` only; L4 expansion: `full_model` |
+| **Documentation / runnable examples** | `tests/examples/offline_inference/`, `tests/examples/online_serving/` | **Offline docs (preferred):** extract Python/Bash blocks from the doc README (e.g. `ReadmeSnippet.extract_readme_snippets`), `pytest.mark.parametrize` each snippet, run via `example_runner.run` with a stable `output_subfolder`. **Online docs:** copy client/request scripts into dedicated tests and keep them in sync with the doc page. | Usually **L4**: `advanced_model`, often `example` plus hardware marks matching the nightly docs-example job (see `.buildkite/cuda/test-nightly.yml`). Full conventions: [docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md](../../../docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md) (introduced in [PR #1910](https://github.com/vllm-project/vllm-omni/pull/1910): naming, output directory layout, skip rules, avoid trimming `num_inference_steps` without a strong CI reason). |
+| **Performance / benchmark** | `tests/dfx/perf/tests/*.json` + `run_*_benchmark.py` | JSON or script-driven server + load config; assert explicit metrics / baselines | L4 Perf: JSON `mark` with `full_model` + `omni`/`tts`/`diffusion`; wire `test-nightly.yml` Perf steps |
+| **Invalid parameter / negative HTTP validation** | `tests/dfx/reliability/invalid_param_test/` | Live `omni_server` + low-level `send_*_http_request` with `err_code` / `err_message` | `pytest.mark.slow` + `omni` / `tts` / `diffusion` + `@hardware_marks` (`H100` or `L4`); CI in **`test-weekly.yml`** (not ready/merge/nightly) |
+
+**If the user’s test plan includes invalid parameter validation / invalid params / negative HTTP / 400 validation:** do **not** add those `test_*` functions to `tests/e2e/online_serving/test_*.py` or `*_expansion.py`. **Move or author them** under `tests/dfx/reliability/invalid_param_test/` in the **endpoint-matching script** (see **Invalid parameter validation** below). Success-path e2e and invalid-param dfx tests must stay in separate modules.
+
+**1b. Model type — `omni` vs `tts` vs `diffusion`**
+
+After choosing offline/online/docs/perf, classify the **product under test** and attach **exactly one** model-type marker. All three can live under the same `tests/e2e/` trees; conventions diverge:
+
+| Dimension | **Omni** (`pytest.mark.omni`) | **TTS** (`pytest.mark.tts`) | **Diffusion** (`pytest.mark.diffusion`) |
+|-----------|------------------------------|----------------------------|----------------------------------------|
+| **What it is** | Multimodal LLM pipeline (thinker/talker/stages; text + vision + audio I/O) | Speech synthesis / voice models | Generative diffusion (noise → image, audio, text, or video) |
+| **Examples** | Qwen2.5-Omni, Qwen3-Omni | Qwen3-TTS, VoxCPM2, Higgs-Audio, Step-Audio2 | Qwen-Image, BAGEL, Wan2.2, HunyuanVideo |
+| **Offline runner** | `omni_runner` + `offline_client`, `generate_multimodal` | `Omni(...).generate` with deploy YAML + TTS params | `Omni(...).generate` + `OmniDiffusionSamplingParams` |
+| **Online client** | `online_client.send_omni_request` (chat completions, modalities) | `online_client.send_audio_speech_request` (`/v1/audio/speech`) | `send_diffusion_request` (chat/T2I), `send_video_diffusion_request` (`/v1/videos`, X2V), or `send_images_generations_http_request` / `send_images_edits_http_request` (DALL-E routes) |
+| **Typical assertions** | Stage outputs, text/audio keywords via `OfflineOmniClient` / response handler | WAV bytes, stream chunks, speech endpoint contract | Image/video dimensions, `final_output_type`, `assert_diffusion_response` |
+| **Deploy YAML** | Per-model deploy configs (`ci/qwen3_omni_moe.yaml`, …) | `qwen3_tts.yaml`, `voxcpm2.yaml`, … | Often default serve; parallel/offload YAML for heavy DiT |
+| **Nightly group** (`test-nightly.yml`) | **Omni Model Test** — `-m "full_model and omni"` | **TTS Model Test** — `-m "full_model and tts"` | **Diffusion X2I(&A&T)** *or* **Diffusion X2V** (see Step 3 table; same `diffusion` marker, different YAML group / file shard) |
+| **L4 pressure** | Expansion per modality/model as needed | Expansion + accuracy/perf in TTS group | X2I: merge feature combos per [#1832](https://github.com/vllm-project/vllm-omni/issues/1832); X2V: separate nightly group |
+
+Do **not** mix fixtures across types (e.g. do not use `omni_runner` layout for a pure diffusion or TTS model without mirroring an in-tree test in that family).
+
+**Diffusion only — X2I(&A&T) vs X2V (nightly routing, not extra markers):**
+
+- **X2I(&A&T)**: image / audio / text generation — Qwen-Image*, FLUX, SD3, Z-Image, BAGEL (expansion), LongCat, audio diffusion, etc.
+- **X2V**: video generation only — Wan2.2, HunyuanVideo 1.5, Wan VACE, LTX video similarity paths.
+
+When adding a new `test_*_expansion.py`, place it in the matching **nightly group** step (explicit file list in `test-nightly.yml`), not only by marker expression.
+
+**2. Use the narrowest deterministic skeleton for the scenario**
+
+*L1 unit / logic (CPU-first):*
+
+**Mocking rule (L1 only):** use **pytest** integration — **`mocker`** (`pytest-mock`) or **`monkeypatch`** (built-in). **Do not** import or call **`unittest.mock`** (`patch`, `MagicMock`, `@patch`, `with patch(...)`, etc.) in L1 tests; patches must auto-revert with the test lifecycle.
+
+```python
+import pytest
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+
+def test_<scenario_name>(mocker):
+    # Prefer mocker.patch / mocker.spy / mocker.Mock — not unittest.mock.patch
+    fake_fn = mocker.patch("vllm_omni.some.module.expensive_call", return_value=...)
+    # Act
+    # Assert
+    fake_fn.assert_called_once()
+
+
+def test_<env_or_attr>(monkeypatch):
+    # For simple env / attribute substitution without a Mock object
+    monkeypatch.setenv("SOME_FLAG", "1")
+    monkeypatch.setattr("vllm_omni.some.module.CONST", 42)
+    ...
+```
+
+**Avoid in L1:**
+
+```python
+# BAD — do not use in L1 unit tests
+from unittest.mock import patch, MagicMock
+
+@patch("vllm_omni.some.module.fn")
+def test_bad(mock_fn): ...
+
+def test_also_bad():
+    with patch("...") as m: ...
+```
+
+See **L1 unit test constraints (mocking)** below for the full do/don't list.
+
+*Offline multimodal e2e — **Omni** (representative):*
+
+```python
+@pytest.mark.core_model
+@pytest.mark.omni
+@hardware_test(...)
+@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
+def test_<scenario>(omni_runner, offline_client) -> None:
+    request_config = {"prompts": ..., "modalities": [...]}  # optional: images, videos, audios
+    offline_client.send_omni_request(request_config)
+```
+
+*Offline generative e2e — **Diffusion** (representative):*
+
+```python
+@pytest.mark.core_model
+@pytest.mark.diffusion
+@hardware_test(...)
+@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
+def test_text_to_image_001(offline_client) -> None:
+    offline_client.send_diffusion_request({"prompt": "...", "extra_body": {"num_inference_steps": 4, ...}})
+```
+
+*Offline TTS e2e — **Qwen3-TTS** (two-stage; representative):*
+
+```python
+@pytest.mark.advanced_model
+@pytest.mark.tts
+@hardware_test(...)
+@pytest.mark.parametrize("omni_runner", tts_server_params, indirect=True)
+def test_text_to_audio_001(omni_runner, offline_client) -> None:
+    offline_client.send_audio_speech_request({
+        "input": "...",
+        "task_type": "Base",
+        "ref_audio": REF_AUDIO_URL,
+        "ref_text": REF_TEXT,
+    })
+```
+
+*Offline TTS e2e — **single-stage** (Coqui XTTS, MOSS-TTS-Nano; representative):*
+
+```python
+@pytest.mark.advanced_model
+@pytest.mark.tts
+@hardware_test(...)
+@pytest.mark.parametrize("omni_runner", tts_server_params, indirect=True)
+def test_voice_clone_001(offline_client) -> None:
+    offline_client.send_single_stage_tts_request({
+        "input": "...",
+        "language": "en",
+        "prompt_audio_path": REF_AUDIO_PATH,
+        "response_format": "wav",
+        "run_level": "advanced_model",
+    })
+```
+
+*Online serving e2e — **Omni** (representative):*
+
+```python
+@pytest.mark.core_model
+@pytest.mark.omni
+@hardware_test(...)
+@pytest.mark.parametrize("omni_server", test_params, indirect=True)
+def test_text_to_text_001(omni_server, online_client) -> None:
+    request_config = {"model": omni_server.model, "messages": ..., "modalities": ["text"]}
+    online_client.send_omni_request(request_config)
+```
+
+*Online serving e2e — **TTS** (representative):*
+
+```python
+@pytest.mark.core_model
+@pytest.mark.advanced_model
+@pytest.mark.tts
+@hardware_test(...)
+@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
+def test_text_to_audio_001(omni_server, online_client) -> None:
+    request_config = {"model": omni_server.model, "input": "...", "response_format": "wav", ...}
+    online_client.send_audio_speech_request(request_config)
+```
+
+*Online serving e2e — **Diffusion** X2I (representative):*
+
+```python
+@pytest.mark.core_model
+@pytest.mark.diffusion
+@pytest.mark.parametrize("omni_server", _get_default_case(MODEL), indirect=True)
+def test_text_to_image_001(omni_server, online_client) -> None:
+    online_client.send_diffusion_request({...})  # chat completions + extra_body
+```
+
+*Online serving e2e — **Diffusion** X2V (representative):*
+
+```python
+@pytest.mark.core_model
+@pytest.mark.diffusion
+def test_text_to_video_001(omni_server, online_client) -> None:
+    online_client.send_video_diffusion_request({"model": ..., "form_data": {...}})  # /v1/videos
+```
+
+*Documentation example tests:* follow the **Preferred Test Strategy** in [l4_doc_example_tests.inc.md](../../../docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md): dynamic extraction for offline READMEs; explicit copied client code for online pages until extraction is justified; use the documented **naming**, **output directory** (page folder + case id), and **skipping** rules (e.g. Gradio-only scripts).
+
+*Performance tests:* add or extend entries under `tests/dfx/perf/tests/` (and JSON configs where the project uses them), with **explicit baselines**, **`mark`** on each case (`hardware_marks` + `full_model` + type marker), and the same nightly Perf step pattern as in-tree configs.
+
+**3. Cross-cutting rules**
+
+- New `.py` files need the Omni SPDX header (`Copyright contributors to the vLLM-Omni project`, not `vLLM project`). `tests/` may keep stdlib `re`/`base64`; pickle is still banned unless the file is already on the pickle allowlist. Every collected `tests/**/test_*.py` needs a CI **level** mark and a **hardware** mark/helper (`check-test-ci-coverage` runs locally; GHA skips it). Do not grow `allowed_files` to land a test. Policy: [docs/contributing/README.md](../../../docs/contributing/README.md#linting).
+- Reuse existing fixtures for the chosen scenario; do not mix “online client” assumptions into offline `OmniRunner` tests without a clear reason.
+- Avoid external network dependency in assertions unless the scenario is explicitly “online serving” or doc examples that require a model hub (then align with CI secrets/cache).
+- Keep **one test function = one intent** (one modality combo, one endpoint contract, or one acceleration combo).
+- **E2E test function layout**: **one case → one `test_<scenario>` function** with a name that states what is validated (endpoint, size/`n`, server flag, or route). **Do not** merge multiple cases into a single test that branches on `request.node.callspec.id`, `param.id`, or `if case_id == ...`. Use `@pytest.mark.parametrize("omni_server", [...], indirect=True)` per function (usually **one** `OmniServerParams` per test). A loop inside one test is OK only when it serves **that function’s single intent** (e.g. three standard sizes in `test_*_sizes_256_512_1024`).
+- **Runtime fixture scope** (`tests/helpers/fixtures/runtime.py`): default **`omni_server` / `omni_runner`** (module) + matching client/handler; use **`omni_server_function` / `omni_runner_function`** only when each `test_*` must start a fresh instance (see below).
+- **L1 mocks**: never `unittest.mock`; use `mocker` or `monkeypatch` only (see below).
+- **API calls (L2+ e2e, online and offline)**: **reuse** `send_*_request` in `tests/helpers/runtime.py` when it exists; **otherwise add** the helper in `runtime.py` first, then call it from the test. General `assert_*` inside `send_*_request`; special `assert_*` only in the test. See **Runtime send helpers** below — do not call `omni.generate`, raw HTTP, or SDK clients from `test_*.py`.
+- **Response assertions**: reusable checks on API bodies / decoded media belong in **`tests/helpers/assertions.py`** — not as private `_assert_*` helpers inside `test_*.py` (see below).
+- **Model-specific payloads stay in test modules** — per-model `MODEL`, deploy path, `REF_AUDIO_URL`, `get_prompt()`, `_build_request()`, and inline `request_config` dicts live in `test_{slug}.py` / `test_{slug}_expansion.py` (and offline/L1 siblings). **Do not** create `tests/helpers/{slug}.py` to deduplicate them; a little copy across files is preferred (see `test_glm_tts.py`, `test_cosyvoice3_tts_expansion.py`). `tests/helpers/` is repo-wide harness only (`mark`, `media`, `runtime`, `stage_config`, `assertions`, `fixtures/`).
+
+#### L1 unit test constraints (mocking)
+
+L1 tests (`core_model and cpu`, under `tests/diffusion/`, `tests/engine/`, `tests/model_executor/`, etc.) must follow the repo’s **pytest-mock** convention (`pytest-mock>=3.10.0` in `pyproject.toml` `[project.optional-dependencies] dev`).
+
+| Do | Don't |
+|----|-------|
+| `def test_foo(mocker):` + `mocker.patch(...)`, `mocker.spy(...)`, `mocker.Mock()`, `mocker.MagicMock()`, `mocker.AsyncMock()` | `from unittest.mock import patch, MagicMock, Mock, AsyncMock` |
+| `def test_bar(monkeypatch):` + `monkeypatch.setattr` / `setenv` / `delenv` / `setitem` | `@patch(...)` decorator |
+| Let `mocker` auto-stop patches after the test | `with patch(...):` / `patch.object(...)` context managers |
+| Mirror neighboring L1 tests in the same directory | `unittest.mock.create_autospec` unless an existing file already documents an exception |
+
+**Rationale:** `mocker` ties patch lifecycle to pytest fixtures (no leaked patches across tests). `unittest.mock` decorators/context managers are easy to compose incorrectly with parametrized or async tests and are inconsistent with in-tree L1 style.
+
+**Minimal patterns:**
+
+```python
+def test_returns_cached_config(mocker):
+    loader = mocker.patch(
+        "vllm_omni.foo.load_yaml",
+        return_value={"stages": []},
+    )
+    result = get_deploy_config("ci/foo.yaml")
+    assert result["stages"] == []
+    loader.assert_called_once_with("ci/foo.yaml")
+
+
+def test_skips_when_env_unset(monkeypatch):
+    monkeypatch.delenv("VLLM_OMNI_FEATURE", raising=False)
+    assert should_enable_feature() is False
+```
+
+E2E levels (L2+) generally **avoid mocks**; if a rare L2 stub is unavoidable, still prefer `mocker` over `unittest.mock` for consistency.
+
+#### Runtime fixtures — scope (`tests/helpers/fixtures/runtime.py`)
+
+vLLM-Omni e2e tests start a real **OmniServer** (online) or **OmniRunner** (offline). Pick scope by **how often the process must be recreated**, not by test level alone.
+
+| Scope | Online fixtures | Offline fixtures | When to use |
+|-------|-----------------|------------------|-------------|
+| **Module** (default) | `omni_server` → `online_client` | `omni_runner` → `offline_client` | **Default for L2/L3/L4 expansion** — amortize model/server init across `test_*` in the same module. Same `OmniServerParams` / runner config can be reused by multiple tests. |
+| **Function** | `omni_server_function` → `online_client_function` | `omni_runner_function` → `offline_client_function` | **Only when required** — each `test_*` must get a **clean** server/runner (no shared engine/GPU state). Typical: `tests/dfx/reliability/`, sleep/wakeup, crash/restart, tests that mutate global server state. |
+
+**Rules:**
+
+1. **Default to module scope** (`omni_server` / `omni_runner`) unless the scenario **explicitly** needs a fresh instance per test function.
+2. **Indirect parametrize name must match the fixture name**: `@pytest.mark.parametrize("omni_server", ...)` with `omni_server` + `online_client`; `@pytest.mark.parametrize("omni_server_function", ...)` with `omni_server_function` + `online_client_function`. Do not mix module fixture with function client (or vice versa).
+3. Different `OmniServerParams` per `test_*` (e.g. default vs `--enable-cpu-offload`) is still OK with **module** `omni_server` — pytest parametrizes per test node; only switch to `_function` when isolation between tests matters, not merely because `server_args` differ.
+4. One `test_*` with **many** server configs (expansion matrix) → single function + `@pytest.mark.parametrize("omni_server", [...], indirect=True)` + module `omni_server` (see in-tree `test_qwen_image_expansion.py`).
+
+```python
+# Default — module-scoped server (L4 expansion)
+@pytest.mark.parametrize("omni_server", [pytest.param(OmniServerParams(model=MODEL), marks=H100)], indirect=True)
+def test_foo_images_generations_default_1024(omni_server, online_client) -> None:
+    online_client.send_images_generations_request({...})
+
+# Function-scoped — reliability / per-test clean state only
+@pytest.mark.parametrize(
+    "omni_server_function",
+    [pytest.param(OmniServerParams(model=MODEL), marks=H100)],
+    indirect=True,
+)
+def test_foo_sleep_wakeup_cycle(omni_server_function, online_client_function) -> None:
+    online_client_function.send_omni_sleep_http_request({...})
+    online_client_function.send_omni_wakeup_http_request({...})
+```
+
+#### Runtime send helpers — online **and** offline (`tests/helpers/runtime.py`)
+
+**L2+ e2e (online serving and offline inference) must call APIs through `tests/helpers/runtime.py`.** Fixtures live in `tests/helpers/fixtures/runtime.py`; **send/assert implementation** lives in `tests/helpers/runtime.py` + `tests/helpers/assertions.py`.
+
+| Principle | Action |
+|-----------|--------|
+| **Reuse first** | Grep `runtime.py` for an existing **`send_*_request`** on `OnlineOmniClient` (online) or `OfflineOmniClient` (offline) that matches the endpoint / pipeline shape. |
+| **Extend when close** | If an existing `send_*_request` almost fits (e.g. missing one optional field), extend it in `runtime.py` — do not fork logic in the test file. |
+| **Add when missing** | No suitable helper → add **`send_<feature>_request`** (high-level: call + general `assert_*`) or **`send_<route>_<verb>_http_request`** (low-level HTTP for negative/dfx) in **`runtime.py` first**, then call it from tests. |
+| **Test module owns payload only** | In `test_*.py`: `MODEL`, deploy path, vendored media, `get_prompt()`, and inline **`request_config` dicts** only. **No** `omni.generate(...)`, raw `requests.post`, OpenAI SDK calls, or `_collect_audio()` / `_process_output()` in e2e tests. |
+
+**Online** (`online_client` from `omni_server`): `OnlineOmniClient.send_*_request`.
+
+**Offline** (`offline_client` from `omni_runner`): `OfflineOmniClient.send_*_request`.
+
+| Do | Don't |
+|----|-------|
+| Online: `online_client.send_omni_request`, `send_diffusion_request`, `send_audio_speech_request`, `send_video_diffusion_request`, `send_images_generations_request`, … | `requests.post(f"{base_url}/v1/...", json=…)` or `client.chat.completions.create(...)` inside a test |
+| Offline: `offline_client.send_omni_request`, `send_diffusion_request`, `send_audio_speech_request`, `send_single_stage_tts_request`, `send_single_stage_tts_batch_request`, … | `omni_runner.omni.generate(...)` + hand-rolled tensor/WAV extraction in `test_*.py` |
+| Add missing `send_*` to **`runtime.py`** first; bundle general `assert_*` inside the send helper | A one-off `def _post_*` / `def _collect_audio` at the bottom of a test module |
+| Mirror naming/style of neighboring `send_*` (docstring, `request_config` dict, optional `run_level`, `err_code` / `err_message` for negative cases) | Different parameter shapes per test file for the same endpoint |
+
+**Workflow when generating online/offline e2e tests:**
+
+1. Decide whether the needed check is **general** (every success call of this `send_*`) or **special** (one case / one parameter combo only). See **General vs special assert placement** below.
+2. **Search `runtime.py`** for a matching **`send_*_request`**; if missing, add it (with general `assert_*` inside) before writing the test body.
+3. In the test module: build `request_config` → call **`send_*_request` only** for the general contract; call **`assert_*` in the test file only** for special, case-specific checks (import from `assertions.py`).
+4. Reserve **low-level** `send_*_http_request` for negative/dfx tests (`err_code` / `err_message`) — not for ordinary L2+ success-path e2e.
+5. When wiring Buildkite `source_file_dependencies`, include **`tests/helpers/runtime.py`** and/or **`tests/helpers/assertions.py`** when new helpers were added.
+
+**Exceptions (document in the test docstring why):** models whose offline prompt path cannot go through existing handlers yet (e.g. `test_higgs_audio_v2.py`, `test_voxtral_tts.py` with custom tokenizer compose) may call `omni.generate` directly until a `send_*_request` is added to `runtime.py` — treat as **debt**, not the default for new TTS/diffusion/omni e2e.
+
+**General vs special assert placement:**
+
+| Kind | Definition | Where it lives | Test file calls |
+|------|------------|----------------|-----------------|
+| **General** | Default success contract for this endpoint/client method — e.g. HTTP 200, `data[]` shape, `n` count, `size` dimensions, decodable image/audio, bundled omni/diffusion fields | Implement in `assertions.py`, invoke **inside** `send_*_request` in `runtime.py` (after low-level send / SDK call, when `err_code` is not set) | **`send_*_request` only** — do **not** repeat the same `assert_*` |
+| **Special** | Extra check tied to **this test case only** — e.g. seed byte-identical replay, accuracy/CLIP threshold, perf ceiling, model-specific optional field | Add or reuse `assert_*` in `assertions.py` (never inline in test) | `send_*_request` **then** `assert_<special>(...)` **once** for that case |
+
+Changing `request_config` fields (`size`, `n`, `seed`, server flags) is **not** special validation — the general `assert_*` should read those from `request_config` inside `send_*`.
+
+**`send_*` ↔ `assert_*` pairing (online `OnlineOmniClient`):**
+
+| High-level `send_*` (prefer in L2+ e2e) | Assert already invoked inside `runtime.py` | Low-level HTTP-only sibling |
+|----------------------------------------|--------------------------------------------|-----------------------------|
+| `send_omni_request` | `assert_omni_response` | `send_chat_completions_http_request` → `assert_http_error` only |
+| `send_diffusion_request` | `assert_diffusion_response` | — |
+| `send_audio_speech_request` | `assert_audio_speech_response` | `send_audio_speech_http_request` → `assert_http_error` only |
+| `send_video_diffusion_request` | `assert_diffusion_response` | `send_videos_*_http_request` → `assert_http_error` only |
+| `send_images_generations_request` *(add when needed)* | **`assert_images_generations_response`** *(general — inside send)* | `send_images_generations_http_request` → `assert_http_error` only |
+| `send_images_edits_request` *(add when needed)* | **`assert_images_edits_response`** *(general — inside send)* | `send_images_edits_http_request` → `assert_http_error` only |
+
+**Rule:** Tests call high-level `send_*_request` for the general contract. **Never** call the same bundled `assert_*` again. Call an extra `assert_*` in the test **only** for special, case-specific validation.
+
+**Common `OnlineOmniClient` entry points** (non-exhaustive — grep `runtime.py` before adding):
+
+| Area | High-level (SDK + assert) | Low-level HTTP (`*_http_request`) |
+|------|---------------------------|-----------------------------------|
+| Omni chat | `send_omni_request` | `send_chat_completions_http_request` |
+| Diffusion T2I (chat route) | `send_diffusion_request` | — |
+| Diffusion X2V | `send_video_diffusion_request` | `send_videos_create_http_request`, `send_video_content_http_request`, … |
+| DALL-E T2I / edit | `send_images_generations_request`, `send_images_edits_request` | `send_images_generations_http_request`, `send_images_edits_http_request` |
+| TTS (online) | `send_audio_speech_request` | `send_audio_speech_http_request`, `send_audio_generate_http_request`, … |
+| Ops / meta | — | `send_health_http_request`, `send_models_http_request`, `send_omni_sleep_http_request`, … |
+
+**Common `OfflineOmniClient` entry points** (offline — grep `runtime.py` before adding):
+
+| Area | High-level (`send_*_request` + assert) | Notes |
+|------|----------------------------------------|-------|
+| Omni multimodal | `send_omni_request` | `generate_multimodal` path |
+| Diffusion offline | `send_diffusion_request` | chat-route / `OmniTextPrompt` offline |
+| Qwen-style TTS | `send_audio_speech_request` | two-stage, `generate_multimodal` + `mm_processor_kwargs` |
+| Single-stage TTS (Coqui XTTS, MOSS-TTS-Nano, …) | `send_single_stage_tts_request`, `send_single_stage_tts_batch_request` | `prompt` + `additional_information` + `omni.generate` — **do not** duplicate in tests |
+| New model family | **Add** `send_<family>_request` here first | Then call from `tests/e2e/offline_inference/test_*.py` |
+
+L1 tests under `tests/entrypoints/` may use **FastAPI `TestClient`** or direct handler calls with mocks; they do **not** need `OnlineOmniClient` / `OfflineOmniClient`. **L2+ online and offline e2e** must use `runtime.py` `send_*` helpers.
+
+#### Invalid parameter validation (`tests/dfx/reliability/invalid_param_test/`)
+
+When the user asks for **invalid parameter validation**, **invalid request bodies**, **HTTP 4xx contract tests**, or any case that sends **malformed / out-of-range / mismatched** API payloads against a **live** server:
+
+1. **Do not** place these in `tests/e2e/online_serving/` or `*_expansion.py`. If already drafted there, **move** the `test_*` into the correct `invalid_param_test` script and delete the duplicate from e2e.
+2. **Pick the script by HTTP route** (extend an existing file; add a new `test_invalid_<area>.py` only when no in-tree script covers that route family):
+
+| API / area | Script |
+|------------|--------|
+| Omni chat completions, WebSocket video/realtime paths | `test_invalid_omni_chat.py` |
+| `POST /v1/audio/speech`, stream, batch, voices | `test_invalid_audio_speech.py` |
+| Audio diffusion endpoints | `test_invalid_audio_diffusion.py` |
+| `POST /v1/images/generations` | `test_invalid_image_generation.py` |
+| `POST /v1/images/edits` | `test_invalid_image_editing.py` |
+| `POST/GET/DELETE /v1/videos*` | `test_invalid_video_generation.py` |
+| Sleep / wakeup / server control | `test_invalid_server_control.py` |
+
+3. **Match in-tree style** in the chosen script:
+
+| Element | Convention |
+|---------|------------|
+| **Module markers** | `pytestmark = [pytest.mark.slow, pytest.mark.<omni or tts or diffusion>]` |
+| **Server fixture** | `_PARAMS` / `_QWEN3_TTS_SPEECH`-style list of `pytest.param(OmniServerParams(...), id="...", marks=hardware_marks(...))`; `@pytest.mark.parametrize("omni_server", _PARAMS, indirect=True)` |
+| **Hardware** | `hardware_marks(res={"cuda": "H100"})` for heavy diffusion/omni/video; `hardware_marks(res={"cuda": "L4"})` for smaller TTS models (must match weekly `-m "slow and L4"` step) |
+| **HTTP client** | **Low-level** `online_client.send_*_http_request({..., "err_code": 400, "err_message": (...)} )` — **not** `send_*_request` (success path) |
+| **Case shape** | Prefer **one `test_*` per route family** + `@pytest.mark.parametrize("body_spec, err_message", [...])` with stable `id=` per case; or dedicated `test_<route>_malformed_json` when not parametrized |
+| **Body helpers** | `_minimal_<endpoint>_json(omni_server)` / `_minimal_*_form_data()`; merge overrides with `body.update(body_spec)` |
+| **Known gaps** | `pytest.mark.skip(reason="…#3649")` as `_SKIP_ISSUE_3649` when server validation is not yet strict (mirror neighboring cases) |
+| **Sections** | Route banner comments (`# ─── POST /v1/images/generations ───`) like existing files |
+| **Shared fixtures** | Reuse `tests/dfx/reliability/invalid_param_test/conftest.py` (`tiny_png_bytes`, env defaults) |
+
+4. **Adding a new model** to invalid-param coverage: append a `pytest.param(OmniServerParams(model="...", stage_config_path=..., server_args=...), ...)` entry to the script’s `_PARAMS` list — do **not** create `test_invalid_<model>.py` unless the route is new.
+
+5. **CI — `.buildkite/cuda/test-weekly.yml` only** (not `test-ready.yml` / `test-merge.yml` / `test-nightly.yml`):
+
+| Weekly step | Command | When your cases run |
+|-------------|---------|---------------------|
+| **Invalid parameters Test · H100** | `pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100"` | Diffusion / omni / video invalid-param tests with `H100` hardware mark |
+| **Invalid parameters Test · L4** | `pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and L4"` | TTS / lighter models marked `L4` |
+
+- **Trigger:** `build.env("WEEKLY") == "1"` or PR label `weekly-test`.
+- **Default:** extending an existing `invalid_param_test` script needs **no YAML edit** — the weekly steps already sweep the whole directory.
+- **Edit YAML only** when adding a **new hardware queue**, a **new top-level script** that must run in isolation, or a **model-specific weekly shard** (mirror neighboring reliability steps).
+- Weekly steps **do not** use `source_file_dependencies`.
+
+**Example — append to `test_invalid_image_generation.py`:**
+
+```python
+@pytest.mark.parametrize(
+    "body_spec, err_message",
+    [
+        pytest.param({"seed": -1}, ("seed", "greater_than_equal", "0"), id="seed_negative"),
+    ],
+)
+@pytest.mark.parametrize("omni_server", _PARAMS, indirect=True)
+def test_images_generations_invalid_requests(
+    omni_server: OmniServer,
+    online_client: OnlineOmniClient,
+    body_spec: dict[str, object],
+    err_message: str | tuple[str, ...],
+) -> None:
+    body = _minimal_images_gen_json(omni_server)
+    body.update(body_spec)
+    online_client.send_images_generations_http_request(
+        {"json": body, "timeout": 300, "err_code": 400, "err_message": err_message}
+    )
+```
+
+See [references/test-routing.md](references/test-routing.md) **Invalid parameter / weekly CI**.
+
+#### Assertion helpers (`tests/helpers/assertions.py`)
+
+**Do not** add module-local helpers such as `_assert_images_generations_payload` or `_send_and_assert_*` in e2e test files. Response/media validation belongs in **`tests/helpers/assertions.py`**, grouped by **category**:
+
+| Category | Existing anchors | When to extend vs add |
+|----------|------------------|------------------------|
+| **Image** (chat diffusion + DALL-E JSON) | `assert_image_diffusion_response`, `assert_image_valid` | DALL-E `/v1/images/generations` JSON → add **`assert_images_generations_response`** beside image helpers (reuses `assert_image_valid`); do not fork decode logic into tests |
+| **Video** | `assert_video_diffusion_response`, `assert_video_valid` | Extend these for new video contracts |
+| **Audio** | `assert_audio_diffusion_response`, `assert_audio_speech_response`, `assert_audio_valid` | Extend for new TTS/audio endpoints |
+| **Omni multimodal** | `assert_omni_response` | Extend for new modality combos |
+| **HTTP errors** | `assert_http_error`, `assert_err_message_in_text` | Used inside low-level `send_*_http_request` and negative tests |
+
+| Do | Don't |
+|----|-------|
+| Put **general** `assert_*` inside `send_*_request` in `runtime.py`; tests call `send_*_request` only | Call `assert_diffusion_response` after `send_diffusion_request` (general assert already bundled) |
+| Put **special** `assert_*` in `assertions.py` and call it **in the test** after `send_*_request` when that case needs extra checks | Put general decode/count/size logic in the test because “this case uses `n=4`” (that belongs in general assert reading `request_config`) |
+| Extend the matching category function, or add `assert_<endpoint>_response` next to its category | `_send_and_assert_*` or PIL/base64 loops in `test_*.py` |
+| Use low-level `send_*_http_request` in dfx/negative tests (`err_code`) | Use low-level HTTP send + manual general assert in every L2+ expansion file |
+
+**Workflow when generating tests:**
+
+1. Implement or reuse **general** `assert_*` in `assertions.py` (by category).
+2. Wire it into **`send_*_request`** in `runtime.py` if not already bundled.
+3. In the test: `request_config` → `send_*_request`. Add a separate `assert_*` import/call **only** for special case validation.
+4. Update `source_file_dependencies` when `runtime.py` or `assertions.py` changes.
+
+**Example — DALL-E `/v1/images/generations`:**
+
+```python
+# tests/helpers/assertions.py — general contract (image category)
+def assert_images_generations_response(resp_body: dict, request_config: dict, *, run_level: str | None = None) -> None:
+    ...  # data[], n, size → assert_image_valid
+
+# tests/helpers/runtime.py — general assert INSIDE send
+def send_images_generations_request(self, request_config: dict[str, Any], ...) -> list[HttpResponse]:
+    responses = self.send_images_generations_http_request(request_config, ...)
+    if request_config.get("err_code") is None:
+        body = responses[0].json_body
+        assert isinstance(body, dict)
+        assert_images_generations_response(body, request_config, run_level=self.run_level)
+    return responses
+
+# tests/e2e/.../test_foo_expansion.py — one case per test_* (no case_id branching)
+@pytest.mark.parametrize("omni_server", [pytest.param(OmniServerParams(model=MODEL), marks=SINGLE_L4)], indirect=True)
+def test_foo_images_generations_default_1024(omni_server, online_client) -> None:
+    online_client.send_images_generations_request({"json": body, "timeout": 300})
+
+@pytest.mark.parametrize("omni_server", [pytest.param(OmniServerParams(model=MODEL), marks=SINGLE_L4)], indirect=True)
+def test_foo_images_generations_sizes_256_512_1024(omni_server, online_client) -> None:
+    for size in ("256x256", "512x512", "1024x1024"):
+        online_client.send_images_generations_request({"json": {**body, "size": size}, "timeout": 300})
+
+# chat diffusion — separate test; send_diffusion_request bundles assert_diffusion_response
+@pytest.mark.parametrize("omni_server", [pytest.param(OmniServerParams(model=MODEL), marks=SINGLE_L4)], indirect=True)
+def test_foo_chat_completions_t2i_fallback(omni_server, online_client) -> None:
+    online_client.send_diffusion_request({...})
+```
+
+L1 tests under `tests/entrypoints/` may keep **minimal** asserts next to the handler under test when validating a single branch; still prefer `assertions.py` when the same JSON/media check appears in more than one file or level (e.g. L1 protocol test + L4 e2e).
+
+#### Omni Test Writing Guidance (L1-L4 Layering)
+
+When the goal is a general Omni/multimodal test case, prioritize mapping the test to the correct purpose, directory, and resource assumptions aligned with the layers (see [test_system_overview.md](../../../docs/contributing/ci/test_system_overview.md) and [test_writing_guide.md](../../../docs/contributing/ci/test_writing_guide.md)):
+
+- **L1**: Unit/logic validation on CPU (`core_model and cpu`). Cover input validation, branches, and exception paths (`tests/<component>/test_*.py`). **Mock with `mocker` / `monkeypatch` only — not `unittest.mock`.**
+- **L2**: Basic e2e (online/offline basic scenarios). Prefer dummy/lightweight models to validate the end-to-end request-to-output-structure/streaming chain (typically `tests/e2e/online_serving/` and `tests/e2e/offline_inference/`).
+- **L3/L4**: Important integration, performance, and accuracy validation. L4 emphasizes “full functional scenarios + performance/stress + runnable doc examples” (typically `*_expansion.py` plus related expansion cases).
+
+Also keep markers consistent with the run level: use `core_model` for L1/L2 and `advanced_model` for L3/L4, and pair with `--run-level` to select the intended CI strategy.
+
+#### Diffusion Test Writing Guidance (L4 Coverage Combinations)
+
+When the task involves diffusion models/features, organize L4 test cases following [`#1832`](https://github.com/vllm-project/vllm-omni/issues/1832): combine multiple diffusion features into as few test cases as possible to fit limited CI GPU resources.
+
+Implementation strategy:
+
+1. If “full L4 is too heavy”: first provide a **reduced local validation case in L1/L2** (so the key assertion and the contract fix point are covered deterministically).
+2. Then provide **CI/nightly-ready L3/L4 high-marked cases** (e.g. `advanced_model`) to broaden coverage under resource constraints.
+
+#### L4 nightly sub-pillars — Function vs Accuracy vs Perf
+
+**L4 is not only `test_*_expansion.py`.** In `test-nightly.yml`, each model-type group typically runs **separate jobs**:
+
+| Sub-pillar | Nightly step label pattern | What you add | Typical paths |
+|------------|----------------------------|--------------|---------------|
+| **Function** | `· Function Test with …` | E2e expansion / feature matrix | `tests/e2e/online_serving/test_<model>_expansion.py` (or offline) |
+| **Accuracy** | `· Accuracy Test` | Quality / similarity vs baseline | `tests/e2e/accuracy/test_<model>*.py` |
+| **Perf** | `· Perf Test · <Model>` | Throughput / latency / memory benchmarks | `tests/dfx/perf/tests/test_<model>_vllm_omni.json` + runner script |
+| **Doc** (optional) | `· Doc Test` | Runnable doc examples | `tests/examples/*/test_text_to_image.py`, … |
+
+**Default when the user asks for “L4 functional cases”:** deliver **Function** pillar only (`*_expansion.py` + Function Test shard in `test-nightly.yml`). **Do not** silently add Perf or Accuracy unless the user also asks for **performance** / **benchmark** / **accuracy** / **full L4** / **full L4 coverage**.
+
+**When the user explicitly asks for L4 perf (or full L4 including perf):**
+
+1. **Do not** put throughput/latency assertions inside `test_*_expansion.py` — perf uses the **dfx benchmark harness**, not `omni_server` e2e fixtures.
+2. Add or extend a **JSON config** under `tests/dfx/perf/tests/` (mirror in-tree names: `test_qwen_image_vllm_omni.json`, `test_cosmos3_vllm_omni.json`, `test_qwen3_omni_async_chunk.json`).
+3. Each JSON **case**: `test_name`, optional **`mark`** (`hardware_marks` required when present; `marks`: `full_model` + `omni`/`tts`/`diffusion`), `server_params` (`model`, `serve_args` or `stage_config_name`), `benchmark_params[]` with **`name`** and explicit **`baseline`** (`throughput_qps`, `latency_mean`, `peak_memory_mb_mean`, …). Diffusion cases also set `server_type` (e.g. `"vllm-omni"`) and usually `benchmark_endpoint`.
+4. Run locally via the matching script (model type → runner):
+
+| Model type | Runner script | Example config |
+|------------|---------------|----------------|
+| **Diffusion X2I/X2V** | `tests/dfx/perf/scripts/run_diffusion_benchmark.py` | `tests/dfx/perf/tests/test_qwen_image_vllm_omni.json` |
+| **TTS** | `tests/dfx/perf/scripts/run_benchmark.py` | `tests/dfx/perf/tests/test_tts.json` (shared) or `test_{slug}.json` (dedicated; use when the model must not join the shared nightly matrix before integration — e.g. `test_voxcpm2.json`, `test_coqui_tts.json`) |
+| **Omni** | `tests/dfx/perf/scripts/run_benchmark.py` | `test_qwen3_omni_no_async_chunk.json`, `test_qwen3_omni_async_chunk.json`, `test_qwen3_omni_vllm_text.json`, `test_qwen3_omni_multi_replicas.json` |
+
+5. Wire **`test-nightly.yml` · Perf Test** step for that model (separate from Function Test): export `DIFFUSION_BENCHMARK_DIR` / `BENCHMARK_DIR`, run pytest on the script with **`--test-config-file`** (nightly perf steps do not use `-m`), **upload artifacts** (`buildkite-agent artifact upload`), often **multi-GPU H100** for diffusion perf.
+6. **One benchmark scenario → one `test_name` block** in JSON (same spirit as one `test_*` per function case). Combine server `serve_args` + workload in one entry; use multiple `benchmark_params` rows for size/step sweeps under the same server config.
+
+**Perf local commands:**
+
+```bash
+cd tests
+export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
+# Single file (CI-like)
+pytest -s -v dfx/perf/scripts/run_diffusion_benchmark.py \
+  --test-config-file dfx/perf/tests/test_<model>_vllm_omni.json
+# Bulk load + filter by JSON mark
+pytest -sv dfx/perf/scripts/run_diffusion_benchmark.py -m "full_model and diffusion and H100"
+pytest -sv dfx/perf/scripts/run_benchmark.py -m "full_model and omni and H100"
+```
+
+**Clarify in the test plan** which L4 pillars you deliver: `Function only` | `Function + Perf` | `Function + Accuracy + Perf`.
+
+See [references/test-routing.md](references/test-routing.md) **L4 nightly pillars** for Buildkite step patterns.
+
+### Step 5: Wire Buildkite (when CI must run the new test)
+
+If the test is not already collected by an existing pipeline command (for example, L1 tests marked `core_model and cpu` are already covered by the **Simple Unit Test** step in ready/merge), update the appropriate pipeline under `.buildkite/`:
+
+| Test level | Edit this file | Typical trigger / intent |
+|------------|----------------|---------------------------|
+| **L1** and **L2** | [`.buildkite/cuda/test-ready.yml`](../../../.buildkite/cuda/test-ready.yml) | PR **ready** label; L1 CPU + L2 GPU/basic e2e (steps labeled **Omni ·**, **TTS ·**, **Diffusion ·**) |
+| **L3** | [`.buildkite/cuda/test-merge.yml`](../../../.buildkite/cuda/test-merge.yml) | Post-merge; `advanced_model` integration per model type |
+| **L4** | [`.buildkite/cuda/test-nightly.yml`](../../../.buildkite/cuda/test-nightly.yml) | Nightly; grouped by model type (see below) |
+| **Invalid param / reliability (weekly)** | [`.buildkite/cuda/test-weekly.yml`](../../../.buildkite/cuda/test-weekly.yml) | Weekly; `tests/dfx/reliability/invalid_param_test/` — **not** L1–L4 e2e pipelines |
+
+**Level-specific Buildkite delivery (follow the user’s requested level):**
+
+| User asks for | Document / edit **only** | Do **not** add by default |
+|---------------|---------------------------|---------------------------|
+| **L4** (`*_expansion.py`, `full_model`) | `test-nightly.yml` — append file to the matching nightly shard (X2I / X2V / Omni / TTS) **or** note it is already collected by an existing `-m` / `-k` sweep | `test-merge.yml` (L3 / `advanced_model`) or `test-ready.yml` (L2) |
+| **L3** | `test-merge.yml` with `advanced_model` + `source_file_dependencies` | `test-nightly.yml` unless user also wants nightly |
+| **L2** | `test-ready.yml` with `core_model` + `source_file_dependencies` | merge / nightly pipelines |
+| **Invalid param** | `test-weekly.yml` — **Invalid parameters Test** group (`-m "slow and H100"` / `-m "slow and L4"`). Usually **no YAML edit** when appending cases to existing `invalid_param_test` scripts | `test-ready.yml`, `test-merge.yml`, `test-nightly.yml` |
+
+`test-nightly.yml` shards use **explicit pytest file paths** in `commands` (and PR labels like `diffusion-x2iat-test`); they generally **do not** use `source_file_dependencies`. Only suggest merge/ready YAML when the requested level is L3/L2 (or the user explicitly asks for multi-level CI).
+
+**`test-nightly.yml` top-level groups** (each uses `full_model` + type marker in pytest `-m`):
+
+| Group | Type marker | Notes |
+|-------|-------------|--------|
+| **Omni Model Test** | `omni` | Function / doc / accuracy / **perf** for Qwen-Omni family |
+| **TTS Model Test** | `tts` | `-m "full_model and L4 and tts"` (and H100 variants when enabled); **Perf Test** uses `run_benchmark.py` + `test_tts.json` |
+| **Diffusion X2I(&A&T) Model Test** | `diffusion` | Image/audio/text diffusion; explicit file shards for **Function**; **Perf Test · &lt;Model&gt;** uses `run_diffusion_benchmark.py` + JSON config |
+| **Diffusion X2V Model Test** | `diffusion` | Video-only (Wan, HunyuanVideo, …); PR label `diffusion-x2v-test`; separate **Perf** steps when in-tree |
+
+Guidelines when editing YAML:
+
+- **Match markers and `--run-level`** to the test: L1/L2 in `test-ready.yml` use `core_model` + `omni` / `tts` / `diffusion` as appropriate; L3 merge uses `advanced_model`; L4 nightly uses `full_model` with the same type marker.
+- **Diffusion L4**: add x2i/x2a/x2t expansion files to an **X2I(&A&T)** step; add x2v expansion files to **X2V** — never rely on a broad `tests/e2e/` sweep alone (shards exist to save GPU and avoid wrong queue).
+- **`source_file_dependencies` (required for new E2E jobs in `test-ready.yml` and `test-merge.yml`)**: every step under the **`:card_index_dividers: E2E Test`** group **must** declare `source_file_dependencies` so Buildkite only schedules the job when relevant paths change. Omitting it breaks path-based CI skipping for GPU e2e jobs.
+- **Reuse or extend an existing step** when the new test shares the same marker expression, queue, and timeout; otherwise add a new `steps:` entry with the correct `agents.queue`, `timeout_in_minutes`, and docker/kubernetes plugin blocks consistent with neighboring jobs.
+- **Job timeout — use `timeout_in_minutes` on the step, not `timeout` before pytest**: when adding or documenting a new E2E job in `test-ready.yml` / `test-merge.yml`, set `timeout_in_minutes` on the step and run `pytest` directly in `commands`. **Do not** wrap pytest in `timeout 40m bash -c "..."` / `timeout 20m bash -c "..."`; Buildkite already enforces the step deadline via `timeout_in_minutes`.
+- **Never omit `plugins` in skill examples or PR YAML snippets**: H100 E2E steps on `mithril-h100-pool` use the full `kubernetes` block (`resources`, `volumeMounts`, `env` with `HF_TOKEN` secret, `nodeSelector: gpu-h100-sxm`, `devshm` + `hf-cache` volumes). L4/docker steps on `gpu_1_queue` use the full `docker#v5.2.0` block (`shm-size`, `HF_HOME`, volumes). Do not write `# ... plugins unchanged` — paste the complete block from a sibling step.
+- **Platform forks** (e.g. AMD-ready / AMD-merge) live alongside these files; apply the same level → file mapping for those pipelines when the test is platform-specific.
+
+#### `source_file_dependencies` for E2E Test jobs (`test-ready.yml`, `test-merge.yml`)
+
+When adding or splitting a step inside the **E2E Test** group, **always** include `source_file_dependencies` on that step. List paths the job actually depends on — at minimum:
+
+1. **Test module(s)** — every `tests/e2e/...` file the `pytest` command runs (online and offline if both are in one step).
+2. **Model implementation** — `vllm_omni/model_executor/models/<family>/` and/or `vllm_omni/diffusion/models/<family>/` as applicable.
+3. **Stage / input plumbing** (omni & multi-stage TTS) — `vllm_omni/model_executor/stage_input_processors/<family>.py` when present.
+4. **Deploy / stage YAML** — `vllm_omni/deploy/<config>.yaml` or `vllm_omni/deploy/ci/<config>.yaml` referenced by the test’s `OmniServerParams` / `get_deploy_config_path(...)`.
+5. **Test helpers** (when changed in the same PR) — `tests/helpers/runtime.py` for new `send_*` client methods; `tests/helpers/assertions.py` for new shared `assert_*` helpers.
+
+Use **repo-relative paths** (no leading `./`). Prefer **directory** entries (`.../models/bagel/`) when the whole package matters; use **file** entries for single test modules and YAML configs. Mirror a neighboring step for the same model family (e.g. **Diffusion · Bagel Test** in `test-ready.yml`).
+
+**Example** (L2 ready — diffusion online smoke, H100 kubernetes):
+
+```yaml
+- label: "Diffusion · Bagel Test"
+  source_file_dependencies:
+    - tests/e2e/online_serving/test_bagel.py
+    - vllm_omni/model_executor/models/bagel/
+    - vllm_omni/diffusion/models/bagel/
+    - vllm_omni/model_executor/stage_input_processors/bagel.py
+    - vllm_omni/deploy/bagel.yaml
+  timeout_in_minutes: 40
+  commands:
+    - |
+      export VLLM_IMAGE_FETCH_TIMEOUT=60
+      pytest -s -v tests/e2e/online_serving/test_bagel.py -m 'core_model' --run-level 'core_model'
+  agents:
+    queue: "mithril-h100-pool"
+  plugins:
+    - kubernetes:
+        podSpec:
+          containers:
+            - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+              resources:
+                limits:
+                  nvidia.com/gpu: 1
+              volumeMounts:
+                - name: devshm
+                  mountPath: /dev/shm
+                - name: hf-cache
+                  mountPath: /root/.cache/huggingface
+              env:
+                - name: HF_HOME
+                  value: /root/.cache/huggingface
+                - name: HF_TOKEN
+                  valueFrom:
+                    secretKeyRef:
+                      name: hf-token-secret
+                      key: token
+          nodeSelector:
+            node.kubernetes.io/instance-type: gpu-h100-sxm
+          volumes:
+            - name: devshm
+              emptyDir:
+                medium: Memory
+            - name: hf-cache
+              hostPath:
+                path: /mnt/hf-cache
+                type: DirectoryOrCreate
+```
+
+**Example** (L3 merge — TTS online + offline, L4 docker):
+
+```yaml
+- label: "TTS · Qwen3-TTS Base Test"
+  source_file_dependencies:
+    - tests/e2e/online_serving/test_qwen3_tts_base.py
+    - tests/e2e/offline_inference/test_qwen3_tts_base.py
+    - vllm_omni/model_executor/models/qwen3_tts/
+    - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
+    - vllm_omni/deploy/qwen3_tts.yaml
+  timeout_in_minutes: 20
+  commands:
+    - |
+      export VLLM_LOGGING_LEVEL=DEBUG
+      export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
+      pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+  agents:
+    queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
+  plugins:
+    - docker#v5.2.0:
+        image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+        always-pull: true
+        propagate-environment: true
+        shm-size: "8gb"
+        environment:
+          - "HF_HOME=/fsx/hf_cache"
+        volumes:
+          - "/fsx/hf_cache:/fsx/hf_cache"
+```
+
+**Example** (L3 merge — diffusion online + offline, H100 kubernetes):
+
+```yaml
+- label: "Diffusion · Qwen Image Test"
+  source_file_dependencies:
+    - tests/e2e/online_serving/test_qwen_image.py
+    - tests/e2e/offline_inference/test_qwen_image.py
+    - vllm_omni/diffusion/models/qwen_image/
+  timeout_in_minutes: 40
+  commands:
+    - |
+      export VLLM_IMAGE_FETCH_TIMEOUT=60
+      pytest -s -v \
+        tests/e2e/online_serving/test_qwen_image.py \
+        tests/e2e/offline_inference/test_qwen_image.py \
+        -m 'advanced_model and cuda' --run-level 'advanced_model'
+  agents:
+    queue: "mithril-h100-pool"
+  plugins:
+    - kubernetes:
+        podSpec:
+          containers:
+            - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+              resources:
+                limits:
+                  nvidia.com/gpu: 1
+              volumeMounts:
+                - name: devshm
+                  mountPath: /dev/shm
+                - name: hf-cache
+                  mountPath: /root/.cache/huggingface
+              env:
+                - name: HF_HOME
+                  value: /root/.cache/huggingface
+                - name: HF_TOKEN
+                  valueFrom:
+                    secretKeyRef:
+                      name: hf-token-secret
+                      key: token
+          nodeSelector:
+            node.kubernetes.io/instance-type: gpu-h100-sxm
+          volumes:
+            - name: devshm
+              emptyDir:
+                medium: Memory
+            - name: hf-cache
+              hostPath:
+                path: /mnt/hf-cache
+                type: DirectoryOrCreate
+```
+
+When extending an **existing** E2E step to run an additional test file, **append** that file (and any new model/deploy paths) to `source_file_dependencies` on the same step. When authoring a PR, mention the dependency list in the **Buildkite change** section of your test plan.
+
+#### L3 / L4: validating Buildkite from a feature branch (`cuda/pipeline.yml`)
+
+Production behavior is driven by [`.buildkite/cuda/pipeline.yml`](../../../.buildkite/cuda/pipeline.yml): it builds the CI image, then uploads **L2** (`test-ready.yml` on non-`main`), **L3** (`test-merge.yml` when `build.branch == "main"`), and **L4** (`test-nightly.yml` when `build.env("NIGHTLY") == "1"`).
+
+When you add or debug **L3** or **L4** tests and need those child pipelines to run **off `main`** or **without** `NIGHTLY=1`, apply **temporary** edits (revert before merge):
+
+1. **Point the upload at the right file (optional shortcut)**  
+   In the **Upload Ready Pipeline** step, the command is `buildkite-agent pipeline upload .buildkite/cuda/test-ready.yml`. For a one-off run you may change that path to `.buildkite/cuda/test-merge.yml` (L3) or `.buildkite/cuda/test-nightly.yml` (L4), and adjust the step’s `if:` so it does not conflict with the other upload steps (avoid double-uploading unless intentional).
+
+2. **L3 — merge pipeline**  
+   The gate `if: build.branch == "main"` lives on the **Upload Merge Pipeline** step in `cuda/pipeline.yml` (not inside `test-merge.yml`). **Comment out that `if` line** so `test-merge.yml` is uploaded on your feature branch. Alternatively rely on step (1) instead of the dedicated merge upload step.
+
+3. **L4 — nightly pipeline**  
+   **Comment out** `if: build.env("NIGHTLY") == "1"` on the **Upload Nightly Pipeline** step in `cuda/pipeline.yml` so the nightly definition is uploaded without setting the env var.  
+   Child steps in [`test-nightly.yml`](../../../.buildkite/cuda/test-nightly.yml) often repeat `if: build.env("NIGHTLY") == "1"`; **comment out those lines on the steps you need to run**, otherwise they will still be skipped after upload.
+
+### Step 6: Run Tests
+
+Pick one command path:
+
+- **Quick local regression (preferred first)**: single file or `file.py::test_name`
+- **CI-like level run**: marker expression + `--run-level`
+
+Full templates live in [references/test-routing.md](references/test-routing.md). **After authoring tests, always emit concrete commands** (see **Output Format**).
+
+**Typical copy-paste examples** (run from repo `tests/` directory; requires matching vLLM/vllm-omni install and hardware):
+
+| Area | Example |
+|------|---------|
+| **Omni offline L2** | `pytest -s -v e2e/offline_inference/test_qwen2_5_omni.py -m "core_model and omni and not cpu" --run-level=core_model` |
+| **Omni online L2** | `pytest -s -v e2e/online_serving/test_qwen3_omni.py -m "core_model and omni" --run-level=core_model` |
+| **TTS online L2** | `pytest -s -v e2e/online_serving/test_qwen3_tts_base.py -m "core_model and tts" --run-level=core_model` |
+| **Diffusion X2I L2 online** | `pytest -s -v e2e/online_serving/test_qwen_image.py -m "core_model and diffusion" --run-level=core_model` |
+| **Diffusion X2V L2 online** | `pytest -s -v e2e/online_serving/test_wan22_t2v.py -m "core_model and diffusion" --run-level=core_model` |
+| **Diffusion X2I L4 nightly** | `pytest -s -v e2e/online_serving/test_qwen_image_expansion.py -m "full_model and diffusion and H100" --run-level=full_model` |
+| **Diffusion X2I L4 perf (nightly)** | `pytest -s -v dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file dfx/perf/tests/test_qwen_image_vllm_omni.json` |
+| **Diffusion X2V L4 nightly** | `pytest -s -v e2e/online_serving/test_wan22_expansion.py -m "full_model and cuda" --run-level=full_model` |
+| **Invalid param (weekly H100)** | `pytest -s -v dfx/reliability/invalid_param_test/ -m "slow and H100"` |
+| **Invalid param (weekly L4)** | `pytest -s -v dfx/reliability/invalid_param_test/ -m "slow and L4"` |
+| **L1 CPU** | `pytest -s -v -m "core_model and cpu"` |
+
+**Prerequisites to mention when relevant**: GPU model (e.g. L4 vs H100), `HF_HOME` / token for hub weights, module-level `skipif` (NPU/XPU-only gaps), and whether CI already collects the path (e.g. `test_*_expansion.py` glob in `test-nightly.yml`).
+
+### Step 7: Validate Result Quality
+
+Before finishing:
+
+- Is the new assertion directly tied to the bug/feature contract?
+- Are API calls made via **`runtime.py` `send_*_request`** (not inline HTTP/SDK in the test file)?
+- Is the **general** `assert_*` bundled inside `send_*_request` (not duplicated in the test)?
+- Is each scenario a **separate `test_*` function** (no `case_id` / `if param.id` branching across cases)?
+- **Fixture scope**: default **`omni_server` / `omni_runner`** (module); **`_function`** variants only when each test must restart the instance?
+- Is the test deterministic (no fragile timing/network coupling)?
+- Is runtime appropriate for the selected level?
+- Are markers and `--run-level` consistent?
+- **Invalid-param cases** in `tests/dfx/reliability/invalid_param_test/` (not e2e), using `send_*_http_request` + `err_code`, with `pytest.mark.slow` and correct `H100` / `L4` hardware mark?
+
+## Output Format
+
+When completing a request, return:
+
+1. **Test plan** (level, markers, file target, **L4 pillar(s)** if applicable: Function / Accuracy / Perf / Doc; **or** **Invalid param** pillar with target `invalid_param_test/test_invalid_<area>.py`; module basename for e2e: `test_{slug}.py` / `test_{slug}_expansion.py`)
+2. **Generated/updated test file(s)** — model-specific constants and request payloads stay **inside** those test modules (**never** `tests/helpers/{slug}.py`). Extend **`tests/helpers/runtime.py`** / **`tests/helpers/assertions.py`** only when new **repo-wide** `send_*` or shared assert helpers are required (do not leave ad-hoc HTTP or `_assert_*` in test modules). **If L4 Perf was requested**, also list **`tests/dfx/perf/tests/*.json`**. **If invalid-param cases were requested**, list the **`invalid_param_test/` script** and parametrized `test_*` / new `id=` rows — **not** e2e paths.
+3. **Buildkite change** — **match the requested level and pillar**:
+   - **L4 Function**: `test-nightly.yml` **Function Test** shard (file list / note existing sweep). **No** `test-merge.yml` unless user also asked for L3.
+   - **L4 Perf**: `test-nightly.yml` **Perf Test · &lt;Model&gt;** step (new step or extend commands); include env exports + artifact upload pattern from a sibling perf job.
+   - **L4 Accuracy**: `test-nightly.yml` **Accuracy Test** step under the same model-type group.
+   - **L3**: `test-merge.yml` + full `source_file_dependencies` + `agents` + `plugins`.
+   - **L2**: `test-ready.yml` + same E2E block requirements.
+   - **Invalid param**: `test-weekly.yml` — note **Invalid parameters Test · H100/L4** group; usually **no YAML change** when only extending existing scripts.
+   If the user asked only for **L4 functional cases**, state explicitly that **Perf / Accuracy / Invalid param** were not included (offer to add if needed). If the plan mixes **success e2e** and **invalid param**, split deliverables across e2e vs `invalid_param_test/` and call out both CI files.
+4. **Run commands (required)** — always include, in fenced `bash` blocks:
+   - **Local — whole file**: `cd tests` then `pytest -s -v <path> …`
+   - **Local — single test** (optional but preferred when the change is one function): `pytest -s -v path::test_func …`
+   - **CI-like** (when not L1 CPU): the same **marker + `--run-level`** pairing the level uses (see [references/test-routing.md](references/test-routing.md) and **Step 6** table above)
+   - **Prerequisites** (one line): e.g. “Requires CUDA L4 + weights cached locally”; “Requires H100 + `HF_TOKEN`”; “Syntax-only validation when vllm is not installed locally”
+5. **Result summary** (pass/fail if you executed tests; if not executed, state that explicitly and what the user should run)
+
+## Additional Resources
+
+- Marker and command routing: [references/test-routing.md](references/test-routing.md)
+- CI pipelines (vllm-omni): [test-ready.yml](../../../.buildkite/cuda/test-ready.yml) (L1/L2), [test-merge.yml](../../../.buildkite/cuda/test-merge.yml) (L3), [test-nightly.yml](../../../.buildkite/cuda/test-nightly.yml) (L4), [test-weekly.yml](../../../.buildkite/cuda/test-weekly.yml) (invalid param / reliability)
+- L4 documentation example tests (naming, extraction vs copied scripts, output dirs, skips): [docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md](../../../docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md) — see also [PR #1910](https://github.com/vllm-project/vllm-omni/pull/1910)

.claude/skills/vllm-omni-test/references/test-routing.md

@@ -0,0 +1,395 @@
+# Test Routing Reference
+
+Use this reference to map testing goals to levels, markers, and runnable commands.
+
+**Repo paths** (`.buildkite/`, `docs/contributing/…`): link with repo-relative paths from this file (e.g. `../../../../.buildkite/cuda/test-ready.yml`, `../../../../docs/contributing/ci/test_system_overview.md`).
+
+## Model-centric e2e filename convention (L2–L4)
+
+When generating a **new** test module tied to a **specific model** under `tests/e2e/offline_inference/` or `tests/e2e/online_serving/`:
+
+| Level | Pattern | Example (`Qwen/Qwen2.5-Omni-7B`) |
+|-------|---------|----------------------------------|
+| **L2**, **L3** | `test_{lowercase_model_slug}.py` | `test_qwen2_5_omni.py` |
+| **L4** | `test_{lowercase_model_slug}_expansion.py` | `test_qwen2_5_omni_expansion.py` |
+
+**Slug rules for `{lowercase_model_slug}`** (canonical: [SKILL.md](../SKILL.md) § *Naming: generated test module files*):
+
+1. Start from the HuggingFace-style id (e.g. `Qwen/Qwen2.5-Omni-7B`), but **do not** put the org into the filename: use the **repo segment only** (`Qwen2.5-Omni-7B`), not `Qwen_...` / `qwen_qwen2_5_...`.
+2. **Lowercase**; replace `.`, `-`, and whitespace with a single `_` (e.g. `Qwen2.5-Omni-7B` → `qwen2_5_omni`). **Omit** trailing size tokens such as `7b` / `30b` in the basename when a single file covers that model line in the directory (matches `test_qwen2_5_omni.py` in-tree).
+3. If two checkpoints in the same folder need separate modules, add a **minimal** disambiguator (e.g. `_7b` vs `_3b`) **only then**.
+4. **L1** unit tests are **not** bound to this pattern; use `tests/<area>/test_<feature>.py` as today.
+
+## Model type markers (`omni` / `tts` / `diffusion`)
+
+Every model-centric e2e test must declare **exactly one** type marker:
+
+| Marker | Model family | Typical APIs |
+|--------|--------------|--------------|
+| `pytest.mark.omni` | Multimodal LLM (Qwen-Omni, …) | `send_omni_request`, `omni_runner`, stage YAML |
+| `pytest.mark.tts` | TTS / speech synthesis | `send_audio_speech_request`, `/v1/audio/speech` |
+| `pytest.mark.diffusion` | Diffusion generative models | `send_diffusion_request`, `send_video_diffusion_request`, `send_images_generations_http_request`, `OmniDiffusionSamplingParams` |
+
+**Diffusion** uses one marker for all output modalities. Nightly CI splits diffusion by **group**, not by extra markers:
+
+| Nightly group (`test-nightly.yml`) | Diffusion scope | Examples |
+|-----------------------------------|-----------------|----------|
+| **Diffusion X2I(&A&T) Model Test** | x2**i** / x2**a** / x2**t** — image, audio, text (non-video) | Qwen-Image*, BAGEL, FLUX, SD3, Z-Image, LongCat, DreamZero |
+| **Diffusion X2V Model Test** | x2**v** — video only | Wan2.2, HunyuanVideo 1.5, Wan VACE |
+
+PR labels for selective nightly runs: `diffusion-x2iat-test`, `diffusion-x2v-test` (plus `nightly-test` / `NIGHTLY=1`).
+
+## Level and Marker Mapping
+
+| Goal | Suggested Level | Marker baseline | Typical location |
+|------|------------------|-----------------|------------------|
+| Unit logic, regression on pure code path | L1 | `core_model and cpu` | `tests/<component>/test_*.py` |
+
+### L1 unit tests — mocking (`mocker`, not `unittest.mock`)
+
+L1 modules run in **Simple Test** (`test-ready.yml` / `test-merge.yml`: `-m 'core_model and cpu'`). When isolation requires doubles:
+
+- **Use** the **`mocker`** fixture (`pytest-mock`): `mocker.patch`, `mocker.spy`, `mocker.Mock`, `mocker.MagicMock`, `mocker.AsyncMock`.
+- **Use** **`monkeypatch`** for env vars and simple `setattr` / `delattr` without mock objects.
+- **Do not use** `unittest.mock` — no `from unittest.mock import ...`, no `@patch`, no `with patch(...)`.
+
+```python
+# Good (L1)
+def test_handler(mocker):
+    mocker.patch("vllm_omni.pkg.fn", return_value=0)
+
+# Bad (L1)
+from unittest.mock import patch
+
+@patch("vllm_omni.pkg.fn")
+def test_handler(mock_fn): ...
+```
+
+E2E (L2+) should not rely on mocks unless documenting a rare exception; prefer real (or lightweight) execution paths.
+| Basic integration/e2e | L2 | `core_model` + **one of** `omni` / `tts` / `diffusion` | `tests/e2e/...` |
+| Advanced integration | L3 | `advanced_model` + type marker | `tests/e2e/...` |
+| Full function / nightly | L4 | `full_model` (nightly) + type marker | **Function:** `tests/e2e/*_expansion.py`; **Perf:** `tests/dfx/perf/tests/*.json`; **Accuracy:** `tests/e2e/accuracy/` |
+| Invalid HTTP / param validation | Weekly (dfx) | `pytest.mark.slow` + type marker + `H100` or `L4` | `tests/dfx/reliability/invalid_param_test/test_invalid_*.py` |
+
+## Marker Selection Rules
+
+1. **Level** (pick one):
+   - `core_model` — L1/L2 (`test-ready.yml`)
+   - `advanced_model` — L3 (`test-merge.yml`)
+   - `full_model` — L4 nightly (`test-nightly.yml`); some expansion tests still carry both `advanced_model` and `full_model` during migration
+2. **Model type** (pick one): `omni`, `tts`, or `diffusion`
+3. **Cross-cutting** when relevant: `parallel`, `cache`, `example`, `benchmark`
+4. **Hardware**: `cpu`, `cuda`, `rocm`, `npu`, `L4`, `H100`, `distributed_cuda`, …
+5. Multi-card: `@hardware_test(...)` in `tests/helpers/mark.py`
+
+## Command Templates
+
+### Quick local checks
+
+```bash
+cd tests
+pytest -s -v test_xxxx.py
+```
+
+### L1
+
+```bash
+cd tests
+pytest -s -v -m "core_model and cpu"
+```
+
+### L2
+
+```bash
+cd tests
+pytest -s -v -m "core_model and not cpu" --run-level=core_model
+```
+
+### L3 (merge)
+
+```bash
+cd tests
+pytest -s -v -m "advanced_model" --run-level=advanced_model
+```
+
+### L4 (nightly)
+
+**Function** (e2e expansion — default when user asks for “L4 functional cases”):
+
+```bash
+cd tests
+pytest -s -v e2e/online_serving/test_qwen_image_expansion.py -m "full_model and diffusion and H100" --run-level=full_model
+```
+
+**Perf** (dfx benchmark harness — separate pillar; not `omni_server` fixtures):
+
+```bash
+cd tests
+export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
+# CI-like: single JSON file (nightly Perf Test steps)
+pytest -s -v dfx/perf/scripts/run_diffusion_benchmark.py \
+  --test-config-file dfx/perf/tests/test_qwen_image_vllm_omni.json
+# Local bulk load: all *.json under tests/dfx/perf/tests/, filter by JSON mark
+pytest -sv dfx/perf/scripts/run_diffusion_benchmark.py -m "full_model and diffusion and H100"
+pytest -sv dfx/perf/scripts/run_benchmark.py -m "full_model and omni and H100"
+```
+
+Broad marker sweep (when no explicit file shard):
+
+```bash
+cd tests
+pytest -s -v -m "full_model" --run-level=full_model
+```
+
+## L4 nightly pillars (Function / Accuracy / Perf)
+
+Nightly **L4** for a model is often **multiple jobs** in `test-nightly.yml`, not a single pytest module:
+
+| Pillar | User says | Deliver | Nightly step |
+|--------|-----------|---------|--------------|
+| **Function** | “L4 functional cases” / “functional” / `*_expansion.py` | `tests/e2e/.../test_<model>_expansion.py` | `· Function Test with H100/L4` |
+| **Perf** | “performance” / “perf” / “benchmark” / “full L4” | `tests/dfx/perf/tests/test_<model>_vllm_omni.json` + runner | `· Perf Test · <Model>` |
+| **Accuracy** | “accuracy” / “similarity” | `tests/e2e/accuracy/test_<model>*.py` | `· Accuracy Test` |
+
+**Rule:** “L4 functional cases” → **Function only** by default. State that Perf/Accuracy are separate pillars; add them only when requested.
+
+**Diffusion perf JSON** (`tests/dfx/perf/tests/test_<slug>_vllm_omni.json`): array of **case objects**, each with:
+
+| Field | Required | Notes |
+|-------|----------|-------|
+| `test_name` | Yes | Unique server/workload id |
+| `mark` | Recommended | When present: **`hardware_marks` required** + optional `marks` (`full_model`, `diffusion`, `local_model`, …). Parsed by `resolve_pytest_marks`; enables local `-m` filtering. |
+| `server_type` | Diffusion | e.g. `"vllm-omni"` — routes case to `run_diffusion_benchmark.py` |
+| `benchmark_endpoint` | Diffusion | e.g. `/v1/videos`, `/v1/images/generations` |
+| `server_params` | Yes | `model`, `serve_args`, … |
+| `benchmark_params[]` | Yes | Each row: **`name`** (pytest id suffix), workload fields, **`baseline`** |
+
+Copy structure from `test_qwen_image_vllm_omni.json` or `test_cosmos3_vllm_omni.json` (2-GPU mark example).
+
+**Omni perf JSON**: `test_qwen3_omni_*.json` — same `mark` shape with `"marks": ["full_model", "omni"]`; run via `run_benchmark.py`.
+
+**Runner split**: `is_diffusion_perf_config()` → diffusion when `server_type` is set or `"diffusion"` ∈ `mark.marks`; otherwise omni/tts → `run_benchmark.py`.
+
+**Param ids**: `{test_name}-{benchmark_params.name}` (e.g. `test_tts-p0`, not `test_tts-0`).
+
+**Result filenames**: runtime hardware from `get_runtime_resource_label()`; `H100` omitted on default CI pool.
+
+**Diffusion perf nightly step** (under **Diffusion X2I(&A&T)**; copy full `kubernetes` plugins from `· Perf Test · Qwen-Image`):
+
+```yaml
+      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · <Model-Display-Name>"
+        key: nightly-diffusion-x2iat-performance-<slug>
+        timeout_in_minutes: 180
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
+          - export CACHE_DIT_VERSION=1.3.0
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_<slug>_vllm_omni.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          # … full kubernetes block (often multi-GPU) — copy from Perf Test · Qwen-Image
+```
+
+| Model type | Perf runner | Config example |
+|------------|-------------|----------------|
+| Diffusion X2I/X2V | `dfx/perf/scripts/run_diffusion_benchmark.py` | `test_qwen_image_vllm_omni.json`, `test_cosmos3_vllm_omni.json` |
+| TTS | `dfx/perf/scripts/run_benchmark.py` | `test_tts.json`, `test_voxcpm2.json`, `test_higgs_audio_v3.json` |
+| Omni | `dfx/perf/scripts/run_benchmark.py` | `test_qwen3_omni_async_chunk.json`, `test_qwen3_omni_no_async_chunk.json`, … |
+
+Do **not** put throughput/latency baselines inside `test_*_expansion.py` — that belongs in the dfx perf JSON + nightly Perf job.
+
+## Invalid parameter validation (weekly / dfx)
+
+When the user’s test plan includes **invalid parameter validation**, **invalid request bodies**, or **HTTP 4xx** validation against a live server:
+
+1. **Do not** author these in `tests/e2e/online_serving/test_*.py` or `*_expansion.py`. **Move** any drafted `test_*` into `tests/dfx/reliability/invalid_param_test/`.
+2. **Pick script by route** (extend in-tree file):
+
+| Route family | Script |
+|--------------|--------|
+| Omni chat / WS video·realtime | `test_invalid_omni_chat.py` |
+| `/v1/audio/speech` (+ stream / batch / voices) | `test_invalid_audio_speech.py` |
+| Audio diffusion | `test_invalid_audio_diffusion.py` |
+| `/v1/images/generations` | `test_invalid_image_generation.py` |
+| `/v1/images/edits` | `test_invalid_image_editing.py` |
+| `/v1/videos*` | `test_invalid_video_generation.py` |
+| Sleep / wakeup / server control | `test_invalid_server_control.py` |
+
+3. **Style:** `pytestmark = [pytest.mark.slow, pytest.mark.<type>]`; `_PARAMS` + `hardware_marks`; `send_*_http_request` with `err_code` + `err_message`; parametrized `body_spec` rows with `id=`; `_minimal_*_json()` helpers; `_SKIP_ISSUE_3649` when tracked in [#3649](https://github.com/vllm-project/vllm-omni/issues/3649).
+4. **CI:** [`.buildkite/cuda/test-weekly.yml`](../../../../.buildkite/cuda/test-weekly.yml) group **Reliability Test - Invalid parameters Test** — **not** ready/merge/nightly.
+
+```bash
+# Weekly H100 (diffusion / omni / video invalid-param)
+cd tests
+pytest -s -v dfx/reliability/invalid_param_test/ -m "slow and H100"
+
+# Weekly L4 (e.g. Qwen3-TTS speech invalid-param)
+pytest -s -v dfx/reliability/invalid_param_test/ -m "slow and L4"
+
+# Single script / case
+pytest -s -v dfx/reliability/invalid_param_test/test_invalid_image_generation.py::test_images_generations_invalid_requests -m "slow and H100"
+```
+
+**Trigger:** `WEEKLY=1` or PR label `weekly-test`. Appending cases to existing scripts usually needs **no YAML edit** (directory sweep). No `source_file_dependencies` on weekly steps.
+
+### Platform-targeted examples
+
+```bash
+cd tests
+pytest -s -v -m "core_model and distributed_cuda and L4" --run-level=core_model
+```
+
+### Concrete e2e paths (common in-tree)
+
+Paths are relative to `tests/` after `cd tests`.
+
+| Scenario | Example command |
+|----------|------------------|
+| **Omni** offline L2 | `pytest -s -v e2e/offline_inference/test_qwen2_5_omni.py -m "core_model and omni and not cpu" --run-level=core_model` |
+| **Omni** offline L2 — one test | `pytest -s -v e2e/offline_inference/test_qwen2_5_omni.py::test_text_to_text -m "core_model and omni and not cpu" --run-level=core_model` |
+| **Omni** online L2 | `pytest -s -v e2e/online_serving/test_qwen3_omni.py -m "core_model and omni" --run-level=core_model` |
+| **Omni** offline L2 (Qwen3.5-9B VL) | `pytest -s -v e2e/offline_inference/test_qwen3_5_9b.py -m "core_model and omni and not cpu" --run-level=core_model` |
+| **TTS** online L2 | `pytest -s -v e2e/online_serving/test_qwen3_tts_base.py -m "core_model and tts" --run-level=core_model` |
+| **TTS** online L2 — one test | `pytest -s -v e2e/online_serving/test_qwen3_tts_base.py::test_text_to_audio_001 -m "core_model and tts" --run-level=core_model` |
+| **Diffusion X2I** offline L2 | `pytest -s -v e2e/offline_inference/test_t2i_model.py -m "core_model and diffusion and not cpu" --run-level=core_model` |
+| **Diffusion X2I** L2 online smoke | `pytest -s -v e2e/online_serving/test_qwen_image.py e2e/online_serving/test_bagel.py -m "core_model and diffusion" --run-level=core_model` |
+| **Diffusion X2V** L2 online smoke | `pytest -s -v e2e/online_serving/test_wan22_t2v.py -m "core_model and diffusion" --run-level=core_model` |
+| **Diffusion X2I** L4 online expansion (H100) | `pytest -s -v e2e/online_serving/test_qwen_image_expansion.py -m "full_model and diffusion and H100" --run-level=full_model` |
+| **Diffusion X2I** L4 perf (nightly) | `pytest -s -v dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file dfx/perf/tests/test_qwen_image_vllm_omni.json` |
+| **Diffusion X2I** L4 online expansion (Edit) | `pytest -s -v e2e/online_serving/test_qwen_image_edit_expansion.py -m "full_model and diffusion and H100" --run-level=full_model` |
+| **Diffusion X2V** L4 nightly | `pytest -s -v e2e/online_serving/test_wan22_expansion.py -m "full_model and cuda" --run-level=full_model` |
+| **Invalid param** weekly H100 | `pytest -s -v dfx/reliability/invalid_param_test/ -m "slow and H100"` |
+| **Invalid param** weekly L4 | `pytest -s -v dfx/reliability/invalid_param_test/ -m "slow and L4"` |
+
+### Agent / author completion checklist
+
+When adding or modifying tests, do not stop at “where the file lives” — also deliver:
+
+1. **Local**: `cd tests` + `pytest -s -v <path>` (and `path::test_func` when a single case is enough).
+2. **CI-like**: marker string + `--run-level` matching the pipeline (`core_model` / `advanced_model` / `full_model`).
+3. **L1 mocks**: `mocker` / `monkeypatch` only; no `unittest.mock`.
+4. **API client + assert placement**: **General** validation → implement in `assertions.py`, call **inside** `send_*_request` in `runtime.py`; tests call **`send_*_request` only**. **Special** case validation → `assert_*` in `assertions.py`, called **in the test** after `send_*_request`. Low-level `send_*_http_request` is for negative/dfx tests (`err_code`), not ordinary L2+ success e2e.
+5. **Shared assertions**: logic in **`tests/helpers/assertions.py`**; general checks inside `send_*_request`; special checks only in the test. No `_assert_*` in `test_*.py`.
+6. **One case → one `test_*`**: function name reflects what is validated; no `if case_id == ...` mega-test merging multiple scenarios.
+7. **Fixture scope**: default **`omni_server` + `online_client`** / **`omni_runner` + `offline_client`** (module). Use **`omni_server_function` + `online_client_function`** / **`omni_runner_function` + `offline_client_function`** only when each `test_*` must spawn a fresh instance. Parametrize name must match fixture (`omni_server` vs `omni_server_function`).
+8. **Type marker**: `omni`, `tts`, or `diffusion` on every model e2e module.
+9. **Diffusion L4 Function**: wire `*_expansion.py` into **X2I(&A&T)** or **X2V** **Function Test** in **`test-nightly.yml` only** — do not add `test-merge.yml` unless the user also requested L3.
+10. **Diffusion L4 Perf** (only when requested): add `tests/dfx/perf/tests/test_<slug>_vllm_omni.json` + **Perf Test · &lt;Model&gt;** step (artifact upload); not part of “L4 functional cases” by default.
+11. **E2E Buildkite (L2/L3 only)**: `test-ready.yml` / `test-merge.yml` steps need **`source_file_dependencies`** + full **`agents` + `plugins`**. **L4 nightly** uses explicit file lists or perf scripts in `test-nightly.yml` (no merge job).
+12. **Invalid param**: cases in **`tests/dfx/reliability/invalid_param_test/`** (route-matching script), `send_*_http_request` + `err_code`, `pytest.mark.slow` + `H100`/`L4`; CI = **`test-weekly.yml`** only — do not put in e2e or nightly.
+13. **Prerequisites**: GPU tier, HF cache/token, and any module `skipif` / platform-only YAML.
+
+## Buildkite pipeline mapping
+
+| Level | Repo file | Model-type grouping |
+|-------|-----------|---------------------|
+| L1, L2 | [`.buildkite/cuda/test-ready.yml`](../../../../.buildkite/cuda/test-ready.yml) | Steps prefixed **Omni ·**, **TTS ·**, **Diffusion ·** under **E2E Test** — **`source_file_dependencies` required** |
+| L3 | [`.buildkite/cuda/test-merge.yml`](../../../../.buildkite/cuda/test-merge.yml) | Per-model E2E steps; `-m "advanced_model and …"` — **`source_file_dependencies` required** |
+| L4 | [`.buildkite/cuda/test-nightly.yml`](../../../../.buildkite/cuda/test-nightly.yml) | **Omni / TTS / Diffusion X2I(&A&T) / X2V** — each group may have **Function**, **Accuracy**, **Perf**, **Doc** steps |
+| Invalid param (weekly) | [`.buildkite/cuda/test-weekly.yml`](../../../../.buildkite/cuda/test-weekly.yml) | **Invalid parameters Test · H100** / **· L4** — sweeps `tests/dfx/reliability/invalid_param_test/` |
+
+### `source_file_dependencies` (E2E Test only — ready & merge)
+
+Required on each step inside the **E2E Test** group in `test-ready.yml` and `test-merge.yml`. Typical entries:
+
+| Category | Path pattern |
+|----------|----------------|
+| Test module | `tests/e2e/online_serving/test_<slug>.py`, `tests/e2e/offline_inference/test_<slug>.py` |
+| API client helpers | `tests/helpers/runtime.py` (when the e2e job uses newly added `send_*` methods) |
+| Shared assert helpers | `tests/helpers/assertions.py` (when the e2e job uses newly added `assert_*` helpers) |
+| AR / omni model | `vllm_omni/model_executor/models/<family>/` |
+| Diffusion model | `vllm_omni/diffusion/models/<family>/` |
+| Stage processor | `vllm_omni/model_executor/stage_input_processors/<family>.py` |
+| Deploy config | `vllm_omni/deploy/<name>.yaml` or `vllm_omni/deploy/ci/<name>.yaml` |
+
+Copy the dependency block from the closest in-tree E2E step for the same model family. Update the list whenever the pytest command or server YAML changes.
+
+### E2E `agents` + `plugins` blocks (required — do not truncate)
+
+When generating or documenting E2E steps, **always include the complete `plugins` section**. Set **`timeout_in_minutes`** on the step for the job deadline; run **`pytest` directly** in `commands` (no `timeout 40m bash -c` wrapper around pytest).
+
+Common patterns in `test-ready.yml` / `test-merge.yml`:
+
+| Queue | Plugin | Required fields |
+|-------|--------|-----------------|
+| `mithril-h100-pool` | `kubernetes` | `resources.limits.nvidia.com/gpu`, `volumeMounts` (`devshm`, `hf-cache`), `env` (`HF_HOME`, `HF_TOKEN` via `secretKeyRef`), `nodeSelector: gpu-h100-sxm`, `volumes` (`devshm` emptyDir, `hf-cache` hostPath `/mnt/hf-cache`) |
+| `gpu_1_queue` / `gpu_4_queue` | `docker#v5.2.0` | `image`, `always-pull`, `propagate-environment`, `shm-size: "8gb"`, `environment: HF_HOME`, `volumes: /fsx/hf_cache` (add `HF_TOKEN` when the step needs hub access) |
+
+**H100 kubernetes template** (copy from **Diffusion · Bagel Test** / **Diffusion · Qwen Image Test** in-tree):
+
+```yaml
+  agents:
+    queue: "mithril-h100-pool"
+  plugins:
+    - kubernetes:
+        podSpec:
+          containers:
+            - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+              resources:
+                limits:
+                  nvidia.com/gpu: 1
+              volumeMounts:
+                - name: devshm
+                  mountPath: /dev/shm
+                - name: hf-cache
+                  mountPath: /root/.cache/huggingface
+              env:
+                - name: HF_HOME
+                  value: /root/.cache/huggingface
+                - name: HF_TOKEN
+                  valueFrom:
+                    secretKeyRef:
+                      name: hf-token-secret
+                      key: token
+          nodeSelector:
+            node.kubernetes.io/instance-type: gpu-h100-sxm
+          volumes:
+            - name: devshm
+              emptyDir:
+                medium: Memory
+            - name: hf-cache
+              hostPath:
+                path: /mnt/hf-cache
+                type: DirectoryOrCreate
+```
+
+**L4 docker template** (copy from **TTS · Qwen3-TTS Base Test** in `test-merge.yml`):
+
+```yaml
+  agents:
+    queue: "gpu_1_queue"
+  plugins:
+    - docker#v5.2.0:
+        image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+        always-pull: true
+        propagate-environment: true
+        shm-size: "8gb"
+        environment:
+          - "HF_HOME=/fsx/hf_cache"
+        volumes:
+          - "/fsx/hf_cache:/fsx/hf_cache"
+```
+
+When extending an existing step (e.g. add `tests/e2e/offline_inference/test_qwen_image.py` to **Diffusion · Qwen Image Test**), update `source_file_dependencies` and `commands` only; **keep the existing `agents` + `plugins` block unchanged** unless the hardware tier changes.
+
+[`.buildkite/cuda/pipeline.yml`](../../../../.buildkite/cuda/pipeline.yml) decides **which** child file is uploaded. To run **L3** on a feature branch, comment out `if: build.branch == "main"` on the merge upload step. To run **L4** without `NIGHTLY=1`, comment out the nightly upload step’s `if: build.env("NIGHTLY") == "1"` and the same `if` on relevant steps inside `test-nightly.yml`. Revert such edits before merging.
+
+Platform-specific pipelines (e.g. AMD) follow the same level → file pairing under `.buildkite/`.
+
+## Diffusion RFC (#1832) Alignment Tips
+
+For **X2I(&A&T)** coverage planning:
+
+- Prioritize high-value feature combinations with minimal case count.
+- Split into lightweight L1/L2 validation plus L4 `*_expansion.py` under the **X2I** nightly shards.
+- **X2V** models (Wan, HunyuanVideo) stay in the **X2V** group — do not merge into X2I sweeps.
+
+If hardware is insufficient, provide an executable reduced case plus a deferred full CI/nightly plan.

.github/CODEOWNERS

@@ -1,6 +1,11 @@
 # CODEOWNERS is used to automatically assign PR reviewers
 # Format: path pattern @username or @team
 # More info: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners
+#
+# The vllm_omni/** source-tree rules below are derived from the `owners` and
+# `primary_code_paths` front matter in docs/design/module/*.md. When updating
+# ownership, update the design doc first, then regenerate the matching rule here.
+# Last-match-wins: more specific rules must come AFTER broader ones.
 
 # Default owners (fallback reviewers for all files)
 * @Gaohan123 @hsliuustc0106 @ywang96
@@ -16,51 +21,126 @@
 /.claude/ @hsliuustc0106
 
 # Tests
-/tests/ @yenuo26
+/tests/ @yenuo26 @NickCao
 
 # CI
-/.buildkite/ @yenuo26 @congw729
+/.buildkite/ @yenuo26 @congw729 @NickCao
 
-# Engine and model_executor
-/vllm_omni/core/ @tzhouam @yinpeiqi @fake0fan
-/vllm_omni/engine/ @tzhouam @yinpeiqi @fake0fan
-/vllm_omni/entrypoints/ @tzhouam @yinpeiqi @fake0fan
-/vllm_omni/model_executor/ @tzhouam @gcanlin
-/vllm_omni/worker/ @tzhouam @gcanlin
+# ComfyUI / external apps
+/apps/ @fhfuih @alex-jw-brooks
 
-# Diffusion
-/vllm_omni/diffusion/ @Isotr0py @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa @david6666666
-/vllm_omni/lora/ @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa @yuanheng-zhao
-
-# Diffusion CPU offloading
-/vllm_omni/diffusion/offloader/ @Isotr0py @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa @yuanheng-zhao
-
-# Omni and TTS support
-/vllm_omni/transformers_utils/ @tzhouam @yuanheng-zhao
-/vllm_omni/model_executor/models/ @linyueqian @ZeldaHuang @princepride @yuanheng-zhao @amy-why-3459 @Bounty-hunter
-
-# Benchmarks
+# Benchmarks (out-of-tree)
 /benchmarks/accuracy/ @david6666666
 /benchmarks/build_dataset/ @linyueqian @yuanheng-zhao
-/benchmarks/diffusion/ @Isotr0py @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa
+/benchmarks/diffusion/ @Isotr0py @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa @xuechendi
 /benchmarks/distributed/ @princepride
 /benchmarks/glm_image/ @princepride @JaredforReal
-/benchmarks/tts/ @linyueqian @ZeldaHuang @princepride
+/benchmarks/tts/ @linyueqian @ZeldaHuang @princepride @Sy0307 @gcanlin
 
-# Usability and Observability
-/vllm_omni/deploy/ @tzhouam @lishunyang12
-/vllm_omni/metrics/ @lishunyang12
-/vllm_omni/profiler/ @gcanlin
+# =====================================================================
+# Module design ownership (docs/design/module/*.md)
+# Each section header cites the source design doc.
+# =====================================================================
 
-# ComfyUI
-/apps/ @fhfuih
+# Autoregressive runtime — docs/design/module/ar_runtime.md
+/vllm_omni/core/ @tzhouam @yinpeiqi @fake0fan @Sy0307 @Gaohan123
+/vllm_omni/worker/ @tzhouam @yinpeiqi @fake0fan @Sy0307 @Gaohan123
 
-# Large Scale Serving
+# Engine orchestration & stage runtime — docs/design/module/engine_orchestration.md, stage_runtime.md
+/vllm_omni/engine/ @tzhouam @fake0fan
+# File-level overrides: input/output modality contracts own these engine files
+# — docs/design/module/input_output_modality_contracts.md
+/vllm_omni/engine/messages.py @Sy0307 @amy-why-3459 @Gaohan123 @alex-jw-brooks
+/vllm_omni/engine/serialization.py @Sy0307 @amy-why-3459 @Gaohan123 @alex-jw-brooks
+/vllm_omni/engine/mm_outputs.py @Sy0307 @amy-why-3459 @Gaohan123 @alex-jw-brooks
+/vllm_omni/engine/output_modality.py @Sy0307 @amy-why-3459 @Gaohan123 @alex-jw-brooks
+/vllm_omni/engine/output_processor.py @Sy0307 @amy-why-3459 @Gaohan123 @alex-jw-brooks
+
+# Input/output modality contracts — docs/design/module/input_output_modality_contracts.md
+/vllm_omni/inputs/ @Sy0307 @amy-why-3459 @Gaohan123 @alex-jw-brooks
+/vllm_omni/outputs/ @Sy0307 @amy-why-3459 @Gaohan123 @alex-jw-brooks
+/vllm_omni/request.py @Sy0307 @amy-why-3459 @Gaohan123 @alex-jw-brooks
+/vllm_omni/data_entry_keys.py @Sy0307 @amy-why-3459 @Gaohan123 @alex-jw-brooks
+
+# Entrypoints and serving boundaries — docs/design/module/entrypoints.md
+/vllm_omni/entrypoints/ @alex-jw-brooks @linyueqian @NickCao
+# File-level overrides within entrypoints/
+/vllm_omni/entrypoints/openai/errors.py @alex-jw-brooks @NickCao
+/vllm_omni/entrypoints/cli/benchmark/ @alex-jw-brooks @Bounty-hunter
+
+# Error contracts — docs/design/module/error_contracts.md
+/vllm_omni/errors.py @alex-jw-brooks @NickCao
+
+# OmniConnector (inter-stage transport) — docs/design/module/omni_connector.md
 /vllm_omni/distributed/ @princepride @yuanheng-zhao @xuechendi @natureofnature @fake0fan
 
-# Quantization and configuration
-/vllm_omni/config/ @lishunyang12
+# Model integration — docs/design/module/model_integration.md
+/vllm_omni/model_executor/ @tzhouam @gcanlin
+/vllm_omni/model_executor/models/ @tzhouam @gcanlin @Sy0307 @amy-why-3459 @linyueqian
+/vllm_omni/model_extras/ @tzhouam @gcanlin @princepride
+/vllm_omni/plugins/ @gcanlin @tjtanaa @xuechendi
+
+# Execution platforms (hardware backends) — docs/design/module/execution_platforms.md
+/vllm_omni/platforms/ @gcanlin @tjtanaa @xuechendi
+/vllm_omni/attention/ @gcanlin @tjtanaa @xuechendi @david6666666 @lishunyang12
+
+# Configuration & deploy — docs/design/module/vllm_omni_config.md
+/vllm_omni/config/ @lishunyang12 @alex-jw-brooks
+/vllm_omni/deploy/ @lishunyang12 @alex-jw-brooks
+
+# Quantization — docs/design/module/quantization.md
 /vllm_omni/quantization/ @david6666666 @Isotr0py @lishunyang12
 
-# Hardware plugins and accelerators
-/vllm_omni/platforms/ @gcanlin @tjtanaa @xuechendi
+# Observability — docs/design/module/observability.md
+/vllm_omni/metrics/ @lishunyang12 @vraiti
+
+# Profiling — docs/design/module/profiling.md
+/vllm_omni/profiler/ @gcanlin
+
+# Benchmarking (in-tree) — docs/design/module/benchmarking.md
+/vllm_omni/benchmarks/ @alex-jw-brooks @Bounty-hunter
+
+# LoRA (no dedicated design doc; ownership intentionally narrowed)
+/vllm_omni/lora/ @SamitHuang @wtomin
+
+# Transformers utils (related path only in design docs; preserves prior ownership)
+/vllm_omni/transformers_utils/ @tzhouam @yuanheng-zhao
+
+# =====================================================================
+# Diffusion module family — docs/design/module/diffusion/
+# Umbrella rule first; sub-module rules after it win via last-match.
+# =====================================================================
+
+# Diffusion umbrella (top-level files: data.py, forward_context.py, etc.)
+# — docs/design/module/diffusion/index.md
+/vllm_omni/diffusion/ @Isotr0py @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa @david6666666 @xuechendi
+
+# Cache management — docs/design/module/cache_management.md
+/vllm_omni/diffusion/cache/ @Isotr0py @princepride @SamitHuang
+
+# Diffusion quantization — docs/design/module/quantization.md
+/vllm_omni/diffusion/quantization/ @david6666666 @Isotr0py @lishunyang12
+
+# Diffusion profiling — docs/design/module/profiling.md
+/vllm_omni/diffusion/profiler/ @gcanlin
+
+# Diffusion runtime (scheduler, executor, worker)
+# — docs/design/module/diffusion/diffusion_runtime.md, continuous_batching.md
+/vllm_omni/diffusion/sched/ @Isotr0py @princepride @SamitHuang @fhfuih
+/vllm_omni/diffusion/executor/ @Isotr0py @princepride @SamitHuang @fhfuih
+/vllm_omni/diffusion/worker/ @Isotr0py @princepride @SamitHuang @fhfuih
+
+# Diffusion model integration (pipelines, registration, checkpoint loading)
+# — docs/design/module/diffusion/diffusion_model_integration.md
+/vllm_omni/diffusion/models/ @fhfuih @Bounty-hunter @wtomin
+/vllm_omni/diffusion/model_loader/ @fhfuih @Bounty-hunter @wtomin @RuixiangMa
+/vllm_omni/diffusion/lora/ @fhfuih @Bounty-hunter @wtomin @RuixiangMa
+
+# Diffusion parallelism (rank topology, TP/SP/PP/VAE patch)
+# — docs/design/module/diffusion/parallelism.md
+/vllm_omni/diffusion/distributed/ @princepride @yuanheng-zhao @xuechendi @Bounty-hunter
+/vllm_omni/diffusion/attention/parallel/ @princepride @yuanheng-zhao @xuechendi @Bounty-hunter
+
+# Diffusion offloader (CPU offloading, DLO)
+# — docs/design/module/diffusion/offloader.md
+/vllm_omni/diffusion/offloader/ @yuanheng-zhao @david6666666 @lishunyang12