feat(diffusion): add QwenImage spatial-shard decode

Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com>

.buildkite/amd/scripts/bootstrap-amd-omni.sh

@@ -6,6 +6,8 @@
 
 set -euo pipefail
 
+source .buildkite/common/scripts/resolve_skip_ci.sh
+
 if [[ -z "${RUN_ALL:-}" ]]; then
     RUN_ALL=0
 fi
@@ -22,10 +24,6 @@ if [[ -z "${AMD_MIRROR_HW:-}" ]]; then
     AMD_MIRROR_HW="amdproduction"
 fi
 
-if [[ -z "${DOCS_ONLY_DISABLE:-}" ]]; then
-    DOCS_ONLY_DISABLE=0
-fi
-
 fail_fast() {
     DISABLE_LABEL="ci-no-fail-fast"
     # If BUILDKITE_PULL_REQUEST != "false", then we check the PR labels using curl and jq
@@ -56,88 +54,6 @@ check_run_all_label() {
     fi
 }
 
-is_docs_only_change() {
-    local file_path
-    local has_any=0
-
-    while IFS= read -r file_path; do
-        [[ -z "${file_path}" ]] && continue
-        has_any=1
-
-        if [[ "${file_path}" == docs/* ]]; then
-            continue
-        fi
-        if [[ "${file_path}" == *.md ]]; then
-            continue
-        fi
-        if [[ "${file_path}" == "mkdocs.yml" ]]; then
-            continue
-        fi
-        return 1
-    done
-
-    [[ "${has_any}" -eq 1 ]]
-}
-
-resolve_skip_ci() {
-    local is_pr_build=0
-    local files
-    local base_branch base_ref
-
-    if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" && -n "${BUILDKITE_PULL_REQUEST:-}" ]]; then
-        is_pr_build=1
-    fi
-
-    if [[ "${is_pr_build}" -eq 1 ]]; then
-        base_branch="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}"
-        if ! git rev-parse --verify "origin/${base_branch}" >/dev/null 2>&1; then
-            echo "resolve_skip_ci: origin/${base_branch} not found locally; trying fetch" >&2
-            git fetch --depth=200 origin "${base_branch}" >/dev/null 2>&1 || true
-        fi
-
-        base_ref=""
-        if git rev-parse --verify "origin/${base_branch}" >/dev/null 2>&1; then
-            base_ref="origin/${base_branch}"
-        elif git rev-parse --verify "${base_branch}" >/dev/null 2>&1; then
-            base_ref="${base_branch}"
-        else
-            echo "resolve_skip_ci: cannot resolve PR base ${base_branch}; skip-ci=0" >&2
-            echo -n 0
-            return 0
-        fi
-
-        if ! files="$(git diff --name-only "${base_ref}...${BUILDKITE_COMMIT}" 2>/dev/null)"; then
-            echo "resolve_skip_ci: failed to compute PR changed files; skip-ci=0" >&2
-            echo -n 0
-            return 0
-        fi
-    elif [[ "${BUILDKITE_BRANCH:-}" == "main" ]]; then
-        if ! git rev-parse --verify "${BUILDKITE_COMMIT}^" >/dev/null 2>&1; then
-            echo "resolve_skip_ci: commit has no parent on main; skip-ci=0" >&2
-            echo -n 0
-            return 0
-        fi
-        if ! files="$(git diff --name-only "${BUILDKITE_COMMIT}^..${BUILDKITE_COMMIT}" 2>/dev/null)"; then
-            echo "resolve_skip_ci: failed to compute main changed files; skip-ci=0" >&2
-            echo -n 0
-            return 0
-        fi
-    else
-        echo "resolve_skip_ci: not PR/main build; skip-ci=0" >&2
-        echo -n 0
-        return 0
-    fi
-
-    if is_docs_only_change <<< "${files}"; then
-        echo "resolve_skip_ci: docs-only change detected; skip-ci=1" >&2
-        echo -n 1
-        return 0
-    fi
-
-    echo "resolve_skip_ci: non-doc changes detected; skip-ci=0" >&2
-    echo -n 0
-}
-
 if [[ -z "${COV_ENABLED:-}" ]]; then
     COV_ENABLED=0
 fi
@@ -154,14 +70,14 @@ upload_pipeline() {
     fi
 
     # Use local template file for vllm-omni
-    cp .buildkite/test-template-amd-omni.j2 .buildkite/test-template.j2
+    cp .buildkite/amd/test-template-amd-omni.j2 .buildkite/amd/test-template.j2
 
 
     # (WIP) Use pipeline generator instead of jinja template
-    if [ -e ".buildkite/pipeline_generator/pipeline_generator.py" ]; then
+    if [ -e ".buildkite/amd/pipeline_generator/pipeline_generator.py" ]; then
         python -m pip install click pydantic
-        python .buildkite/pipeline_generator/pipeline_generator.py --run_all=$RUN_ALL --list_file_diff="$LIST_FILE_DIFF" --nightly="$NIGHTLY" --mirror_hw="$AMD_MIRROR_HW"
-        buildkite-agent pipeline upload .buildkite/pipeline.yaml
+        python .buildkite/amd/pipeline_generator/pipeline_generator.py --run_all=$RUN_ALL --list_file_diff="$LIST_FILE_DIFF" --nightly="$NIGHTLY" --mirror_hw="$AMD_MIRROR_HW"
+        buildkite-agent pipeline upload .buildkite/amd/pipeline.yaml
         exit 0
     fi
     echo "List file diff: $LIST_FILE_DIFF"
@@ -171,7 +87,7 @@ upload_pipeline() {
 
     FAIL_FAST=$(fail_fast)
 
-    cd .buildkite
+    cd .buildkite/amd
 
     # Select test definition file: merge suite for main, ready suite for PRs.
     # For debugging, DEBUG_TEST_YAML can override the selection — accepts
@@ -182,7 +98,7 @@ upload_pipeline() {
                 TEST_YAML="test-amd-merge.yml"
                 ;;
             ready)
-                TEST_YAML="test-amd-ready.yaml"
+                TEST_YAML="test-amd-ready.yml"
                 ;;
             *)
                 echo "ERROR: DEBUG_TEST_YAML must be 'merge' or 'ready', got '$DEBUG_TEST_YAML'" >&2
@@ -193,7 +109,7 @@ upload_pipeline() {
     elif [[ $BUILDKITE_BRANCH == "main" ]]; then
         TEST_YAML="test-amd-merge.yml"
     else
-        TEST_YAML="test-amd-ready.yaml"
+        TEST_YAML="test-amd-ready.yml"
     fi
 
     (
@@ -234,23 +150,13 @@ if [[ $BUILDKITE_BRANCH == "main" ]]; then
     file_diff=$(get_diff_main)
 fi
 
-# ----------------------------------------------------------------------
-# Early exit start: skip pipeline if conditions are met
-# ----------------------------------------------------------------------
-
-# Match CUDA Buildkite skip-ci behavior for docs/markdown-only changes.
-if [[ "${DOCS_ONLY_DISABLE}" != "1" ]]; then
-    SKIP_CI="$(resolve_skip_ci)"
-    if [[ "${SKIP_CI}" == "1" ]]; then
-        echo "[docs-only] Docs/markdown-only changes detected. Exiting before pipeline upload."
-        exit 0
-    fi
+# Early exit: unified skip-ci (docs / skip marks) and CI-yaml-only level targeting.
+if [[ $BUILDKITE_BRANCH == "main" ]]; then
+    gate_bootstrap_ci amd l3
+else
+    gate_bootstrap_ci amd l2
 fi
 
-# ----------------------------------------------------------------------
-# Early exit end
-# ----------------------------------------------------------------------
-
 patterns=(
     "docker/Dockerfile"
     "CMakeLists.txt"

.buildkite/amd/scripts/run-amd-test.sh

@@ -56,7 +56,7 @@ container_name="rocm_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | hea
 # # Install AWS CLI to authenticate to ECR Public Gallery to get higher rate limit for pulling images
 # sudo apt-get update && sudo apt-get install -y awscli
 ##  Use safe docker login helper to prevent race conditions
-# source "$(dirname "${BASH_SOURCE[0]}")/../docker_login_ecr_public.sh"
+# source "$(dirname "${BASH_SOURCE[0]}")/../../common/scripts/docker_login_ecr_public.sh"
 # safe_docker_login_ecr_public
 
 ## Pull the container from AMD Docker Hub
@@ -64,7 +64,8 @@ container_name="rocm_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | hea
 docker pull "${image_name}"
 
 remove_docker_container() {
-   docker rm -f "${container_name}" || docker image rm -f "${image_name}" || true
+   docker ps -aq --filter "name=${container_name}" | xargs -r docker rm -f || true
+   docker image rm -f "${image_name}" || true
 }
 trap remove_docker_container EXIT
 
@@ -107,7 +108,6 @@ if [[ $commands == *"--shard-id="* ]]; then
         --network=host \
         --shm-size=16gb \
         --group-add "$render_gid" \
-        --rm \
         -e MIOPEN_DEBUG_CONV_DIRECT=0 \
         -e MIOPEN_DEBUG_CONV_GEMM=0 \
         -e VLLM_ROCM_USE_AITER=0 \
@@ -140,7 +140,6 @@ else
           --network=host \
           --shm-size=16gb \
           --group-add "$render_gid" \
-          --rm \
           -e MIOPEN_DEBUG_CONV_DIRECT=0 \
           -e MIOPEN_DEBUG_CONV_GEMM=0 \
           -e VLLM_ROCM_USE_AITER=0 \

.buildkite/amd/test-amd-merge.yml

@@ -0,0 +1,233 @@
+env:
+  VLLM_WORKER_MULTIPROC_METHOD: spawn
+  HF_HUB_DOWNLOAD_TIMEOUT: 300
+  HF_HUB_ETAG_TIMEOUT: 60
+
+steps:
+
+- group: ":card_index_dividers: Simple Test"
+  steps:
+    - label: "Simple · Diffusion & Model Executor Test (1/2)"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      timeout_in_minutes: 70
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
+        # that is not supported on CPU
+        # CPU tests on ROCm gpu machines are very slow, so we give them more time.
+        - "timeout 60m pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+
+    - label: "Simple · Diffusion & Model Executor Test (2/2)"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      timeout_in_minutes: 70
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        # CPU tests on ROCm gpu machines are very slow, so we give them more time.
+        - "timeout 60m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+
+    - label: "Simple · Other Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      timeout_in_minutes: 50
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
+        # that is not supported on CPU
+        - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+
+- group: ":card_index_dividers: Diffusion Test"
+  steps:
+    - label: "Diffusion · LoRA Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - timeout 20m pytest -s -v tests/diffusion/lora/test_images_generations_lora.py
+
+    - label: "Diffusion Model CPU offloading Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - export VLLM_LOGGING_LEVEL=DEBUG
+        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+        - |
+          timeout 20m bash -c '
+            set +e
+            pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
+            EXIT1=\$?
+            pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
+            EXIT2=\$?
+            exit \$((EXIT1 | EXIT2))
+          '
+
+    - label: "Diffusion Sequence Parallelism Test (Need 4 GPUs)"
+      agent_pool: mi325_4
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - export VLLM_LOGGING_LEVEL=DEBUG
+        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+        - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py
+        - timeout 20m pytest -s -v tests/diffusion/distributed/test_ulysses_uaa_perf.py
+
+    # merge-only tests
+    - label: "Diffusion Tensor Parallelism Test"
+      agent_pool: mi325_2
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - pytest -s -v tests/diffusion/distributed/test_tensor_parallel.py
+
+    - label: "Diffusion Model Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - timeout 30m pytest -s -v tests/e2e/offline_inference/test_z_image.py
+
+    - label: "Diffusion GPU Worker Test"
+      agent_pool: mi325_2
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - timeout 20m pytest -s -v tests/diffusion/test_diffusion_worker.py
+
+    - label: "Diffusion · Wan22 Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - |
+          timeout 40m bash -c "
+            export VLLM_IMAGE_FETCH_TIMEOUT=60
+            pytest -s -v tests/e2e/offline_inference/test_wan22_t2v.py tests/e2e/online_serving/test_wan22_t2v.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
+          "
+
+## ISSUE depends on `diffusers` package: https://github.com/huggingface/diffusers/issues/13274
+# - label: "Audio Generation Model Test"
+#   agent_pool: mi325_1
+#   depends_on: amd-build
+#   mirror_hardwares: [amdproduction]
+#   grade: Blocking
+#   commands:
+#     - export GPU_ARCHS=gfx942
+#     - export VLLM_LOGGING_LEVEL=DEBUG
+#     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+#     - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_expansion.py -m "advanced_model and diffusion and L4" --run-level advanced_model
+
+- label: "Engine Test"
+  agent_pool: mi325_1
+  depends_on: amd-build
+  mirror_hardwares: [amdproduction]
+  grade: Blocking
+  commands:
+    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+    - timeout 20m pytest -s -v tests/engine/test_async_omni_engine_abort.py
+
+# This is from test-nightly.yml.
+# This model seems to be important, so I am trying to run it here.
+# Since this is from test-nightly.yml,
+# I keep this non-blocking.
+- group: ":card_index_dividers: E2E Test"
+  steps:
+    - label: "Omni Model Test Qwen2-5-Omni"
+      agent_pool: mi325_4
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      commands:
+        - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+        - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model"
+
+    - label: "Omni Model Test Qwen3-Omni"
+      agent_pool: mi325_2
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+        - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+
+    - label: "Qwen3-TTS CustomVoice E2E Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - |
+          timeout 50m bash -c '
+            export VLLM_LOGGING_LEVEL=DEBUG
+            export VLLM_WORKER_MULTIPROC_METHOD=spawn
+            export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+            pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m "advanced_model" --run-level "advanced_model"
+          '
+
+    - label: "Qwen3-TTS Base E2E Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      timeout_in_minutes: 50
+      commands:
+        - |
+          timeout 40m bash -c '
+            export VLLM_LOGGING_LEVEL=DEBUG
+            export VLLM_WORKER_MULTIPROC_METHOD=spawn
+            export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+            pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+          '
+
+# TODO: Bagel test on ROCm is very unstable. @tjtanaa
+# Need to debug before reneable numerical changes across large PRs
+# # split Bagel Model Test with H100 (Real Weights) into three tests
+# - label: "Bagel Text2Img Model Test (1/3)"
+#   agent_pool: mi325_1
+#   depends_on: amd-build
+#   mirror_hardwares: [amdproduction]
+#   grade: Blocking
+#   commands:
+#     - export GPU_ARCHS=gfx942
+#     - export VLLM_LOGGING_LEVEL=DEBUG
+#     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+#     - export VLLM_ROCM_USE_AITER_RMSNORM=0
+#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "advanced_model" --run-level "advanced_model" -k "shared_memory" -k "rocm"
+
+# - label: "Bagel Img2Img Model Test (2/3)"
+#   agent_pool: mi325_1
+#   depends_on: amd-build
+#   mirror_hardwares: [amdproduction]
+#   grade: Blocking
+#   commands:
+#     - export GPU_ARCHS=gfx942
+#     - export VLLM_LOGGING_LEVEL=DEBUG
+#     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+#     - export VLLM_ROCM_USE_AITER_RMSNORM=0
+#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "advanced_model" --run-level "advanced_model" -k "rocm"
+
+# - label: "Bagel Online Serving Test (3/3)"
+#   agent_pool: mi325_1
+#   depends_on: amd-build
+#   mirror_hardwares: [amdproduction]
+#   grade: Blocking
+#   commands:
+#     - export GPU_ARCHS=gfx942
+#     - export VLLM_IMAGE_FETCH_TIMEOUT=60
+#     - export VLLM_LOGGING_LEVEL=DEBUG
+#     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+#     - export VLLM_ROCM_USE_AITER_RMSNORM=0
+#     - timeout 40m pytest -s -v tests/e2e/online_serving/test_bagel.py -m "advanced_model" --run-level "advanced_model" -k "rocm"

.buildkite/amd/test-amd-ready.yml

@@ -0,0 +1,223 @@
+env:
+  VLLM_WORKER_MULTIPROC_METHOD: spawn
+  HF_HUB_DOWNLOAD_TIMEOUT: 300
+  HF_HUB_ETAG_TIMEOUT: 60
+
+steps:
+
+- group: ":card_index_dividers: Simple Test"
+  steps:
+    - label: "Simple · Diffusion & Model Executor Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
+        # that is not supported on CPU
+        - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+
+    - label: "Simple · Other Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
+        # that is not supported on CPU
+        - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+
+- label: "Custom Pipeline Test"
+  agent_pool: mi325_1
+  depends_on: amd-build
+  mirror_hardwares: [amdproduction]
+  grade: Blocking
+  commands:
+    - timeout 20m pytest -s -v tests/e2e/features/custom_pipeline/ -m "core_model"
+
+
+- label: "Voxtral TTS CUDA Unit Test"
+  agent_pool: mi325_1
+  depends_on: amd-build
+  mirror_hardwares: [amdproduction]
+  grade: Blocking
+  commands:
+    - timeout 10m pytest -s -v tests/model_executor/models/voxtral_tts/test_cuda_graph_acoustic_transformer.py
+
+- group: ":card_index_dividers: Diffusion Test"
+  steps:
+    - label: "Diffusion · Batch Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - timeout 20m pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model"
+
+    - label: "Diffusion · Offloader Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - export VLLM_LOGGING_LEVEL=DEBUG
+        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+        - |
+          timeout 20m bash -c '
+            set +e
+            pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
+            EXIT1=\$?
+            pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
+            EXIT2=\$?
+            exit \$((EXIT1 | EXIT2))
+          '
+
+    - label: "Diffusion · Cache Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - export GPU_ARCHS=gfx942
+        - export VLLM_LOGGING_LEVEL=DEBUG
+        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+        - timeout 15m pytest -s -v -m "core_model and cache and diffusion and not distributed_cuda and L4"
+
+    - label: "Diffusion Sequence Parallelism Test"
+      agent_pool: mi325_2
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - export VLLM_LOGGING_LEVEL=DEBUG
+        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+        - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py -m core_model
+
+    - label: "Diffusion GPU Worker Test"
+      agent_pool: mi325_2
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - timeout 20m pytest -s -v tests/diffusion/test_diffusion_worker.py
+
+    - label: "Diffusion · Model Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model"
+
+    - label: "Diffusion · Wan22 Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - |
+          timeout 40m bash -c "
+            export VLLM_TEST_CLEAN_GPU_MEMORY=1
+            export VLLM_IMAGE_FETCH_TIMEOUT=60
+            pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model' --run-level 'core_model'
+          "
+
+## ISSUE depends on `diffusers` package: https://github.com/huggingface/diffusers/issues/13274
+# - label: "Audio Generation Model Test"
+#   agent_pool: mi325_1
+#   depends_on: amd-build
+#   mirror_hardwares: [amdproduction]
+#   grade: Blocking
+#   commands:
+#     - export GPU_ARCHS=gfx942
+#     - export VLLM_LOGGING_LEVEL=DEBUG
+#     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+#     - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_expansion.py -m "advanced_model and diffusion and L4" --run-level advanced_model
+
+- label: "Engine Test"
+  agent_pool: mi325_1
+  depends_on: amd-build
+  mirror_hardwares: [amdproduction]
+  grade: Blocking
+  commands:
+    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+    - |
+      timeout 15m bash -c '
+              pytest -s -v tests/engine/test_async_omni_engine_abort.py
+      '
+
+# - label: "Omni Model Test Qwen3-Omni"
+#   agent_pool: mi325_2
+#   depends_on: amd-build
+#   mirror_hardwares: [amdproduction]
+#   grade: Blocking
+#   commands:
+#     - export VLLM_LOGGING_LEVEL=DEBUG
+#     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+#     - timeout 10m pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py
+#     - timeout 20m pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m "core_model" --run-level "core_model"
+
+- group: ":card_index_dividers: E2E Test"
+  steps:
+    - label: "Qwen3-TTS E2E Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - export VLLM_LOGGING_LEVEL=DEBUG
+        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+        - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+        - timeout 30m pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m "core_model" --run-level "core_model"
+
+    - label: "CosyVoice3-TTS E2E Test"
+      agent_pool: mi325_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - |
+          timeout 20m bash -c '
+            pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "slow" --run-level "core_model"
+          '
+
+# TODO: Bagel test on ROCm is very unstable. @tjtanaa
+# Need to debug before reneable numerical changes across large PRs
+# - label: "Bagel Text2Img Model Test"
+#   agent_pool: mi325_1
+#   depends_on: amd-build
+#   mirror_hardwares: [amdproduction]
+#   grade: Blocking
+#   commands:
+#     - export GPU_ARCHS=gfx942
+#     - export VLLM_LOGGING_LEVEL=DEBUG
+#     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+#     - export VLLM_ROCM_USE_AITER_RMSNORM=0
+#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"
+
+# - label: "Bagel Img2Img Model Test"
+#   agent_pool: mi325_1
+#   depends_on: amd-build
+#   mirror_hardwares: [amdproduction]
+#   grade: Blocking
+#   commands:
+#     - export GPU_ARCHS=gfx942
+#     - export VLLM_LOGGING_LEVEL=DEBUG
+#     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+#     - export VLLM_ROCM_USE_AITER_RMSNORM=0
+#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"
+
+# - label: "Bagel Online Serving Test"
+#   agent_pool: mi325_1
+#   depends_on: amd-build
+#   mirror_hardwares: [amdproduction]
+#   grade: Blocking
+#   commands:
+#     - export GPU_ARCHS=gfx942
+#     - export VLLM_IMAGE_FETCH_TIMEOUT=60
+#     - export VLLM_LOGGING_LEVEL=DEBUG
+#     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+#     - export VLLM_ROCM_USE_AITER_RMSNORM=0
+#     - timeout 40m pytest -s -v tests/e2e/online_serving/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"

.buildkite/amd/test-template-amd-omni.j2

@@ -0,0 +1,141 @@
+{# vllm-omni customized version
+   Based on: https://github.com/vllm-project/ci-infra/blob/main/buildkite/test-template-amd.j2
+   Last synced: 2025-12-15
+   Modifications: Removed unused CUDA/NVIDIA logic, keeping only AMD tests
+#}
+{% set docker_image_amd = "rocm/vllm-omni:$BUILDKITE_COMMIT" %}
+{% set default_working_dir = "/app/vllm-omni" %}
+
+{% if env %}
+env:
+{% for name in env %}
+  {{ name }}: "{{ env[name] }}"
+{% endfor %}
+{% endif %}
+steps:
+  - group: "AMD Tests"
+    depends_on: ~
+    steps:
+      - label: "AMD: :docker: build image"
+        depends_on: ~
+        soft_fail: false
+        commands:
+          - "docker build -f docker/Dockerfile.rocm -t {{ docker_image_amd }} --target test --build-arg ARG_PYTORCH_ROCM_ARCH=gfx942 --progress plain ."
+          - "docker push {{ docker_image_amd }}"
+        key: "amd-build"
+        env:
+          DOCKER_BUILDKIT: "1"
+        retry:
+          automatic:
+            - exit_status: -1  # Agent was lost
+              limit: 2
+            - exit_status: -10  # Agent was lost
+              limit: 2
+            - exit_status: 128  # Git connectivity issues
+              limit: 2
+            - exit_status: 1  # Machine occasionally fail
+              limit: 1
+        agents:
+          queue: amd-cpu
+
+{% for step_group in steps %}
+{% if step_group.group %}
+  - group: "{{ step_group.group }}"
+{% if step_group.depends_on %}
+    depends_on: {{ step_group.depends_on }}
+{% endif %}
+    steps:
+{% for step in step_group.steps %}
+{% if step.mirror_hardwares and mirror_hw in step.mirror_hardwares %}
+      - label: "{{ step.agent_pool }}: {{ step.label }}"
+        depends_on: amd-build
+        agents:
+          {% if step.agent_pool %}
+          queue: amd_{{ step.agent_pool }}
+          {% else %}
+          queue: amd_mi325_1
+          {% endif %}
+{% set cmd_body = (step.command or (step.commands | join("\n"))) | trim %}
+{% set indented_cmd = cmd_body | replace("\n", "\n            ") %}
+        command: bash .buildkite/amd/scripts/run-amd-test.sh
+        env:
+          DOCKER_BUILDKIT: "1"
+{% if step.env %}
+{% for name in step.env %}
+          {{ name }}: "{{ step.env[name] }}"
+{% endfor %}
+{% endif %}
+          TEST_COMMAND: |-
+            (command rocm-smi || true) && cd {{ (step.working_dir or default_working_dir) | safe }}
+{% if env %}
+{% for name in env %}
+            export {{ name }}="{{ env[name] }}"
+{% endfor %}
+{% endif %}
+{% if step.env %}
+{% for name in step.env %}
+            export {{ name }}="{{ step.env[name] }}"
+{% endfor %}
+{% endif %}
+{% if "mi250" in step.agent_pool %}
+            python3 -m pip uninstall -y amd-aiter
+{% endif %}
+            {{ indented_cmd | safe }}
+        priority: 100
+        {% if step.timeout_in_minutes %}
+        timeout_in_minutes: {{ step.timeout_in_minutes }}
+        {% endif %}
+        {% if step.grade and step.grade == "Blocking" %}
+        soft_fail: false
+        {% else %}
+        soft_fail: true
+        {% endif%}
+{% endif %}
+{% endfor %}
+{% elif step_group.mirror_hardwares and mirror_hw in step_group.mirror_hardwares %}
+{% set step = step_group %}
+  - label: "{{ step.agent_pool }}: {{ step.label }}"
+    depends_on: amd-build
+    agents:
+      {% if step.agent_pool %}
+      queue: amd_{{ step.agent_pool }}
+      {% else %}
+      queue: amd_mi325_1
+      {% endif %}
+{% set cmd_body = (step.command or (step.commands | join("\n"))) | trim %}
+{% set indented_cmd = cmd_body | replace("\n", "\n        ") %}
+    command: bash .buildkite/amd/scripts/run-amd-test.sh
+    env:
+      DOCKER_BUILDKIT: "1"
+{% if step.env %}
+{% for name in step.env %}
+      {{ name }}: "{{ step.env[name] }}"
+{% endfor %}
+{% endif %}
+      TEST_COMMAND: |-
+        (command rocm-smi || true) && cd {{ (step.working_dir or default_working_dir) | safe }}
+{% if env %}
+{% for name in env %}
+        export {{ name }}="{{ env[name] }}"
+{% endfor %}
+{% endif %}
+{% if step.env %}
+{% for name in step.env %}
+        export {{ name }}="{{ step.env[name] }}"
+{% endfor %}
+{% endif %}
+{% if "mi250" in step.agent_pool %}
+        python3 -m pip uninstall -y amd-aiter
+{% endif %}
+        {{ indented_cmd | safe }}
+    priority: 100
+    {% if step.timeout_in_minutes %}
+    timeout_in_minutes: {{ step.timeout_in_minutes }}
+    {% endif %}
+    {% if step.grade and step.grade == "Blocking" %}
+    soft_fail: false
+    {% else %}
+    soft_fail: true
+    {% endif%}
+{% endif %}
+{% endfor %}

.buildkite/bootstrap-intel-omni.sh

@@ -1,51 +0,0 @@
-#!/bin/bash
-# vllm-omni Intel bootstrap
-# Uses static pipeline-intel.yaml for Intel XPU tests
-# Based on: bootstrap-amd-omni.sh
-
-set -euo pipefail
-
-if [[ -z "${DOCS_ONLY_DISABLE:-}" ]]; then
-    DOCS_ONLY_DISABLE=0
-fi
-
-upload_pipeline() {
-    echo "--- 🛠 Preparing Intel pipeline"
-    buildkite-agent pipeline upload .buildkite/pipeline-intel.yaml
-}
-
-get_diff() {
-    $(git add .)
-    echo $(git diff --name-only --diff-filter=ACMDR $(git merge-base origin/main HEAD))
-}
-
-get_diff_main() {
-    $(git add .)
-    echo $(git diff --name-only --diff-filter=ACMDR HEAD~1)
-}
-
-file_diff=$(get_diff)
-if [[ $BUILDKITE_BRANCH == "main" ]]; then
-    file_diff=$(get_diff_main)
-fi
-
-# ----------------------------------------------------------------------
-# Early exit: skip pipeline if all changed files are under docs/
-# ----------------------------------------------------------------------
-if [[ "${DOCS_ONLY_DISABLE}" != "1" ]] && [[ -n "${file_diff:-}" ]]; then
-    docs_only=1
-    while IFS= read -r f; do
-        [[ -z "$f" ]] && continue
-        if [[ "$f" != docs/* ]]; then
-            docs_only=0
-            break
-        fi
-    done < <(printf '%s\n' "$file_diff" | tr ' ' '\n' | tr -d '\r')
-
-    if [[ "$docs_only" -eq 1 ]]; then
-        buildkite-agent annotate ":memo: CI skipped — docs only" --style "info"
-        exit 0
-    fi
-fi
-
-upload_pipeline

.buildkite/common/ci_mirror_hardwares.yml

@@ -0,0 +1,239 @@
+# Uploader-only hardware presets for CUDA / NPU test YAML.
+# Referenced as ``mirror_hardwares: l4_1`` on a step; ``upload_pipeline.py`` expands to
+# ``agents`` (+ optional top-level ``image`` for NPU) + ``plugins``, then strips
+# ``mirror_hardwares`` before Buildkite upload.
+#
+# CUDA naming: {chip}_{count}
+#   l4_1, l4_4, h100_1 … h100_4
+# NPU naming: {queue}_{resource_class}
+#   a2b3_npu_1  — ascend-a2b3, resource_class npu-1 (A2/B3 CI image)
+#   a2b3_npu_4  — ascend-a2b3, resource_class npu-4
+#   a3_npu_2    — ascend-a3, resource_class npu-2 (A3 CI image tag …:$BUILDKITE_COMMIT-a3)
+
+x-npu-k8s-plugins: &npu_k8s_plugins
+  - kubernetes:
+      podSpecPatch:
+        imagePullSecrets:
+          - name: swr-secret
+
+
+mirror_hardwares:
+  l4_1:
+    agents:
+      queue: gpu_1_queue
+    plugins:
+      - docker#v5.2.0:
+          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+          always-pull: true
+          propagate-environment: true
+          shm-size: "8gb"
+          environment:
+            - HF_HOME=/fsx/hf_cache
+            - HF_TOKEN
+          volumes:
+            - /fsx/hf_cache:/fsx/hf_cache
+
+  l4_4:
+    agents:
+      queue: gpu_4_queue
+    plugins:
+      - docker#v5.2.0:
+          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+          always-pull: true
+          propagate-environment: true
+          shm-size: "8gb"
+          environment:
+            - HF_HOME=/fsx/hf_cache
+            - HF_TOKEN
+          volumes:
+            - /fsx/hf_cache:/fsx/hf_cache
+
+  h100_1:
+    agents:
+      queue: mithril-h100-pool
+    plugins:
+      - kubernetes:
+          podSpec:
+            containers:
+              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 1
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: hf-cache
+                    mountPath: /root/.cache/huggingface
+                env:
+                  - name: HF_HOME
+                    value: /root/.cache/huggingface
+                  - name: HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            nodeSelector:
+              node.kubernetes.io/instance-type: gpu-h100-sxm
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+              - name: hf-cache
+                hostPath:
+                  path: /mnt/hf-cache
+                  type: DirectoryOrCreate
+
+  h100_2:
+    agents:
+      queue: mithril-h100-pool
+    plugins:
+      - kubernetes:
+          podSpec:
+            containers:
+              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 2
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: hf-cache
+                    mountPath: /root/.cache/huggingface
+                env:
+                  - name: HF_HOME
+                    value: /root/.cache/huggingface
+                  - name: HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            nodeSelector:
+              node.kubernetes.io/instance-type: gpu-h100-sxm
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+              - name: hf-cache
+                hostPath:
+                  path: /mnt/hf-cache
+                  type: DirectoryOrCreate
+
+  h100_3:
+    agents:
+      queue: mithril-h100-pool
+    plugins:
+      - kubernetes:
+          podSpec:
+            containers:
+              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 3
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: hf-cache
+                    mountPath: /root/.cache/huggingface
+                env:
+                  - name: HF_HOME
+                    value: /root/.cache/huggingface
+                  - name: HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            nodeSelector:
+              node.kubernetes.io/instance-type: gpu-h100-sxm
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+              - name: hf-cache
+                hostPath:
+                  path: /mnt/hf-cache
+                  type: DirectoryOrCreate
+
+  h100_4:
+    agents:
+      queue: mithril-h100-pool
+    plugins:
+      - kubernetes:
+          podSpec:
+            containers:
+              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 4
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: hf-cache
+                    mountPath: /root/.cache/huggingface
+                env:
+                  - name: HF_HOME
+                    value: /root/.cache/huggingface
+                  - name: HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            nodeSelector:
+              node.kubernetes.io/instance-type: gpu-h100-sxm
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+              - name: hf-cache
+                hostPath:
+                  path: /mnt/hf-cache
+                  type: DirectoryOrCreate
+
+  a2b3_npu_1:
+    agents:
+      queue: "ascend-a2b3"
+      resource_class: "npu-1"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}"
+    plugins: *npu_k8s_plugins
+
+  a2b3_npu_4:
+    agents:
+      queue: "ascend-a2b3"
+      resource_class: "npu-4"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}"
+    plugins: *npu_k8s_plugins
+
+
+  a2b3_npu_8:
+    agents:
+      queue: "ascend-a2b3"
+      resource_class: "npu-8"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}"
+    plugins: *npu_k8s_plugins
+
+  a3_npu_2:
+    agents:
+      queue: "ascend-a3"
+      resource_class: "npu-2"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3"
+    plugins: *npu_k8s_plugins
+
+  a3_npu_4:
+    agents:
+      queue: "ascend-a3"
+      resource_class: "npu-4"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3"
+    plugins: *npu_k8s_plugins
+
+  a3_npu_8:
+    agents:
+      queue: "ascend-a3"
+      resource_class: "npu-8"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3"
+    plugins: *npu_k8s_plugins
+
+  a3_npu_16:
+    agents:
+      queue: "ascend-a3"
+      resource_class: "npu-16"
+    image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3"
+    plugins: *npu_k8s_plugins

.buildkite/common/scripts/resolve_skip_ci.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Shared Buildkite skip-ci helpers for AMD / Intel bootstrap scripts.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SKIP_CI_PY="${SCRIPT_DIR}/skip_ci.py"
+
+# Resolve git diff once. Exit 0 when this bootstrap should stop uploading.
+# Decision details are logged by skip_ci.py; this helper only annotates skip-all.
+# Usage: gate_bootstrap_ci <platform> <l2|l3>
+gate_bootstrap_ci() {
+    local platform="$1"
+    local level="$2"
+    local kind
+    if kind="$(python3 "${SKIP_CI_PY}" gate "${platform}" "${level}")"; then
+        if [[ "${kind}" == "skip-all" ]]; then
+            buildkite-agent annotate \
+                ":memo: CI skipped — docs or pytest skip-mark changes only" \
+                --style "info" 2>/dev/null || true
+        fi
+        exit 0
+    fi
+}

.buildkite/common/scripts/skip_ci.py

@@ -0,0 +1,617 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+"""Shared Buildkite skip-ci and CI-YAML level targeting logic.
+
+``resolve_ci_decision(changed_files, ...)`` buckets explicit file lists (tests, dry-run).
+``resolve_ci_context_from_git()`` resolves git diff once and returns decision + changed files
+for Buildkite bootstrap / upload scripts.
+
+CLI (for bootstrap scripts; exit code is the signal):
+  python3 skip_ci.py gate <platform> <level>
+      Exit 0 when bootstrap should stop (skip-all, or that L2/L3 target is off).
+      Prints ``skip-all`` or ``skip-l23`` on stdout; logs the decision once.
+  python3 skip_ci.py check-skip-all
+      Exit 0 when only docs/skip-mark changes → bootstrap skips entire CI upload.
+  python3 skip_ci.py check-skip-l2-l3 <platform> <level>
+      Exit 0 when that L2/L3 pipeline should be skipped (CI config YAML-only diffs).
+  python3 skip_ci.py print-annotate
+      Print decision message for Buildkite annotation.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import re
+import subprocess
+import sys
+from dataclasses import dataclass, field, replace
+from pathlib import Path
+
+LOG = "[skip-ci]"
+ROOT = Path(__file__).resolve().parent.parent.parent.parent
+
+PLATFORMS = ("cuda", "amd", "intel", "npu")
+L23_LEVELS = frozenset({"l2", "l3"})
+
+L2_YAML_FILES: dict[str, str] = {
+    ".buildkite/cuda/test-ready.yml": "cuda",
+    ".buildkite/npu/test-npu-ready.yml": "npu",
+    ".buildkite/amd/test-amd-ready.yml": "amd",
+    ".buildkite/intel/pipeline-intel.yml": "intel",
+}
+
+L3_YAML_FILES: dict[str, str] = {
+    ".buildkite/cuda/test-merge.yml": "cuda",
+    ".buildkite/amd/test-amd-merge.yml": "amd",
+}
+
+L45_YAML_FILES: dict[str, str] = {
+    ".buildkite/cuda/test-nightly.yml": "cuda",
+    ".buildkite/cuda/test-weekly.yml": "cuda",
+    ".buildkite/npu/test-npu-nightly.yml": "npu",
+}
+
+
+def _empty_platform_buckets() -> dict[str, list[str]]:
+    return {platform: [] for platform in PLATFORMS}
+
+
+@dataclass(frozen=True)
+class CiDevice:
+    """Per-platform / per-level skip flags for yaml-gated bootstrap (``skip_l2_l3``)."""
+
+    skip_cuda_l2: bool = False
+    skip_cuda_l3: bool = False
+    skip_amd_l2: bool = False
+    skip_amd_l3: bool = False
+    skip_intel_l2: bool = False
+    skip_intel_l3: bool = False
+    skip_npu_l2: bool = False
+    skip_npu_l3: bool = False
+
+    def is_skip(self, platform: str, level: str) -> bool:
+        return bool(getattr(self, f"skip_{platform}_{level}", True))
+
+
+@dataclass
+class DiffBuckets:
+    """Changed files grouped by doc / skip-mark / CI yaml / other."""
+
+    docs: list[str] = field(default_factory=list)
+    skip_tests: list[str] = field(default_factory=list)
+    other: list[str] = field(default_factory=list)
+    l2: dict[str, list[str]] = field(default_factory=_empty_platform_buckets)
+    l3: dict[str, list[str]] = field(default_factory=_empty_platform_buckets)
+    l45: dict[str, list[str]] = field(default_factory=_empty_platform_buckets)
+
+    @property
+    def has_doc_changes(self) -> bool:
+        return bool(self.docs)
+
+    @property
+    def has_skip_test_changes(self) -> bool:
+        return bool(self.skip_tests)
+
+    @property
+    def has_other_changes(self) -> bool:
+        """Product code or paths outside skip-ci scope."""
+        return bool(self.other)
+
+    @property
+    def has_l23_yaml_changes(self) -> bool:
+        """At least one whitelisted L2/L3 CI yaml file changed."""
+        return any(self.l2[platform] or self.l3[platform] for platform in PLATFORMS)
+
+    @property
+    def has_l45_yaml_changes(self) -> bool:
+        """At least one whitelisted L4/L5 (nightly/weekly) CI yaml file changed."""
+        return any(self.l45[platform] for platform in PLATFORMS)
+
+    @property
+    def has_skip_ci_scope_changes(self) -> bool:
+        """At least one doc / skip-mark / whitelisted CI yaml change."""
+        return (
+            self.has_doc_changes or self.has_skip_test_changes or self.has_l23_yaml_changes or self.has_l45_yaml_changes
+        )
+
+    def l2_platforms(self) -> frozenset[str]:
+        return frozenset(platform for platform in PLATFORMS if self.l2[platform])
+
+    def l3_platforms(self) -> frozenset[str]:
+        return frozenset(platform for platform in PLATFORMS if self.l3[platform])
+
+
+@dataclass(frozen=True)
+class SkipL2L3Basis:
+    skip_all_l2: bool
+    skip_all_l3: bool
+
+    @classmethod
+    def from_buckets(cls, buckets: DiffBuckets) -> SkipL2L3Basis | None:
+        """L4/L5 yaml → skip both L2 and L3 (L2/L3 buckets may rescue).
+
+        Docs / skip-mark changes do not block this path; they are ignored for
+        L2/L3 targeting when CI YAML is also present.
+        """
+        if buckets.has_other_changes:
+            return None
+        if not buckets.has_l45_yaml_changes:
+            return None
+        return cls(
+            skip_all_l2=not buckets.l2_platforms(),
+            skip_all_l3=not buckets.l3_platforms(),
+        )
+
+
+@dataclass(frozen=True)
+class SkipL3Basis:
+    enable_l2_platforms: frozenset[str]
+
+    @classmethod
+    def from_buckets(cls, buckets: DiffBuckets) -> SkipL3Basis | None:
+        """L2 yaml changed → skip L3; keep L2 on touched platforms."""
+        platforms = buckets.l2_platforms()
+        if not platforms:
+            return None
+        return cls(enable_l2_platforms=platforms)
+
+
+@dataclass(frozen=True)
+class SkipL2Basis:
+    enable_l3_platforms: frozenset[str]
+
+    @classmethod
+    def from_buckets(cls, buckets: DiffBuckets) -> SkipL2Basis | None:
+        """L3 yaml changed → skip L2; keep L3 on touched platforms."""
+        platforms = buckets.l3_platforms()
+        if not platforms:
+            return None
+        return cls(enable_l3_platforms=platforms)
+
+
+@dataclass(frozen=True)
+class CiDecision:
+    """Final CI skip decision for one diff."""
+
+    skip_all: bool = False
+    skip_l2_l3: bool = False
+    device: CiDevice = field(default_factory=CiDevice)
+    message: str = ""
+
+    @classmethod
+    def skip_all_basis(cls, buckets: DiffBuckets, *, diff_range: str | None) -> bool:
+        """Docs / pytest skip-mark only → suppress entire default CI."""
+        if buckets.has_other_changes:
+            return False
+        if (buckets.has_l23_yaml_changes or buckets.has_l45_yaml_changes) and (
+            buckets.has_doc_changes or buckets.has_skip_test_changes
+        ):
+            return False
+        if not buckets.has_doc_changes and not buckets.has_skip_test_changes:
+            return False
+        if diff_range is None and not buckets.has_doc_changes:
+            return False
+        return True
+
+    @classmethod
+    def from_yaml_basis(
+        cls,
+        l2_l3_basis: SkipL2L3Basis | None,
+        l3_basis: SkipL3Basis | None,
+        l2_basis: SkipL2Basis | None,
+    ) -> CiDecision:
+        """Build a yaml-gated decision by merging L2/L3 skip bases.
+
+        Bases are applied as a union of enables: L2 yaml keeps L2 on touched
+        platforms, L3 yaml keeps L3 on touched platforms. Cross-platform mixes
+        (e.g. CUDA ready + AMD merge) must preserve both enables. The resulting
+        per-platform / per-level matrix is stored on ``device`` as ``skip_*``.
+        """
+        run = {platform: {"l2": True, "l3": True} for platform in PLATFORMS}
+
+        if l2_l3_basis is not None:
+            if l2_l3_basis.skip_all_l2:
+                for platform in PLATFORMS:
+                    run[platform]["l2"] = False
+            if l2_l3_basis.skip_all_l3:
+                for platform in PLATFORMS:
+                    run[platform]["l3"] = False
+
+        if l3_basis is not None:
+            for platform in PLATFORMS:
+                run[platform]["l3"] = False
+                run[platform]["l2"] = platform in l3_basis.enable_l2_platforms
+
+        if l2_basis is not None:
+            for platform in PLATFORMS:
+                if platform in l2_basis.enable_l3_platforms:
+                    run[platform]["l3"] = True
+                    if l3_basis is None or platform not in l3_basis.enable_l2_platforms:
+                        run[platform]["l2"] = False
+                elif l3_basis is None:
+                    # L3-yaml-only: disable platforms that did not change L3.
+                    run[platform]["l2"] = False
+                    run[platform]["l3"] = False
+                # else: leave L2 enables from SkipL3Basis; keep this platform's L3 off.
+
+        return cls(
+            skip_l2_l3=True,
+            device=cls._device_from_run(run),
+        )
+
+    @staticmethod
+    def _device_from_run(run: dict[str, dict[str, bool]]) -> CiDevice:
+        return CiDevice(
+            **{f"skip_{platform}_{level}": not run[platform][level] for platform in PLATFORMS for level in ("l2", "l3")}
+        )
+
+    def is_run(self, platform: str, level: str) -> bool:
+        if self.skip_all:
+            return False
+        if not self.skip_l2_l3:
+            return True
+        if level not in L23_LEVELS:
+            return True
+        return not self.device.is_skip(platform, level)
+
+
+@dataclass(frozen=True)
+class CiContext:
+    """Skip-ci decision plus git diff inputs used to compute it."""
+
+    decision: CiDecision
+    changed_files: list[str] | None
+    diff_range: str | None
+
+
+def _log(message: str) -> None:
+    print(f"{LOG}: {message}", file=sys.stderr)
+
+
+def _git(*args: str) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(
+        ["git", *args],
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+
+
+# --- Diff helpers ---
+
+
+def _resolve_diff_range() -> str | None:
+    """Return a git diff range for PR or main builds, or None when unavailable."""
+    is_pr = os.environ.get("BUILDKITE_PULL_REQUEST", "false") != "false" and os.environ.get(
+        "BUILDKITE_PULL_REQUEST", ""
+    )
+    commit = os.environ.get("BUILDKITE_COMMIT", "")
+
+    if is_pr:
+        base_branch = os.environ.get("BUILDKITE_PULL_REQUEST_BASE_BRANCH", "main")
+        base_ref = f"origin/{base_branch}"
+        if _git("rev-parse", "--verify", base_ref).returncode != 0:
+            _log(f"origin/{base_branch} not found locally; trying fetch")
+            _git("fetch", "--depth=200", "origin", base_branch)
+        if _git("rev-parse", "--verify", base_ref).returncode != 0:
+            if _git("rev-parse", "--verify", base_branch).returncode == 0:
+                base_ref = base_branch
+            else:
+                _log(f"cannot resolve PR base {base_branch}; using safe defaults")
+                return None
+        return f"{base_ref}...{commit}"
+
+    if os.environ.get("BUILDKITE_BRANCH", "") == "main":
+        if _git("rev-parse", "--verify", f"{commit}^").returncode != 0:
+            _log("main commit has no parent; using safe defaults")
+            return None
+        return f"{commit}^..{commit}"
+
+    _log("not PR/main build; using safe defaults")
+    return None
+
+
+def _changed_files_for_diff_range(diff_range: str | None) -> list[str] | None:
+    """Return changed file paths for *diff_range*, or None when unavailable."""
+    if diff_range is None:
+        return None
+
+    result = _git("diff", "--name-only", diff_range)
+    if result.returncode != 0:
+        _log(f"git diff failed ({diff_range}); using safe defaults")
+        return None
+
+    files = [line for line in result.stdout.splitlines() if line.strip()]
+    _log(f"{len(files)} changed file(s)")
+    return files
+
+
+def resolve_ci_context_from_git() -> CiContext:
+    """Resolve git diff once and return skip-ci decision plus changed files."""
+    diff_range = _resolve_diff_range()
+    changed_files = _changed_files_for_diff_range(diff_range)
+    decision = resolve_ci_decision(changed_files, diff_range=diff_range)
+    return CiContext(decision=decision, changed_files=changed_files, diff_range=diff_range)
+
+
+# --- Doc / skip-mark helpers ---
+
+
+_SKIP_MARK_RE = re.compile(r"pytest\.mark\.skip(?:if)?\b|pytest\.skip\s*\(")
+_PYTESTMARK_SKIP_RE = re.compile(r"pytest\.mark\.skip\b")
+_PYTEST_MARK_ONLY_RE = re.compile(r"pytest\.mark\.\w+")
+
+
+def _diff_only_contains_skip_mark_changes(diff_text: str) -> bool:
+    """Return True when a unified diff only *adds* pytest skip marks.
+
+    Deleting a skip mark re-enables the test and must not suppress CI. Editing an
+    existing skip (``-`` old / ``+`` new) still qualifies when at least one skip
+    mark is added.
+    """
+
+    def paren_balance(line: str) -> int:
+        return line.count("(") - line.count(")")
+
+    def is_blank_or_comment(line: str) -> bool:
+        stripped = line.strip()
+        return not stripped or stripped.startswith("#")
+
+    def is_skip_mark_line(line: str) -> bool:
+        return _SKIP_MARK_RE.search(line.strip()) is not None
+
+    def is_pytestmark_adjacent_line(line: str) -> bool:
+        stripped = line.strip().rstrip(",")
+        if not stripped or stripped in {"[", "]"} or stripped.startswith("#"):
+            return True
+        if stripped.startswith("pytestmark"):
+            return True
+        return _PYTEST_MARK_ONLY_RE.search(stripped) is not None
+
+    pending_depth = 0
+    saw_change = False
+    has_added_skip_mark = False
+    for raw_line in diff_text.splitlines():
+        if raw_line.startswith("@@"):
+            pending_depth = 0
+            continue
+        if not (raw_line.startswith("+") or raw_line.startswith("-")):
+            continue
+        if raw_line.startswith("+++") or raw_line.startswith("---"):
+            continue
+
+        saw_change = True
+        is_add = raw_line.startswith("+")
+        content = raw_line[1:]
+
+        if pending_depth > 0:
+            pending_depth += paren_balance(content)
+            if pending_depth < 0:
+                pending_depth = 0
+            continue
+
+        if is_blank_or_comment(content):
+            continue
+
+        if is_skip_mark_line(content):
+            if is_add:
+                has_added_skip_mark = True
+            pending_depth = max(0, paren_balance(content))
+            continue
+
+        if is_pytestmark_adjacent_line(content):
+            continue
+
+        return False
+
+    return saw_change and has_added_skip_mark
+
+
+def _is_skip_testcase_change(file_path: str, *, diff_range: str) -> bool:
+    """Return True when a test file diff is skip-mark-only (including new-file cases)."""
+    diff_result = _git("diff", diff_range, "--", file_path)
+    if diff_result.returncode != 0:
+        _log(f"skip-mark-only: git diff failed for {file_path}")
+        return False
+
+    diff_text = diff_result.stdout
+    if not diff_text.strip():
+        _log(f"skip-mark-only: empty diff for {file_path}")
+        return False
+
+    if _diff_only_contains_skip_mark_changes(diff_text):
+        return True
+
+    if "new file mode" not in diff_text and "--- /dev/null" not in diff_text:
+        _log(f"skip-mark-only: non-skip changes in {file_path}")
+        return False
+
+    commit = os.environ.get("BUILDKITE_COMMIT", "")
+    if not commit:
+        return False
+
+    content_result = _git("show", f"{commit}:{file_path}")
+    if content_result.returncode != 0:
+        _log(f"skip-mark-only: cannot read {file_path} at {commit}")
+        return False
+
+    content = content_result.stdout
+    match = re.search(r"^pytestmark\s*=\s*\[(.*?)^\s*\]", content, re.MULTILINE | re.DOTALL)
+    if match is not None and _PYTESTMARK_SKIP_RE.search(match.group(1)) is not None:
+        _log(f"skip-mark-only: new test file with module-level pytest.mark.skip: {file_path}")
+        return True
+    if re.search(r"^pytestmark\s*=\s*pytest\.mark\.skip\b", content, re.MULTILINE):
+        _log(f"skip-mark-only: new test file with module-level pytest.mark.skip: {file_path}")
+        return True
+
+    _log(f"skip-mark-only: new test file without module-level pytest.mark.skip: {file_path}")
+    return False
+
+
+def _classify_changed_files_into_buckets(changed_files: list[str], *, diff_range: str | None) -> DiffBuckets:
+    """Classify each changed file path into a ``DiffBuckets`` category."""
+    buckets = DiffBuckets()
+    for file_path in changed_files:
+        if not file_path:
+            continue
+        if file_path.startswith("docs/") or file_path.endswith(".md") or file_path == "mkdocs.yml":
+            buckets.docs.append(file_path)
+            continue
+        path = Path(file_path)
+        if path.suffix == ".py" and path.parts and path.parts[0] == "tests":
+            if diff_range is not None and _is_skip_testcase_change(file_path, diff_range=diff_range):
+                buckets.skip_tests.append(file_path)
+            else:
+                buckets.other.append(file_path)
+            continue
+        if file_path in L2_YAML_FILES:
+            platform = L2_YAML_FILES[file_path]
+            buckets.l2[platform].append(file_path)
+        elif file_path in L3_YAML_FILES:
+            platform = L3_YAML_FILES[file_path]
+            buckets.l3[platform].append(file_path)
+        elif file_path in L45_YAML_FILES:
+            platform = L45_YAML_FILES[file_path]
+            buckets.l45[platform].append(file_path)
+        else:
+            buckets.other.append(file_path)
+    return buckets
+
+
+# --- Decision ---
+
+
+def _finish(decision: CiDecision, message: str) -> CiDecision:
+    finished = replace(decision, message=message)
+    _log(message)
+    return finished
+
+
+def _format_yaml_gated_message(buckets: DiffBuckets, decision: CiDecision) -> str:
+    """Human-readable summary of which CI YAML changed and which L2/L3 targets run."""
+    changed_parts: list[str] = []
+    for level_name, by_platform in (("L2", buckets.l2), ("L3", buckets.l3), ("L4/L5", buckets.l45)):
+        files = [path for platform in PLATFORMS for path in by_platform[platform]]
+        if files:
+            changed_parts.append(f"{level_name}=[{', '.join(files)}]")
+
+    run_targets = [
+        f"{platform}/{level}" for platform in PLATFORMS for level in ("l2", "l3") if decision.is_run(platform, level)
+    ]
+    skip_targets = [
+        f"{platform}/{level}"
+        for platform in PLATFORMS
+        for level in ("l2", "l3")
+        if not decision.is_run(platform, level)
+    ]
+
+    changed = "; ".join(changed_parts) if changed_parts else "none"
+    run = ", ".join(run_targets) if run_targets else "none"
+    skip = ", ".join(skip_targets) if skip_targets else "none"
+    if buckets.has_doc_changes or buckets.has_skip_test_changes:
+        return (
+            "CI config YAML + docs/skip-mark changed — follow CI YAML gating; "
+            f"changed: {changed}; run: {run}; skip: {skip}"
+        )
+    return f"only CI config YAML changed — changed: {changed}; run: {run}; skip: {skip}"
+
+
+def resolve_ci_decision(
+    changed_files: list[str] | None = None,
+    *,
+    diff_range: str | None = None,
+) -> CiDecision:
+    if changed_files is None:
+        return _finish(CiDecision(), "could not resolve changed files; run normal CI")
+
+    buckets = _classify_changed_files_into_buckets(changed_files, diff_range=diff_range)
+    if buckets.has_other_changes or not buckets.has_skip_ci_scope_changes:
+        return _finish(CiDecision(), "product or non-whitelisted changes present; run normal CI")
+
+    if CiDecision.skip_all_basis(buckets, diff_range=diff_range):
+        return _finish(
+            CiDecision(skip_all=True),
+            "CI skipped — docs or pytest skip-mark changes only",
+        )
+
+    # Docs / skip-mark mixed with whitelisted CI YAML → same yaml-gated path
+    # as CI-YAML-only (docs/skip-mark do not widen to normal CI).
+    if not buckets.has_l23_yaml_changes and not buckets.has_l45_yaml_changes:
+        if buckets.has_skip_test_changes and diff_range is None:
+            message = (
+                "only pytest skip-mark test changes detected, but no git diff range "
+                "is available to verify them (skip-all requires diff_range); run normal CI"
+            )
+        elif buckets.has_skip_test_changes:
+            message = "pytest skip-mark test changes did not qualify for skip-all and no ci yaml changed; run normal CI"
+        else:
+            message = "doc/skip-mark changes did not qualify for skip-all; run normal CI"
+        return _finish(CiDecision(), message)
+
+    decision = CiDecision.from_yaml_basis(
+        SkipL2L3Basis.from_buckets(buckets),
+        SkipL3Basis.from_buckets(buckets),
+        SkipL2Basis.from_buckets(buckets),
+    )
+    return _finish(decision, _format_yaml_gated_message(buckets, decision))
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    gate_parser = subparsers.add_parser(
+        "gate",
+        help="exit 0 when bootstrap should stop (skip-all or PLATFORM/LEVEL disabled); logs once",
+    )
+    gate_parser.add_argument("platform", help="cuda, amd, intel, or npu")
+    gate_parser.add_argument("level", choices=("l2", "l3"), help="test level")
+
+    subparsers.add_parser(
+        "check-skip-all",
+        help="exit 0 when docs/skip-mark-only diff should skip entire CI upload",
+    )
+
+    skip_parser = subparsers.add_parser(
+        "check-skip-l2-l3",
+        help="exit 0 when PLATFORM l2/l3 pipeline should be skipped (CI config YAML-only diffs)",
+    )
+    skip_parser.add_argument("platform", help="cuda, amd, intel, or npu")
+    skip_parser.add_argument("level", choices=("l2", "l3"), help="test level")
+
+    subparsers.add_parser(
+        "print-annotate",
+        help="print human-readable skip reason for Buildkite annotation",
+    )
+
+    args = parser.parse_args()
+    decision = resolve_ci_context_from_git().decision
+
+    if args.command == "gate":
+        if decision.skip_all:
+            print("skip-all")
+            return 0
+        if not decision.is_run(args.platform, args.level):
+            print("skip-l23")
+            return 0
+        return 1
+
+    if args.command == "check-skip-all":
+        return 0 if decision.skip_all else 1
+
+    if args.command == "check-skip-l2-l3":
+        return 0 if not decision.is_run(args.platform, args.level) else 1
+
+    if args.command == "print-annotate":
+        print(decision.message or "CI will run normally")
+        return 0
+
+    return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.buildkite/common/scripts/upload_pipeline.py

@@ -0,0 +1,449 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+"""Render and optionally upload Buildkite pipeline YAML with diff-aware logic.
+
+Bootstrap mode (``bootstrap-upload-steps.yml``):
+  - Hook uploads the entry YAML (``pipeline.yml`` / ``pipeline-npu.yml``) with one step that runs
+    ``upload_pipeline.py --upload <platform>/bootstrap-upload-steps.yml``.
+  - Injects ``if`` by step ``key`` from skip-ci and uploads child steps (image build, L2–L5 upload).
+  - Detect docs-only, pytest skip-mark-only, or combined skip-ci from git diff.
+  - When only CI level YAML changes, enable **L2/L3** upload steps for affected levels only.
+
+Test pipeline mode (e.g. test-merge.yml):
+  - Drop steps whose ``source_file_dependencies`` do not match changed files.
+  - Expand uploader-only ``mirror_hardwares: l4_1`` into ``agents`` (+ optional ``image``
+    for NPU) + ``plugins`` (see ci_mirror_hardwares.yml).
+
+Usage:
+  python3 upload_pipeline.py [--upload] [--all | --e2e] <pipeline.yml>
+
+Requires PyYAML (``pip install pyyaml``); installs it automatically when missing.
+"""
+
+from __future__ import annotations
+
+import argparse
+import copy
+import subprocess
+import sys
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+try:
+    import yaml
+except ModuleNotFoundError:
+    subprocess.run(
+        [sys.executable, "-m", "pip", "install", "-q", "pyyaml"],
+        check=True,
+    )
+    import yaml
+
+from skip_ci import (
+    ROOT,
+    resolve_ci_context_from_git,
+)
+
+# --- Constants ---
+
+LOG = "upload_pipeline"
+BOOTSTRAP_STEPS_FILENAME = "bootstrap-upload-steps.yml"
+BOOTSTRAP_IMAGE_BUILD_KEYS = frozenset({"image-build", "image-build-a2", "image-build-a3"})
+BOOTSTRAP_UPLOAD_IF_KEYS = {
+    "upload-ready-pipeline": "ready",
+    "upload-merge-pipeline": "merge",
+    "upload-nightly-pipeline": "nightly",
+    "upload-weekly-pipeline": "weekly",
+}
+E2E_GROUP_MARKER = "E2E Test"
+CI_MIRROR_HARDWARES_PATH = ROOT / ".buildkite/common/ci_mirror_hardwares.yml"
+
+CUDA_NIGHTLY_ONLY = (
+    '(build.pull_request.labels includes "nightly-test") || (build.branch == "main" && build.env("NIGHTLY") == "1")'
+)
+NPU_NIGHTLY_ONLY = (
+    '(build.branch == "main" && build.env("NIGHTLY") == "1") || '
+    '(build.branch != "main" && ('
+    'build.pull_request.labels includes "nightly-test" || '
+    'build.pull_request.labels includes "omni-test" || '
+    'build.pull_request.labels includes "tts-test" || '
+    'build.pull_request.labels includes "diffusion-x2iat-test" || '
+    'build.pull_request.labels includes "diffusion-x2v-test"'
+    "))"
+)
+
+
+# --- Logging ---
+
+
+def _log(message: str) -> None:
+    print(f"{LOG}: {message}", file=sys.stderr)
+
+
+# --- Bootstrap pipeline (bootstrap-upload-steps.yml) ---
+
+
+def _get_bootstrap_platform(path: Path) -> str:
+    parts = path.as_posix().split("/")
+    return "npu" if "npu" in parts else "cuda"
+
+
+def _load_bootstrap_steps(path: Path) -> str:
+    if path.name != BOOTSTRAP_STEPS_FILENAME:
+        raise ValueError(f"expected {BOOTSTRAP_STEPS_FILENAME}, got {path.name}")
+    return path.read_text(encoding="utf-8")
+
+
+def _format_bootstrap_if(expr: str) -> str:
+    """Return a Buildkite ``if`` string. Buildkite rejects YAML bool ``if`` values."""
+    if expr in ("true", "false"):
+        return expr
+    return f"({expr})"
+
+
+def _compute_bootstrap_if_exprs(*, decision, platform: str) -> dict[str, str]:
+    disabled = "false"
+    nightly_main = 'build.branch == "main" && build.env("NIGHTLY") == "1"'
+    nightly_only = NPU_NIGHTLY_ONLY if platform == "npu" else CUDA_NIGHTLY_ONLY
+
+    if platform == "npu":
+        ready_pr = (
+            'build.branch != "main" && ('
+            'build.pull_request.labels includes "npu-test" || '
+            'build.pull_request.labels includes "ready"'
+            ")"
+        )
+        nightly_label_if = nightly_only
+        weekly_label_if = disabled
+        merge_base = disabled
+    else:
+        ready_pr = 'build.branch != "main" && build.pull_request.labels includes "ready"'
+        merge_main = 'build.branch == "main" && build.env("NIGHTLY") != "1" && build.env("WEEKLY") != "1"'
+        merge_pr = 'build.branch != "main" && build.pull_request.labels includes "merge-test"'
+        nightly_label_if = (
+            '(build.branch == "main" && build.env("NIGHTLY") == "1") || '
+            '(build.branch != "main" && ('
+            'build.pull_request.labels includes "nightly-test" || '
+            'build.pull_request.labels includes "omni-test" || '
+            'build.pull_request.labels includes "tts-test" || '
+            'build.pull_request.labels includes "diffusion-x2iat-test" || '
+            'build.pull_request.labels includes "diffusion-x2v-test"'
+            "))"
+        )
+        weekly_label_if = (
+            '(build.branch == "main" && build.env("WEEKLY") == "1") || '
+            '(build.branch != "main" && build.pull_request.labels includes "weekly-test")'
+        )
+        merge_base = f"({nightly_main}) || (({merge_main}) || ({merge_pr}))"
+
+    ready_base = f"({nightly_main}) || ({ready_pr})"
+    weekly_main = 'build.branch == "main" && build.env("WEEKLY") == "1"'
+
+    if decision.skip_all:
+        # Docs / skip-mark only: no PR-label escape hatch. Main scheduled
+        # NIGHTLY=1 still runs L4 + L2/L3 (--e2e); WEEKLY=1 still runs L5.
+        image_expr = f"({nightly_main}) || ({weekly_main})" if platform == "cuda" else nightly_main
+        ready_expr = nightly_main
+        merge_expr = nightly_main if platform == "cuda" else disabled
+        nightly_expr = nightly_main
+        weekly_expr = weekly_main if platform == "cuda" else disabled
+    elif decision.skip_l2_l3:
+        l2_enabled = decision.is_run("npu", "l2") if platform == "npu" else decision.is_run("cuda", "l2")
+        l3_enabled = platform == "cuda" and decision.is_run("cuda", "l3")
+
+        ready_expr = ready_base if l2_enabled else disabled
+        merge_expr = merge_base if l3_enabled else disabled
+        nightly_expr = nightly_label_if
+        weekly_expr = weekly_label_if if platform == "cuda" else disabled
+
+        image_parts = [f"({nightly_label_if})"]
+        if platform == "cuda":
+            image_parts.append(f"({weekly_label_if})")
+        if l2_enabled:
+            image_parts.insert(0, f"({ready_base})")
+        if l3_enabled:
+            image_parts.insert(1 if l2_enabled else 0, f"({merge_base})")
+        image_expr = " || ".join(image_parts)
+    else:
+        image_expr = "true"
+        ready_expr = ready_base
+        merge_expr = merge_base if platform == "cuda" else disabled
+        nightly_expr = nightly_label_if
+        weekly_expr = weekly_label_if if platform == "cuda" else disabled
+
+    return {
+        "image": _format_bootstrap_if(image_expr),
+        "ready": _format_bootstrap_if(ready_expr),
+        "merge": _format_bootstrap_if(merge_expr),
+        "nightly": _format_bootstrap_if(nightly_expr),
+        "weekly": _format_bootstrap_if(weekly_expr),
+    }
+
+
+def _apply_bootstrap_if(steps: list[Any], if_exprs: dict[str, str]) -> list[Any]:
+    """Inject ``if`` by step key; drop steps that are unconditionally disabled."""
+    kept: list[Any] = []
+    for step in steps:
+        if not isinstance(step, dict):
+            kept.append(step)
+            continue
+        nested = step.get("steps")
+        if isinstance(nested, list):
+            nested_kept = _apply_bootstrap_if(nested, if_exprs)
+            if nested_kept:
+                kept.append({**step, "steps": nested_kept})
+            continue
+        key = step.get("key")
+        if key in BOOTSTRAP_IMAGE_BUILD_KEYS:
+            step["if"] = if_exprs["image"]
+        elif key in BOOTSTRAP_UPLOAD_IF_KEYS:
+            step["if"] = if_exprs[BOOTSTRAP_UPLOAD_IF_KEYS[key]]
+        if_expr = step.get("if")
+        if if_expr == "false":
+            _log(f"omit disabled bootstrap step {key!r}")
+            continue
+        if if_expr == "true":
+            # Unconditional step: omit ``if`` (Buildkite requires string ``if``, not YAML bool).
+            step.pop("if", None)
+        kept.append(step)
+    return kept
+
+
+def _render_bootstrap_pipeline(
+    steps_yaml: str,
+    *,
+    decision,
+    path: Path,
+) -> str:
+    """Load bootstrap steps YAML and inject ``if`` expressions from skip-ci decision."""
+    doc = yaml.safe_load(steps_yaml)
+    if not isinstance(doc, dict):
+        raise ValueError(f"invalid bootstrap steps YAML for {path}")
+    steps = doc.get("steps")
+    if not isinstance(steps, list):
+        raise ValueError(f"bootstrap steps YAML must contain steps: list in {path}")
+
+    platform = _get_bootstrap_platform(path)
+    if_exprs = _compute_bootstrap_if_exprs(decision=decision, platform=platform)
+    doc["steps"] = _apply_bootstrap_if(steps, if_exprs)
+    return yaml.safe_dump(doc, sort_keys=False)
+
+
+# --- Test pipeline (test-ready.yml, test-merge.yml) ---
+
+
+@lru_cache(maxsize=1)
+def _load_mirror_hardwares() -> dict[str, dict[str, Any]]:
+    if not CI_MIRROR_HARDWARES_PATH.is_file():
+        raise FileNotFoundError(f"missing CI mirror_hardwares registry: {CI_MIRROR_HARDWARES_PATH}")
+    doc = yaml.safe_load(CI_MIRROR_HARDWARES_PATH.read_text(encoding="utf-8"))
+    if not isinstance(doc, dict):
+        raise ValueError(f"invalid CI mirror_hardwares registry: {CI_MIRROR_HARDWARES_PATH}")
+    presets = doc.get("mirror_hardwares")
+    if not isinstance(presets, dict):
+        raise ValueError(f"mirror_hardwares must be a mapping in {CI_MIRROR_HARDWARES_PATH}")
+    return presets
+
+
+def _expand_mirror_hardwares(step: dict[str, Any]) -> dict[str, Any]:
+    """Replace uploader-only ``mirror_hardwares`` with preset fields from ci_mirror_hardwares.yml."""
+    hardware = step.get("mirror_hardwares")
+    if hardware is None:
+        return step
+
+    if not isinstance(hardware, str) or not hardware.strip():
+        raise ValueError(
+            f"mirror_hardwares must be a non-empty string in step {_get_step_label(step)!r}",
+        )
+
+    preset = _load_mirror_hardwares().get(hardware)
+    if preset is None:
+        known = ", ".join(sorted(_load_mirror_hardwares()))
+        raise ValueError(
+            f"unknown mirror_hardwares {hardware!r} in step {_get_step_label(step)!r}; known: {known}",
+        )
+
+    if step.get("agents") is not None or step.get("plugins") is not None or step.get("image") is not None:
+        raise ValueError(
+            f"step {_get_step_label(step)!r} sets mirror_hardwares together with agents/plugins/image; "
+            "use mirror_hardwares only",
+        )
+
+    expanded = copy.deepcopy(preset)
+    return {key: value for key, value in step.items() if key != "mirror_hardwares"} | expanded
+
+
+def _match_source_file(changed_files: list[str], prefixes: list[str]) -> bool:
+    for path in changed_files:
+        for prefix in prefixes:
+            normalized = prefix.rstrip("/")
+            if path == normalized or path.startswith(f"{normalized}/"):
+                return True
+    return False
+
+
+def _get_step_label(step: dict[str, Any]) -> str:
+    return str(step.get("group") or step.get("label") or "<step>")
+
+
+def _process_test_steps(
+    steps: list[Any],
+    changed_files: list[str] | None,
+) -> list[Any]:
+    """Drop steps by ``source_file_dependencies`` when *changed_files* is set; always strip that field."""
+    processed: list[Any] = []
+    for step in steps:
+        if not isinstance(step, dict):
+            processed.append(step)
+            continue
+
+        deps = step.get("source_file_dependencies")
+        if deps is not None and not isinstance(deps, list):
+            raise ValueError(
+                f"source_file_dependencies must be a list in step {_get_step_label(step)!r}",
+            )
+        if changed_files is not None and deps is not None and not _match_source_file(changed_files, deps):
+            _log(f"skip {_get_step_label(step)!r} (no changes under {deps})")
+            continue
+
+        nested = step.get("steps")
+        if nested is not None:
+            kept_nested = _process_test_steps(nested, changed_files)
+            if changed_files is not None and not kept_nested:
+                _log(f"omit empty group {_get_step_label(step)!r}")
+                continue
+            new_step = {key: value for key, value in step.items() if key != "source_file_dependencies"}
+            new_step["steps"] = kept_nested
+            processed.append(new_step)
+            continue
+
+        if deps is not None:
+            processed.append(
+                _expand_mirror_hardwares(
+                    {key: value for key, value in step.items() if key != "source_file_dependencies"},
+                ),
+            )
+        else:
+            processed.append(_expand_mirror_hardwares(step))
+
+    return processed
+
+
+def _select_e2e_group_steps(steps: list[Any]) -> list[Any]:
+    """Keep only top-level groups whose name contains ``E2E_GROUP_MARKER``."""
+    selected = [
+        step
+        for step in steps
+        if isinstance(step, dict) and isinstance(step.get("group"), str) and E2E_GROUP_MARKER in step["group"]
+    ]
+    if not selected:
+        _log(f"no group matching {E2E_GROUP_MARKER!r} found")
+    else:
+        _log(f"keep {len(selected)} group(s) matching {E2E_GROUP_MARKER!r}")
+    return selected
+
+
+def _render_test_pipeline(
+    doc: dict[str, Any],
+    changed_files: list[str] | None,
+    *,
+    e2e_only: bool = False,
+) -> dict[str, Any]:
+    """Filter steps by PR diff and strip uploader-only ``source_file_dependencies`` metadata."""
+    steps = doc.get("steps")
+    if not isinstance(steps, list):
+        return doc
+    if e2e_only:
+        steps = _select_e2e_group_steps(steps)
+    steps = _process_test_steps(steps, changed_files)
+    return {**doc, "steps": steps}
+
+
+# --- Entry (read file → bootstrap or test render → YAML string) ---
+
+
+def _render_pipeline(
+    path: Path,
+    *,
+    force_all: bool = False,
+    e2e_only: bool = False,
+) -> str:
+    if path.name == BOOTSTRAP_STEPS_FILENAME:
+        ctx = resolve_ci_context_from_git()
+        continuation = _load_bootstrap_steps(path)
+        return _render_bootstrap_pipeline(
+            continuation,
+            decision=ctx.decision,
+            path=path,
+        )
+
+    text = path.read_text(encoding="utf-8")
+    ctx = resolve_ci_context_from_git()
+    if force_all or e2e_only:
+        changed_files = None
+    else:
+        changed_files = ctx.changed_files
+
+    doc = yaml.safe_load(text)
+    if not isinstance(doc, dict):
+        raise ValueError(f"invalid pipeline YAML: {path}")
+
+    doc = _render_test_pipeline(doc, changed_files, e2e_only=e2e_only)
+    return yaml.safe_dump(doc, sort_keys=False)
+
+
+def _upload_to_buildkite(content: str) -> None:
+    subprocess.run(
+        ["buildkite-agent", "pipeline", "upload"],
+        input=content,
+        text=True,
+        check=True,
+    )
+
+
+# --- CLI ---
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "pipeline",
+        nargs="?",
+        default=".buildkite/cuda/bootstrap-upload-steps.yml",
+        help="Pipeline YAML path (default: .buildkite/cuda/bootstrap-upload-steps.yml)",
+    )
+    parser.add_argument(
+        "--upload",
+        action="store_true",
+        help="Pipe rendered YAML to buildkite-agent pipeline upload",
+    )
+    mode = parser.add_mutually_exclusive_group()
+    mode.add_argument(
+        "--all",
+        action="store_true",
+        help="Keep all steps (disable diff-aware skipping)",
+    )
+    mode.add_argument(
+        "--e2e",
+        action="store_true",
+        help="Keep only the E2E Test group",
+    )
+    args = parser.parse_args()
+
+    path = Path(args.pipeline)
+    if not path.is_absolute():
+        path = ROOT / path
+    if not path.is_file():
+        _log(f"missing pipeline file: {path}")
+        return 1
+
+    rendered = _render_pipeline(path, force_all=args.all, e2e_only=args.e2e)
+    if args.upload:
+        _upload_to_buildkite(rendered)
+    else:
+        sys.stdout.write(rendered)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.buildkite/cuda/bootstrap-upload-steps.yml

@@ -0,0 +1,55 @@
+# Uploaded by upload_pipeline.py --upload (not the Buildkite hook).
+# ``if`` is injected by step ``key`` from skip-ci at upload time.
+steps:
+  - label: ":docker: Build image"
+    key: image-build
+    commands:
+      - "bash .buildkite/cuda/scripts/build-ci-image.sh"
+    agents:
+      queue: "cpu_queue_premerge"
+
+  # L2 Test — PR ready label (diff-aware) or main+NIGHTLY=1 (--e2e, full E2E group)
+  - label: "Upload Ready Pipeline"
+    depends_on: image-build
+    key: upload-ready-pipeline
+    commands:
+      - |
+        EXTRA=""
+        if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then
+          EXTRA="--e2e"
+        fi
+        python3 .buildkite/common/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-ready.yml
+    agents:
+      queue: "cpu_queue_premerge"
+
+  # L3 Test — PR merge-test label (diff-aware) or main+NIGHTLY=1 (--e2e, full E2E group)
+  - label: "Upload Merge Pipeline"
+    depends_on: image-build
+    key: upload-merge-pipeline
+    commands:
+      - |
+        EXTRA=""
+        if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then
+          EXTRA="--e2e"
+        fi
+        python3 .buildkite/common/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-merge.yml
+    agents:
+      queue: "cpu_queue_premerge"
+
+  # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
+  - label: "Upload Nightly Pipeline"
+    depends_on: image-build
+    key: upload-nightly-pipeline
+    commands:
+      - python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/cuda/test-nightly.yml
+    agents:
+      queue: "cpu_queue_premerge"
+
+  # L5 Test — main+WEEKLY=1 (scheduled), or PR with weekly-test label
+  - label: "Upload Weekly Pipeline"
+    depends_on: image-build
+    key: upload-weekly-pipeline
+    commands:
+      - python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/cuda/test-weekly.yml
+    agents:
+      queue: "cpu_queue_premerge"

.buildkite/cuda/pipeline.yml

@@ -0,0 +1,9 @@
+# Hook uploads this entry YAML; upload_pipeline.py --upload reads bootstrap-upload-steps.yml
+# and injects ``if`` by step ``key`` from skip-ci.
+steps:
+  - label: ":github: Resolve skip-ci & upload pipeline"
+    key: upload-ci-pipeline
+    commands:
+      - "python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/cuda/bootstrap-upload-steps.yml"
+    agents:
+      queue: "cpu_queue_premerge"

.buildkite/cuda/rebase-pipeline.yml

@@ -2,10 +2,7 @@ steps:
   - label: ":docker: Build image"
     key: image-build
     commands:
-      - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
-      - "docker build --progress=plain --file docker/Dockerfile.ci -t vllm-omni-ci ."
-      - "docker tag vllm-omni-ci public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
-      - "docker push public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
+      - "bash .buildkite/cuda/scripts/build-ci-image.sh"
     agents:
       queue: "cpu_queue_premerge"
 
@@ -14,7 +11,7 @@ steps:
     depends_on: image-build
     key: upload-ready-pipeline
     commands:
-      - buildkite-agent pipeline upload .buildkite/test-ready.yml
+      - python3 .buildkite/common/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-ready.yml
     agents:
       queue: "cpu_queue_premerge"
 
@@ -23,7 +20,7 @@ steps:
     depends_on: image-build
     key: upload-merge-pipeline
     commands:
-      - buildkite-agent pipeline upload .buildkite/test-merge.yml
+      - python3 .buildkite/common/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-merge.yml
     agents:
       queue: "cpu_queue_premerge"
 
@@ -33,6 +30,6 @@ steps:
     key: upload-nightly-pipeline
     if: build.env("NIGHTLY") == "1"
     commands:
-      - buildkite-agent pipeline upload .buildkite/test-nightly.yml
+      - python3 .buildkite/common/scripts/upload_pipeline.py --upload --all .buildkite/cuda/test-nightly.yml
     agents:
       queue: "cpu_queue_premerge"

.buildkite/cuda/scripts/build-ci-image.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+set -euo pipefail
+
+# Build and push the CI Docker image with registry-based layer caching.
+#
+# Usage: build-ci-image.sh
+#
+# Expects the following environment variables (set by Buildkite):
+#   BUILDKITE_COMMIT          - commit SHA used as the image tag
+ECR_NAMESPACE="public.ecr.aws/q9t5s3a7"
+REGISTRY="${ECR_NAMESPACE}/vllm-ci-test-repo"
+REGION="us-east-1"
+DOCKERFILE="docker/Dockerfile.ci"
+BUILDER_NAME="vllm-omni-builder"
+
+# Ensure that the env vars are actually set, otherwise exit early
+if [ -z "${BUILDKITE_COMMIT:-}" ]; then
+    echo "ERROR: BUILDKITE_COMMIT is not set"
+    exit 1
+fi
+echo "BUILDKITE_COMMIT: ${BUILDKITE_COMMIT}"
+
+# Authenticate to ECR Public
+aws ecr-public get-login-password --region "$REGION" \
+    | docker login --username AWS --password-stdin "$ECR_NAMESPACE"
+
+# Compute the cache tag for the dependencies image as the hash of the contents.
+# This is because dependencies don't change often, so most PRs will have the same hash.
+#
+# The Dockerfile is also included to avoid churn in the cached image based on the vLLM version.
+DEP_FILES="pyproject.toml setup.py requirements/*.txt docker/Dockerfile.ci"
+CACHE_KEY=$(cat ${DEP_FILES} | sha256sum | cut -c1-16)
+CACHE_TAG="deps-cache-${CACHE_KEY}"
+echo "Cache key: ${CACHE_TAG}"
+
+# Set up buildx with docker-container driver; we need to do this
+# since cache export is not supported for the default docker driver
+# that is running in the CI.
+docker buildx inspect $BUILDER_NAME >/dev/null 2>&1 \
+|| docker buildx create --name $BUILDER_NAME --driver docker-container
+docker buildx use $BUILDER_NAME
+
+echo "Building image tag: ${REGISTRY}:${BUILDKITE_COMMIT}"
+
+# Build + push the image to the registry, using the hash from the dependencies.
+# Note that the caching here is layerwise, so we're just making sure we always
+# can cache hit on everything but the last copy at the moment.
+docker buildx build --push --progress=plain \
+    --cache-from "type=registry,ref=${REGISTRY}:${CACHE_TAG}" \
+    --cache-to "type=registry,ref=${REGISTRY}:${CACHE_TAG},mode=max,compression=zstd" \
+    --file "$DOCKERFILE" \
+    -t "${REGISTRY}:${BUILDKITE_COMMIT}" .

.buildkite/cuda/test-merge.yml

@@ -0,0 +1,259 @@
+env:
+  VLLM_WORKER_MULTIPROC_METHOD: spawn
+  VLLM_USE_FLASHINFER_MOE_FP16: 0
+  HF_HUB_DOWNLOAD_TIMEOUT: 300
+  HF_HUB_ETAG_TIMEOUT: 60
+
+steps:
+  - group: ":card_index_dividers: Simple Test"
+    depends_on: upload-merge-pipeline
+    steps:
+      - label: "Simple · Model Executor Test"
+        commands:
+          - "timeout 20m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+        mirror_hardwares: l4_1
+
+      - label: "Simple · Diffusion Test"
+        commands:
+          - "timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+        mirror_hardwares: l4_1
+
+      - label: "Simple · Engine&Entrypoints Test"
+        commands:
+          - "timeout 20m pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+        mirror_hardwares: l4_1
+
+      - label: "Simple · Other Test"
+        commands:
+          - "timeout 20m pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+        mirror_hardwares: l4_1
+
+  - group: ":card_index_dividers: Diffusion Test"
+    depends_on: upload-merge-pipeline
+    steps:
+      - label: "Diffusion · LoRA Test"
+        timeout_in_minutes: 30
+        commands:
+          - pytest -sv tests/diffusion/lora/ -m 'advanced_model and cuda' --run-level "advanced_model"
+        mirror_hardwares: l4_1
+
+  - label: "Distributed Test"
+    depends_on: upload-merge-pipeline
+    timeout_in_minutes: 60
+    commands:
+      - |
+        timeout 55m bash -c "
+          set -e
+          export VLLM_IMAGE_FETCH_TIMEOUT=60
+          pytest -s -v tests/distributed -m 'advanced_model and cuda and H100' --run-level 'advanced_model'
+        "
+    mirror_hardwares: h100_1
+
+  - group: ":card_index_dividers: Entrypoints Test"
+    depends_on: upload-merge-pipeline
+    steps:
+      - label: "Entrypoint Test with H100"
+        # TEMPORARY 60 -> 90: engine startup/sleep-wake is ~13-15 min slower per
+        # run on the vLLM 0.26 image (model loading ~2x, init ~2x; see builds
+        # 2915 vs 2916), which pushed this suite past 60 min. Revert once the
+        # 0.26 startup regression is understood and fixed.
+        timeout_in_minutes: 90
+        commands:
+          - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and H100" --run-level "advanced_model"
+        mirror_hardwares: h100_2
+
+      - label: "Entrypoint Test with L4"
+        timeout_in_minutes: 60
+        commands:
+          - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and L4" --run-level "advanced_model"
+        mirror_hardwares: l4_1
+
+  - group: ":card_index_dividers: E2E Test"
+    depends_on: upload-merge-pipeline
+    steps:
+      - label: "TTS · Qwen3-TTS CustomVoice Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen3_tts_customvoice.py
+          - tests/e2e/offline_inference/test_qwen3_tts_customvoice.py
+          - vllm_omni/model_executor/models/qwen3_tts/
+          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
+          - vllm_omni/deploy/qwen3_tts.yaml
+        commands:
+          - |
+            timeout 40m bash -c "
+              export VLLM_LOGGING_LEVEL=DEBUG
+              export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
+              pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+            "
+        mirror_hardwares: l4_1
+
+      - label: "TTS · Qwen3-TTS Base Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen3_tts_base.py
+          - tests/e2e/offline_inference/test_qwen3_tts_base.py
+          - vllm_omni/model_executor/models/qwen3_tts/
+          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
+          - vllm_omni/deploy/qwen3_tts.yaml
+        commands:
+          - |
+            timeout 40m bash -c "
+              export VLLM_LOGGING_LEVEL=DEBUG
+              export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
+              pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+            "
+        mirror_hardwares: l4_1
+
+      - label: "TTS · Higgs-Audio-V3 Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_higgs_audio_v3.py
+          - vllm_omni/model_executor/models/higgs_audio_v3/
+          - vllm_omni/model_executor/stage_input_processors/higgs_audio_v3.py
+          - vllm_omni/deploy/higgs_multimodal_qwen3.yaml
+        commands:
+          - |
+            timeout 20m bash -c "
+              export VLLM_WORKER_MULTIPROC_METHOD=spawn
+              export VLLM_USE_DEEP_GEMM=0
+              export VLLM_MOE_USE_DEEP_GEMM=0
+              pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py -m 'advanced_model' --run-level 'advanced_model'
+            "
+        mirror_hardwares: l4_1
+
+      - label: "Omni · Qwen3-Omni Test"
+        source_file_dependencies:
+          - tests/e2e/offline_inference/test_qwen3_omni.py
+          - tests/e2e/online_serving/test_qwen3_omni.py
+          - vllm_omni/model_executor/models/qwen3_omni/
+          - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
+          - vllm_omni/deploy/qwen3_omni_moe.yaml
+        timeout_in_minutes: 50
+        commands:
+          - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+        mirror_hardwares: h100_2
+
+      - label: "Omni · MiniCPM-o 4.5 Test"
+        source_file_dependencies:
+          - tests/e2e/offline_inference/test_minicpmo_4_5.py
+          - tests/e2e/online_serving/test_minicpmo_4_5.py
+          - vllm_omni/deploy/minicpmo_4_5.yaml
+        timeout_in_minutes: 50
+        commands:
+          - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+        mirror_hardwares: h100_2
+
+      - label: "Omni · MiniCPM-o 4.5 Duplex Test"
+        source_file_dependencies:
+          - tests/assets/minicpmo_4_5/response_required_16k.wav
+          - tests/assets/minicpmo_4_5/soft_interrupt_16k.wav
+          - tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
+          - tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
+          - tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py
+          - tests/e2e/online_serving/minicpmo_realtime_duplex_scenarios.py
+          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py
+          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
+          - tests/helpers/minicpmo_4_5_duplex.py
+          - vllm_omni/deploy/minicpmo_4_5_duplex.yaml
+          - vllm_omni/experimental/fullduplex/
+          - vllm_omni/model_executor/models/minicpmo_4_5/
+          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
+          - vllm_omni/core/sched/
+          - vllm_omni/outputs/multimodal_accumulation.py
+          - vllm_omni/engine/stage_init_utils.py
+          - vllm_omni/engine/async_omni_engine.py
+          - vllm_omni/entrypoints/cli/serve.py
+        timeout_in_minutes: 50
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+        mirror_hardwares: h100_2
+
+      - label: "TTS · VoxCPM2 Test"
+        source_file_dependencies:
+          - tests/e2e/offline_inference/test_voxcpm2_tts.py
+          - tests/e2e/online_serving/test_voxcpm2_tts.py
+          - vllm_omni/model_executor/models/voxcpm2/
+          - vllm_omni/deploy/voxcpm2.yaml
+        timeout_in_minutes: 30
+        commands:
+          - |
+            timeout 30m bash -c "
+              export VLLM_LOGGING_LEVEL=DEBUG
+              export VLLM_WORKER_MULTIPROC_METHOD=spawn
+              pytest -s -v tests/e2e/offline_inference/test_voxcpm2_tts.py tests/e2e/online_serving/test_voxcpm2_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+            "
+        mirror_hardwares: l4_1
+
+      - label: "Diffusion · Qwen Image Edit Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen_image_edit.py
+          - vllm_omni/diffusion/models/qwen_image/
+        timeout_in_minutes: 20
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_qwen_image_edit.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+        mirror_hardwares: h100_1
+      - label: "Diffusion · Bagel Test"
+        source_file_dependencies:
+          - tests/e2e/offline_inference/test_bagel.py
+          - tests/e2e/online_serving/test_bagel.py
+          - vllm_omni/model_executor/models/bagel/
+          - vllm_omni/diffusion/models/bagel/
+          - vllm_omni/model_executor/stage_input_processors/bagel.py
+          - vllm_omni/deploy/bagel.yaml
+        timeout_in_minutes: 20
+        commands:
+          - pytest -s -v tests/e2e/offline_inference/test_bagel.py tests/e2e/online_serving/test_bagel.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · Qwen Image Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen_image.py
+          - vllm_omni/diffusion/models/qwen_image/
+        commands:
+          - |
+            timeout 40m bash -c '
+              export VLLM_IMAGE_FETCH_TIMEOUT=60
+              pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+            '
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · Qwen Image Layered Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen_image_layered.py
+          - vllm_omni/diffusion/models/qwen_image/
+        commands:
+          - |
+            timeout 40m bash -c "
+              export VLLM_IMAGE_FETCH_TIMEOUT=60
+              pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+            "
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · Wan22 Test"
+        source_file_dependencies:
+          - tests/e2e/offline_inference/test_wan22_t2v.py
+          - tests/e2e/online_serving/test_wan22_t2v.py
+          - vllm_omni/diffusion/models/wan2_2/
+        commands:
+          - |
+            timeout 40m bash -c "
+              export VLLM_IMAGE_FETCH_TIMEOUT=60
+              pytest -s -v tests/e2e/offline_inference/test_wan22_t2v.py tests/e2e/online_serving/test_wan22_t2v.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
+            "
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · Z Image Test"
+        source_file_dependencies:
+          - tests/e2e/offline_inference/test_z_image.py
+          - tests/e2e/online_serving/test_z_image.py
+          - vllm_omni/diffusion/models/z_image/
+        timeout_in_minutes: 30
+        commands:
+          - pytest -s -v tests/e2e/offline_inference/test_z_image.py tests/e2e/online_serving/test_z_image.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
+        mirror_hardwares: l4_1
+      - label: "Diffusion · FLUX.2 Klein Test"
+        source_file_dependencies:
+          - tests/e2e/offline_inference/test_flux2_klein.py
+          - vllm_omni/diffusion/models/flux2_klein/
+        timeout_in_minutes: 30
+        commands:
+          - pytest -s -v tests/e2e/offline_inference/test_flux2_klein.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
+        mirror_hardwares: l4_1

.buildkite/cuda/test-nightly.yml

@@ -0,0 +1,572 @@
+env:
+  VLLM_WORKER_MULTIPROC_METHOD: spawn
+  VLLM_USE_FLASHINFER_MOE_FP16: 0
+  HF_HUB_DOWNLOAD_TIMEOUT: 300
+  HF_HUB_ETAG_TIMEOUT: 60
+
+steps:
+  # Group: collapses under one heading in the Buildkite UI; child steps still run in parallel.
+  - group: ":card_index_dividers: Omni Model Test"
+    key: nightly-omni-test-group
+    depends_on: upload-nightly-pipeline
+    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "omni-test"
+    steps:
+
+      - label: ":full_moon: Omni · Function Test with H100"
+        timeout_in_minutes: 120
+        commands:
+          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Omni · MiniCPM-o 4.5 Duplex Test"
+        timeout_in_minutes: 50
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and omni" --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Omni · Doc Test with L4"
+        timeout_in_minutes: 90
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - pytest -s -v tests/examples/ -m "full_model and omni and L4" --run-level "full_model"
+        mirror_hardwares: l4_4
+
+      - label: ":full_moon: Omni · Doc Test with H100"
+        timeout_in_minutes: 90
+        commands:
+          - pytest -s -v tests/examples/ -m "full_model and omni and H100" --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Omni · Accuracy Test"
+        timeout_in_minutes: 180
+        commands:
+          - export SEED_TTS_WER_EVAL=1
+          - export SEED_TTS_EVAL_DEVICE=cuda:1
+          - |
+            set +e
+            pytest -s -v tests/e2e/accuracy/qwen3_omni/test_qwen3_omni.py -m "full_model" --run-level full_model
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/e2e/accuracy/qwen3_omni/results/qwen_omni_acc/*.json"
+            exit $$EXIT
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Omni · Perf Test · No Async Chunk"
+        key: nightly-omni-performance-no-async-chunk
+        timeout_in_minutes: 300
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
+            exit $$EXIT
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Omni · Perf Test · Async Chunk"
+        key: nightly-omni-performance-async-chunk
+        timeout_in_minutes: 180
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
+            exit $$EXIT
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Omni · Perf Test · vLLM Text"
+        key: nightly-omni-performance-vllm-text
+        timeout_in_minutes: 180
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
+            exit $$EXIT
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Omni · Perf Test · Multi-Replica"
+        key: nightly-omni-performance-multi-replicas
+        timeout_in_minutes: 180
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
+            exit $$EXIT
+        mirror_hardwares: h100_3
+
+      - label: ":full_moon: Omni · Multi-Replica Startup Test with 4x H100"
+        timeout_in_minutes: 45
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py -m "full_model" --run-level "core_model"
+        mirror_hardwares: h100_4
+
+  - group: ":card_index_dividers: TTS Model Test"
+    key: nightly-tts-test-group
+    depends_on: upload-nightly-pipeline
+    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test"
+    steps:
+
+      - label: ":full_moon: TTS · Function Test with L4"
+        timeout_in_minutes: 120
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - export VLLM_USE_DEEP_GEMM="0"
+          - export VLLM_MOE_USE_DEEP_GEMM="0"
+          - pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_voxcpm2_tts_expansion.py
+        mirror_hardwares: l4_1
+
+      - label: ":full_moon: TTS · Accuracy Test · VoxCPM2"
+        timeout_in_minutes: 30
+        commands:
+          - |
+            timeout 30m bash -c "
+              export VLLM_LOGGING_LEVEL=DEBUG
+              export VLLM_WORKER_MULTIPROC_METHOD=spawn
+              export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
+              pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts_expansion.py -m 'full_model and cuda and tts' --run-level 'full_model'
+            "
+        mirror_hardwares: l4_1
+
+      - label: ":full_moon: TTS · Perf Test"
+        key: nightly-tts-performance
+        timeout_in_minutes: 180
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
+            exit $$EXIT
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: TTS · VoxCPM2 · Perf Test"
+        key: nightly-tts-performance-voxcpm2
+        timeout_in_minutes: 180
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_voxcpm2.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
+            exit $$EXIT
+        mirror_hardwares: h100_1
+
+      - label: ":full_moon: TTS · Higgs-Audio-V3 · Perf Test"
+        key: nightly-tts-performance-higgs-audio-v3
+        timeout_in_minutes: 180
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - export VLLM_USE_DEEP_GEMM="0"
+          - export VLLM_MOE_USE_DEEP_GEMM="0"
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_higgs_audio_v3.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
+            exit $$EXIT
+        mirror_hardwares: h100_1
+
+  - group: ":card_index_dividers: Tiny Model Tests [Multi-GPU]"
+    key: nightly-tiny-model-test-group
+    depends_on: upload-nightly-pipeline
+    if: >-
+      build.env("NIGHTLY") == "1" ||
+      build.pull_request.labels includes "nightly-test"
+    steps:
+
+      - label: ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations)"
+        timeout_in_minutes: 30
+        # NOTE: Single-GPU (core_model) tiny model tests run on the ready label,
+        # so nightly only adds the multi-GPU parallelism configurations.
+        #
+        # This is a bit confusing, but `full_model` is used to mark the multigpu tests, and
+        # --run-level `core_model` is used to indicate that we should use tiny model weights
+        # for the CI, since these tests should pass irrespective of whether or not we use
+        # the full weights or not.
+        commands:
+          - timeout 25m pytest -sv tests/model_tests/diffusion -m 'full_model and cuda' --run-level "core_model"
+        mirror_hardwares: l4_4
+
+  - group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test"
+    key: nightly-diffusion-x2iat-group
+    depends_on: upload-nightly-pipeline
+    if: >-
+      build.env("NIGHTLY") == "1" ||
+      build.pull_request.labels includes "nightly-test" ||
+      build.pull_request.labels includes "diffusion-x2iat-test"
+    steps:
+      # Use explicit file shards so single-card cases do not reserve 2+ GPUs,
+      # and Bagel feature coverage is not duplicated via a broad tests/e2e sweep.
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Single-GPU"
+        timeout_in_minutes: 120
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_qwen_image_expansion.py
+            tests/e2e/online_serving/test_qwen_image_edit_expansion.py
+            tests/e2e/online_serving/test_qwen_image_layered_expansion.py
+            tests/e2e/online_serving/test_boogu_image.py
+            tests/e2e/online_serving/test_boogu_image_edit.py
+            tests/e2e/online_serving/test_boogu_image_expansion.py
+            tests/e2e/online_serving/test_boogu_image_edit_expansion.py
+            tests/e2e/online_serving/test_bagel_expansion.py
+            tests/e2e/online_serving/test_krea2_expansion.py
+            tests/e2e/offline_inference/test_krea2_expansion.py
+            -m "full_model and diffusion and H100 and not distributed_cuda"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Qwen-Image"
+        timeout_in_minutes: 120
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_qwen_image_expansion.py
+            tests/e2e/online_serving/test_qwen_image_edit_expansion.py
+            tests/e2e/offline_inference/test_krea2_expansion.py
+            -m "full_model and diffusion and H100 and distributed_cuda"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Layered/LongCat/FLUX"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_qwen_image_layered_expansion.py
+            -m "full_model and diffusion and H100 and distributed_cuda"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Bagel"
+        timeout_in_minutes: 120
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_bagel_expansion.py
+            -m "full_model and diffusion and H100 and distributed_cuda"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · GLM-Image Function Test with H100"
+        timeout_in_minutes: 120
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_glm_image_expansion.py
+            tests/e2e/offline_inference/test_glm_image_autoround_w4a16_expansion.py
+            -m "full_model and diffusion and H100"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4"
+        timeout_in_minutes: 120
+        commands:
+          - pytest -sv tests/e2e/ -k "not test_wan and not test_bagel_expansion and not hunyuan" -m "full_model and diffusion and L4" --run-level "full_model" --ignore=tests/e2e/accuracy
+        mirror_hardwares: l4_4
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · Doc Test"
+        timeout_in_minutes: 60
+        commands:
+          - pytest -s -v tests/examples/*/test_text_to_image.py -m "full_model and example and H100" --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · Accuracy Test"
+        timeout_in_minutes: 180
+        commands:
+          - export VLLM_HTTP_TIMEOUT_KEEP_ALIVE=120
+          - pytest -s -v tests/e2e/accuracy/test_qwen_image*.py --run-level full_model
+          - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2i_matches_diffusers' -m full_model --run-level full_model
+        mirror_hardwares: h100_1
+
+      - label: ":full_moon: HunyuanImage3-DIT · Accuracy Test"
+        timeout_in_minutes: 180
+        commands:
+          - export HUNYUAN_IMAGE3_MODEL="tencent/HunyuanImage-3.0-Instruct"
+          - export HUNYUAN_IMAGE3_DEVICES="0,1,2,3"
+          - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m full_model --run-level full_model
+        mirror_hardwares: h100_4
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image"
+        key: nightly-diffusion-x2iat-performance-qwen-image
+        timeout_in_minutes: 180
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
+          - export CACHE_DIT_VERSION=1.3.0
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+        mirror_hardwares: h100_4
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit"
+        key: nightly-diffusion-x2iat-performance-qwen-image-edit
+        timeout_in_minutes: 180
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
+          - export CACHE_DIT_VERSION=1.3.0
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+        mirror_hardwares: h100_4
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit-2511"
+        key: nightly-diffusion-x2iat-performance-qwen-image-edit-2511
+        timeout_in_minutes: 180
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
+          - export CACHE_DIT_VERSION=1.3.0
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+        mirror_hardwares: h100_4
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Layered"
+        key: nightly-diffusion-x2iat-performance-qwen-image-layered
+        timeout_in_minutes: 120
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
+          - export CACHE_DIT_VERSION=1.3.0
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+        mirror_hardwares: h100_1
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · BAGEL"
+        key: nightly-diffusion-x2iat-performance-bagel
+        timeout_in_minutes: 180
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
+          - export CACHE_DIT_VERSION=1.3.0
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_bagel_vllm_omni.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+        mirror_hardwares: h100_1
+
+  - group: ":card_index_dividers: Diffusion X2V Model Test"
+    key: nightly-diffusion-x2v-group
+    depends_on: upload-nightly-pipeline
+    if: >-
+      build.env("NIGHTLY") == "1" ||
+      build.pull_request.labels includes "nightly-test" ||
+      build.pull_request.labels includes "diffusion-x2v-test"
+    steps:
+
+      - label: ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test"
+        timeout_in_minutes: 90
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda" --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test"
+        timeout_in_minutes: 90
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda" --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Diffusion X2V · LingBot Video MoE Function Test"
+        timeout_in_minutes: 90
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_lingbot_video_moe.py -k "video_generation" -m "full_model and cuda" --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: ":full_moon: Diffusion X2V · Doc Test"
+        timeout_in_minutes: 60
+        commands:
+          - pytest -s -v tests/examples/offline_inference/test_image_to_video.py -m "full_model and example and H100" --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: ":full_moon: Diffusion X2V · Other Function Test"
+        timeout_in_minutes: 90
+        commands:
+          - pytest -s -v tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py -m "full_model and cuda" --run-level "full_model"
+          - pytest -s -v tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_ltx2_expansion.py -m "full_model and cuda" --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Diffusion X2V · Wan2.2 Accuracy Test"
+        timeout_in_minutes: 180
+        commands:
+          - pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py -m full_model --run-level full_model
+          - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m full_model --run-level full_model
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Diffusion X2V · MiniMax H3 T2VA Accuracy Test"
+        timeout_in_minutes: 180
+        commands:
+          - pytest -s -v tests/e2e/accuracy/minimax_h3/test_minimax_h3_t2va_similarity.py -m full_model --run-level full_model
+        mirror_hardwares: h100_4
+
+      - label: ":full_moon: Diffusion X2V · Other Accuracy Test"
+        timeout_in_minutes: 180
+        commands:
+          - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py -m full_model --run-level full_model
+          - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m full_model --run-level full_model
+          - pytest -s -v 'tests/e2e/accuracy/ltx/test_ltx_official_similarity.py::test_ltx_one_stage_matches_official[ltx2_3_i2v]' -m full_model --run-level full_model
+          - pytest -s -v tests/e2e/accuracy/test_video_streaming_output_similarity.py -m full_model --run-level full_model
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Diffusion X2V · Perf Test"
+        key: nightly-diffusion-x2v-performance
+        timeout_in_minutes: 180
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json
+            EXIT1=$$?
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_cosmos3_vllm_omni.json
+            EXIT2=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            if [ $$EXIT1 -ne 0 ]; then exit $$EXIT1; fi
+            exit $$EXIT2
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Diffusion X2V · Perf Test · MiniMax-H3"
+        key: nightly-diffusion-x2v-performance-minimax-h3
+        timeout_in_minutes: 180
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_minimax_h3_vllm_omni.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+        mirror_hardwares: h100_4
+
+  - group: ":card_index_dividers: Diffusion Test"
+    key: nightly-diffusion-test-group
+    depends_on: upload-nightly-pipeline
+    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
+    steps:
+      - label: ":full_moon: Diffusion · Distributed Test with L4"
+        timeout_in_minutes: 60
+        commands:
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4' --run-level "full_model"
+        mirror_hardwares: l4_4
+
+      - label: ":full_moon: Diffusion · Distributed Test with H100"
+        timeout_in_minutes: 60
+        commands:
+          - pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100' --run-level "full_model"
+        mirror_hardwares: h100_4
+
+      - label: ":full_moon: Diffusion · Offloader Test"
+        timeout_in_minutes: 60
+        commands:
+          - pytest -sv tests/diffusion/offloader -m "full_model and cuda"
+        mirror_hardwares: l4_1
+
+      - label: ":full_moon: Diffusion · Quantization Test with H100 · Single-GPU"
+        timeout_in_minutes: 60
+        commands:
+          - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100' --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: ":full_moon: Diffusion · MiniMax H3 FP8 Quality Test with H100 · 2-GPU"
+        timeout_in_minutes: 60
+        commands:
+          - pytest -sv tests/diffusion/models/minimax_h3/test_minimax_h3_quantization_quality.py -m 'full_model and cuda and H100' --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: ":full_moon: Diffusion · Quantization Test with L4"
+        timeout_in_minutes: 60
+        commands:
+          - pip install "vllm-gguf-plugin>=0.0.3"
+          - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4' --run-level "full_model"
+        mirror_hardwares: l4_1
+
+  - label: ":bar_chart: Testcase Statistics"
+    key: nightly-testcase-statistics
+    timeout_in_minutes: 120
+    depends_on: upload-nightly-pipeline
+    if: build.env("NIGHTLY") == "1"
+    commands:
+      - python tools/nightly/buildkite_testcase_statistics.py -o tests/dfx/perf/results/buildkite_testcase_statistics.html
+      - buildkite-agent artifact upload "tests/dfx/perf/results/*.html"
+    mirror_hardwares: h100_1
+
+  # NOTE: ":email: Nightly Collection & Email" is deprecated and will be removed soon.
+  # Do not add new depends_on entries or extend this step.
+  - label: ":email: Nightly Collection & Email"
+    key: nightly-perf-distribution
+    depends_on:
+      - nightly-omni-performance-no-async-chunk
+      - nightly-omni-performance-async-chunk
+      - nightly-omni-performance-vllm-text
+      - nightly-omni-performance-multi-replicas
+      - nightly-tts-performance
+      - nightly-tts-performance-higgs-audio-v3
+      - nightly-diffusion-x2iat-performance-qwen-image
+      - nightly-diffusion-x2iat-performance-qwen-image-edit
+      - nightly-diffusion-x2iat-performance-qwen-image-edit-2511
+      - nightly-diffusion-x2iat-performance-qwen-image-layered
+      - nightly-diffusion-x2iat-performance-bagel
+      - nightly-diffusion-x2v-performance
+      - nightly-testcase-statistics
+    if: build.env("NIGHTLY") == "1" && build.env("EMAIL_DISTRIBUTION") == "1"
+    commands:
+      - pip install openpyxl
+      - export DEFAULT_INPUT_DIR=tests/dfx/perf/results
+      - export DEFAULT_OUTPUT_DIR=tests/dfx/perf/results
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance-higgs-audio-v3
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-no-async-chunk
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-async-chunk
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-vllm-text
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-multi-replicas
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit-2511
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-layered
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-bagel
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.html" . --step nightly-testcase-statistics
+      - python tools/nightly/generate_nightly_perf_excel.py
+      - python tools/nightly/generate_nightly_perf_html.py
+      - python tools/nightly/send_nightly_email.py --report-file "tests/dfx/perf/results/*.xlsx, tests/dfx/perf/results/*.html"
+      - buildkite-agent artifact upload "tests/dfx/perf/results/*.xlsx"
+      - buildkite-agent artifact upload "tests/dfx/perf/results/*.html"
+    agents:
+      queue: "cpu_queue_premerge"

.buildkite/cuda/test-ready.yml

@@ -0,0 +1,348 @@
+env:
+  VLLM_WORKER_MULTIPROC_METHOD: spawn
+  VLLM_USE_FLASHINFER_MOE_FP16: 0
+  HF_HUB_DOWNLOAD_TIMEOUT: 300
+  HF_HUB_ETAG_TIMEOUT: 60
+
+steps:
+  - group: ":card_index_dividers: Simple Test"
+    depends_on: upload-ready-pipeline
+    steps:
+      - label: "Simple · Model Executor Test"
+        commands:
+          - "timeout 20m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+        mirror_hardwares: l4_1
+
+      - label: "Simple · Diffusion Test"
+        commands:
+          - "timeout 20m pytest -sv tests/diffusion -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+        mirror_hardwares: l4_1
+
+      - label: "Simple · Engine&Entrypoints Test"
+        commands:
+          - "timeout 20m pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+        mirror_hardwares: l4_1
+
+      - label: "Simple · Other Test"
+        commands:
+          - "timeout 20m pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+        mirror_hardwares: l4_1
+
+  - group: ":card_index_dividers: Diffusion Test"
+    depends_on: upload-ready-pipeline
+    steps:
+
+      - label: "Diffusion · Offloader Test"
+        commands:
+          - timeout 40m pytest -s -v tests/diffusion/offloader -m "core_model and cuda" --run-level "core_model"
+        mirror_hardwares: l4_1
+
+      - label: "Diffusion · Cache Test"
+        commands:
+          - timeout 15m pytest -sv tests/diffusion/cache -m 'core_model and cuda' --run-level "core_model"
+        mirror_hardwares: l4_1
+
+      - label: "Diffusion · Batch Test"
+        commands:
+          - timeout 15m pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model"
+        mirror_hardwares: l4_1
+
+      - label: "Diffusion · Distributed & Attention Test"
+        source_file_dependencies:
+          - tests/diffusion/distributed/
+          - tests/diffusion/attention/test_attention_sp.py
+          - vllm_omni/diffusion/distributed/
+          - vllm_omni/diffusion/attention/
+          - vllm_omni/diffusion/data.py
+          - vllm_omni/diffusion/models/
+        commands:
+          - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda' --run-level "core_model"
+          - timeout 20m pytest -sv tests/diffusion/attention/test_attention_sp.py -m 'core_model and cuda' --run-level "core_model"
+        mirror_hardwares: l4_4
+
+      - label: "Diffusion · Model Test"
+        commands:
+          - timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model"
+        mirror_hardwares: l4_1
+
+      - label: "Diffusion · Other Test"
+        commands:
+          - timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda' --run-level "core_model"
+        mirror_hardwares: l4_1
+
+  - group: ":card_index_dividers: Tiny Model Tests [Single GPU]"
+    depends_on: upload-ready-pipeline
+    steps:
+
+      - label: "Tiny Model Tests · Diffusion (base)"
+        commands:
+          - timeout 15m pytest -sv tests/model_tests/diffusion -m 'core_model and cuda' --run-level "core_model"
+        mirror_hardwares: l4_1
+
+  - label: "Engine&Model Executor Test"
+    depends_on: upload-ready-pipeline
+    commands:
+      - |
+        timeout 15m bash -c "
+                  pytest -sv tests/engine/ tests/model_executor/ -m 'core_model and cuda' --run-level 'core_model'
+        "
+    mirror_hardwares: l4_1
+
+  - label: "Distributed Test"
+    depends_on: upload-ready-pipeline
+    commands:
+      - |
+        timeout 20m bash -c "
+          pytest -sv tests/distributed/ -m 'core_model and cuda and L4' --run-level 'core_model'
+        "
+    mirror_hardwares: l4_1
+
+  - label: "Entrypoints Test"
+    depends_on: upload-ready-pipeline
+    commands:
+      - |
+        timeout 30m bash -c "
+          export VLLM_WORKER_MULTIPROC_METHOD=spawn
+          pytest -sv tests/entrypoints/ -m 'core_model and cuda' --run-level 'core_model'
+        "
+    mirror_hardwares: l4_1
+
+  - group: ":card_index_dividers: E2E Test"
+    depends_on: upload-ready-pipeline
+    steps:
+      - label: "Custom Pipeline Test"
+        source_file_dependencies:
+          - tests/e2e/features/custom_pipeline/
+          - tests/e2e/features/helpers/custom_pipeline.py
+          - vllm_omni/diffusion/
+          - vllm_omni/engine/
+          - vllm_omni/entrypoints/
+          - vllm_omni/inputs/
+          - vllm_omni/outputs/
+        commands:
+          - timeout 20m pytest -s -v tests/e2e/features/custom_pipeline/ -m "core_model"
+        agents:
+          queue: "gpu_1_queue"
+        plugins:
+          - docker#v5.2.0:
+              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+              always-pull: true
+              propagate-environment: true
+              environment:
+                - "HF_HOME=/fsx/hf_cache"
+              volumes:
+                - "/fsx/hf_cache:/fsx/hf_cache"
+
+      - label: "RLHF · VeRL-Omni E2E Test"
+        source_file_dependencies:
+          - tests/e2e/features/rlhf_test/test_verl_omni_e2e.py
+          - tests/e2e/features/helpers/verl_omni_server.py
+          - tests/e2e/features/helpers/custom_pipeline.py
+          - vllm_omni/diffusion/
+          - vllm_omni/engine/
+          - vllm_omni/entrypoints/
+          - vllm_omni/inputs/
+          - vllm_omni/outputs/
+        commands:
+          - |
+            timeout 20m bash -c "
+              pytest -s -v tests/e2e/features/rlhf_test/test_verl_omni_e2e.py
+            "
+        mirror_hardwares: l4_1
+
+      - label: "Omni · Qwen3-Omni Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen3_omni.py
+          - vllm_omni/model_executor/models/qwen3_omni/
+          - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
+          - vllm_omni/deploy/qwen3_omni_moe.yaml
+        commands:
+          - |
+            timeout 20m bash -c "
+              pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m 'core_model' --run-level 'core_model'
+            "
+        mirror_hardwares: h100_2
+
+      - label: "Omni · MiniCPM-o 4.5 Offline Test"
+        source_file_dependencies:
+          - tests/e2e/offline_inference/test_minicpmo_4_5.py
+          - vllm_omni/deploy/minicpmo_4_5.yaml
+        commands:
+          - |
+            timeout 20m bash -c "
+              pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model'
+            "
+        mirror_hardwares: h100_2
+
+      - label: "Omni · MiniCPM-o 4.5 Online Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_minicpmo_4_5.py
+          - vllm_omni/deploy/minicpmo_4_5.yaml
+        commands:
+          - |
+            timeout 20m bash -c "
+              pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model'
+            "
+        mirror_hardwares: h100_2
+
+      - label: "Omni · MiniCPM-o 4.5 Duplex Test"
+        source_file_dependencies:
+          - tests/assets/minicpmo_4_5/response_required_16k.wav
+          - tests/assets/minicpmo_4_5/soft_interrupt_16k.wav
+          - tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
+          - tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
+          - tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py
+          - tests/e2e/online_serving/minicpmo_realtime_duplex_scenarios.py
+          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py
+          - tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
+          - tests/helpers/minicpmo_4_5_duplex.py
+          - vllm_omni/deploy/minicpmo_4_5_duplex.yaml
+          - vllm_omni/experimental/fullduplex/
+          - vllm_omni/model_executor/models/minicpmo_4_5/
+          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
+          - vllm_omni/core/sched/
+          - vllm_omni/outputs/multimodal_accumulation.py
+          - vllm_omni/engine/stage_init_utils.py
+          - vllm_omni/engine/async_omni_engine.py
+          - vllm_omni/entrypoints/cli/serve.py
+        commands:
+          - |
+            timeout 20m bash -c "
+              pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'core_model and cuda' --run-level 'core_model'
+            "
+        mirror_hardwares: h100_2
+
+      - label: "TTS · VoxCPM2 Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_voxcpm2_tts.py
+          - vllm_omni/model_executor/models/voxcpm2/
+          - vllm_omni/deploy/voxcpm2.yaml
+        timeout_in_minutes: 30
+        commands:
+          - |
+            timeout 30m bash -c "
+              export VLLM_LOGGING_LEVEL=DEBUG
+              export VLLM_WORKER_MULTIPROC_METHOD=spawn
+              pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts.py -m 'core_model' --run-level 'core_model'
+            "
+        mirror_hardwares: l4_1
+
+      - label: "TTS · Qwen3-TTS CustomVoice Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen3_tts_customvoice.py
+          - vllm_omni/model_executor/models/qwen3_tts/
+          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
+          - vllm_omni/deploy/qwen3_tts.yaml
+        commands:
+          - |
+            timeout 20m bash -c "
+              export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
+              pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'
+            "
+        mirror_hardwares: l4_1
+
+      - label: "TTS · Qwen3-TTS Base Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen3_tts_base.py
+          - vllm_omni/model_executor/models/qwen3_tts/
+          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
+          - vllm_omni/deploy/qwen3_tts.yaml
+        commands:
+          - |
+            timeout 20m bash -c "
+              export VLLM_WORKER_MULTIPROC_METHOD=spawn
+              export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
+              export VLLM_OMNI_USE_V2_RUNNER=1
+              pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py -m 'core_model' --run-level 'core_model'
+            "
+        mirror_hardwares: l4_1
+
+      - label: "TTS · Higgs-Audio-V3 Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_higgs_audio_v3.py
+          - vllm_omni/model_executor/models/higgs_audio_v3/
+          - vllm_omni/model_executor/stage_input_processors/higgs_audio_v3.py
+          - vllm_omni/deploy/higgs_multimodal_qwen3.yaml
+        commands:
+          - |
+            timeout 20m bash -c "
+              export VLLM_WORKER_MULTIPROC_METHOD=spawn
+              export VLLM_USE_DEEP_GEMM=0
+              export VLLM_MOE_USE_DEEP_GEMM=0
+              pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py -m 'core_model' --run-level 'core_model'
+            "
+        mirror_hardwares: l4_1
+
+      - label: "Diffusion · Qwen Image Edit Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen_image_edit.py
+          - vllm_omni/diffusion/models/qwen_image/
+        commands:
+          - |
+            timeout 20m bash -c "
+              pytest -s -v tests/e2e/online_serving/test_qwen_image_edit.py -m 'core_model' --run-level 'core_model'
+            "
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · Bagel Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_bagel.py
+          - vllm_omni/model_executor/models/bagel/
+          - vllm_omni/diffusion/models/bagel/
+          - vllm_omni/model_executor/stage_input_processors/bagel.py
+          - vllm_omni/deploy/bagel.yaml
+        commands:
+          - |
+            timeout 40m bash -c "
+              export VLLM_IMAGE_FETCH_TIMEOUT=60
+              pytest -s -v tests/e2e/online_serving/test_bagel.py -m 'core_model' --run-level 'core_model'
+            "
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · Wan22 Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_wan22_t2v.py
+          - vllm_omni/diffusion/models/wan2_2/
+        commands:
+          - |
+            timeout 40m bash -c "
+              export VLLM_IMAGE_FETCH_TIMEOUT=60
+              pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model' --run-level 'core_model'
+            "
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · Cosmos3 Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_cosmos3.py
+          - vllm_omni/diffusion/models/cosmos3/
+        commands:
+          - |
+            timeout 40m bash -c "
+              export VLLM_IMAGE_FETCH_TIMEOUT=60
+              pytest -s -v tests/e2e/online_serving/test_cosmos3.py -m 'core_model' --run-level 'core_model'
+            "
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · Qwen Image Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen_image.py
+          - vllm_omni/diffusion/models/qwen_image/
+        commands:
+          - |
+            timeout 40m bash -c "
+              export VLLM_IMAGE_FETCH_TIMEOUT=60
+              pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model' --run-level 'core_model'
+            "
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · Qwen Image Layered Test"
+        source_file_dependencies:
+          - tests/e2e/online_serving/test_qwen_image_layered.py
+          - vllm_omni/diffusion/models/qwen_image/
+        commands:
+          - |
+            timeout 40m bash -c "
+              export VLLM_IMAGE_FETCH_TIMEOUT=60
+              pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'core_model' --run-level 'core_model'
+            "
+        mirror_hardwares: h100_1

.buildkite/cuda/test-weekly.yml

@@ -0,0 +1,136 @@
+env:
+  VLLM_WORKER_MULTIPROC_METHOD: spawn
+  VLLM_USE_FLASHINFER_MOE_FP16: 0
+  HF_HUB_DOWNLOAD_TIMEOUT: 300
+  HF_HUB_ETAG_TIMEOUT: 60
+
+steps:
+
+  - label: "Reliability Test - qwen3-omni"
+    timeout_in_minutes: 180
+    depends_on: upload-weekly-pipeline
+    if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
+    commands:
+      - pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m "slow"
+    mirror_hardwares: h100_2
+
+  - label: "Reliability Test - wan22"
+    timeout_in_minutes: 180
+    depends_on: upload-weekly-pipeline
+    if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
+    commands:
+      - pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m "slow"
+    mirror_hardwares: h100_2
+
+  - label: "Reliability Test - hunyuan-image"
+    timeout_in_minutes: 180
+    depends_on: upload-weekly-pipeline
+    if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
+    commands:
+      - pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m "slow"
+    mirror_hardwares: h100_4
+
+  - label: "Reliability Test - voxcpm2"
+    timeout_in_minutes: 180
+    depends_on: upload-weekly-pipeline
+    if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
+    commands:
+      - pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m "slow"
+    mirror_hardwares: l4_1
+
+  - group: "Reliability Test - Invalid parameters Test"
+    depends_on: upload-weekly-pipeline
+    if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
+    steps:
+
+      - label: "Invalid parameters Test · H100"
+        timeout_in_minutes: 120
+        commands:
+          - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100"
+        mirror_hardwares: h100_2
+
+      - label: "Invalid parameters Test · L4"
+        timeout_in_minutes: 120
+        commands:
+          - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and L4"
+        mirror_hardwares: l4_1
+
+  - group: ":card_index_dividers: E2E Tests"
+    depends_on: upload-weekly-pipeline
+    if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
+    steps:
+
+      - label: "Omni · L4"
+        timeout_in_minutes: 90
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model"
+          - >-
+            pytest -s -v tests/e2e/
+            --ignore=tests/e2e/online_serving/test_qwen2_5_omni_expansion.py
+            --ignore=tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py
+            -m "slow and L4 and omni"
+            --run-level "full_model"
+        mirror_hardwares: l4_4
+
+      - label: "Omni · H100"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/ -m "slow and H100 and omni"
+            --run-level "full_model"
+        mirror_hardwares: h100_2
+
+      - label: "TTS · L4"
+        timeout_in_minutes: 90
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - export VLLM_USE_DEEP_GEMM="0"
+          - export VLLM_MOE_USE_DEEP_GEMM="0"
+          - |
+            failed=0
+            for f in $(pytest --collect-only -q tests/e2e/ -m "slow and L4 and tts" --run-level "full_model" | grep '\.py::' | sed -E 's|::.*||' | sort -u); do
+              pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1
+            done
+            exit $$failed
+        mirror_hardwares: l4_1
+
+      - label: "TTS · H100"
+        timeout_in_minutes: 90
+        commands:
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - export VLLM_USE_DEEP_GEMM="0"
+          - export VLLM_MOE_USE_DEEP_GEMM="0"
+          - >-
+            pytest -sv
+            tests/e2e/ -m "slow and H100 and tts"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · L4"
+        timeout_in_minutes: 60
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/ -m "slow and diffusion and L4"
+            --run-level "full_model"
+        mirror_hardwares: l4_4
+
+      - label: "Diffusion · H100 · Single-GPU"
+        timeout_in_minutes: 120
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/ -m "slow and diffusion and H100 and not distributed_cuda"
+            --run-level "full_model"
+        mirror_hardwares: h100_1
+
+      - label: "Diffusion · H100 · Multi-GPU"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/ -m "slow and diffusion and H100 and distributed_cuda"
+            --run-level "full_model"
+        mirror_hardwares: h100_2

.buildkite/intel/pipeline-intel.yml

@@ -1,16 +1,16 @@
-# .buildkite/pipeline-intel.yaml
+# .buildkite/intel/pipeline-intel.yml
 steps:
   - group: "INTEL Tests"
     steps:
       - label: "Intel: XPU Test"
-        command: "bash .buildkite/scripts/hardware_ci/run-xpu-test.sh"
+        command: "bash .buildkite/intel/scripts/run-xpu-test.sh"
         agents:
           queue: "intel-gpu-omni"
         env:
           DOCKER_BUILDKIT: "1"
           # Buildkite will automatically replace this with the actual commit hash
           VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
-          VLLM_VERSION: "v0.21.0"
+          VLLM_VERSION: "v0.26.0"
         priority: 100
         timeout_in_minutes: 60
         soft_fail: false

.buildkite/intel/scripts/bootstrap-intel-omni.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+# vllm-omni Intel bootstrap
+# Uses static pipeline-intel.yml for Intel XPU tests
+
+set -euo pipefail
+
+source .buildkite/common/scripts/resolve_skip_ci.sh
+
+upload_pipeline() {
+    echo "--- 🛠 Preparing Intel pipeline"
+    buildkite-agent pipeline upload .buildkite/intel/pipeline-intel.yml
+}
+
+gate_bootstrap_ci intel l2
+
+upload_pipeline

.buildkite/intel/scripts/run-xpu-test.sh

@@ -13,12 +13,33 @@ container_name="xpu_${BUILDKITE_COMMIT}_$(
 )"
 
 cd "${omni_source_dir}"
+
+# The XPU base image is ~37GB; the default gzip layer exporter is single-threaded
+# and takes ~16min to compress it. zstd is multi-threaded (uses all cores) at a
+# similar ratio; set EXPORT_COMPRESSION=uncompressed to skip compression entirely
+# for local-only images. This requires the containerd image store (buildx docker
+# driver), which is the default here.
+EXPORT_COMPRESSION="${EXPORT_COMPRESSION:-zstd}"
+if [ "${EXPORT_COMPRESSION}" = "uncompressed" ]; then
+    export_args=(--output "type=image,name={{IMAGE}},compression=uncompressed")
+else
+    export_args=(--output "type=image,name={{IMAGE}},compression=${EXPORT_COMPRESSION},compression-level=3,force-compression=true")
+fi
+
+docker_build() {
+    # $1 = image name; remaining args passed through to docker build.
+    local image="$1"
+    shift
+    local out=("${export_args[@]/'{{IMAGE}}'/${image}}")
+    docker build "${out[@]}" "$@" -f docker/Dockerfile.xpu .
+}
+
 if [ -z "$(docker images -q "${base_image_name}")" ]; then
-    docker build --target vllm-base -t "${base_image_name}" --build-arg "VLLM_VERSION=${VLLM_VERSION}" -f docker/Dockerfile.xpu .
+    docker_build "${base_image_name}" --target vllm-base --build-arg "VLLM_VERSION=${VLLM_VERSION}"
 fi
 
 # Try building the docker image
-docker build --build-arg "VLLM_BASE=${base_image_name}" --build-arg "VLLM_VERSION=${VLLM_VERSION}" -t "${image_name}" -f docker/Dockerfile.xpu .
+docker_build "${image_name}" --build-arg "VLLM_BASE=${base_image_name}" --build-arg "VLLM_VERSION=${VLLM_VERSION}"
 
 # Setup cleanup
 remove_docker_container() {
@@ -53,6 +74,6 @@ time timeout -k 30 30m docker run \
     cd /workspace/vllm-omni
     pytest -v -s -m "core_model and xpu and B60"
     pytest -v -s tests/diffusion/quantization/test_mxfp8_config.py
-    export VLLM_XPU_USE_SAMPLER_KERNEL=0    # NOTE: Remove this after vLLM v0.21.1 is merged. Fixes Qwen2-5 omni-expansion tests.
     pytest -v -s -m "advanced_model and xpu and B60"
+    pytest -v -s -m "omni and xpu and B60"
 '

.buildkite/npu/bootstrap-upload-steps.yml

@@ -0,0 +1,181 @@
+# Uploaded by upload_pipeline.py --upload (not the Buildkite hook).
+# ``if`` is injected by step ``key`` from skip-ci at upload time.
+steps:
+  - label: ":buildkit: Build and Push NPU Test Image - A2"
+    key: image-build-a2
+    agents:
+      queue: "ascend-a2b3"
+      resource_class: "npu-1"
+    plugins:
+      - kubernetes:
+          metadata:
+            annotations:
+              vault.hashicorp.com/agent-init-first: "true"
+              vault.hashicorp.com/agent-inject: "true"
+              vault.hashicorp.com/agent-inject-perms-ca.pem: "0400"
+              vault.hashicorp.com/agent-inject-perms-cert.pem: "0400"
+              vault.hashicorp.com/agent-inject-perms-config.json: "0400"
+              vault.hashicorp.com/agent-inject-perms-key.pem: "0400"
+              vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd
+              vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd
+              vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd
+              vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd
+              vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}"
+              vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}"
+              vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}"
+              vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}"
+              vault.hashicorp.com/agent-pre-populate-only: "true"
+              vault.hashicorp.com/agent-run-as-group: "1000"
+              vault.hashicorp.com/agent-run-as-user: "1000"
+              vault.hashicorp.com/agent-service-account-token-volume-name: token-vol
+              vault.hashicorp.com/role: ascend-gha-runners
+              vault.hashicorp.com/secret-volume-path: /home/user/.docker/
+              vault.hashicorp.com/tls-skip-verify: "true"
+          podSpecPatch:
+            volumes:
+            - name: token-vol
+              projected:
+                defaultMode: 420
+                sources:
+                - serviceAccountToken:
+                    audience: api
+                    expirationSeconds: 600
+                    path: token
+    env:
+      VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
+      IMAGE_NAME: "vllm-omni-ci-npu"
+      IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
+      BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
+    command: |
+      set -ex
+
+      echo "--- Building and pushing NPU Test Image"
+      echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
+      echo "buildkitd address: $${BUILDKITD_ADDR}"
+
+      if ! command -v buildctl &> /dev/null; then
+        echo "Installing buildctl..."
+        mkdir -p /tmp/buildkit
+        BUILDKIT_VERSION="v0.29.0"
+        wget -q "https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz
+        tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
+        cp /tmp/buildkit/bin/buildctl /usr/local/bin/
+      fi
+
+      export DOCKER_CONFIG=/home/user/.docker
+      buildctl \
+        --addr="$${BUILDKITD_ADDR}" \
+        --tlscacert=/home/user/.docker/ca.pem \
+        --tlscert=/home/user/.docker/cert.pem \
+        --tlskey=/home/user/.docker/key.pem \
+        build \
+        --frontend dockerfile.v0 \
+        --local context=. \
+        --local dockerfile=./docker \
+        --opt filename=Dockerfile.npu.ci \
+        --secret id=dockerconfig,src=/home/user/.docker/config.json \
+        --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \
+        --progress=plain
+
+      echo "--- Image pushed successfully"
+      echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
+
+  - label: ":buildkit: Build and Push NPU Test Image - A3"
+    key: image-build-a3
+    agents:
+      queue: "ascend-a2b3"
+      resource_class: "npu-1"
+    plugins:
+      - kubernetes:
+          metadata:
+            annotations:
+              vault.hashicorp.com/agent-init-first: "true"
+              vault.hashicorp.com/agent-inject: "true"
+              vault.hashicorp.com/agent-inject-perms-ca.pem: "0400"
+              vault.hashicorp.com/agent-inject-perms-cert.pem: "0400"
+              vault.hashicorp.com/agent-inject-perms-config.json: "0400"
+              vault.hashicorp.com/agent-inject-perms-key.pem: "0400"
+              vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd
+              vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd
+              vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd
+              vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd
+              vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}"
+              vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}"
+              vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}"
+              vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}"
+              vault.hashicorp.com/agent-pre-populate-only: "true"
+              vault.hashicorp.com/agent-run-as-group: "1000"
+              vault.hashicorp.com/agent-run-as-user: "1000"
+              vault.hashicorp.com/agent-service-account-token-volume-name: token-vol
+              vault.hashicorp.com/role: ascend-gha-runners
+              vault.hashicorp.com/secret-volume-path: /home/user/.docker/
+              vault.hashicorp.com/tls-skip-verify: "true"
+          podSpecPatch:
+            volumes:
+            - name: token-vol
+              projected:
+                defaultMode: 420
+                sources:
+                - serviceAccountToken:
+                    audience: api
+                    expirationSeconds: 600
+                    path: token
+    env:
+      VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}-a3"
+      IMAGE_NAME: "vllm-omni-ci-npu"
+      IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
+      BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
+    command: |
+      set -ex
+
+      echo "--- Building and pushing NPU Test Image"
+      echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
+      echo "buildkitd address: $${BUILDKITD_ADDR}"
+
+      if ! command -v buildctl &> /dev/null; then
+        echo "Installing buildctl..."
+        mkdir -p /tmp/buildkit
+        BUILDKIT_VERSION="v0.29.0"
+        wget -q "https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz
+        tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
+        cp /tmp/buildkit/bin/buildctl /usr/local/bin/
+      fi
+
+      export DOCKER_CONFIG=/home/user/.docker
+      buildctl \
+        --addr="$${BUILDKITD_ADDR}" \
+        --tlscacert=/home/user/.docker/ca.pem \
+        --tlscert=/home/user/.docker/cert.pem \
+        --tlskey=/home/user/.docker/key.pem \
+        build \
+        --frontend dockerfile.v0 \
+        --local context=. \
+        --local dockerfile=./docker \
+        --opt filename=Dockerfile.npu.ci.a3 \
+        --secret id=dockerconfig,src=/home/user/.docker/config.json \
+        --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \
+        --progress=plain
+
+      echo "--- Image pushed successfully"
+      echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
+
+  - label: "Upload Ready Pipeline"
+    depends_on:
+      - image-build-a2
+      - image-build-a3
+    key: upload-ready-pipeline
+    agents:
+      queue: "cpu_queue_premerge"
+    command: |
+      python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/npu/test-npu-ready.yml
+
+  # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
+  - label: "Upload Nightly Pipeline"
+    depends_on:
+      - image-build-a2
+      - image-build-a3
+    key: upload-nightly-pipeline
+    commands:
+      - python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/npu/test-npu-nightly.yml
+    agents:
+      queue: "cpu_queue_premerge"

.buildkite/npu/pipeline-npu-a3.yml

@@ -88,7 +88,7 @@
 #   #     queue: "ascend-a3"
 #   #     resource_class: "npu-2"
 #   #   command: |
-#   #     buildkite-agent pipeline upload .buildkite/test-npu-ready.yaml
+#   #     buildkite-agent pipeline upload .buildkite/npu/test-npu-ready.yml
 
   # # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
   # - label: "Upload Nightly Pipeline"
@@ -104,7 +104,7 @@
   #       build.pull_request.labels includes "diffusion-x2v-test"
   #     ))
   #   commands:
-  #     - buildkite-agent pipeline upload .buildkite/test-npu-nightly.yaml
+  #     - buildkite-agent pipeline upload .buildkite/npu/test-npu-nightly.yml
   #   agents:
   #     queue: "ascend-a3"
   #     resource_class: "npu-2"

.buildkite/npu/pipeline-npu.yml

@@ -0,0 +1,9 @@
+# Hook uploads this entry YAML; upload_pipeline.py --upload reads bootstrap-upload-steps.yml
+# and injects ``if`` by step ``key`` from skip-ci.
+steps:
+  - label: ":github: Resolve skip-ci & upload pipeline"
+    key: upload-ci-pipeline
+    commands:
+      - "python3 .buildkite/common/scripts/upload_pipeline.py --upload .buildkite/npu/bootstrap-upload-steps.yml"
+    agents:
+      queue: "cpu_queue_premerge"

.buildkite/npu/test-npu-nightly.yml

@@ -0,0 +1,205 @@
+steps:
+  - group: ":card_index_dividers: Omni Model Test"
+    key: nightly-omni-test-group
+    depends_on: upload-nightly-pipeline
+    if: >-
+      build.env("NIGHTLY") == "1" ||
+      build.pull_request.labels includes "nightly-test" ||
+      build.pull_request.labels includes "omni-test"
+    steps:
+      - label: ":full_moon: Omni · Perf Test · No Async Chunk"
+        key: nightly-omni-performance-no-async-chunk
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_4
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
+            exit $$EXIT
+      - label: ":full_moon: Omni · Perf Test · Async Chunk"
+        key: nightly-omni-performance-async-chunk
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_4
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
+            exit $$EXIT
+      - label: ":robot: Omni · MiniCPM-o 4.5 Test"
+        mirror_hardwares: a3_npu_2
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+        commands:
+          - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5_expansion.py -m 'full_model' --run-level 'full_model'
+  - group: ":card_index_dividers: TTS Model Test"
+    key: nightly-tts-test-group
+    depends_on: upload-nightly-pipeline
+    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test"
+    steps:
+      - label: ":full_moon: TTS · Perf Test"
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_2
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_tts.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
+            exit $$EXIT
+  - group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test"
+    key: nightly-diffusion-x2iat-group
+    depends_on: upload-nightly-pipeline
+    if: >-
+      build.env("NIGHTLY") == "1" ||
+      build.pull_request.labels includes "nightly-test" ||
+      build.pull_request.labels includes "diffusion-x2iat-test"
+    steps:
+      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit-2511"
+        key: nightly-diffusion-x2iat-performance-qwen-image-edit-2511
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_8
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - export CACHE_DIT_VERSION=1.3.0
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+  - group: ":card_index_dividers: Diffusion I2I Model Test"
+    key: nightly-HunyuanImage3-x2i-test-group
+    depends_on: upload-nightly-pipeline
+    if: >-
+      build.env("NIGHTLY") == "1" ||
+      build.pull_request.labels includes "nightly-test" ||
+      build.pull_request.labels includes "diffusion-x2iat-test"
+    steps:
+      - label: ":full_moon: HunyuanImage3-DIT · Accuracy Test"
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_4
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          HUNYUAN_IMAGE3_MODEL: "tencent/HunyuanImage-3.0-Instruct"
+          HUNYUAN_IMAGE3_DEVICES: "0,1,2,3"
+          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
+        commands:
+          - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m "full_model and A3" --run-level full_model
+  - group: ":card_index_dividers: Diffusion Benchmark Perf Test"
+    key: nightly-diffusion-benchmark-group
+    depends_on: upload-nightly-pipeline
+    if: >-
+      build.env("NIGHTLY") == "1" ||
+      build.pull_request.labels includes "nightly-test" ||
+      build.pull_request.labels includes "diffusion-x2iat-test"
+    steps:
+      - label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=2 CFGP=2"
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_4
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          HF_TOKEN: "${HF_TOKEN}"
+          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m A3 --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+      - label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=2 SP=2"
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_4
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          HF_TOKEN: "${HF_TOKEN}"
+          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m A3 --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+      - label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=4"
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_4
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          HF_TOKEN: "${HF_TOKEN}"
+          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m A3 --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp4.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+  - group: ":card_index_dividers: Diffusion X2V Model Test"
+    key: nightly-diffusion-x2v-group
+    depends_on: upload-nightly-pipeline
+    if: >-
+      build.env("NIGHTLY") == "1" ||
+      build.pull_request.labels includes "nightly-test" ||
+      build.pull_request.labels includes "diffusion-x2v-test"
+    steps:
+      - label: ":full_moon: Diffusion X2V · Function Test"
+        mirror_hardwares: a2b3_npu_4
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k npu_i2v --run-level "full_model"
+      - label: ":full_moon: Diffusion X2V · Perf Test"
+        key: nightly-diffusion-x2v-performance
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_4
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT

.buildkite/npu/test-npu-ready.yml

@@ -0,0 +1,34 @@
+steps:
+  - group: ":card_index_dividers: Omni Model Test"
+    key: ready-omni-group
+    depends_on: upload-ready-pipeline
+    steps:
+      - label: ":robot: Omni · MiniCPM-o 4.5 Test"
+        agents:
+          queue: "ascend-a3"
+          resource_class: "npu-2"
+        image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3"
+        plugins:
+          - kubernetes:
+              podSpecPatch:
+                imagePullSecrets:
+                  - name: swr-secret
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+        commands:
+          - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model'
+
+  - group: ":card_index_dividers: TTS Model Test"
+    key: ready-tts-group
+    depends_on: upload-ready-pipeline
+    steps:
+      - label: ":sound: TTS · Qwen3-TTS CustomVoice Test"
+        mirror_hardwares: a2b3_npu_1
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model and npu' --run-level 'core_model'

.buildkite/pipeline-npu.yaml

@@ -1,112 +0,0 @@
-steps:
-  - label: ":buildkit: Build and Push NPU Test Image - A2"
-    key: image-build
-    agents:
-      queue: "ascend-a2b3"
-      resource_class: "npu-1"
-    plugins:
-      - kubernetes:
-          metadata:
-            annotations:
-              vault.hashicorp.com/agent-init-first: "true"
-              vault.hashicorp.com/agent-inject: "true"
-              vault.hashicorp.com/agent-inject-perms-ca.pem: "0400"
-              vault.hashicorp.com/agent-inject-perms-cert.pem: "0400"
-              vault.hashicorp.com/agent-inject-perms-config.json: "0400"
-              vault.hashicorp.com/agent-inject-perms-key.pem: "0400"
-              vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}"
-              vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}"
-              vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}"
-              vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}"
-              vault.hashicorp.com/agent-pre-populate-only: "true"
-              vault.hashicorp.com/agent-run-as-group: "1000"
-              vault.hashicorp.com/agent-run-as-user: "1000"
-              vault.hashicorp.com/agent-service-account-token-volume-name: token-vol
-              vault.hashicorp.com/role: ascend-gha-runners
-              vault.hashicorp.com/secret-volume-path: /home/user/.docker/
-              vault.hashicorp.com/tls-skip-verify: "true"
-          podSpecPatch:
-            volumes:
-            - name: token-vol
-              projected:
-                defaultMode: 420
-                sources:
-                - serviceAccountToken:
-                    audience: api
-                    expirationSeconds: 600
-                    path: token
-    env:
-      VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
-      IMAGE_NAME: "vllm-omni-ci-npu"
-      IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
-      BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
-    command: |
-      set -ex
-
-      echo "--- Building and pushing NPU Test Image"
-      echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
-      echo "buildkitd address: $${BUILDKITD_ADDR}"
-
-      if ! command -v buildctl &> /dev/null; then
-        echo "Installing buildctl..."
-        mkdir -p /tmp/buildkit
-        BUILDKIT_VERSION="v0.29.0"
-        wget -q "https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz
-        tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
-        cp /tmp/buildkit/bin/buildctl /usr/local/bin/
-      fi
-
-      export DOCKER_CONFIG=/home/user/.docker
-      buildctl \
-        --addr="$${BUILDKITD_ADDR}" \
-        --tlscacert=/home/user/.docker/ca.pem \
-        --tlscert=/home/user/.docker/cert.pem \
-        --tlskey=/home/user/.docker/key.pem \
-        build \
-        --frontend dockerfile.v0 \
-        --local context=. \
-        --local dockerfile=./docker \
-        --opt filename=Dockerfile.npu.ci \
-        --secret id=dockerconfig,src=/home/user/.docker/config.json \
-        --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \
-        --progress=plain
-
-      echo "--- Image pushed successfully"
-      echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
-
-  - label: "Upload Ready Pipeline"
-    depends_on: image-build
-    key: upload-ready-pipeline
-    if: >-
-      build.branch != "main" && (
-        build.pull_request.labels includes "npu-test" ||
-        build.pull_request.labels includes "ready"
-      )
-    agents:
-      queue: "ascend-a2b3"
-      resource_class: "npu-1"
-    command: |
-      buildkite-agent pipeline upload .buildkite/test-npu-ready.yaml
-
-  # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
-  - label: "Upload Nightly Pipeline"
-    depends_on: image-build
-    key: upload-nightly-pipeline
-    if: >-
-      (build.branch == "main" && build.env("NIGHTLY") == "1") ||
-      (build.branch != "main" && (
-        build.pull_request.labels includes "nightly-test" ||
-        build.pull_request.labels includes "omni-test" ||
-        build.pull_request.labels includes "tts-test" ||
-        build.pull_request.labels includes "diffusion-x2iat-test" ||
-        build.pull_request.labels includes "diffusion-x2v-test"
-      ))
-    commands:
-      - buildkite-agent pipeline upload .buildkite/test-npu-nightly.yaml
-    agents:
-      queue: "ascend-a2b3"
-      resource_class: "npu-1"

.buildkite/pipeline.yml

@@ -1,76 +0,0 @@
-# Document 1: Buildkite loads only this block on first parse. The next step resolves docs-only skip-ci
-# from git diff, then uploads document 2. When docs-only skip applies, image-build still runs for the nightly
-# exception in upload_pipeline_with_skip_ci.sh (nightly-test label or main NIGHTLY=1); otherwise upload-nightly
-# can be skipped together with test-ready/test-merge. Weekly uploads also depend on image-build.
-
-# Document 2: appended after `---`; same file, read by upload_pipeline_with_skip_ci.sh (not evaluated as a second pipeline by Buildkite). Child uploads include test-ready.yml, test-merge.yml, test-nightly.yml, and test-weekly.yml.
-steps:
-  - label: ":github: Resolve skip-ci & upload pipeline"
-    key: upload-ci-pipeline
-    commands:
-      - "bash .buildkite/scripts/upload_pipeline_with_skip_ci.sh"
-    agents:
-      queue: "cpu_queue_premerge"
-
----
-steps:
-  - label: ":docker: Build image"
-    key: image-build
-    if: __IMAGE_BUILD_IF__
-    commands:
-      - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
-      - "docker build --progress=plain --file docker/Dockerfile.ci -t vllm-omni-ci ."
-      - "docker tag vllm-omni-ci public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
-      - "docker push public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
-    agents:
-      queue: "cpu_queue_premerge"
-
-  # L2 Test
-  - label: "Upload Ready Pipeline"
-    depends_on: image-build
-    key: upload-ready-pipeline
-    if: __UPLOAD_READY_IF__
-    commands:
-      - buildkite-agent pipeline upload .buildkite/test-ready.yml
-    agents:
-      queue: "cpu_queue_premerge"
-
-  # L3 Test
-  - label: "Upload Merge Pipeline"
-    depends_on: image-build
-    key: upload-merge-pipeline
-    if: __UPLOAD_MERGE_IF__
-    commands:
-      - buildkite-agent pipeline upload .buildkite/test-merge.yml
-    agents:
-      queue: "cpu_queue_premerge"
-
-  # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
-  - label: "Upload Nightly Pipeline"
-    depends_on: image-build
-    key: upload-nightly-pipeline
-    if: >-
-      (build.branch == "main" && build.env("NIGHTLY") == "1") ||
-      (build.branch != "main" && (
-        build.pull_request.labels includes "nightly-test" ||
-        build.pull_request.labels includes "omni-test" ||
-        build.pull_request.labels includes "tts-test" ||
-        build.pull_request.labels includes "diffusion-x2iat-test" ||
-        build.pull_request.labels includes "diffusion-x2v-test"
-      ))
-    commands:
-      - buildkite-agent pipeline upload .buildkite/test-nightly.yml
-    agents:
-      queue: "cpu_queue_premerge"
-
-  # L5 Test — main+WEEKLY=1 (scheduled), or PR with weekly-test label
-  - label: "Upload Weekly Pipeline"
-    depends_on: image-build
-    key: upload-weekly-pipeline
-    if: >-
-      (build.branch == "main" && build.env("WEEKLY") == "1") ||
-      (build.branch != "main" && build.pull_request.labels includes "weekly-test")
-    commands:
-      - buildkite-agent pipeline upload .buildkite/test-weekly.yml
-    agents:
-      queue: "cpu_queue_premerge"

.buildkite/release/release-pipeline.yml

@@ -16,7 +16,7 @@ steps:
       - "uv venv --python=3.12 && source .venv/bin/activate"
       - "uv pip install --upgrade build"
       - "python3 -m build"
-      - "bash .buildkite/scripts/upload-nightly-wheels.sh"
+      - "bash .buildkite/release/scripts/upload-nightly-wheels.sh"
 
   - label: "Generate and upload wheel indices"
     depends_on: "build-wheel"
@@ -24,7 +24,7 @@ steps:
     agents:
       queue: small_cpu_queue_release
     commands:
-      - "bash .buildkite/scripts/generate-and-upload-nightly-index.sh"
+      - "bash .buildkite/release/scripts/generate-and-upload-nightly-index.sh"
 
   - block: "Upload release wheels to PyPI"
     depends_on: "build-wheel"
@@ -38,7 +38,7 @@ steps:
     agents:
       queue: small_cpu_queue_release
     commands:
-      - "bash .buildkite/scripts/publish-release-wheels-pypi.sh"
+      - "bash .buildkite/release/scripts/publish-release-wheels-pypi.sh"
 
   - block: "Unblock to build release Docker images"
     depends_on: ~
@@ -100,7 +100,7 @@ steps:
     agents:
       queue: small_cpu_queue_release
     commands:
-      - "bash .buildkite/scripts/publish-release-images.sh"
+      - "bash .buildkite/release/scripts/publish-release-images.sh"
     plugins:
       - docker-login#v3.0.0:
           username: vllmbot

.buildkite/release/scripts/generate-and-upload-nightly-index.sh

@@ -36,12 +36,12 @@ mkdir -p "$INDICES_OUTPUT_DIR"
 
 # HACK: we do not need regex module here, but it is required by pre-commit hook
 # To avoid any external dependency, we simply replace it back to the stdlib re module
-sed -i.bak 's/import regex as re/import re/g' .buildkite/scripts/generate-nightly-index.py && rm -f .buildkite/scripts/generate-nightly-index.py.bak
+sed -i.bak 's/import regex as re/import re/g' .buildkite/release/scripts/generate-nightly-index.py && rm -f .buildkite/release/scripts/generate-nightly-index.py.bak
 
 # Generate indices -- the version is just the commit hash (not omni/{commit})
 # because relative paths are computed between the index and wheel directories,
 # both of which live under the omni/ prefix in S3.
-$PYTHON .buildkite/scripts/generate-nightly-index.py \
+$PYTHON .buildkite/release/scripts/generate-nightly-index.py \
     --version "$BUILDKITE_COMMIT" \
     --current-objects "$obj_json" \
     --output-dir "$INDICES_OUTPUT_DIR" \
@@ -78,7 +78,7 @@ if [[ "$version" != *"dev"* ]]; then
     rm -rf "${INDICES_OUTPUT_DIR:?}"
     mkdir -p "$INDICES_OUTPUT_DIR"
     # wheel-dir is overridden to be the commit directory, so that the indices point to the correct wheel path
-    $PYTHON .buildkite/scripts/generate-nightly-index.py \
+    $PYTHON .buildkite/release/scripts/generate-nightly-index.py \
         --version "$s3_version" \
         --wheel-dir "$BUILDKITE_COMMIT" \
         --current-objects "$obj_json" \

.buildkite/scripts/docker_login_ecr_public.sh

@@ -1,73 +0,0 @@
-#!/bin/bash
-# Helper function to safely login to ECR Public with per-job config isolation
-# Uses DOCKER_CONFIG environment variable to prevent race conditions
-#
-# This script prevents the "device or resource busy" error by giving each
-# Buildkite job its own isolated Docker config directory.
-#
-# Usage:
-#   source docker_login_ecr_public.sh && safe_docker_login_ecr_public
-
-set -euo pipefail
-
-# Configuration
-ECR_REGISTRY="public.ecr.aws"
-
-setup_isolated_docker_config() {
-    # Use BUILDKITE_JOB_ID for job-specific isolation
-    # Fallback to PID if running outside Buildkite
-    local job_id="${BUILDKITE_JOB_ID:-$$}"
-
-    # Set Docker config to job-specific directory
-    export DOCKER_CONFIG="/tmp/docker-config-${job_id}"
-
-    # Create directory if it doesn't exist
-    mkdir -p "$DOCKER_CONFIG"
-
-    echo "[docker-config] Using isolated Docker config: $DOCKER_CONFIG"
-}
-
-check_docker_auth() {
-    # Check if already authenticated to the given registry
-    # Returns 0 if authenticated, 1 if not
-    local registry="$1"
-
-    # Check if credentials exist in the isolated config
-    if [[ -f "$DOCKER_CONFIG/config.json" ]]; then
-        # Check if registry is present in config
-        if grep -q "$registry" "$DOCKER_CONFIG/config.json" 2>/dev/null; then
-            return 0
-        fi
-    fi
-
-    return 1
-}
-
-safe_docker_login_ecr_public() {
-    # Setup isolated config first
-    setup_isolated_docker_config
-
-    local registry="$ECR_REGISTRY"
-
-    # Check if already authenticated (within this job)
-    if check_docker_auth "$registry"; then
-        echo "[docker-login] Already authenticated to $registry in this job"
-        return 0
-    fi
-
-    # Perform login to isolated config directory
-    echo "[docker-login] Logging in to $ECR_REGISTRY (isolated config)..."
-    if aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$ECR_REGISTRY"; then
-        echo "[docker-login] Login successful (config: $DOCKER_CONFIG)"
-        return 0
-    else
-        local exit_code=$?
-        echo "[docker-login] ERROR: Login failed with exit code $exit_code" >&2
-        return $exit_code
-    fi
-}
-
-# Execute if run as script (not sourced)
-if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
-    safe_docker_login_ecr_public
-fi

.buildkite/scripts/hardware_ci/run_npu_test.sh

@@ -1,145 +0,0 @@
-#!/bin/bash
-
-# This script build the Ascend NPU docker image and run the offline inference inside the container.
-# It serves a sanity check for compilation and basic model usage.
-set -ex
-
-# Base ubuntu image with basic ascend development libraries and python installed
-VLLM_OMNI_REPO="https://github.com/vllm-project/vllm-omni.git"
-BASE_IMAGE_NAME="quay.nju.edu.cn/ascend/vllm-ascend:v0.11.0rc2"
-image_name="npu/vllm-omni-ci:${BUILDKITE_COMMIT}_${EPOCHSECONDS}"
-# image_name="npu/vllm-ci:${BUILDKITE_COMMIT}_${EPOCHSECONDS}"
-container_name="npu_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)"
-
-# BUILDKITE_AGENT_NAME format is {hostname}-{agent_idx}-{npu_card_num}cards
-agent_idx=$(echo "${BUILDKITE_AGENT_NAME}" | awk -F'-' '{print $(NF-1)}')
-echo "agent_idx: ${agent_idx}"
-builder_name="cachebuilder${agent_idx}"
-builder_cache_dir="/mnt/docker-cache${agent_idx}"
-mkdir -p ${builder_cache_dir}
-
-# Try building the docker image
-cat <<EOF | DOCKER_BUILDKIT=1 docker build \
-    --add-host pypi-cache:${PYPI_CACHE_HOST} \
-    --builder ${builder_name} --cache-from type=local,src=${builder_cache_dir} \
-                           --cache-to type=local,dest=${builder_cache_dir},mode=max \
-    --build-arg BUILDKITE_PULL_REQUEST="${BUILDKITE_PULL_REQUEST}" \
-    --build-arg BUILDKITE_PULL_REQUEST_REPO="${BUILDKITE_PULL_REQUEST_REPO}" \
-    --progress=plain --load -t ${image_name} -f - .
-FROM ${BASE_IMAGE_NAME}
-
-# Define environments
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN pip config set global.index-url http://pypi-cache:${PYPI_CACHE_PORT}/pypi/simple && \
-    pip config set global.trusted-host pypi-cache && \
-    apt-get update -y && \
-    apt-get install -y python3-pip git vim wget net-tools gcc g++ cmake libnuma-dev && \
-    rm -rf /var/cache/apt/* && \
-    rm -rf /var/lib/apt/lists/*
-
-# Install for pytest to make the docker build cache layer always valid
-RUN --mount=type=cache,target=/root/.cache/pip \
-    pip install pytest>=6.0  pytest-cov modelscope
-
-COPY . .
-
-# Install vllm-omni
-WORKDIR /workspace
-ARG VLLM_OMNI_REPO=https://github.com/vllm-project/vllm-omni.git
-ARG VLLM_OMNI_TAG=main
-ARG BUILDKITE_PULL_REQUEST
-ARG BUILDKITE_PULL_REQUEST_REPO
-RUN git config --global url."https://gh-proxy.test.osinfra.cn/https://github.com/".insteadOf "https://github.com/" && \
-    if [ "\$BUILDKITE_PULL_REQUEST" != "false" ] && [ -n "\$BUILDKITE_PULL_REQUEST" ]; then \
-        echo "Cloning and checking out PR #\$BUILDKITE_PULL_REQUEST..." && \
-        git clone \$VLLM_OMNI_REPO /workspace/vllm-omni && \
-        cd /workspace/vllm-omni && \
-        git fetch origin pull/\$BUILDKITE_PULL_REQUEST/head:pr-\$BUILDKITE_PULL_REQUEST && \
-        git checkout pr-\$BUILDKITE_PULL_REQUEST; \
-    else \
-        echo "Not a PR build, using main branch" && \
-        git clone --depth 1 \$VLLM_OMNI_REPO /workspace/vllm-omni; \
-    fi
-
-RUN --mount=type=cache,target=/root/.cache/pip \
-    export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi && \
-    source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
-    source /usr/local/Ascend/nnal/atb/set_env.sh && \
-    export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
-    python3 -m pip install -v -e /workspace/vllm-omni/
-
-ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-WORKDIR /workspace/vllm-omni
-CMD ["/bin/bash"]
-
-EOF
-
-# Setup cleanup
-remove_docker_container() {
-  docker rm -f "${container_name}" || true;
-  docker image rm -f "${image_name}" || true;
-  docker system prune -f || true;
-}
-trap remove_docker_container EXIT
-
-# Generate corresponding --device args based on BUILDKITE_AGENT_NAME
-# Ascend NPU BUILDKITE_AGENT_NAME format is {hostname}-{agent_idx}-{npu_card_num}cards, and agent_idx starts from 1.
-#   e.g. atlas-a2-001-1-2cards means this is the 1-th agent on atlas-a2-001 host, and it has 2 NPU cards.
-#   returns --device /dev/davinci0 --device /dev/davinci1
-parse_and_gen_devices() {
-    local input="$1"
-    local index cards_num
-    if [[ "$input" =~ ([0-9]+)-([0-9]+)cards$ ]]; then
-        index="${BASH_REMATCH[1]}"
-        cards_num="${BASH_REMATCH[2]}"
-    else
-        echo "parse error" >&2
-        return 1
-    fi
-
-    local devices=""
-    local i=0
-    while (( i < cards_num )); do
-        local dev_idx=$(((index - 1)*cards_num + i ))
-        devices="$devices --device /dev/davinci${dev_idx}"
-        ((i++))
-    done
-
-    # trim leading space
-    devices="${devices#"${devices%%[![:space:]]*}"}"
-    # Output devices: assigned to the caller variable
-    printf '%s' "$devices"
-}
-
-devices=$(parse_and_gen_devices "${BUILDKITE_AGENT_NAME}") || exit 1
-
-# Run the image and execute the Out-Of-Tree (OOT) platform interface test case on Ascend NPU hardware.
-# This test checks whether the OOT platform interface is functioning properly in conjunction with
-# the hardware plugin vllm-ascend.
-hf_model_cache_dir=/mnt/hf_cache${agent_idx}
-ms_model_cache_dir=/mnt/modelscope${agent_idx}
-mkdir -p ${hf_model_cache_dir}
-mkdir -p ${ms_model_cache_dir}
-docker run \
-    --init \
-    ${devices} \
-    --device /dev/davinci_manager \
-    --device /dev/devmm_svm \
-    --device /dev/hisi_hdc \
-    -v /usr/local/dcmi:/usr/local/dcmi \
-    -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
-    -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
-    -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
-    -v /etc/ascend_install.info:/etc/ascend_install.info \
-    -v ${hf_model_cache_dir}:/root/.cache/huggingface \
-    -v ${ms_model_cache_dir}:/root/.cache/modelscope \
-    --network host \
-    --entrypoint="" \
-    --name "${container_name}" \
-    "${image_name}" \
-    bash -c '
-    set -e
-    VLLM_USE_MODELSCOPE=True pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py
-'

.buildkite/scripts/upload_pipeline_with_skip_ci.sh

@@ -1,146 +0,0 @@
-#!/usr/bin/env bash
-# Evaluate docs-only skip-ci and upload continuation steps from the given pipeline YAML
-# (YAML document after the first `---`). Buildkite `if` is evaluated at upload time.
-#
-# Usage: upload_pipeline_with_skip_ci.sh [pipeline_yaml]
-#   pipeline_yaml: path relative to repo root or absolute (default: .buildkite/pipeline.yml)
-set -euo pipefail
-
-ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
-PIPELINE_ARG="${1:-.buildkite/pipeline.yml}"
-if [[ "${PIPELINE_ARG}" = /* ]]; then
-  PIPELINE_YML="${PIPELINE_ARG}"
-else
-  PIPELINE_YML="${ROOT}/${PIPELINE_ARG}"
-fi
-
-# Prints a single digit to stdout: 1 = skip image CI, 0 = run. Logs go to stderr.
-is_docs_only_change() {
-  local file_path
-  local has_any=0
-
-  while IFS= read -r file_path; do
-    [[ -z "${file_path}" ]] && continue
-    has_any=1
-
-    if [[ "${file_path}" == docs/* ]]; then
-      continue
-    fi
-    if [[ "${file_path}" == *.md ]]; then
-      continue
-    fi
-    if [[ "${file_path}" == "mkdocs.yml" ]]; then
-      continue
-    fi
-    return 1
-  done
-
-  [[ "${has_any}" -eq 1 ]]
-}
-
-resolve_skip_ci() {
-  local is_pr_build=0
-  local files
-  local base_branch base_ref
-
-  if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" && -n "${BUILDKITE_PULL_REQUEST:-}" ]]; then
-    is_pr_build=1
-  fi
-
-  if [[ "${is_pr_build}" -eq 1 ]]; then
-    base_branch="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}"
-    if ! git rev-parse --verify "origin/${base_branch}" >/dev/null 2>&1; then
-      echo "resolve_skip_ci: origin/${base_branch} not found locally; trying fetch" >&2
-      git fetch --depth=200 origin "${base_branch}" >/dev/null 2>&1 || true
-    fi
-
-    base_ref=""
-    if git rev-parse --verify "origin/${base_branch}" >/dev/null 2>&1; then
-      base_ref="origin/${base_branch}"
-    elif git rev-parse --verify "${base_branch}" >/dev/null 2>&1; then
-      base_ref="${base_branch}"
-    else
-      echo "resolve_skip_ci: cannot resolve PR base ${base_branch}; skip-ci=0" >&2
-      echo -n 0
-      return 0
-    fi
-
-    if ! files="$(git diff --name-only "${base_ref}...${BUILDKITE_COMMIT}" 2>/dev/null)"; then
-      echo "resolve_skip_ci: failed to compute PR changed files; skip-ci=0" >&2
-      echo -n 0
-      return 0
-    fi
-  elif [[ "${BUILDKITE_BRANCH:-}" == "main" ]]; then
-    if ! git rev-parse --verify "${BUILDKITE_COMMIT}^" >/dev/null 2>&1; then
-      echo "resolve_skip_ci: commit has no parent on main; skip-ci=0" >&2
-      echo -n 0
-      return 0
-    fi
-    if ! files="$(git diff --name-only "${BUILDKITE_COMMIT}^..${BUILDKITE_COMMIT}" 2>/dev/null)"; then
-      echo "resolve_skip_ci: failed to compute main changed files; skip-ci=0" >&2
-      echo -n 0
-      return 0
-    fi
-  else
-    echo "resolve_skip_ci: not PR/main build; skip-ci=0" >&2
-    echo -n 0
-    return 0
-  fi
-
-  if is_docs_only_change <<< "${files}"; then
-    echo "resolve_skip_ci: docs-only change detected; skip-ci=1" >&2
-    echo -n 1
-    return 0
-  fi
-
-  echo "resolve_skip_ci: non-doc changes detected; skip-ci=0" >&2
-  echo -n 0
-}
-
-SKIP_CI="$(resolve_skip_ci)"
-
-if [[ ! -f "${PIPELINE_YML}" ]]; then
-  echo "upload_pipeline_with_skip_ci: missing ${PIPELINE_YML}" >&2
-  exit 1
-fi
-
-export ROOT SKIP_CI PIPELINE_YML
-python3 <<'PY' | buildkite-agent pipeline upload
-import os
-import pathlib
-
-path = pathlib.Path(os.environ["PIPELINE_YML"])
-text = path.read_text(encoding="utf-8")
-sep = "\n---\n"
-# Two supported layouts:
-#   - multi-doc: doc 1 = bootstrap (loaded by Buildkite), doc 2 (after `---`) = uploaded steps.
-#   - single-doc: whole file is the uploaded steps; caller (UI init step) is the bootstrap.
-if sep in text:
-    _, continuation = text.split(sep, 1)
-else:
-    continuation = text
-
-skip = os.environ.get("SKIP_CI") == "1"
-# When docs-only skip-ci: skip default CI image, but still build for L4 nightly (PR label nightly-test or
-# main NIGHTLY=1), otherwise upload-nightly (depends_on image-build) would be skipped too.
-nightly_only = (
-    '(build.pull_request.labels includes "nightly-test") '
-    '|| (build.branch == "main" && build.env("NIGHTLY") == "1")'
-)
-# Placeholder in pipeline.yml is `if: __IMAGE_BUILD_IF__` (valid YAML); replace value only.
-if skip:
-    rep = f"'{nightly_only}'"
-    ready_rep = "'false'"
-    merge_rep = "'false'"
-else:
-    rep = "'true'"
-    ready_rep = "'build.branch != \"main\" && build.pull_request.labels includes \"ready\"'"
-    merge_rep = "'(build.branch == \"main\" && build.env(\"NIGHTLY\") != \"1\" && build.env(\"WEEKLY\") != \"1\") || (build.branch != \"main\" && build.pull_request.labels includes \"merge-test\")'"
-rendered = (
-    continuation
-    .replace("__IMAGE_BUILD_IF__", rep)
-    .replace("__UPLOAD_READY_IF__", ready_rep)
-    .replace("__UPLOAD_MERGE_IF__", merge_rep)
-)
-print(rendered, end="")
-PY

.buildkite/test-amd-merge.yml

@@ -1,219 +0,0 @@
-steps:
-
-- label: "Simple Unit Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export VLLM_ROCM_USE_AITER=0
-    # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
-    # that is not supported on CPU
-    - "timeout 60m pytest -v -s -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
-
-- label: "Diffusion Images API LoRA E2E"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - timeout 20m pytest -s -v tests/diffusion/lora/test_images_generations_lora.py
-
-- label: "Diffusion Model CPU offloading Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - |
-      timeout 20m bash -c '
-        set +e
-        pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
-        EXIT1=\$?
-        pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
-        EXIT2=\$?
-        exit \$((EXIT1 | EXIT2))
-      '
-
-## ISSUE depends on `diffusers` package: https://github.com/huggingface/diffusers/issues/13274
-# - label: "Audio Generation Model Test"
-#   agent_pool: mi325_1
-#   depends_on: amd-build
-#   mirror_hardwares: [amdproduction]
-#   grade: Blocking
-#   commands:
-#     - export GPU_ARCHS=gfx942
-#     - export VLLM_LOGGING_LEVEL=DEBUG
-#     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-#     - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_expansion.py -m "advanced_model and diffusion and L4" --run-level advanced_model
-
-- label: "Diffusion Cache Backend Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - timeout 15m pytest -s -v -m "core_model and cache and diffusion and not distributed_cuda and L4"
-
-- label: "Diffusion Sequence Parallelism Test (Need 4 GPUs)"
-  agent_pool: mi325_4
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py
-    - timeout 20m pytest -s -v tests/diffusion/distributed/test_ulysses_uaa_perf.py
-
-# merge-only tests
-- label: "Diffusion Tensor Parallelism Test"
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - pytest -s -v tests/diffusion/distributed/test_zimage_parallelism.py
-
-- label: "Diffusion Model Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - timeout 30m pytest -s -v tests/e2e/offline_inference/test_z_image.py
-
-- label: "Diffusion GPU Worker Test"
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - timeout 20m pytest -s -v tests/diffusion/test_diffusion_worker.py
-
-- label: "Engine Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - timeout 20m pytest -s -v tests/engine/test_async_omni_engine_abort.py
-
-# This is from test-nightly.yml.
-# This model seems to be important, so I am trying to run it here.
-# Since this is from test-nightly.yml,
-# I keep this non-blocking.
-- label: "Omni Model Test Qwen2-5-Omni"
-  agent_pool: mi325_4
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  commands:
-    - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-    - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model"
-
-- label: "Omni Model Test Qwen3-Omni"
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/online_serving/test_mimo_audio.py -m "advanced_model" --run-level "advanced_model"
-
-- label: "Qwen3-TTS CustomVoice E2E Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - |
-      timeout 20m bash -c '
-        export VLLM_LOGGING_LEVEL=DEBUG
-        export VLLM_WORKER_MULTIPROC_METHOD=spawn
-        export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-        pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m "advanced_model" --run-level "advanced_model"
-      '
-
-- label: "Qwen3-TTS Base E2E Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - |
-      timeout 30m bash -c '
-        export VLLM_LOGGING_LEVEL=DEBUG
-        export VLLM_WORKER_MULTIPROC_METHOD=spawn
-        export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-        pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m "advanced_model" --run-level "advanced_model"
-      '
-
-# TODO: Bagel test on ROCm is very unstable. @tjtanaa
-# Need to debug before reneable numerical changes across large PRs
-# # split Bagel Model Test with H100 (Real Weights) into three tests
-# - label: "Bagel Text2Img Model Test (1/3)"
-#   agent_pool: mi325_1
-#   depends_on: amd-build
-#   mirror_hardwares: [amdproduction]
-#   grade: Blocking
-#   commands:
-#     - export GPU_ARCHS=gfx942
-#     - export VLLM_LOGGING_LEVEL=DEBUG
-#     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-#     - export VLLM_ROCM_USE_AITER_RMSNORM=0
-#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "advanced_model" --run-level "advanced_model" -k "shared_memory" -k "rocm"
-
-# - label: "Bagel Img2Img Model Test (2/3)"
-#   agent_pool: mi325_1
-#   depends_on: amd-build
-#   mirror_hardwares: [amdproduction]
-#   grade: Blocking
-#   commands:
-#     - export GPU_ARCHS=gfx942
-#     - export VLLM_LOGGING_LEVEL=DEBUG
-#     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-#     - export VLLM_ROCM_USE_AITER_RMSNORM=0
-#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "advanced_model" --run-level "advanced_model" -k "rocm"
-
-# - label: "Bagel Online Serving Test (3/3)"
-#   agent_pool: mi325_1
-#   depends_on: amd-build
-#   mirror_hardwares: [amdproduction]
-#   grade: Blocking
-#   commands:
-#     - export GPU_ARCHS=gfx942
-#     - export VLLM_IMAGE_FETCH_TIMEOUT=60
-#     - export VLLM_LOGGING_LEVEL=DEBUG
-#     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-#     - export VLLM_ROCM_USE_AITER_RMSNORM=0
-#     - timeout 40m pytest -s -v tests/e2e/online_serving/test_bagel.py -m "advanced_model" --run-level "advanced_model" -k "rocm"
-
-- label: "Diffusion · Wan22 Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - |
-      timeout 40m bash -c "
-        export VLLM_IMAGE_FETCH_TIMEOUT=60
-        pytest -s -v tests/e2e/offline_inference/test_wan22_t2v.py tests/e2e/online_serving/test_wan22_t2v.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
-      "
-
-- label: "Voxtral-TTS E2E Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - |
-      timeout 20m bash -c '
-        export VLLM_LOGGING_LEVEL=DEBUG
-        export VLLM_WORKER_MULTIPROC_METHOD=spawn
-        pytest -s -v tests/e2e/online_serving/test_voxtral_tts.py tests/e2e/offline_inference/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model"
-      '

.buildkite/test-amd-ready.yaml

@@ -1,227 +0,0 @@
-steps:
-
-- label: "Simple Unit Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export VLLM_ROCM_USE_AITER=0
-    # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
-    # that is not supported on CPU
-    - "timeout 60m pytest -v -s -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
-
-- label: "Voxtral TTS CUDA Unit Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - timeout 10m pytest -s -v tests/model_executor/models/voxtral_tts/test_cuda_graph_acoustic_transformer.py
-
-- label: "Diffusion Batching Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - timeout 20m pytest -s -v tests/diffusion/batching/test_diffusion_batching.py -m "core_model and diffusion" --run-level "core_model"
-
-- label: "Custom Pipeline Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - timeout 20m pytest -s -v tests/e2e/offline_inference/custom_pipeline/ -m "core_model"
-
-- label: "Diffusion Model CPU offloading Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - |
-      timeout 20m bash -c '
-        set +e
-        pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
-        EXIT1=\$?
-        pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
-        EXIT2=\$?
-        exit \$((EXIT1 | EXIT2))
-      '
-
-## ISSUE depends on `diffusers` package: https://github.com/huggingface/diffusers/issues/13274
-# - label: "Audio Generation Model Test"
-#   agent_pool: mi325_1
-#   depends_on: amd-build
-#   mirror_hardwares: [amdproduction]
-#   grade: Blocking
-#   commands:
-#     - export GPU_ARCHS=gfx942
-#     - export VLLM_LOGGING_LEVEL=DEBUG
-#     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-#     - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_expansion.py -m "advanced_model and diffusion and L4" --run-level advanced_model
-
-- label: "Diffusion Cache Backend Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - timeout 15m pytest -s -v -m "core_model and cache and diffusion and not distributed_cuda and L4"
-
-- label: "Diffusion Sequence Parallelism Test"
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py -m core_model
-
-- label: "Diffusion GPU Worker Test"
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - timeout 20m pytest -s -v tests/diffusion/test_diffusion_worker.py
-
-- label: "Diffusion · Model Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model"
-
-- label: "Diffusion · Wan22 Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - |
-      timeout 40m bash -c "
-        export VLLM_TEST_CLEAN_GPU_MEMORY=1
-        export VLLM_IMAGE_FETCH_TIMEOUT=60
-        pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model' --run-level 'core_model'
-      "
-
-- label: "Engine Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - |
-      timeout 15m bash -c '
-              pytest -s -v tests/engine/test_async_omni_engine_abort.py
-      '
-
-# - label: "Omni Model Test Qwen3-Omni"
-#   agent_pool: mi325_2
-#   depends_on: amd-build
-#   mirror_hardwares: [amdproduction]
-#   grade: Blocking
-#   commands:
-#     - export VLLM_LOGGING_LEVEL=DEBUG
-#     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-#     - timeout 10m pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py
-#     - timeout 20m pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m "core_model" --run-level "core_model"
-
-- label: "MiMo-Audio E2E Test with H100"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - |
-      timeout 30m bash -c '
-        export VLLM_LOGGING_LEVEL=DEBUG
-        export VLLM_WORKER_MULTIPROC_METHOD=spawn
-        pytest -s -v tests/e2e/online_serving/test_mimo_audio.py -m "core_model" --run-level "core_model"
-      '
-
-- label: "Qwen3-TTS E2E Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-    - timeout 30m pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m "core_model" --run-level "core_model"
-
-- label: "Voxtral-TTS E2E Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - |
-      timeout 20m bash -c '
-        export VLLM_LOGGING_LEVEL=DEBUG
-        export VLLM_WORKER_MULTIPROC_METHOD=spawn
-        pytest -s -v tests/e2e/online_serving/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model"
-        pytest -s -v tests/e2e/offline_inference/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model"
-      '
-
-# TODO: Bagel test on ROCm is very unstable. @tjtanaa
-# Need to debug before reneable numerical changes across large PRs
-# - label: "Bagel Text2Img Model Test"
-#   agent_pool: mi325_1
-#   depends_on: amd-build
-#   mirror_hardwares: [amdproduction]
-#   grade: Blocking
-#   commands:
-#     - export GPU_ARCHS=gfx942
-#     - export VLLM_LOGGING_LEVEL=DEBUG
-#     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-#     - export VLLM_ROCM_USE_AITER_RMSNORM=0
-#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"
-
-# - label: "Bagel Img2Img Model Test"
-#   agent_pool: mi325_1
-#   depends_on: amd-build
-#   mirror_hardwares: [amdproduction]
-#   grade: Blocking
-#   commands:
-#     - export GPU_ARCHS=gfx942
-#     - export VLLM_LOGGING_LEVEL=DEBUG
-#     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-#     - export VLLM_ROCM_USE_AITER_RMSNORM=0
-#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"
-
-# - label: "Bagel Online Serving Test"
-#   agent_pool: mi325_1
-#   depends_on: amd-build
-#   mirror_hardwares: [amdproduction]
-#   grade: Blocking
-#   commands:
-#     - export GPU_ARCHS=gfx942
-#     - export VLLM_IMAGE_FETCH_TIMEOUT=60
-#     - export VLLM_LOGGING_LEVEL=DEBUG
-#     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-#     - export VLLM_ROCM_USE_AITER_RMSNORM=0
-#     - timeout 40m pytest -s -v tests/e2e/online_serving/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"
-
-- label: "CosyVoice3-TTS E2E Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - |
-      timeout 20m bash -c '
-        pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "full_model" --run-level "core_model"
-      '

.buildkite/test-amd.yaml

@@ -1,118 +0,0 @@
-steps:
-
-- label: "Diffusion Model Test"
-  timeout_in_minutes: 30
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - pytest -s -v tests/e2e/offline_inference/test_z_image.py
-
-- label: "Diffusion Images API LoRA E2E"
-  timeout_in_minutes: 30
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/diffusion/lora/test_images_generations_lora.py
-
-- label: "Diffusion Model CPU offloading Test"
-  timeout_in_minutes: 20
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
-    - pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
-
-- label: "Diffusion Cache Backend Test"
-  timeout_in_minutes: 15
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v -m 'core_model and cache and diffusion and not distributed_rocm and MI325'
-
-- label: "Diffusion Sequence Parallelism Test"
-  timeout_in_minutes: 20
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py -m core_model
-
-- label: "Diffusion Tensor Parallelism Test"
-  timeout_in_minutes: 20
-  agent_pool: mi325_2
-  depends_on: amd-build
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/diffusion/distributed/test_zimage_parallelism.py
-
-- label: "Diffusion GPU Worker Test"
-  timeout_in_minutes: 20
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - pytest -s -v tests/diffusion/test_diffusion_worker.py
-
-- label: "Omni Model Test Qwen2-5-Omni"
-  timeout_in_minutes: 15
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py
-    - pytest -s -v tests/engine/test_async_omni_engine_abort.py
-
-- label: "Omni Model Test Qwen3-Omni"
-  timeout_in_minutes: 15
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - export VLLM_ROCM_USE_AITER=0
-    - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py
-    - pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py
-
-
-- label: "Omni Sleep Mode Test"
-  timeout_in_minutes: 40
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/entrypoints/test_omni_sleep_mode.py -m "advanced_model and omni and MI325" --run-level "advanced_model"

.buildkite/test-merge.yml

@@ -1,808 +0,0 @@
-env:
-  VLLM_WORKER_MULTIPROC_METHOD: spawn
-  VLLM_USE_FLASHINFER_MOE_FP16: 0
-  HF_HUB_DOWNLOAD_TIMEOUT: 300
-  HF_HUB_ETAG_TIMEOUT: 60
-
-steps:
-  - group: ":card_index_dividers: Simple Test"
-    depends_on: upload-merge-pipeline
-    steps:
-      - label: "Simple · Diffusion & Model Executor Test"
-        commands:
-          - "timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "Simple · Other Test"
-        commands:
-          - "timeout 20m pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - group: ":card_index_dividers: Diffusion Test"
-    depends_on: upload-merge-pipeline
-    steps:
-      - label: "Diffusion · LoRA Test"
-        timeout_in_minutes: 30
-        commands:
-          - pytest -sv tests/diffusion/lora/ -m 'advanced_model and cuda' --run-level "advanced_model"
-        agents:
-          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Distributed Test"
-    timeout_in_minutes: 60
-    commands:
-      - |
-        timeout 55m bash -c "
-          set -e
-          export VLLM_IMAGE_FETCH_TIMEOUT=60
-          pytest -s -v tests/distributed -m 'advanced_model and cuda' --run-level 'advanced_model'
-        "
-    agents:
-      queue: "mithril-h100-pool"
-    plugins:
-      - kubernetes:
-          podSpec:
-            containers:
-              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                resources:
-                  limits:
-                    nvidia.com/gpu: 1
-                volumeMounts:
-                  - name: devshm
-                    mountPath: /dev/shm
-                  - name: hf-cache
-                    mountPath: /root/.cache/huggingface
-                env:
-                  - name: HF_HOME
-                    value: /root/.cache/huggingface
-                  - name: HF_TOKEN
-                    valueFrom:
-                      secretKeyRef:
-                        name: hf-token-secret
-                        key: token
-            nodeSelector:
-              node.kubernetes.io/instance-type: gpu-h100-sxm
-            volumes:
-              - name: devshm
-                emptyDir:
-                  medium: Memory
-              - name: hf-cache
-                hostPath:
-                  path: /mnt/hf-cache
-                  type: DirectoryOrCreate
-
-  - group: ":card_index_dividers: Entrypoints Test"
-    depends_on: upload-merge-pipeline
-    steps:
-      - label: "Entrypoint Test with H100"
-        timeout_in_minutes: 35
-        commands:
-          - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and H100" --run-level "advanced_model"
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 2
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: "Entrypoint Test with L4"
-        timeout_in_minutes: 30
-        commands:
-          - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and L4" --run-level "advanced_model"
-        agents:
-          queue: "gpu_4_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - group: ":card_index_dividers: E2E Test"
-    depends_on: upload-merge-pipeline
-    steps:
-      - label: "TTS · Qwen3-TTS CustomVoice Test"
-        commands:
-          - |
-            timeout 20m bash -c "
-              export VLLM_LOGGING_LEVEL=DEBUG
-              export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
-              pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-            "
-        agents:
-          queue: "gpu_4_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "TTS · Qwen3-TTS Base Test"
-        commands:
-          - |
-            timeout 20m bash -c "
-              export VLLM_LOGGING_LEVEL=DEBUG
-              export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
-              pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-            "
-        agents:
-          queue: "gpu_4_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "TTS · Voxtral-TTS Test"
-        timeout_in_minutes: 20
-        commands:
-          - |
-            timeout 20m bash -c "
-              export VLLM_LOGGING_LEVEL=DEBUG
-              pytest -s -v tests/e2e/offline_inference/test_voxtral_tts.py tests/e2e/online_serving/test_voxtral_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-            "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: "TTS · GLM-TTS Test"
-        timeout_in_minutes: 20
-        commands:
-          - |
-            timeout 20m bash -c "
-              export VLLM_LOGGING_LEVEL=DEBUG
-              pytest -s -v tests/e2e/online_serving/test_glm_tts.py tests/e2e/offline_inference/test_glm_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-            "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: "Omni · Qwen3-Omni Test"
-        timeout_in_minutes: 30
-        commands:
-          - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 2
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: "Omni · Mimo Audio Test"
-        timeout_in_minutes: 30
-        commands:
-          - pytest -s -v tests/e2e/online_serving/test_mimo_audio.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 2
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: "Omni · VoxCPM2 Test"
-        timeout_in_minutes: 20
-        commands:
-          - |
-            timeout 20m bash -c "
-              export VLLM_LOGGING_LEVEL=DEBUG
-              export VLLM_WORKER_MULTIPROC_METHOD=spawn
-              pytest -s -v tests/e2e/offline_inference/test_voxcpm2.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-            "
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "Omni · Higgs-Audio V2 Test"
-        timeout_in_minutes: 30
-        commands:
-          - |
-            timeout 30m bash -c "
-              export VLLM_LOGGING_LEVEL=DEBUG
-              export VLLM_WORKER_MULTIPROC_METHOD=spawn
-              export VLLM_USE_DEEP_GEMM=0
-              export VLLM_MOE_USE_DEEP_GEMM=0
-              pytest -s -v tests/e2e/online_serving/test_higgs_audio_v2.py tests/e2e/offline_inference/test_higgs_audio_v2.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-            "
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "Diffusion · Qwen Image Edit Test"
-        timeout_in_minutes: 20
-        commands:
-          - pytest -s -v tests/e2e/online_serving/test_qwen_image_edit.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: "Diffusion · Longcat Image Edit Test"
-        timeout_in_minutes: 20
-        commands:
-          - pytest -s -v tests/e2e/online_serving/test_longcat_image_edit.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: "Diffusion · Longcat Image Test"
-        timeout_in_minutes: 20
-        commands:
-          - pytest -s -v tests/e2e/online_serving/test_longcat_image.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: "Diffusion · Bagel Test"
-        timeout_in_minutes: 20
-        commands:
-          - pytest -s -v tests/e2e/offline_inference/test_bagel.py tests/e2e/online_serving/test_bagel.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-
-      - label: "Diffusion · Qwen Image Test"
-        commands:
-          - |
-            timeout 40m bash -c '
-              export VLLM_IMAGE_FETCH_TIMEOUT=60
-              pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-            '
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: "Diffusion · Qwen Image Layered Test"
-        commands:
-          - |
-            timeout 40m bash -c "
-              export VLLM_IMAGE_FETCH_TIMEOUT=60
-              pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-            "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: "Diffusion · Wan22 Test"
-        commands:
-          - |
-            timeout 40m bash -c "
-              export VLLM_IMAGE_FETCH_TIMEOUT=60
-              pytest -s -v tests/e2e/offline_inference/test_wan22_t2v.py tests/e2e/online_serving/test_wan22_t2v.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
-            "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: "Diffusion · HunyuanVideo 1.5 Test"
-        timeout_in_minutes: 40
-        commands:
-          - |
-            timeout 40m bash -c "
-              pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
-            "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: "Diffusion · Z Image Test"
-        timeout_in_minutes: 30
-        commands:
-          - pytest -s -v tests/e2e/offline_inference/test_z_image.py tests/e2e/online_serving/test_z_image.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
-        agents:
-          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "Diffusion · FLUX.2 Klein Test"
-        timeout_in_minutes: 30
-        commands:
-          - pytest -s -v tests/e2e/offline_inference/test_flux2_klein.py tests/e2e/online_serving/test_flux2_klein.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
-        agents:
-          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "Diffusion · SD3 Test"
-        timeout_in_minutes: 45
-        commands:
-          - |
-            timeout 45m bash -c "
-              export VLLM_IMAGE_FETCH_TIMEOUT=60
-              pytest -s -v tests/e2e/online_serving/test_sd3.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
-            "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 4
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate

.buildkite/test-npu-nightly.yaml

@@ -1,24 +0,0 @@
-steps:
-  - group: ":card_index_dividers: Diffusion X2V Model Test"
-    key: nightly-diffusion-x2v-group
-    depends_on: upload-nightly-pipeline
-    if: >-
-      build.env("NIGHTLY") == "1" ||
-      build.pull_request.labels includes "nightly-test" ||
-      build.pull_request.labels includes "diffusion-x2v-test"
-    steps:
-      - label: ":full_moon: Diffusion X2V · Function Test"
-        agents:
-          queue: "ascend-a2b3"
-          resource_class: "npu-4"
-        image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}"
-        plugins:
-          - kubernetes:
-              podSpecPatch:
-                imagePullSecrets:
-                  - name: swr-secret
-        env:
-          VLLM_WORKER_MULTIPROC_METHOD: spawn
-          HF_TOKEN: "${HF_TOKEN}"
-        commands:
-          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k npu_i2v --run-level "full_model"

.buildkite/test-npu-ready.yaml

@@ -1,21 +0,0 @@
-steps:
-  - group: ":card_index_dividers: TTS Model Test"
-    key: ready-tts-group
-    depends_on: upload-ready-pipeline
-    steps:
-      - label: ":sound: TTS · Qwen3-TTS CustomVoice Test"
-        agents:
-          queue: "ascend-a2b3"
-          resource_class: "npu-1"
-        image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}"
-        plugins:
-          - kubernetes:
-              podSpecPatch:
-                imagePullSecrets:
-                  - name: swr-secret
-        env:
-          VLLM_WORKER_MULTIPROC_METHOD: spawn
-          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
-          HF_TOKEN: "${HF_TOKEN}"
-        commands:
-          - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'

.buildkite/test-ready.yml

@@ -1,541 +0,0 @@
-env:
-  VLLM_WORKER_MULTIPROC_METHOD: spawn
-  VLLM_USE_FLASHINFER_MOE_FP16: 0
-  HF_HUB_DOWNLOAD_TIMEOUT: 300
-  HF_HUB_ETAG_TIMEOUT: 60
-
-steps:
-  - group: ":card_index_dividers: Simple Test"
-    depends_on: upload-ready-pipeline
-    steps:
-      - label: "Simple · Diffusion & Model Executor Test"
-        commands:
-          - "timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "Simple · Other Test"
-        commands:
-          - "timeout 20m pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Custom Pipeline Test"
-    depends_on: upload-ready-pipeline
-    commands:
-      - timeout 20m pytest -s -v tests/e2e/offline_inference/custom_pipeline/ -m "core_model"
-    agents:
-      queue: "gpu_1_queue"
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - group: ":card_index_dividers: Diffusion Test"
-    depends_on: upload-ready-pipeline
-    steps:
-      - label: "Diffusion · Offloader Test"
-        commands:
-          - timeout 10m pytest -s -v tests/diffusion/offloader -m "core_model and cuda" --run-level "core_model"
-        agents:
-          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "Diffusion · Cache Test"
-        commands:
-          - timeout 15m pytest -sv tests/diffusion/cache -m 'core_model and cuda' --run-level "core_model"
-        agents:
-          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "Diffusion · Batch Test"
-        commands:
-          - timeout 15m pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model"
-        agents:
-          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "Diffusion · Distributed Test"
-        commands:
-          - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda' --run-level "core_model"
-        agents:
-          queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "Diffusion · Model Test"
-        commands:
-          - timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model"
-        agents:
-          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "Diffusion · Other Test"
-        commands:
-          - timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda' --run-level "core_model"
-        agents:
-          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-
-  - label: "Engine&Model Executor Test"
-    depends_on: upload-ready-pipeline
-    commands:
-      - |
-        timeout 15m bash -c "
-                  pytest -sv tests/engine/ tests/model_executor/ -m 'core_model and cuda' --run-level 'core_model'
-        "
-    agents:
-      queue: "gpu_1_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          shm-size: "8gb"
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-            - "HF_TOKEN"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-
-  - label: "Distributed Test"
-    depends_on: upload-ready-pipeline
-    commands:
-      - |
-        timeout 15m bash -c "
-                  pytest -sv tests/distributed/ -m 'core_model and cuda' --run-level 'core_model'
-        "
-    agents:
-      queue: "mithril-h100-pool"
-    plugins:
-      - kubernetes:
-          podSpec:
-            containers:
-              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                resources:
-                  limits:
-                    nvidia.com/gpu: 1
-                volumeMounts:
-                  - name: devshm
-                    mountPath: /dev/shm
-                  - name: hf-cache
-                    mountPath: /root/.cache/huggingface
-                env:
-                  - name: HF_HOME
-                    value: /root/.cache/huggingface
-            nodeSelector:
-              node.kubernetes.io/instance-type: gpu-h100-sxm
-            volumes:
-              - name: devshm
-                emptyDir:
-                  medium: Memory
-              - name: hf-cache
-                hostPath:
-                  path: /mnt/hf-cache
-                  type: DirectoryOrCreate
-
-  - group: ":card_index_dividers: E2E Test"
-    depends_on: upload-ready-pipeline
-    steps:
-      - label: "RLHF · VeRL-Omni E2E Test"
-        commands:
-          - |
-            timeout 20m bash -c "
-              pytest -s -v tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py
-            "
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "Omni · Qwen3-Omni Test"
-        commands:
-          - |
-            timeout 20m bash -c "
-              pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m 'core_model' --run-level 'core_model'
-            "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 2
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: "Omni · VoxCPM2 Test"
-        timeout_in_minutes: 20
-        commands:
-          - |
-            timeout 20m bash -c "
-              export VLLM_LOGGING_LEVEL=DEBUG
-              export VLLM_WORKER_MULTIPROC_METHOD=spawn
-              pytest -s -v tests/e2e/offline_inference/test_voxcpm2.py -m 'core_model' --run-level 'core_model'
-            "
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "TTS · Qwen3-TTS CustomVoice Test"
-        commands:
-          - |
-            timeout 20m bash -c "
-              export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
-              pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'
-            "
-        agents:
-          queue: "gpu_4_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "TTS · Qwen3-TTS Base Test"
-        commands:
-          - |
-            timeout 20m bash -c "
-              export VLLM_WORKER_MULTIPROC_METHOD=spawn
-              export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
-              export VLLM_OMNI_USE_V2_RUNNER=1
-              pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py -m 'core_model' --run-level 'core_model'
-            "
-        agents:
-          queue: "gpu_4_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
-      - label: "Diffusion · Qwen Image Edit Test"
-        commands:
-          - |
-            timeout 20m bash -c "
-              pytest -s -v tests/e2e/online_serving/test_qwen_image_edit.py -m 'core_model' --run-level 'core_model'
-            "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: "Diffusion · Bagel Test"
-        commands:
-          - |
-            timeout 40m bash -c "
-              export VLLM_IMAGE_FETCH_TIMEOUT=60
-              pytest -s -v tests/e2e/online_serving/test_bagel.py -m 'core_model' --run-level 'core_model'
-            "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: "Diffusion · Wan22 Test"
-        commands:
-          - |
-            timeout 40m bash -c "
-              export VLLM_IMAGE_FETCH_TIMEOUT=60
-              pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model' --run-level 'core_model'
-            "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-
-      - label: "Diffusion · Qwen Image Test"
-        commands:
-          - |
-            timeout 40m bash -c "
-              export VLLM_IMAGE_FETCH_TIMEOUT=60
-              pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model' --run-level 'core_model'
-            "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: "Diffusion · Qwen Image Layered Test"
-        commands:
-          - |
-            timeout 40m bash -c "
-              export VLLM_IMAGE_FETCH_TIMEOUT=60
-              pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'core_model' --run-level 'core_model'
-            "
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate

.buildkite/test-template-amd-omni.j2

@@ -1,62 +0,0 @@
-{# vllm-omni customized version
-   Based on: https://github.com/vllm-project/ci-infra/blob/main/buildkite/test-template-amd.j2
-   Last synced: 2025-12-15
-   Modifications: Removed unused CUDA/NVIDIA logic, keeping only AMD tests
-#}
-{% set docker_image_amd = "rocm/vllm-omni:$BUILDKITE_COMMIT" %}
-{% set default_working_dir = "/app/vllm-omni" %}
-
-  - group: "AMD Tests"
-    depends_on: ~
-    steps:
-      - label: "AMD: :docker: build image"
-        depends_on: ~
-        soft_fail: false
-        commands:
-          - "docker build -f docker/Dockerfile.rocm -t {{ docker_image_amd }} --target test --build-arg ARG_PYTORCH_ROCM_ARCH=gfx942 --progress plain ."
-          - "docker push {{ docker_image_amd }}"
-        key: "amd-build"
-        env:
-          DOCKER_BUILDKIT: "1"
-        retry:
-          automatic:
-            - exit_status: -1  # Agent was lost
-              limit: 2
-            - exit_status: -10  # Agent was lost
-              limit: 2
-            - exit_status: 128  # Git connectivity issues
-              limit: 2
-            - exit_status: 1  # Machine occasionally fail
-              limit: 1
-        agents:
-          queue: amd-cpu
-
-    {% for step in steps %}
-    {% if step.mirror_hardwares and mirror_hw in step.mirror_hardwares %}
-      - label: "{{ step.agent_pool }}: {{ step.label }}"
-        depends_on: amd-build
-        agents:
-          {% if step.agent_pool %}
-          queue: amd_{{ step.agent_pool }}
-          {% else %}
-          queue: amd_mi325_1
-          {% endif %}
-{% set cmd_body = (step.command or (step.commands | join("\n"))) | trim %}
-{% set indented_cmd = cmd_body | replace("\n", "\n            ") %}
-        command: bash .buildkite/scripts/hardware_ci/run-amd-test.sh
-        env:
-          DOCKER_BUILDKIT: "1"
-          TEST_COMMAND: |-
-            (command rocm-smi || true) && cd {{ (step.working_dir or default_working_dir) | safe }}
-{% if "mi250" in step.agent_pool %}
-            python3 -m pip uninstall -y amd-aiter
-{% endif %}
-            {{ indented_cmd | safe }}
-        priority: 100
-        {% if step.grade and step.grade == "Blocking" %}
-        soft_fail: false
-        {% else %}
-        soft_fail: true
-        {% endif%}
-    {% endif %}
-    {% endfor %}

.buildkite/test-weekly.yml

@@ -1,148 +0,0 @@
-env:
-  VLLM_WORKER_MULTIPROC_METHOD: spawn
-  VLLM_USE_FLASHINFER_MOE_FP16: 0
-  HF_HUB_DOWNLOAD_TIMEOUT: 300
-  HF_HUB_ETAG_TIMEOUT: 60
-
-steps:
-  - label: "Reliability Test - qwen3-omni"
-    timeout_in_minutes: 180
-    depends_on: upload-weekly-pipeline
-    if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
-    commands:
-      - pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m "slow"
-    agents:
-      queue: "mithril-h100-pool"
-    plugins:
-      - kubernetes:
-          podSpec:
-            containers:
-              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                resources:
-                  limits:
-                    nvidia.com/gpu: 2
-                volumeMounts:
-                  - name: devshm
-                    mountPath: /dev/shm
-                  - name: hf-cache
-                    mountPath: /root/.cache/huggingface
-                env:
-                  - name: HF_HOME
-                    value: /root/.cache/huggingface
-                  - name: HF_TOKEN
-                    valueFrom:
-                      secretKeyRef:
-                        name: hf-token-secret
-                        key: token
-            nodeSelector:
-              node.kubernetes.io/instance-type: gpu-h100-sxm
-            volumes:
-              - name: devshm
-                emptyDir:
-                  medium: Memory
-              - name: hf-cache
-                hostPath:
-                  path: /mnt/hf-cache
-                  type: DirectoryOrCreate
-
-  - label: "Reliability Test - wan22"
-    timeout_in_minutes: 180
-    depends_on: upload-weekly-pipeline
-    if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
-    commands:
-      - pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m "slow"
-    agents:
-      queue: "mithril-h100-pool"
-    plugins:
-      - kubernetes:
-          podSpec:
-            containers:
-              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                resources:
-                  limits:
-                    nvidia.com/gpu: 2
-                volumeMounts:
-                  - name: devshm
-                    mountPath: /dev/shm
-                  - name: hf-cache
-                    mountPath: /root/.cache/huggingface
-                env:
-                  - name: HF_HOME
-                    value: /root/.cache/huggingface
-                  - name: HF_TOKEN
-                    valueFrom:
-                      secretKeyRef:
-                        name: hf-token-secret
-                        key: token
-            nodeSelector:
-              node.kubernetes.io/instance-type: gpu-h100-sxm
-            volumes:
-              - name: devshm
-                emptyDir:
-                  medium: Memory
-              - name: hf-cache
-                hostPath:
-                  path: /mnt/hf-cache
-                  type: DirectoryOrCreate
-
-  # Invalid HTTP / WS payloads vs live Omni servers (Issue #3649 tracks known skips).
-  - group: "Reliability Test - Invalid parameters Test"
-    depends_on: upload-weekly-pipeline
-    if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
-    steps:
-      - label: "Invalid parameters Test · H100"
-        timeout_in_minutes: 120
-        commands:
-          - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100"
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 2
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      # Stable Audio Open, Qwen3-TTS speech/batch/voices paths marked ``L4``.
-      - label: "Invalid parameters Test · L4"
-        timeout_in_minutes: 120
-        commands:
-          - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and L4"
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"

.claude/skills/add-diffusion-model/SKILL.md

@@ -24,7 +24,7 @@ Check the model's HF repo for `model_index.json`. This determines your path:
 
 | Scenario | How to identify | Migration path |
 |----------|----------------|----------------|
-| **Already supported** | `_class_name` in `model_index.json` matches a key in `_DIFFUSION_MODELS` in `registry.py` | Skip to Step 5 (test) and Step 7 (docs) |
+| **Already supported** | `_class_name` in `model_index.json` matches a key in `_DIFFUSION_MODELS` in `registry.py` | Skip implementation, then validate model-specific examples, tests, and docs as needed |
 | **Diffusers-based** | Has standard `model_index.json` with `_diffusers_version`, subfolders for `transformer/`, `vae/`, etc. | Follow **Path A** below |
 | **Custom/private repo** | No diffusers `model_index.json`, weights in non-standard format, custom model code in a separate git repo | Follow **Path B** below |
 | **Hybrid** | Has some diffusers components (VAE) but custom transformer/fusion | Mix of Path A and Path B |
@@ -52,7 +52,7 @@ vllm_omni/diffusion/models/your_model_name/
 2. Replace attention with `vllm_omni.diffusion.attention.layer.Attention` (QKV shape: `[B, seq, heads, head_dim]`).
 3. Add `od_config: OmniDiffusionConfig | None = None` to `__init__`.
 4. Add `load_weights()` method mapping diffusers weight names to vllm-omni names.
-5. Add class attributes: `_repeated_blocks`, `_layerwise_offload_blocks_attr`.
+5. Add class attributes for acceleration features such as `_repeated_blocks` and `_layerwise_offload_blocks_attrs` (see [references/transformer-adaptation.md](references/transformer-adaptation.md) for examples).
 
 ### A4. Adapt pipeline
 
@@ -78,13 +78,34 @@ class YourPipeline(nn.Module):
 
 Add post/pre-process functions in the same pipeline file. Register them in `registry.py`.
 
+### A4.1 Add progress bar support (recommended)
+
+For pipelines with a standard denoising loop, prefer the existing progress bar pattern instead of hand-rolled logging.
+
+```python
+from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin
+
+class YourPipeline(nn.Module, ProgressBarMixin):
+    def forward(self, req: OmniDiffusionRequest) -> DiffusionOutput:
+        # ... prepare timesteps / latents ...
+        with self.progress_bar(total=len(timesteps)) as progress_bar:
+            for i, t in enumerate(timesteps):
+                # predict noise / scheduler step
+                latents = ...
+                progress_bar.update()
+
+        return DiffusionOutput(output=output)
+```
+
+For custom loop structures, follow `vllm_omni/diffusion/models/progress_bar.py` and existing pipelines using `ProgressBarMixin`.
+
 ### A5. Register, test, docs → continue at Step 4 below.
 
 ---
 
 ## Path B: Custom/Private Repo Model
 
-For models without a diffusers pipeline — weights in custom format, model code in a private repo. Real examples: DreamID-Omni, BAGEL, HunyuanImage3.
+For models without a diffusers pipeline — weights in custom format, model code in a private repo. Real examples: DreamID-Omni, BAGEL, HunyuanImage3. See [references/custom-model-patterns.md](references/custom-model-patterns.md) for concrete custom-model integration patterns.
 
 ### B1. Understand the reference repo
 
@@ -134,7 +155,7 @@ Copy the essential model files into `vllm_omni/diffusion/models/<name>/` and ada
 
 ### B4. Handle custom weight loading
 
-Custom models have two patterns for weight loading:
+Custom models have two common patterns for weight loading:
 
 **Pattern 1: Bypass standard loader** (DreamID-Omni style)
 
@@ -149,8 +170,10 @@ class CustomPipeline(nn.Module):
         self.vae = custom_init_vae(model, device=self.device)
         self.text_encoder = custom_init_text_encoder(model, device=self.device)
         self.transformer = CustomFusionModel(CONFIG)
-        load_custom_checkpoint(self.transformer,
-            checkpoint_path=os.path.join(model, "model.safetensors"))
+        load_custom_checkpoint(
+            self.transformer,
+            checkpoint_path=os.path.join(model, "model.safetensors"),
+        )
         # NO weights_sources defined — bypasses standard loader
 
     def load_weights(self, weights):
@@ -162,6 +185,8 @@ class CustomPipeline(nn.Module):
 When weights are in safetensors format but need name remapping:
 
 ```python
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
+
 class CustomPipeline(nn.Module):
     def __init__(self, *, od_config, prefix=""):
         super().__init__()
@@ -187,7 +212,9 @@ class CustomPipeline(nn.Module):
             # Remap original weight names to vllm-omni module names
             name = self._remap_weight_name(name)
             if name in params:
-                default_weight_loader(params[name], tensor)
+                param = params[name]
+                weight_loader = getattr(param, "weight_loader", default_weight_loader)
+                weight_loader(param, tensor)
                 loaded.add(name)
         return loaded
 ```
@@ -281,16 +308,43 @@ For Omni or custom models, create:
 ### Step 7: Update Documentation
 
 Required updates:
-1. `docs/user_guide/diffusion/parallelism_acceleration.md` — parallelism support table
-2. `docs/user_guide/diffusion/cpu_offload_diffusion.md` — if CPU offload supported (add to supported models table)
-3. `docs/user_guide/diffusion/teacache.md` — if TeaCache supported
-4. `docs/user_guide/diffusion/cache_dit_acceleration.md` — if Cache-DiT supported
-5. `examples/offline_inference/xxx/README.md` — offline example docs
-6. `examples/online_serve/xxx/README.md` — online serve docs
+1. `docs/user_guide/diffusion/parallelism/overview.md` — parallelism support overview/table
+2. `docs/user_guide/diffusion/cpu_offload.md` — if CPU offload supported (add to supported models table)
+3. `docs/user_guide/diffusion/cache_acceleration/teacache.md` — if TeaCache supported
+4. `docs/user_guide/diffusion/cache_acceleration/cache_dit.md` — if Cache-DiT supported
+5. Offline example docs under `examples/offline_inference/<name>/` (`README.md` or category-specific `.md`)
+6. `examples/online_serving/<name>/README.md` — online serving docs
 
-### Step 8: Add E2E Tests (Recommended)
+### Step 8: Add E2E Tests
 
-Create `tests/e2e/online_serving/test_your_model_expansion.py`.
+**Follow the [vllm-omni-test skill](../vllm-omni-test/SKILL.md)** for markers, file naming, Buildkite wiring, and run commands. Also read [l4_functionality_tests.inc.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/test_examples/l4_functionality_tests.inc.md), [test_system_overview.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/test_system_overview.md), and [test_writing_guide.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/test_writing_guide.md).
+
+Classify the model's **CI priority** first:
+
+| Priority | Required test levels | Files & markers |
+|----------|---------------------|-----------------|
+| **High** (listed in [#1832](https://github.com/vllm-project/vllm-omni/issues/1832) or on the diffusion hot path) | **L1** · **L2** online · **L3** online + offline · **L4** feature + performance | See table below |
+| **Medium** (*normal priority* in L4 docs) | **L3** online + offline · **L4** feature only | Fewer L4 parametrized rows |
+| **Low** | **L4** feature only | One or two `*_expansion.py` cases |
+
+**Per-level deliverables (diffusion / `pytest.mark.diffusion`):**
+
+| Level | Location | Marker | CI pipeline | Notes |
+|-------|----------|--------|-------------|-------|
+| **L1** | `tests/diffusion/models/{slug}/`, `tests/diffusion/cache/`, transformer unit tests | `core_model` + `cpu` | `test-ready.yml` | Weight remap, `_sp_plan`, cache enabler registration, shape contracts |
+| **L2** | `tests/e2e/online_serving/test_{slug}.py` (and offline if the category is offline-first) | **`core_model` + `advanced_model`** (both on baseline smoke) + `diffusion` + `@hardware_test` / `hardware_marks` | `test-ready.yml` | Default deploy smoke — minimal `num_inference_steps`, single prompt |
+| **L3** | `tests/e2e/online_serving/test_{slug}.py` **and** `tests/e2e/offline_inference/test_{slug}.py` when offline matters | Baseline smoke: **`core_model` + `advanced_model`**; heavier cases: `advanced_model` only (+ `diffusion`) | `test-merge.yml` **or** merged into nightly diffusion function job | Real weights, streaming/API paths, LoRA/offload smoke |
+| **L4** | `tests/e2e/online_serving/test_{slug}_expansion.py` (+ offline expansion if needed) | `full_model` + `diffusion` | `test-nightly.yml` (X2I / X2V / X2A function groups) | Feature combos per [#1832](https://github.com/vllm-project/vllm-omni/issues/1832); perf → `tests/dfx/perf/tests/test_{model}_vllm_omni.json` with per-case **`mark`** (`hardware_marks` + `full_model` + `diffusion`) |
+
+**L2 & L3 online — same file, dual marks on the baseline smoke:** The **first / simplest** case in `test_{slug}.py` (default deploy, minimal steps, single prompt) should carry **both** `@pytest.mark.core_model` and `@pytest.mark.advanced_model` on the **same** function so L2 (`test-ready.yml`) and L3 (`test-merge.yml`) share one smoke test. Heavier deploy variants or API paths in the same file use **`advanced_model` only**. When L3 moves to nightly, migrate those heavier cases into `test_{slug}_expansion.py` with `full_model` and remove the dedicated `test-merge.yml` job (see `test_longcat_image_expansion.py`, `test_qwen_image_expansion.py`).
+
+**L4 design (high priority):** Combine multiple supported features (Cache-DiT, TP, USP, CFG, HSDP, CPU offload, quantization) into **few parametrized** `OmniServerParams` rows so each feature appears in at least one case without exploding GPU jobs. Shard single-GPU vs multi-GPU cases across the nightly X2I/X2V function steps (1-GPU vs `distributed_cuda`).
+
+**L4 design (medium / low):** One or two parametrized rows covering the best quality/perf trade-off; skip perf JSON unless the model is high priority.
+
+**Reference implementations:** `tests/e2e/online_serving/test_qwen_image_edit_expansion.py`, `tests/e2e/online_serving/test_longcat_image_expansion.py`, `tests/e2e/online_serving/test_hunyuan_video_15_expansion.py`.
+
+**Keep model-specific code inside test modules — not `tests/helpers/{slug}.py`:** deploy constants, prompts, sampling dicts, and inline `request_config` / `form_data` belong in each `test_{slug}.py` and `test_{slug}_expansion.py`. Do not add per-model files under `tests/helpers/`; reuse only repo-wide harness (`mark`, `media`, `runtime`, `stage_config`, `assertions`). **L2+ online/offline e2e:** reuse or add `send_*_request` in `tests/helpers/runtime.py` — tests call the handler, not raw `omni.generate` / HTTP. See [vllm-omni-test skill](../vllm-omni-test/SKILL.md) § **Runtime send helpers**.
 
 ### Step 9: Add Cache-DiT Acceleration
 
@@ -364,7 +418,8 @@ def enable_cache_for_your_model(pipeline, cache_config):
 
 #### 9d. Register the custom enabler
 
-Add your enabler to `CUSTOM_DIT_ENABLERS` in `vllm_omni/diffusion/cache/cache_dit_backend.py`:
+Add your enabler to `CUSTOM_DIT_ENABLERS` in
+`vllm_omni/diffusion/cache/cachedit/model_specific.py`:
 
 ```python
 CUSTOM_DIT_ENABLERS = {
@@ -510,7 +565,7 @@ class YourTransformer(nn.Module):
 #### 10e. Update parallelism documentation
 
 After adding parallelism support, update:
-1. `docs/user_guide/diffusion/parallelism_acceleration.md` — add your model to the support table
+1. `docs/user_guide/diffusion/parallelism/overview.md` — add your model to the support overview/table
 2. Record which parallelism methods are supported (USP, Ring, CFG, TP, HSDP, VAE-Patch)
 
 ### Step 11: Add CPU Offload Support
@@ -568,6 +623,7 @@ See the [Profiling Single-Stage Diffusion](../../../docs/contributing/profiling.
 
 ## Reference Files
 
+- [vllm-omni-test skill](../vllm-omni-test/SKILL.md) — L1–L4 markers, naming, Buildkite wiring, run commands
 - [Transformer Adaptation](references/transformer-adaptation.md) — porting transformers from diffusers
 - [Custom Model Patterns](references/custom-model-patterns.md) — patterns for non-diffusers models
 - [Parallelism Patterns](references/parallelism-patterns.md) — TP, SP/USP, CFG parallel, HSDP implementation details

.claude/skills/add-diffusion-model/references/cache-dit-patterns.md

@@ -29,8 +29,13 @@ vLLM-Omni integrates cache-dit through `CacheDiTBackend`:
 | `cache_dit.refresh_context()` | Updates cache context when `num_inference_steps` changes |
 
 **Source files**:
-- `vllm_omni/diffusion/cache/cache_dit_backend.py` — `CacheDiTBackend`, enablers, `CUSTOM_DIT_ENABLERS`
-- `vllm_omni/diffusion/cache/` — cache backend implementations
+- `vllm_omni/diffusion/cache/cachedit/__init__.py` — public package API
+- `vllm_omni/diffusion/cache/cachedit/backend.py` — generic backend lifecycle and integration
+- `vllm_omni/diffusion/cache/cachedit/config.py` — Cache-DiT configuration types
+- `vllm_omni/diffusion/cache/cachedit/model_specific.py` — model-specific adapters, enablers, and registry
+
+Outside `vllm_omni/diffusion/cache`, import Cache-DiT symbols only from
+`vllm_omni.diffusion.cache.cachedit`.
 
 ## Standard Models: Automatic Support
 
@@ -175,7 +180,8 @@ Inspect your block's `forward()` return type and residual connection pattern to
 
 ## Registering Custom Enablers
 
-Add your enabler to `CUSTOM_DIT_ENABLERS` in `vllm_omni/diffusion/cache/cache_dit_backend.py`:
+Add your enabler to `CUSTOM_DIT_ENABLERS` in
+`vllm_omni/diffusion/cache/cachedit/model_specific.py`:
 
 ```python
 CUSTOM_DIT_ENABLERS = {
@@ -248,7 +254,7 @@ Models listed in `_NO_CACHE_ACCELERATION` in `vllm_omni/diffusion/registry.py` d
 
 | Model | Path | Notes |
 |-------|------|-------|
-| Standard DiT | `cache_dit_backend.py::enable_cache_for_dit` | Default enabler, automatic |
-| Wan2.2 | `cache_dit_backend.py::enable_cache_for_wan22` | Dual-transformer, auto-detects mode |
-| LongCat | `cache_dit_backend.py::enable_cache_for_longcat_image` | Multi-block-list |
-| BAGEL | `cache_dit_backend.py::enable_cache_for_bagel` | Complex omni model |
+| Standard DiT | `cachedit.backend::enable_cache_for_dit` | Default enabler, automatic |
+| Wan2.2 | `cachedit.model_specific::enable_cache_for_wan22` | Dual-transformer, auto-detects mode |
+| LongCat | `cachedit.config::CacheDiTAdapterConfig` | Declarative multi-block-list adapter |
+| BAGEL | `cachedit.model_specific::BagelCachedAdapter` | Custom cached adapter |

.claude/skills/add-diffusion-model/references/parallelism-patterns.md

@@ -569,4 +569,4 @@ Check which parallelism methods a model supports:
 | **TP** | Uses `ColumnParallelLinear` / `QKVParallelLinear`. Search: `grep -r 'ParallelLinear\|QKVParallel' vllm_omni/diffusion/models/<model>/` |
 | **HSDP** | Transformer defines `_hsdp_shard_conditions`. Search: `grep -r '_hsdp_shard_conditions' vllm_omni/diffusion/models/` |
 
-The canonical per-model support table is in `docs/user_guide/diffusion/parallelism_acceleration.md`.
+The canonical per-model support overview/table is in `docs/user_guide/diffusion/parallelism/overview.md`.

.claude/skills/add-diffusion-model/references/transformer-adaptation.md

@@ -219,13 +219,14 @@ def load_weights(self, weights):
 
 ### Class-Level Attributes for Features
 
+
 ```python
 class MyTransformer(nn.Module):
     # torch.compile: list block class names that repeat and can be compiled
     _repeated_blocks = ["MyTransformerBlock"]
 
     # CPU offload: attribute name of the nn.ModuleList containing blocks
-    _layerwise_offload_blocks_attr = "blocks"
+    _layerwise_offload_blocks_attrs = ["blocks"]
 
     # LoRA: mapping of fused param names to original param names
     packed_modules_mapping = {"to_qkv": ["to_q", "to_k", "to_v"]}

.claude/skills/add-tts-model/SKILL.md

@@ -220,65 +220,70 @@ See `plan/voxcpm2_native_ar_design.md`.
 
 ### Steps
 
-1. **Register in `serving_speech.py`** — add all 5 points in a **single commit**;
-   partial integration causes hard-to-debug failures. This file is modified by every
-   model PR and is the most common source of rebase conflicts — see conflict note below.
+1. **Write one adapter** under `vllm_omni/entrypoints/openai/tts_adapters/`.
+   `serving_speech.py` should not need an edit — detection, stage discovery and
+   dispatch are all derived from what the adapter declares.
 
-   **Point 1** — stage constant (near the top, alongside the other `_*_TTS_MODEL_STAGES` sets):
+   Create `vllm_omni/entrypoints/openai/tts_adapters/your_model.py`:
    ```python
-   _YOUR_MODEL_TTS_MODEL_STAGES = {"your_stage_key"}
+   @register_tts_adapter
+   class YourModelAdapter(ARTTSAdapter):
+       name = "your_model"                              # registry key + log label
+       stage_keys = frozenset({"your_stage_key"})       # the deploy yaml's model_stage
+
+       def validate(self, request) -> str | None:
+           if not request.input or not request.input.strip():
+               return "Input text cannot be empty"
+           return None
+
+       async def build(self, request, sampling_params_list, has_inline_ref_audio):
+           params = {"text": [request.input]}
+           if request.voice is not None:
+               params["voice"] = [request.voice]
+           return PreparedRequest(
+               prompt={"prompt": request.input},
+               tts_params=params,
+               model_type=self.name,
+           )
    ```
 
-   **Point 2** — union into `_TTS_MODEL_STAGES`:
-   ```python
-   _TTS_MODEL_STAGES: set[str] = (
-       ...
-       | _YOUR_MODEL_TTS_MODEL_STAGES
-   )
-   ```
+   Then add the module to the import block at the bottom of
+   `tts_adapters/__init__.py` so it registers. That import line is the only shared
+   file a new model touches — which is also why the old rebase-conflict hotspot is
+   gone.
 
-   **Point 3** — model type detection in `_detect_tts_model_type()`:
-   ```python
-   if model_stage in _YOUR_MODEL_TTS_MODEL_STAGES:
-       return "your_model"
-   ```
+   > **Pure-diffusion TTS does not go through adapters yet.** Under
+   > `for_diffusion()`, `create_speech()` routes straight to
+   > `_create_diffusion_speech()` and never calls `validate()`/`build()`.
+   > `DiffusionTTSAdapter` is scaffolding with no production subclass, so logic
+   > placed in one would silently never run. Diffusion-engine models follow the
+   > existing diffusion path; wiring it through adapters is open work (#4855).
 
-   **Point 4** — validation dispatch in `_validate_tts_request()`:
-   ```python
-   if self._tts_model_type == "your_model":
-       return self._validate_your_model_request(request)
-   ```
+   **If a stage key alone cannot identify the model**, declare
+   `model_archs = frozenset({"YourModelForConditionalGeneration"})`; add
+   `arch_identifies_entry_stage = True` when the model owns no stage key at all
+   (Ming dense). For a rule that is not set membership, override `matches()` (see
+   `covo_audio.py`). For a genuine overlap with another adapter, give one an
+   explicit `detect_priority` — `test_tts_detection.py` fails on an unordered
+   overlap. Models that only serve speech in some topologies override
+   `stage_serves_speech()` (see `audex.py`).
 
-   **Point 5** — validation + parameter-builder methods:
-   ```python
-   def _validate_your_model_request(self, request) -> str | None:
-       if not request.input or not request.input.strip():
-           return "Input text cannot be empty"
-       return None
+   **Reuse shared helpers via `self.ctx.server`** rather than reimplementing them:
+   `_resolve_ref_audio`, `_apply_uploaded_speaker`, `_validate_ref_audio_format`,
+   `_max_instructions_length`. Read a comparable adapter first — `fish_speech.py`
+   (voice cloning), `higgs_audio_v3.py` (parameter-heavy), `moss_tts.py` (family
+   sharing a base class).
 
-   def _build_your_model_params(self, request) -> dict:
-       params = {"text": [request.input]}
-       if request.voice is not None:
-           params["voice"] = [request.voice]
-       return params
-   ```
-   Wire `_build_your_model_params` into `_create_tts_request()` alongside the other
-   model-specific param builders.
+   > **Do not add `self._tts_model_type == ...` branches to `serving_speech.py`.**
+   > Older models predate the adapter framework and still have them; they are being
+   > migrated out (RFC #4327, #4855). `tools/pre_commit/check_tts_adapter.py` is a
+   > ratchet on the remaining count and fails the commit if it grows. Behaviour that
+   > no adapter hook can express is a missing hook — propose it on the RFC.
 
-   > **Two dispatch patterns coexist**: Fish Speech uses a `self._is_fish_speech` boolean
-   > instance attribute checked before `elif self._is_tts`, while all newer models
-   > (CosyVoice3, MOSS-TTS-Nano) use the `_tts_model_type` string returned by
-   > `_detect_tts_model_type()`. For new models, always use the `_tts_model_type` string
-   > pattern — do not add new `_is_*` flags.
-
-   > **Unused variable rule**: only extract fields in `_build_your_model_params` that
-   > are actually forwarded to the model. Unused extractions fail `ruff F841`.
-   > For voice-cloning fields (`ref_audio` → `prompt_audio_path`, `ref_text` →
-   > `prompt_text`), add them to the param builder and verify they reach the model call.
-
-   **Rebase conflict note**: when rebasing onto `main` after another model was merged,
-   `serving_speech.py` will conflict. Resolution: always keep *both* the upstream
-   model's additions and your own — never discard either side.
+   > **Unused variable rule**: only extract fields in `build()` that are actually
+   > forwarded to the model. Unused extractions fail `ruff F841`. For voice-cloning
+   > fields (`ref_audio` -> `prompt_audio_path`, `ref_text` -> `prompt_text`), add
+   > them to the params and verify they reach the model call.
 
 2. **Handle model-specific parameters**:
    - Voice cloning: `ref_audio` encoding and prompt injection
@@ -295,7 +300,7 @@ import base64
 from pathlib import Path
 
 def build_voice_clone_prompt(ref_audio_path: str, text: str, codec) -> list:
-    """Build prompt with reference audio for voice cloning in serving_speech.py."""
+    """Build prompt with reference audio for voice cloning, called from the adapter."""
     audio_bytes = Path(ref_audio_path).read_bytes()
     codes = codec.encode(audio_bytes)  # Encode on CPU using model's codec (e.g., DAC)
     token_ids = [code + codec.vocab_offset for code in codes.flatten().tolist()]
@@ -305,13 +310,72 @@ def build_voice_clone_prompt(ref_audio_path: str, text: str, codec) -> list:
     ]
 ```
 
+### Test Case Writing (CI Levels)
+
+**Follow the [vllm-omni-test skill](../vllm-omni-test/SKILL.md)** for markers, file naming (`test_{slug}.py` / `test_{slug}_expansion.py`), Buildkite wiring, and copy-paste run commands. Also read [test_system_overview.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/test_system_overview.md) and [test_writing_guide.md](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/ci/test_writing_guide.md).
+
+Classify the model's **CI priority** first (high / medium / low). High-priority TTS models are typically those on the integration hot path or listed in tracking issues such as [#1832](https://github.com/vllm-project/vllm-omni/issues/1832); medium and low tiers cover the long tail. When unsure, ask the reviewer which tier applies.
+
+| Priority | Required test levels | Files & markers |
+|----------|---------------------|-----------------|
+| **High** | **L1** unit/logic · **L2** online smoke · **L3** online + offline integration · **L4** feature + performance | See table below |
+| **Medium** | **L3** online + offline · **L4** feature only | Skip dedicated L1/L2 unless fixing a logic bug |
+| **Low** | **L4** feature only | One or two `*_expansion.py` parametrized cases |
+
+**Per-level deliverables (TTS / `pytest.mark.tts`):**
+
+| Level | Location | Marker | CI pipeline | Notes |
+|-------|----------|--------|-------------|-------|
+| **L1** | `tests/model_executor/…`, `tests/entrypoints/openai_api/…`, stage-processor tests | `core_model` + `cpu` | `test-ready.yml` | Prompt assembly, async_chunk helpers, adapter validation — no GPU |
+| **L2** | `tests/e2e/online_serving/test_{slug}.py` | **`core_model` + `advanced_model`** (both on baseline smoke) + `tts` + `@hardware_test(...)` | `test-ready.yml` (`ready` label) | Default deploy smoke: single `/v1/audio/speech` or offline `OmniRunner` path |
+| **L3** | `tests/e2e/online_serving/test_{slug}.py` **and** `tests/e2e/offline_inference/test_{slug}.py` | Baseline smoke: **`core_model` + `advanced_model`**; heavier cases: `advanced_model` only (+ `tts`) | `test-merge.yml` **or** merged into nightly TTS function job | Streaming, voice clone, batch/queue, async_chunk |
+| **L4** | `tests/e2e/online_serving/test_{slug}_expansion.py`, optional offline expansion | `full_model` + `tts` | `test-nightly.yml` (`:full_moon: TTS · Function Test with L4`) | Feature matrix; perf → `tests/dfx/perf/tests/test_tts.json` |
+
+**L2 & L3 online — same file, dual marks on the baseline smoke:** The **first / simplest** case in `test_{slug}.py` (default deploy, single non-streaming `/v1/audio/speech` or equivalent offline path) should carry **both** `@pytest.mark.core_model` **and** `@pytest.mark.advanced_model` on the **same** function so it runs in L2 (`test-ready.yml`, `--run-level core_model`, basic validation) and L3 (`test-merge.yml`, `--run-level advanced_model`, deeper validation) without duplicating the test. In-tree examples: `test_voxcpm2_tts.py::test_text_to_audio_001`, `test_qwen3_tts_customvoice.py::test_text_to_audio_001`.
+
+Heavier scenarios in the same file use **`advanced_model` only** (streaming, extra languages, concurrency, async_chunk, batch). Example: `test_voice_clone_en_streaming_001` → `advanced_model` only. When migrating L3 to nightly, move those heavier cases into `test_{slug}_expansion.py` with `full_model` and drop the dedicated merge job (see `test_ming_tts_expansion.py`, `test_glm_tts_expansion.py`).
+
+```python
+@pytest.mark.core_model
+@pytest.mark.advanced_model
+@pytest.mark.tts
+@hardware_test(res={"cuda": "L4"}, num_cards=1)
+@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
+def test_voice_clone_en_non_streaming_001(omni_server, openai_client) -> None:
+    openai_client.send_audio_speech_request({...})
+```
+
+**L4 consolidation:** Prefer parametrized `OmniServerParams` rows (`default`, `async_chunk`, feature flags) in one expansion module rather than many merge-only files ([#1832](https://github.com/vllm-project/vllm-omni/issues/1832)).
+
+**L4 performance (high-priority models):** Add latency / throughput / stress rows in `tests/dfx/perf/tests/test_tts.json`, or a dedicated `tests/dfx/perf/tests/test_{slug}.json` when the model must not join the shared nightly server matrix before integration lands (see VoxCPM2 / Coqui XTTS pattern). Register the model in `benchmarks/tts/model_configs.yaml` for local `bench_tts.py`. Wire a **separate** `test-nightly.yml` Perf Test step when the JSON is not merged into `test_tts.json` yet.
+
+**Keep model-specific code inside test modules — not `tests/helpers/{slug}.py`:**
+
+- Put `MODEL`, deploy path, vendored `REF_AUDIO_URL`, `get_prompt()`, and inline `request_config` dicts **in each** `test_{slug}.py`, `test_{slug}_expansion.py`, offline `test_{slug}.py`, and L1 `test_{slug}_*.py` as needed.
+- **Do not** add `tests/helpers/{slug}.py` (or `tests/helpers/{model_name}.py`) to deduplicate constants or request builders across those files. A little duplication is intentional; follow in-tree references such as `tests/e2e/online_serving/test_glm_tts.py` and `tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py`.
+- `tests/helpers/` is for **repo-wide** harness code only (`mark.py`, `media.py`, `runtime.py`, `stage_config.py`, `assertions.py`, `fixtures/`). Import those; do not extend the tree with per-model modules.
+
+**Runtime send helpers (`tests/helpers/runtime.py`) — online and offline e2e:**
+
+| Path | Fixture | Call |
+|------|---------|------|
+| Online `/v1/*` | `openai_client` | `openai_client.send_*_request(request_config)` |
+| Offline inference | `omni_runner_handler` | `omni_runner_handler.send_*_request(request_config)` |
+
+1. **Grep `runtime.py` first** — reuse `send_omni_request`, `send_diffusion_request`, `send_audio_speech_request` (online + offline Qwen-style TTS), `send_single_stage_tts_request` (Coqui XTTS / MOSS-TTS-Nano offline), etc.
+2. **No matching helper** → add `send_<feature>_request` (or `send_<route>_http_request` for negative/dfx) in **`runtime.py`** with general `assert_*` bundled inside, **then** call it from the test.
+3. **Test file** holds `request_config` dicts only — not `omni.generate`, not `_collect_audio()`, not raw HTTP/SDK.
+4. Wire `tests/helpers/runtime.py` into Buildkite `source_file_dependencies` when you add helpers.
+
+See [vllm-omni-test skill](../vllm-omni-test/SKILL.md) § **Runtime send helpers** for full tables and exceptions.
+
 ### Deliverables
 
-- Updated `serving_speech.py` with all 5 integration points (single commit)
+- One adapter file under `tts_adapters/` plus its line in the package import block
 - Client scripts and server launcher under `examples/online_serving/text_to_speech/<model>/`
 - Gradio demo with streaming and voice cloning UI in the same dir
-- E2E online serving test (`tests/e2e/online_serving/test_<model>.py`)
-- Buildkite CI entry in `.buildkite/test-merge.yml`
+- E2E tests per **Test Case Writing (CI Levels)** above (priority tier determines L1–L4 scope)
+- Buildkite wired per level: `test-ready.yml` (L1/L2), `test-merge.yml` or nightly function job (L3), `test-nightly.yml` (L4) — see [vllm-omni-test skill](../vllm-omni-test/SKILL.md)
 - New section in `examples/online_serving/text_to_speech/README.md` (table row + per-model section). Do **not** create a top-level `examples/online_serving/<model>/` dir or a per-model `README.md` inside `text_to_speech/<model>/`.
 
 ### E2E test pitfalls to avoid
@@ -326,9 +390,9 @@ def build_voice_clone_prompt(ref_audio_path: str, text: str, codec) -> list:
 - **Use the harness readiness gate.** The fixture waits for HTTP 200 on
   `/health`; don't add `time.sleep` in tests. If warmup is incomplete, make
   `/health` return non-200 until you're actually ready.
-- **Mark with `@pytest.mark.core_model` + `hardware_test(res={"cuda": "H100"})`**
-  so the test lands in `test-ready.yml` (triggered by the `ready` label) rather
-  than only nightly.
+- **Mark tests per the CI Levels table** — baseline smoke: `core_model` + `advanced_model`; heavier cases: `advanced_model` only; L4 expansion: `full_model`
+- **No per-model helper modules** — do not create `tests/helpers/{slug}.py`; keep constants and `request_config` payloads in the test file
+- **Online and offline e2e go through `runtime.py`** — `openai_client.send_audio_speech_request` (online); `omni_runner_handler.send_audio_speech_request` (Qwen-style offline) or `send_single_stage_tts_request` (single-stage offline). Add a new `send_*_request` in `runtime.py` when none fits; do not embed `omni.generate` or HTTP in tests
 
 ## Phase 4: Async Chunk (Streaming)
 
@@ -455,14 +519,14 @@ Use this checklist when integrating a new TTS model:
 - [ ] README.md written
 
 ### Phase 3: Online Serving
-- [ ] All 5 `serving_speech.py` integration points added in one commit
-- [ ] Only extract params in `_build_*_params` that are forwarded to the model call (ruff F841)
+- [ ] Adapter written under `tts_adapters/` and registered in the import block
+- [ ] Only extract params in `build()` that are forwarded to the model call (ruff F841)
 - [ ] Prompt builder handles text input correctly
 - [ ] Voice cloning works (if supported)
 - [ ] All response formats work (wav, mp3, flac, pcm)
 - [ ] Client scripts and server launcher created
-- [ ] E2E online serving test written (`tests/e2e/online_serving/test_<model>.py`)
-- [ ] Buildkite CI entry added to `.buildkite/test-merge.yml`
+- [ ] E2E tests added per model priority tier (see **Test Case Writing (CI Levels)**)
+- [ ] Buildkite entries match level: `test-ready.yml` / `test-merge.yml` or nightly TTS job / `test-nightly.yml`
 - [ ] Gradio demo working
 - [ ] Documentation added (offline + online docs, nav, supported models)
 
@@ -500,5 +564,6 @@ Project docs and adjacent skills:
 - [TTS audio skill](../vllm-omni-audio-tts/SKILL.md) — supported models and usage
 - [Fish Speech integration](../vllm-omni-audio-tts/references/fish-speech.md) — complete example of Phases 1–3
 - [Qwen3-TTS reference](../vllm-omni-audio-tts/references/qwen-tts.md) — complete example of all 5 phases
+- [vllm-omni-test skill](../vllm-omni-test/SKILL.md) — L1–L4 markers, naming, Buildkite wiring, run commands
 - [Adding a TTS model (developer guide)](https://github.com/vllm-project/vllm-omni/blob/main/docs/contributing/model/adding_tts_model.md)
 - `plan/voxcpm2_native_ar_design.md` — VoxCPM2's vLLM-native AR + side-computation pattern (distinct from the generator-based single-stage described above)

.claude/skills/precheck-pr/SKILL.md

@@ -0,0 +1,100 @@
+---
+name: precheck-pr
+description: Self-check your branch before creating a PR — catch dead code, verify accuracy/perf claims, validate PR title format, and confirm merge readiness. Use when the user says "precheck", "self review", "pre-submit check", or "check my PR before I open it." Never posts to GitHub.
+---
+
+# PR Pre-Check
+
+Self-review your branch before creating a PR against `vllm-project/vllm-omni`. Two modes: **quick** catches showstoppers, **full** does a thorough maintainer-grade review. Never posts to GitHub; the report is for the contributor's terminal only.
+
+## Mode Selection
+
+| Mode | When | Time |
+|------|------|------|
+| **Quick** | About to push, final sanity check | ~3 min |
+| **Full** | Ready for review, want maintainer-level scan | ~10 min |
+
+Default to quick if unsure. Run full before marking a PR "ready for review."
+
+## Workflow
+
+### Step 1: Detect Base Branch
+
+```bash
+BASE_SHA=$(git merge-base HEAD origin/main 2>/dev/null \
+         || git merge-base HEAD main 2>/dev/null \
+         || echo origin/main)
+echo "diffing against ${BASE_SHA}"
+git diff --name-only ${BASE_SHA}...HEAD
+```
+
+### Step 2: Validate PR Title
+
+Check the most recent commit message (or branch name if no commit yet) against the project convention. Valid prefixes:
+
+| Prefix | Applies to |
+|--------|-----------|
+| `[Bugfix]` | Bug fixes |
+| `[CI/Build]` | Build or CI improvements |
+| `[Doc]` | Documentation changes |
+| `[Model]` | New/improved models (include model name) |
+| `[Frontend]` | Frontend changes (API server, OmniLLM class, etc.) |
+| `[Kernel]` | CUDA/kernel changes |
+| `[Core]` | Core logic changes (OmniProcessor, OmniARScheduler, etc.) |
+| `[Hardware][Vendor]` | Hardware-specific (e.g., `[Hardware][Ascend]`) |
+| `[Misc]` | Other changes (use sparingly) |
+
+✗ if: missing prefix, wrong case (`[bugfix]`), or WIP/Draft in title.
+⚠ if: `[Model]` prefix without the model identifier (e.g., `[Model] Add new model` — should be `[Model] Add <ModelName> ...`).
+
+### Step 3: Categorize the PR
+
+| Diff contains | PR type |
+|---------------|---------|
+| New files under `vllm_omni/model_executor/models/<name>/` | **New Model** |
+| Changes to `vllm_omni/diffusion/` | **Diffusion Model** |
+| `[Bugfix]` prefix or single-file fix | **Bug Fix** |
+| Perf/benchmark/throughput claims in commit msg or diff | **Performance** |
+| Everything else | **General** |
+
+If multiple rows apply (e.g., a diffusion model is also a new model), union the checklists.
+
+### Step 4: Run Checklist
+
+Ask: "Quick mode or full mode?" Then walk the checklist for the detected PR type from [references/checklists.md](references/checklists.md). Each item produces ✓, ✗, or ⚠.
+
+**Also run the Code-Quality sweep on every PR**, regardless of type or mode: the five diff-scoped checks in [references/code-quality.md](references/code-quality.md) — kwargs fragility, broad-except swallow, `Any`/wrong type hints, hot-path `.clone()`/`deepcopy`, and event-loop blocking — plus the advisory conventions (log level, structured logging, synchronization, cleanup, dependencies, naming) in [checklists.md](references/checklists.md). These count only lines the PR *adds* — the pre-existing backlog across the repo is out of scope.
+
+### Step 5: Print Report
+
+```
+Pre-check report for <branch>
+
+  Mode: quick | full
+  Type: <new-model | diffusion-model | bug-fix | perf | general>
+
+  Dimension          Result
+  ─────────────────  ──────
+  PR title format    ✓
+  Code quality       ⚠ 1 broad except, 2 Any hints
+  PR desc integrity  ✓
+  Registry/config    ✓
+  Dead code          ⚠ 2 warnings
+  Accuracy           ✓
+  Benchmark          ✗ missing software versions
+
+  Verdict: 1 blocking | 2 warnings | recommend fixing ✗ before PR
+```
+
+**Severity:**
+
+| Mark | Meaning |
+|------|---------|
+| ✗ | Blocking — fix before opening PR |
+| ⚠ | Warning — consider fixing |
+| ✓ | Pass |
+| — | Skipped (not applicable) |
+
+### Stop Here
+
+Do not post comments, open PRs, or modify files. The report is for the contributor's terminal only.

.claude/skills/precheck-pr/references/checklists.md

@@ -0,0 +1,160 @@
+# Pre-Check Checklists
+
+Each item is a concrete check to run against the diff. ✗ = fix before PR. ⚠ = consider fixing.
+
+---
+
+## All PRs: Title Format (check first)
+
+PR title must follow the project convention documented in `docs/contributing/README.md`:
+
+| Prefix | Applies to |
+|--------|-----------|
+| `[Bugfix]` | Bug fixes |
+| `[CI/Build]` | Build or CI improvements |
+| `[Doc]` | Documentation changes |
+| `[Model]` | New/improved models (include model name) |
+| `[Frontend]` | Frontend changes (API server, OmniLLM class, etc.) |
+| `[Kernel]` | CUDA/kernel changes |
+| `[Core]` | Core logic changes (OmniProcessor, OmniARScheduler, etc.) |
+| `[Hardware][Vendor]` | Hardware-specific (e.g., `[Hardware][Ascend]`) |
+| `[Misc]` | Other changes (use sparingly) |
+
+**Examples:**
+```
+✓ [Model] Add MiniCPM-o 4.5 support
+✓ [Bugfix] Fix Ovis image text encoder dtype
+✓ [Core] Qwen3-Omni performance optimization
+✓ [Hardware][Ascend] Fix VoxCPM2 on Ascend
+✗ Fix Ovis image text encoder dtype              ← missing prefix
+✗ [bugfix] Fix Ovis image text encoder           ← wrong case
+✗ [WIP] Add model                                ← WIP in title
+```
+
+- [ ] **Title has valid prefix** — ✗ if missing prefix, wrong case, or WIP/Draft in title. For `[Model]` PRs, the title MUST also include the model identifier (e.g., `[Model] Add <ModelName> ...`). ⚠ if missing the model name.
+
+---
+
+## All PRs: Code Quality (run on every PR, quick & full)
+
+Diff-scoped sweep for five fragility patterns that are pervasive in the existing codebase — **only lines the PR adds count**, not the pre-existing backlog. Detection commands, codebase examples, and full severity rules live in [code-quality.md](code-quality.md).
+
+- [ ] **No new `**kwargs` string-lookup plumbing:** the diff adds no `**kwargs` + `kwargs["..."]` / `kwargs.get("...")` / `"..." in kwargs`. ⚠ any new instance; ✗ if a new string key is duplicated across ≥2 files without a shared constant, or unknown keys are silently dropped on a fail-fast path.
+- [ ] **No new broad exception swallow:** the diff adds no `except Exception:` / bare `except:`. ⚠ any new instance; ✗ for `except: pass` / `except Exception: return None` / `continue` on a fail-fast path (init, config validation, weight loading, request handling).
+- [ ] **No new `Any` / wrong type hints:** the diff adds no `: Any` / `-> Any` / untyped production signature, and no new `SimpleNamespace` in tests faking an object that already has a real typed stub. ⚠ any new instance; ✗ for a *wrong* (actively misleading) annotation or a `SimpleNamespace` test fake of a typed object.
+- [ ] **No new hot-path copy:** the diff adds no `.clone()` / `.copy_()` / `copy.deepcopy(...)` inside a per-step loop or per-request path. ⚠ any new instance; ✗ for `.clone()` of a full latent/activations tensor inside an AR or diffusion step loop without a comment, or `deepcopy` of a scheduler / sampling-config on the request path.
+- [ ] **No new event-loop blocking:** the diff adds no `time.sleep` / blocking HTTP (`requests`, `urllib`) / `await` inside a held lock in `async def` code. ⚠ any new instance; ✗ for blocking sleep or HTTP on the serving path (`engine/`, `entrypoints/`, async `connectors/`), or a lock held across an `await`. Blocking inside a dedicated worker thread is fine.
+
+Roll the ⚠/✗ counts into the report as a single **Code quality** dimension row.
+
+### Conventions (eyeball — no grep)
+
+Judgment calls that don't grep cleanly — apply them while reading the diff, not as a mechanical check:
+
+- **Right log level** — `info` is for end-user-visible events; routine internal events are `debug`; hot paths are `trace` or removed. Logging is not free.
+- **Structured fields over f-strings** — pass dynamic values as structured fields (`logger.info("msg", extra={...})`), not interpolated strings, so they stay greppable and parseable.
+- **No reflexive synchronization** — don't add `threading.Lock` / `asyncio.Lock` / `deepcopy` / re-wrap shared state unless there is real concurrent mutation; the GIL covers most Python code. Owners decide their own synchronization — don't pre-lock in a constructor.
+- **Context managers for cleanup** — use `with` / `contextlib` for files, locks, CUDA streams, and profilers; not raw `open()` + manual `.close()` or ad-hoc `finally`.
+- **No new dependencies without justification** — prefer the stdlib and existing vllm-omni deps; a new top-level dep needs a reason (check `pyproject.toml` / `requirements*.txt`).
+- **Naming / interface discipline** — don't rename or change interfaces just for taste in this PR, but do fix names that actively mislead (e.g. `serve` implying a long-running server, `Instance` where multiple instance kinds coexist).
+
+---
+
+## Bug Fix PRs
+
+### Quick
+
+- [ ] **PR body has 4 sections:** what broke (error message), repro steps, root cause, fix description
+- [ ] **Fix is minimal:** diff touches only files directly related to the bug. No unrelated refactors or style changes.
+- [ ] **Repro is runnable:** the PR body includes a copy-paste command that reproduces the bug
+- [ ] **Branch is rebased:** `git merge-base HEAD origin/main` is recent, no merge conflicts
+
+### Full (adds)
+
+- [ ] **Regression test exists:** `git diff --name-only` includes at least one `tests/` file
+- [ ] **Regression test reproduces the original error:** the test would fail on main, pass on this branch
+- [ ] **Fix matches root cause exactly:** no "fix the symptom + something else" — if the root cause is dtype, the fix is dtype, not dtype + reformatting
+- [ ] **No silent failure risk:** no bare `except: pass`, no `try: ... except: return None`, no empty fallback added. See the broad-except pattern in [code-quality.md](code-quality.md#2-broad-exception-swallow) for the full severity rule (⚠ any new broad catch; ✗ for a swallow on a fail-fast path).
+- [ ] **Upstream pattern match:** the fix follows the same pattern as existing code for similar cases (grep for analogous `from_pretrained` calls, etc.)
+- [ ] **Environment documented:** torch/transformers/vllm versions listed if the bug is version-dependent
+
+---
+
+## Performance PRs
+
+### Quick
+
+- [ ] **Before/after numbers in PR body:** at minimum, one metric (RTF, RPS, latency, VRAM) with main vs PR values
+- [ ] **Hardware specified:** GPU model, count, VRAM. "L4" or "H20" is sufficient. "tested on GPU" is not.
+- [ ] **Concurrency scaling:** if the optimization claims better throughput, numbers at 2+ concurrency levels
+- [ ] **No unexplained regressions:** if any metric goes down at any concurrency level, explain why
+- [ ] **Branch is rebased:** `git merge-base HEAD origin/main` is recent, no merge conflicts
+
+### Full (adds)
+
+- [ ] **Software versions:** torch, CUDA, vllm, vllm-omni versions in PR body
+- [ ] **Warmup stated:** "1 warmup + 3 measured" or equivalent
+- [ ] **Stddev or range:** not just a single best run. "average of 2 rounds" is minimum.
+- [ ] **VRAM measured:** peak GPU memory before and after
+- [ ] **Benchmark script checked in:** under `tests/` or `examples/`, runnable by reviewer
+- [ ] **Exact command line in PR:** not pseudocode, not "run the benchmark"
+- [ ] **Pytest summary included:** `36 passed, 3 skipped, 17 warnings in 2897.66s` — proves the whole suite passes
+- [ ] **Config changes documented:** if `max_num_seqs`, `gpu_memory_utilization`, or deploy YAML changed, old and new values stated with reasoning
+
+---
+
+## New Model PRs
+
+### Quick
+
+- [ ] **PR body file list matches `git diff --name-only`:** every file path in the description exists in the diff
+- [ ] **Registry entries match claimed count:** count `_OMNI_MODELS` or `_DIFFUSION_MODELS` entries added vs claimed in body
+- [ ] **Pipeline config `custom_process_input_func` paths resolve:** each referenced module.function exists
+- [ ] **`__init__.py` exports match registry:** each registered class appears in `__init__.py` and `__all__`
+- [ ] **Shell script modes match README:** `run_curl.sh` supports every mode the README documents
+- [ ] **Branch is rebased:** `git merge-base HEAD origin/main` is recent, no merge conflicts
+
+### Full (adds)
+
+- [ ] **Dead code scan (5 patterns):**
+  1. Dead `forward()` — defined but never called through the pipeline path
+  2. Dead factory/builder functions — instantiated nowhere
+  3. Dead wrappers — thin wrappers with no callers
+  4. Dead branch guards — `if version > X` where X is always satisfied
+  5. Unused parameters — `__init__` args never accessed
+- [ ] **Copy-paste detection:** no string constant defined in 3+ files, no cross-module validation duplication, no near-identical shape coercion fns
+- [ ] **Import hygiene:** no re-export-only imports, no module-level side effects without docs, no `import os` in function body when already at top. Function-body imports are acceptable when the import is only reachable from a closure or factory return value (i.e., `import` would otherwise execute at module load even though the user may never trigger that path). Either lift to top-of-file or annotate with `# noqa: PLC0415`.
+- [ ] **Accuracy:** at least one test compares output against reference; audio = valid WAV at correct sample rate; determinism verified if seed claimed. ⚠ if missing and not explicitly deferred to a follow-up PR or RFC section; — if explicitly deferred (the deferral target SHOULD be linked in the PR body).
+- [ ] **Performance:** hardware, software versions, warmup, and metrics (RTF, VRAM, RPS, latency) stated. ⚠ if missing and not explicitly deferred; — if explicitly deferred.
+- [ ] **Benchmark settings:** model config (TP, PP, max_model_len, enforce_eager, quant), runtime config (batch, input/output spec), environment versions
+- [ ] **Benchmark script checked in + exact command line + pytest summary**
+
+---
+
+## Diffusion Model PRs
+
+Diffusion models live under `vllm_omni/diffusion/`. In addition to the New Model checklist above, verify:
+
+- [ ] **Transformer adapter follows contract:** inherits from `nn.Module`, implements `load_weights()`, uses `vllm_omni.diffusion.attention.layer.Attention`
+- [ ] **Pipeline has both offline and online paths:** `forward()` for batch generation, streaming path for online serving
+- [ ] **Cache-DiT integration:** at least one acceleration (TeaCache, FBCache, etc.) is wired up. ✗ for new **top-level** pipelines. For **subclass** pipelines that inherit `__call__` from a parent where Cache-DiT is already absent, downgrade to ⚠ and recommend filing a follow-up issue against the parent pipeline family.
+- [ ] **Parallelism config:** TP/SP/USP/CFG-Parallel options exposed in pipeline config
+- [ ] **Docs table updated:** model added to `docs/models/supported_models.md` with correct metadata
+- [ ] **E2E test exists:** at least one test exercises the full generate path
+
+---
+
+## General PRs
+
+### Quick
+
+- [ ] **PR body matches diff:** description doesn't claim changes not in the diff
+- [ ] **Branch is rebased:** `git merge-base HEAD origin/main` is recent, no merge conflicts
+- [ ] **CI gates passing:** DCO, pre-commit, build — check `gh pr view --json statusCheckRollup` if a PR already exists
+
+### Full (adds)
+
+- [ ] **No new dead code in changed files:** scan for unreachable paths, unused imports
+- [ ] **Test coverage:** if core code changed (`engine/`, `stages/`, `connectors/`), tests are added or existing tests cover the path
+- [ ] **Import hygiene:** no redundant imports added
+- [ ] **No unrelated changes:** diff scope matches PR description

.claude/skills/precheck-pr/references/code-quality.md

@@ -0,0 +1,179 @@
+# Code-Quality Patterns
+
+Five fragility patterns to sweep on **every** PR, in both quick and full mode. These are pervasive in the existing codebase (hundreds of pre-existing sites) — so the checks are **diff-scoped**: only lines the PR *adds* count. Whole-repo greps light up every PR with the backlog and are useless.
+
+All commands assume:
+```bash
+BASE="$(git merge-base HEAD origin/main)"
+```
+
+**Severity:** ⚠ for any *new* instance introduced by the diff; ✗ for the acute sub-cases defined per pattern. Acute ✗ findings belong in the report's blocking count.
+
+---
+
+## 1. `**kwargs` string-lookup passthrough
+
+### Anti-pattern
+```python
+def forward(self, **kwargs):
+    # BLOCKER-prone: raw dict + string keys duplicated across files
+    if "runtime_additional_information" in kwargs and \
+       "model_intermediate_buffer" not in kwargs:
+        ...
+```
+Real instances of this exact guard are copy-pasted across `cosyvoice3.py`, `indextts2_talker.py` (×2), `indextts2_s2mel_decoder.py`, `qwen3_tts_talker.py`, `qwen3_omni.py`. A typo in any string key silently no-ops; there is no type check, no IDE completion, and unknown keys are silently dropped.
+
+### Why dangerous
+Raw `dict` + `**kwargs` + lists of param-name strings is **fragile by construction**: the compiler can't see a missing or misspelled key, so failures are silent. Each duplicated copy is a new place for the strings to drift.
+
+### Severity
+- ⚠ any *new* `**kwargs` signature whose body does `kwargs["..."]` / `kwargs.get("...")` / `"..." in kwargs`.
+- ✗ when the diff introduces a **new** string key that is duplicated across ≥2 files without a shared module constant, or when `**kwargs` plumbing **silently drops unknown keys** on a fail-fast path (init, config validation, weight loading).
+
+### Detect (diff-scoped)
+```bash
+git diff ${BASE}...HEAD -- "*.py" | grep "^+[^+]" \
+  | grep -E '\*\*kwargs|kwargs(\.get|\[)|"[a-z_]+"\s+in\s+kwargs'
+```
+
+### Fix
+Prefer explicit typed params. The repo already has the right primitive: `msgspec.Struct` with `forbid_unknown_fields=True` rejects unknown keys instead of silently dropping them — see `_StructBase` in `vllm_omni/data_entry_keys.py:105` and `MoriPullRequest` in `vllm_omni/distributed/omni_connectors/connectors/mori_transfer_engine_connector.py:81`. A `@dataclass` or `TypedDict` works too.
+
+If `**kwargs` is unavoidable (vLLM base-class compat), centralize the key strings as module-level constants behind **one** typed accessor, instead of repeating the literal across files.
+
+---
+
+## 2. Broad exception swallow
+
+### Anti-pattern
+```python
+# BLOCKER-prone: catches everything, returns None on any failure
+try:
+    evt = build_event(...)
+    return evt
+except Exception:
+    return None
+```
+Real clusters: `vllm_omni/metrics/stats.py:240,274,367,450` and `vllm_omni/metrics/{modality,utils}.py` — all `except Exception: return None`.
+
+### Why dangerous
+A generic `except Exception:` is **too flexible**. If we catch an exception type we don't expect, it is almost always an unhandled edge case that needs to be *fixed* — not swallowed into `None`/`pass`/`continue`. Broad catches turn fixable bugs into silent wrong behavior and make debugging impossible.
+
+The repo backs this with ruff `BLE001` (broad-exception catch); it is currently suppressed via `# noqa: BLE001` in `vllm_omni/patch.py:413`. Prefer fixing over suppressing.
+
+### Severity
+- ⚠ any *new* `except Exception:` / `except BaseException:` / bare `except:`.
+- ✗ for `except:` / `except Exception: pass|return None|continue` on a **fail-fast path** (init, config validation, weight loading, request handling, connector setup). This extends the Bug-Fix "No silent failure risk" ✗ in [checklists.md](checklists.md).
+
+### Detect (diff-scoped)
+```bash
+# Any new broad catch (⚠ baseline):
+git diff ${BASE}...HEAD -- "*.py" | grep "^+[^+]" \
+  | grep -E 'except\s*(Exception|BaseException)?\s*:'
+# Note: this matches only the immediate-colon swallow forms (`except :`,
+# `except Exception:`) — it deliberately excludes specific catches like
+# `except ValueError:` and the `except Exception as e:` variant. Eyeball
+# the latter; `as e` usually means it is logged, but still verify.
+```
+
+### Fix
+Catch the **specific** types you actually expect (`ValueError`, `KeyError`, `AttributeError`, `OSError`, `torch.cuda.OutOfMemoryError`, `msgspec.ValidationError`, …). At a genuine top-level / best-effort boundary (metrics, signal handlers), at minimum **log** the exception; never swallow into a bare `pass`/`return None` on a path that should fail loudly.
+
+---
+
+## 3. `Any` / wrong type annotations
+
+### Anti-pattern
+```python
+# ⚠ Any param + Any return — no contract, no narrowing
+def _extract_mm_output(engine_outputs: Any) -> dict[str, Any]:
+    ...
+```
+
+### Why dangerous (root cause)
+`Any` disables the type checker for that position, so typos and wrong shapes pass silently. The repo has 636 `: Any` params, 244 `-> Any` returns, and **1,173 `SimpleNamespace` in `tests/`**. The `Any` leak is largely test-driven: vibe-coded unit tests fake objects with `SimpleNamespace`, which forces the production code consuming them to be typed `Any`, and the `Any` then propagates outward into public signatures.
+
+### Severity
+- ⚠ any *new* `: Any` / `-> Any` / fully-untyped signature in production code.
+- ✗ for a **wrong** annotation (actively misleading — e.g. `-> bool` that returns `Optional[bool]`), or a *new* `SimpleNamespace` in a test that mimics an object which already has a real typed stub / `@dataclass` / `TypedDict` / `Protocol`.
+
+### Detect (diff-scoped)
+```bash
+# Any leaking into new signatures:
+git diff ${BASE}...HEAD -- "*.py" | grep "^+[^+]" \
+  | grep -E ':\s*Any\b|->\s*Any\b'
+# SimpleNamespace newly added in tests (the Any-leak source):
+git diff ${BASE}...HEAD -- "tests/" | grep "^+[^+]" | grep "SimpleNamespace"
+```
+
+### Fix
+Replace `Any` with the concrete type, a `Protocol`, or a `Union`. If a type is genuinely dynamic, prefer `object` + `isinstance` narrowing over `Any` — `object` keeps the checker engaged. In tests, replace `SimpleNamespace` with the real class, a small `@dataclass`/`TypedDict`, or a `Protocol` + fake, so the system under test stays fully typed.
+
+---
+
+## 4. Unnecessary copy — `.clone()` / `copy.deepcopy` in hot paths
+
+### Anti-pattern
+```python
+# Per-step in a diffusion/AR loop: clones the full latent every iteration
+for t in timesteps:
+    trajectory_latents.append(x_t.clone())   # O(N) GPU alloc + memcpy each step
+```
+Real clusters: `vllm_omni/diffusion/models/bagel/bagel_transformer.py` calls `x_t.clone()` ~9× across a trajectory loop (lines 1820–2186); the cache hooks `vllm_omni/diffusion/cache/teacache/hook.py:155` and `magcache/hook.py:182` clone `hidden_states` per step. Separately, `copy.deepcopy(self.scheduler)` / `copy.deepcopy(req.sampling_params)` is copy-pasted across diffusion pipelines — `pipeline_qwen_image.py:786`, `pipeline_hunyuan_image3.py:1904`, `pipeline_ltx2*.py` (×3), `pipeline_dreamzero.py:1085`, `diffusion_model_runner.py:398` — a full object-graph deepcopy on the request path. (Contrast `dreamzero/causal_wan_model.py:203`, which carries a `# No .clone() needed.` comment — the aware pattern.)
+
+### Why dangerous
+A `.clone()` of a GPU tensor or a `copy.deepcopy` of a config object inside a per-step / per-request path is a hidden allocator: it reads as one Python line but triggers a GPU memcpy or a deep object-graph walk every iteration. Across 50 diffusion steps or hundreds of AR steps it dominates latency and VRAM bandwidth. `deepcopy` additionally ignores any custom device / `__copy__` logic and can drag tensors through CPU.
+
+### Severity
+- ⚠ any *new* `.clone()` / `.copy_()` / `copy.deepcopy(...)` inside a per-step loop or per-request path.
+- ✗ for `.clone()` of a full latent/activations tensor inside an AR or diffusion step loop without an explanatory comment, or `copy.deepcopy` of a scheduler / sampling-config on the request hot path.
+
+### Detect (diff-scoped)
+```bash
+git diff ${BASE}...HEAD -- "*.py" | grep "^+[^+]" \
+  | grep -E '\.clone\(\)|\.copy_\(|copy\.deepcopy|deepcopy\('
+```
+
+### Fix
+Tensors: prefer a view (`x[..., :k]`, `torch.narrow`, `as_strided`) or write into a pre-allocated buffer with `dst.copy_(src)` instead of `.clone()`; if the caller no longer needs the original, `move`/`del` it rather than copy. Schedulers / configs: build a fresh lightweight instance per request, or split out the mutable per-request bits so the shared immutable part isn't deepcopied. If a copy is genuinely required (e.g. the source is mutated in place later), keep it and add a one-line comment saying why.
+
+---
+
+## 5. Blocking the asyncio event loop / lock held across `await`
+
+### Anti-pattern
+```python
+# BLOCKER-prone: blocking sleep / blocking HTTP / lock held across an await,
+# all inside an async (event-loop) function
+async def handle(self, req):
+    with self._lock:
+        resp = await self._client.call(req)   # lock held across await -> serializes all requests
+    time.sleep(0.1)                           # stalls the single loop for every concurrent request
+```
+
+### Why dangerous
+vLLM-Omni serves over a single asyncio event loop. Any synchronous blocking work on an `async def` path — `time.sleep`, blocking HTTP (`requests`/`urllib`), heavy CPU work, or a lock held across an `await` — stalls the loop for *every* concurrent request, not just the blocking one. A lock held across `await` silently serializes the whole pipeline; blocking I/O tanks throughput and inflates tail latency.
+
+### Severity
+- ⚠ any *new* `time.sleep(`, blocking HTTP call (`requests.get/post`, `urllib.request`, `urlopen`), or `await` inside a `with Lock` / `async with` critical section, in `async def` code.
+- ✗ for `time.sleep` or blocking HTTP on the serving path (`engine/`, `entrypoints/`, async `connectors/`), or a lock acquired and held across an `await` on the request path.
+
+### Detect (diff-scoped)
+```bash
+# New blocking primitives introduced by the diff — then eyeball whether each
+# sits in an async function on the loop vs a dedicated worker thread:
+git diff ${BASE}...HEAD -- "*.py" | grep "^+[^+]" \
+  | grep -E 'time\.sleep|requests\.(get|post|put|delete)|urllib\.request|urlopen'
+# Lock acquired with an await potentially inside the critical section (manual eyeball):
+git diff ${BASE}...HEAD -- "*.py" | grep "^+[^+]" | grep -E 'async with|\.acquire\(\)'
+```
+Note: blocking inside a **dedicated worker thread is correct** — the repo already does this (`diffusion/model_loader/hub_prefetch.py`, `distributed/omni_connectors/kv_transfer_manager.py`, the `mooncake`/`mori` connectors poll with `time.sleep` on their own threads). The check targets the *event-loop* path only.
+
+### Fix
+Move blocking work off the loop with `asyncio.to_thread(...)` / `run_in_executor`, or use a native async client (`httpx.AsyncClient` / `aiohttp`). Never `await` while holding a lock — drop the lock before the `await` and re-acquire after, or restructure so the critical section contains no `await`.
+
+---
+
+## Running the sweep
+
+Run all five detections against `${BASE}...HEAD`, then for each hit decide ⚠ vs ✗ using the severity rules above. Roll the results into the report as a single **Code quality** dimension row (count of ⚠ and ✗), alongside the type-specific checklist from [checklists.md](checklists.md).

.claude/skills/quantization/SKILL.md

@@ -0,0 +1,103 @@
+---
+name: quantization
+description: Work on vLLM-Omni quantization for diffusion, autoregressive, omni, or multi-stage models. Use when choosing or adding methods such as fp8, int8, gguf, mxfp8, mxfp4, mxfp4_dualscale, ModelOpt, AutoRound, INC, msModelSlim, awq, or gptq; debugging quantized loading; or validating memory, speed, and output quality.
+---
+
+# vLLM-Omni Quantization
+
+Use this skill for quantization work in `vllm-omni`. Start from the local
+code and docs, then pick the smallest validated path for the model, method,
+hardware, and quality target.
+
+## First Checks
+
+Before changing code or recommending a command, identify:
+
+- Model and task: AR text, omni/TTS, text-to-image, text-to-video, or multi-stage diffusion.
+- Hardware backend: CUDA generation, Ascend NPU, Intel XPU, or another platform.
+- Quantization mode: online from BF16/FP16 weights, pre-quantized checkpoint, runtime KV/FA quantization, or per-component routing.
+- Scope: transformer/DiT, thinker language model, text encoder, VAE, audio/vision encoder, talker, or a specific stage.
+- Validation target: memory reduction, latency/throughput, output quality, or all three.
+
+Do not assume a method is supported just because the CLI accepts the string.
+Check `docs/user_guide/quantization/` and the closest implementation first.
+
+## Quick Decision
+
+| Task | Start With |
+|------|------------|
+| Choose a method or command | `references/methods.md` and `references/modality-compat.md` |
+| Use `build_quant_config()` or per-component routing | `references/methods.md` |
+| Work on diffusion quantization | `references/diffusion.md` |
+| Add quantization to a new model | `references/adding-models.md` |
+| Convert or load ModelOpt FP8 checkpoints | `references/modelopt-fp8.md` |
+| Debug ModelOpt, GGUF, AutoRound, MXFP, or serialized Int8 loading | `references/diffusion.md` and method docs under `docs/user_guide/quantization/` |
+
+## Current Runtime Shape
+
+The unified entrypoint is:
+
+```python
+from vllm_omni.quantization import build_quant_config
+```
+
+It supports method strings, flat method dictionaries, per-component
+dictionaries, existing `QuantizationConfig` objects, and `None`. The factory
+delegates generic methods to upstream `vllm` and keeps vLLM-Omni overrides for
+diffusion or omni-specific routing:
+
+- `gguf`
+- `int8`
+- `mxfp8`
+- `mxfp4`
+- `mxfp4_dualscale`
+- `inc`, `auto-round`, `auto_round`
+- ModelOpt auto-detected configs: `modelopt`, `modelopt_fp4`, `modelopt_mixed`
+
+Use `ComponentQuantizationConfig` when only one stage or component should be
+quantized. Pre-quantized ModelOpt-style checkpoints should not spill into
+vision/audio encoders that have no corresponding scale tensors.
+
+## Ownership Boundary
+
+- Upstream `vllm` owns generic quantization configs, kernels, loader semantics,
+  hardware capability rules, and generic AR quantization methods.
+- `vllm-omni` owns unified config routing, diffusion-specific wrappers,
+  component scoping, GGUF or ModelOpt checkpoint adapters, model-specific
+  prefix mapping, docs, examples, and validation.
+
+If a new method needs missing generic kernels or loader behavior, fix upstream
+`vllm` first. In `vllm-omni`, add thin integration and model wiring.
+
+## Working Loop
+
+1. Read the local method doc under `docs/user_guide/quantization/`.
+2. Inspect the active factory and config class in `vllm_omni/quantization/`.
+3. Inspect the target model pipeline and transformer for stable prefixes and
+   whether every relevant vLLM linear layer receives `quant_config`.
+4. For pre-quantized checkpoints, inspect `transformer/config.json` and the
+   checkpoint tensor names before running full generation.
+5. Validate with the same prompt, seed, size, scheduler, steps, dtype, eager or
+   non-eager mode, and parallelism as the BF16 baseline.
+6. Report quality, latency, and memory separately. Do not call a method
+   supported until quality is checked.
+
+## Common Mistakes
+
+| Symptom | Likely Cause | Fix |
+|---------|--------------|-----|
+| `--quantization` has no visible effect | Wrong scope or unsupported model path | Check component routing and method docs |
+| Some layers stay BF16 unexpectedly | `quant_config` was not threaded into all vLLM linear layers | Audit transformer constructors and prefixes |
+| Quality collapses but loading succeeds | Too many sensitive layers were quantized | Add model-specific `ignored_layers` and compare to BF16 |
+| ModelOpt checkpoint loads but output is corrupted | Prefixes, packed-module mapping, scale routing, or BF16 fallback is wrong | Use `references/modelopt-fp8.md` |
+| GGUF shape or tensor mismatch | Missing architecture-specific adapter | Add explicit adapter mapping; avoid generic fallback |
+| Online and offline MXFP4 disagree | Wrong `mxfp4` vs `mxfp4_dualscale` mode | Check `docs/user_guide/quantization/mxfp4.md` |
+| Quantized path is slower than BF16 | Kernel path, compile mode, dtype, or shape mismatch | Compare eager-to-eager and non-eager-to-non-eager |
+
+## References
+
+- General methods and config routing: [references/methods.md](references/methods.md)
+- Modality and model scope: [references/modality-compat.md](references/modality-compat.md)
+- Diffusion implementation patterns: [references/diffusion.md](references/diffusion.md)
+- Adding model support: [references/adding-models.md](references/adding-models.md)
+- ModelOpt FP8 conversion and loading: [references/modelopt-fp8.md](references/modelopt-fp8.md)

.claude/skills/quantization/references/adding-models.md

@@ -0,0 +1,70 @@
+# Adding Quantization Support to a Model
+
+Use this checklist when extending quantization to a new architecture or stage.
+
+## Classify Ownership
+
+| Case | Owner |
+|------|-------|
+| New generic AR method, kernel, or loader semantics | upstream `vllm` first |
+| Existing upstream method needs diffusion wiring | `vllm-omni` model and config integration |
+| GGUF or ModelOpt tensor mapping for a diffusion architecture | `vllm-omni` adapter |
+| NPU MXFP or msModelSlim diffusion path | `vllm-omni` platform/method wrapper and docs |
+| Multi-stage omni checkpoint quantizes only the thinker LM | `vllm-omni` component scoping |
+
+Avoid building a private quantization stack in model code.
+
+## Model Wiring Checklist
+
+1. Find the closest supported model implementation.
+2. Confirm the model uses vLLM linear layers that accept `quant_config`.
+3. Add `quant_config` to the transformer constructor.
+4. Pass stable `prefix` values through every linear layer.
+5. Handle fused modules and packed mappings explicitly.
+6. Ensure `load_weights()` and any `WeightsMapper` remap `ignored_layers` or
+   `modules_to_not_convert`.
+7. Keep tokenizer, scheduler, VAE, text encoder, audio/vision encoders, and
+   talker BF16 unless method-specific docs validate them.
+8. Add or update a method doc under `docs/user_guide/quantization/`.
+
+## Checkpoint Adapter Checklist
+
+For pre-quantized checkpoints:
+
+- Inspect `transformer/config.json` first.
+- Check `quant_method`, `method`, `producer.name`, `quant_algo`, serialized
+  flags, `ignored_layers`, and any method-specific fields.
+- Verify tensor name mapping before generation.
+- Verify packed modules such as `to_qkv`, `add_kv_proj`, and `w13`.
+- Confirm scale tensor names and shapes.
+- Allow dequantization back to BF16 for intentionally skipped target layers.
+
+## Sensitive Layer Discovery
+
+Start conservative. Quantize the backbone first and keep output-side or routing
+layers BF16 until quality is proven.
+
+Frequently sensitive areas:
+
+- attention output projections such as `to_out`
+- FFN down projections such as `w2`
+- final projections such as `proj_out`, `final_layer`, and output heads
+- modulation, timestep, position, image, or text embedders
+- refiner blocks
+- MoE routers, experts selected by routing, and layernorm-adjacent modules
+- VAE, text/vision towers, audio encoders, and codec/talker stages
+
+Do not reuse one model family's ignore list blindly.
+
+## Tests and Evidence
+
+Minimum evidence for a PR:
+
+- loader/config test or smoke command
+- fixed-seed BF16 vs quantized output comparison
+- quality metrics and saved outputs for precision-changing work
+- memory and latency comparison under identical execution mode
+- failure behavior for unsupported models or missing checkpoint fields
+
+When the PR adds a method to a support table, include the exact model,
+hardware, command, quality threshold, and artifact path used for validation.

.claude/skills/quantization/references/diffusion.md

@@ -0,0 +1,113 @@
+# Diffusion Quantization Reference
+
+Use this file for DiT quantization, config routing, checkpoint adapters, and
+quality or loader debugging.
+
+## Supported Method Families
+
+The current local diffusion quantization families are:
+
+- CUDA/GPU online or checkpoint: `fp8`, `int8`, `gguf`, ModelOpt
+- Ascend NPU: `int8`, `mxfp8`, `mxfp4`, `mxfp4_dualscale`, msModelSlim paths
+- Intel XPU: AutoRound/INC and online MXFP8 paths where documented
+
+Always cross-check the method doc before claiming model support. The CLI
+accepting a method string is not enough.
+
+## Implementation Starting Points
+
+| Area | Files |
+|------|-------|
+| Unified factory | `vllm_omni/quantization/factory.py` |
+| Per-component routing | `vllm_omni/quantization/component_config.py` |
+| GGUF diffusion method | `vllm_omni/quantization/gguf_config.py` |
+| Int8 diffusion method | `vllm_omni/quantization/int8_config.py` |
+| MXFP8 diffusion method | `vllm_omni/quantization/mxfp8_config.py` |
+| MXFP4 and dual-scale method | `vllm_omni/quantization/mxfp4_config.py` |
+| AutoRound/INC omni extension | `vllm_omni/quantization/inc_config.py` |
+| ModelOpt checkpoint adapters | `vllm_omni/diffusion/model_loader/checkpoint_adapters/modelopt.py` |
+| FP8 scaled-mm runtime patches | `vllm_omni/patch.py` |
+| Similarity tool | `vllm_omni/quantization/tools/compare_diffusion_trajectory_similarity.py` |
+
+## Threading `quant_config`
+
+For a diffusion transformer:
+
+1. Accept `quant_config: QuantizationConfig | None` in the transformer.
+2. Pass `quant_config` and stable `prefix` values to every vLLM linear layer.
+3. Preserve packed-module mappings for fused QKV, FFN, or model-specific fused
+   projections.
+4. Call `apply_vllm_mapper()` on configs that need HF-to-vLLM name remapping.
+5. For component configs, keep prefixes specific enough to avoid quantizing VAE
+   or encoders by accident.
+
+Common skipped areas are `proj_out`, output projections, modulation layers,
+embedders, refiner blocks, MoE routers, and layernorm-adjacent paths.
+
+## Online vs Offline
+
+Online quantization:
+
+- starts from BF16/FP16 checkpoint weights
+- computes or stores quantized weights at load time
+- typically uses `--quantization <method>` or `quantization="<method>"`
+
+Offline or pre-quantized checkpoints:
+
+- include quantized weights and scales on disk
+- usually carry `quantization_config` in `transformer/config.json`
+- should auto-detect when the config is complete
+- may require model-specific adapter logic for tensor names and packed modules
+
+Do not mix online flags with incompatible offline checkpoint configs. If the
+active config and disk config differ, `resolve_quant_config_from_disk()` should
+raise or rebuild rather than silently loading corrupted weights.
+
+## GGUF Rules
+
+- Require `quantization_config.gguf_model`.
+- Use the normal base model for tokenizer, scheduler, text encoder, and VAE.
+- Add explicit architecture adapters for tensor-name mapping.
+- Use `named_parameters()` and `named_buffers()`, not `state_dict()`, to
+  discover loadable names.
+- Guard fused QKV/KV rewrites so tensors are not rewritten twice.
+- Flatten N-D activations around the GGUF matmul and restore shape.
+
+Unsupported architectures should fail clearly instead of falling back to a fake
+generic mapper.
+
+## MXFP Rules
+
+- `mxfp8` is W8A8 microscaling, currently centered on NPU and documented XPU
+  behavior.
+- `mxfp4` is online single-scale W4A4 on NPU.
+- `mxfp4_dualscale` is online/offline dual-scale W4A4 with BF16 fallback; the
+  offline path is the production path when calibrated `mul_scale` exists.
+- Offline dual-scale checkpoints must inject per-transformer `ignored_layers`
+  into config because cascade transformers can have different BF16 fallback
+  sets.
+
+Do not load an offline dual-scale checkpoint with the online single-scale path.
+
+## Common Failures
+
+| Symptom | Likely Cause | Fix |
+|---------|--------------|-----|
+| `Unknown quantization method` | Factory override or upstream registration is missing | Add the right override or upstream method |
+| Missing scale tensors | Wrong scope, wrong adapter, or BF16 layer not allowed to dequantize | Check config scope and adapter scale routing |
+| Output rank collapses from 3-D to 2-D | FP8 scaled-mm kernel ignored output shape | Check the patch in `vllm_omni/patch.py` |
+| Quality degrades silently | Sensitive layers quantized | Add `ignored_layers`; compare to BF16 |
+| Only one cascade transformer works | Disk config differs per transformer but active config is reused | Rebuild from each transformer's config |
+| Quantized path slower than BF16 | Kernel, dtype, compile, or benchmark mode mismatch | Compare eager-to-eager and non-eager-to-non-eager |
+
+## Verification
+
+For a new or changed diffusion quantization path, run:
+
+1. Config and loader smoke test.
+2. One fixed-seed generation against BF16.
+3. Numeric comparison with the similarity tool or equivalent metrics.
+4. Serving test if the path is user-facing online.
+5. Memory and latency check under the same execution mode as the baseline.
+
+Keep output artifacts when precision changes are part of the PR.

.claude/skills/quantization/references/methods.md

@@ -0,0 +1,82 @@
+# General Quantization Methods
+
+Use this file for method selection and unified `vllm-omni` config syntax. For
+diffusion-specific wiring, see `diffusion.md`. For ModelOpt checkpoint export
+and loading details, see `modelopt-fp8.md`.
+
+## Unified Entry Point
+
+`vllm_omni.quantization.build_quant_config()` accepts:
+
+- `None` or `"none"` to disable quantization
+- a method string such as `"fp8"`, `"int8"`, `"mxfp8"`, `"awq"`, or `"auto-round"`
+- a flat dict with `"method"` or `"quant_method"`
+- a per-component dict such as `{"transformer": {"method": "fp8"}, "vae": None}`
+- an already-built `QuantizationConfig`
+
+Per-component dicts are routed by `ComponentQuantizationConfig` using
+longest-prefix matching. Prefix stability matters: if a model remaps HF names
+to vLLM names, apply the mapper to `ignored_layers` or component prefixes.
+
+## Local Overrides
+
+The local factory delegates most methods to upstream `vllm`, but has vLLM-Omni
+overrides for methods that need diffusion, NPU, XPU, or component behavior:
+
+| Method | Local Config | Main Use |
+|--------|--------------|----------|
+| `gguf` | `DiffusionGGUFConfig` | Pre-quantized diffusion transformer weights |
+| `int8` | `DiffusionInt8Config` | Online or serialized W8A8 diffusion transformers |
+| `mxfp8` | `DiffusionMXFP8Config` | W8A8 MXFP8 diffusion on NPU and online XPU path |
+| `mxfp4` | `DiffusionMXFP4Config` | Online W4A4 MXFP4 diffusion on NPU |
+| `mxfp4_dualscale` | `DiffusionMXFP4DualScaleMixedConfig` | Online/offline W4A4 dual-scale plus BF16 fallback on NPU |
+| `inc`, `auto-round`, `auto_round` | `OmniINCConfig` | AutoRound/INC checkpoint loading, including MXFP8 checkpoint metadata |
+| ModelOpt config dicts | upstream ModelOpt configs through auto-detect | FP8, NVFP4, or mixed ModelOpt checkpoints |
+
+Check `SUPPORTED_QUANTIZATION_METHODS` in `vllm_omni/quantization/factory.py`
+before adding a new method name.
+
+## AR and Generic Quantization
+
+For AR-backed models, upstream `vllm` methods usually apply:
+
+- `awq` and `gptq`: 4-bit or 8-bit weight quantization with calibration,
+  commonly used for text and AR stages.
+- `fp8`: FP8 weight path or KV-cache FP8 depending on flags and hardware.
+- `inc` or `auto-round`: pre-quantized checkpoints with Intel Neural
+  Compressor / AutoRound metadata.
+
+Use these for the AR language model stage only when the checkpoint and model
+guide support it. Omni and TTS models often keep audio, vision, talker, and
+codec stages in BF16.
+
+## Diffusion and Offline Methods
+
+For diffusion transformers, prefer the method-specific docs:
+
+- `docs/user_guide/quantization/fp8.md`
+- `docs/user_guide/quantization/int8.md`
+- `docs/user_guide/quantization/gguf.md`
+- `docs/user_guide/quantization/modelopt.md`
+- `docs/user_guide/quantization/mxfp8.md`
+- `docs/user_guide/quantization/mxfp4.md`
+- `docs/user_guide/quantization/autoround.md`
+- `docs/user_guide/quantization/msmodelslim.md`
+
+Pre-quantized checkpoints should normally auto-detect from config. Runtime
+flags such as `--force-cutlass-fp8`, `--linear-backend cutlass`, and
+`--moe-backend cutlass` select kernels; they do not quantize a BF16 checkpoint.
+
+## Validation
+
+Use fixed-seed A/B comparisons and record:
+
+- model path and quantized checkpoint path
+- method config and `ignored_layers`
+- prompt, size, frame count, steps, scheduler, dtype, and seed
+- eager/non-eager mode, compile flags, and parallelism
+- output metrics such as PSNR, MAE, cosine similarity, SSIM, LPIPS, plus visual artifacts
+- latency and peak memory, reported separately from quality
+
+The helper `vllm_omni.quantization.tools.compare_diffusion_trajectory_similarity`
+is the default lightweight image/video comparison tool for diffusion paths.

.claude/skills/quantization/references/modality-compat.md

@@ -0,0 +1,47 @@
+# Quantization Compatibility by Modality
+
+Use this as a routing guide. The source of truth is the local method docs under
+`docs/user_guide/quantization/` and the current model implementation.
+
+## Summary
+
+- AR and omni language-model quantization mostly follows upstream `vllm`, with
+  vLLM-Omni scoping for multi-stage models.
+- Diffusion quantization is method-specific and model-specific. It usually
+  targets the diffusion transformer only.
+- Pre-quantized checkpoints should quantize only the component that has
+  quantized weights and scale tensors.
+
+## Model Type Matrix
+
+| Model Type | Typical Quantization Scope | Notes |
+|------------|----------------------------|-------|
+| AR text models | Language model | Upstream `vllm` methods usually apply |
+| Omni/TTS models | Thinker or AR language-model stage | Audio, vision, talker, and codec stages usually stay BF16 |
+| DiT image models | Diffusion transformer | Check FP8, Int8, GGUF, ModelOpt, AutoRound docs |
+| DiT video models | Diffusion transformer | HunyuanVideo FP8 and Wan2.2 MXFP paths are documented; validate other video paths |
+| Multi-stage diffusion | Selected stage only | Use per-component routing and per-stage validation |
+| Audio diffusion | Usually unsupported | Prefer dtype, offload, cache, or parallelism unless docs say otherwise |
+
+## Method Hints
+
+| Goal | Candidate |
+|------|-----------|
+| AR memory reduction | AWQ, GPTQ, AutoRound, ModelOpt where checkpoint supports it |
+| CUDA diffusion W8A8 | FP8 or Int8 |
+| Pre-quantized CUDA diffusion checkpoint | ModelOpt or GGUF |
+| Ascend NPU diffusion | Int8, MXFP8, MXFP4, MXFP4 DualScale, msModelSlim |
+| Intel XPU checkpoint | AutoRound/INC paths |
+| Conservative image quality | selective FP8/Int8 with `ignored_layers` |
+| Unsupported diffusion model | cache acceleration, CPU/layerwise offload, dtype, TP/SP/CFG parallelism |
+
+## Validation Rule
+
+Compatibility means more than successful loading. A method is compatible only
+after it has:
+
+- correct component scope
+- complete checkpoint/config metadata if offline
+- fixed-seed quality comparison against BF16
+- acceptable latency and memory behavior
+- docs or examples that state model, method, hardware, and caveats

.claude/skills/quantization/references/modelopt-fp8.md

@@ -0,0 +1,122 @@
+# ModelOpt FP8 Conversion and Loading
+
+Use this when exporting or loading ModelOpt FP8 checkpoints for diffusion
+transformers. This reference captures the practical conversion lessons from the
+2026-04-21 ModelOpt FP8 paste and aligns them with the current
+`vllm-omni` ModelOpt adapter.
+
+## Core Rule
+
+Treat ModelOpt FP8 as selective checkpoint quantization, not one-click
+whole-model FP8. Quantize the main transformer or DiT backbone, keep sensitive
+or peripheral modules BF16, serialize a complete ModelOpt checkpoint, and let
+vLLM-Omni auto-detect it from config.
+
+Do not pass `--quantization fp8` to force a ModelOpt checkpoint path. The
+checkpoint's `quantization_config` should select `modelopt`.
+
+## Conversion Strategy
+
+Prefer:
+
+- backbone attention Q/K/V and FFN expansion projections
+- conservative per-model ignore lists
+- MHA quantizers disabled until basic linear quantization is stable
+- full module-name prefixes in ignore/routing logic
+- BF16 fallback for output-side and routing-sensitive layers
+
+Avoid quantizing by default:
+
+- VAE
+- text, vision, or audio encoders
+- embedders and modulation layers
+- final heads and output projections
+- refiner modules
+- MoE routers and layernorm-sensitive blocks
+
+## Model-Specific Starting Points
+
+| Model Family | Quantize First | Keep BF16 First |
+|--------------|----------------|-----------------|
+| Z-Image | attention q/k/v, FFN w1/w3 | `to_out`, `w2`, `noise_refiner`, `context_refiner`, embedders, modulation, final/output |
+| Qwen-Image | transformer backbone linear layers | `proj_out`, `img_in`, `txt_in`, norms, modulation, position/time/embed layers |
+| FLUX2-dev / FLUX2-klein | backbone linear layers | `proj_out`, context/x embedders, `norm_out`, stream modulation, time embeddings |
+| HunyuanImage-3.0 | `model.model` backbone | VAE, vision tower, token/time/image embedders, final image head, LM head, routers, layernorm-sensitive blocks |
+
+These are starting points, not support claims. Validate with BF16 outputs.
+
+## Required Checkpoint Metadata
+
+`transformer/config.json` must include a complete `quantization_config`.
+At minimum it should identify ModelOpt and the algorithm:
+
+```json
+{
+  "quantization_config": {
+    "producer": {"name": "modelopt"},
+    "quant_method": "modelopt",
+    "quant_algo": "FP8"
+  }
+}
+```
+
+Current vLLM-Omni auto-detects:
+
+- `FP8` and `FP8_PER_CHANNEL_PER_TOKEN` -> `modelopt`
+- `NVFP4` -> `modelopt_fp4`
+- `MIXED_PRECISION` -> `modelopt_mixed`
+
+Packed modules and scales must map correctly:
+
+- `to_qkv` from `to_q`, `to_k`, `to_v`
+- `add_kv_proj` from `add_q_proj`, `add_k_proj`, `add_v_proj`
+- `w13` from `w1`, `w3`
+- `.input_scale`, `.weight_scale`, `.weight_scale_2`, `.weight_scale_inv`
+
+For selective FP8, target layers that remain BF16 must be allowed to
+dequantize from FP8 weights using the matching `weight_scale`, or the adapter
+must skip the quantized scale tensors for that BF16 target.
+
+## Runtime Signals
+
+Healthy loading should show ModelOpt auto-detection and a ModelOpt linear
+kernel path. Useful signals include:
+
+- auto-detected quantization from config
+- selected ModelOpt FP8 linear method
+- CUTLASS or FlashInfer FP8 scaled-mm backend when enabled and supported
+- non-eager path compiling the transformer when expected
+
+If online serving fails while offline inference works, inspect warmup and
+serving initialization before changing checkpoint conversion.
+
+## Validation Order
+
+1. Check config completeness.
+2. Run an export checker such as `examples/quantization/check_modelopt_fp8_export.py`.
+3. Confirm actual FP8 tensors and expected disk-size reduction.
+4. Run one offline generation with fixed prompt, seed, size, and steps.
+5. Run serving with `vllm serve --omni` and verify `/v1/images/generations`.
+6. Compare against BF16 using visual review plus metrics such as LPIPS, PSNR,
+   MAE, cosine similarity, and SSIM.
+7. Compare performance only under matched modes:
+   - BF16 eager vs FP8 eager
+   - BF16 non-eager vs FP8 non-eager
+
+FP8 is not automatically faster. Check whether execution reaches FP8 kernels,
+whether compile mode is comparable, and whether the workload shape fits the
+kernel path.
+
+## Triage
+
+| Symptom | First Check |
+|---------|-------------|
+| Startup failure | `config.json`, `model_index.json`, `quantization_config`, missing scales |
+| Loads but output is corrupted | Over-quantized sensitive layers or wrong full-name prefixes |
+| Offline works but online fails | Dummy warmup, serving init, compile path |
+| FP8 is not faster | Kernel selection, dtype, compile mode, workload shape |
+| BF16 fallback fails | Missing `weight_scale` for dequantization or wrong packed-module mapping |
+
+The success criterion is stable, correctly scoped checkpoint loading with
+quality preserved enough for the target use case, not maximizing the number of
+FP8 layers.

.claude/skills/readme.md

@@ -21,11 +21,24 @@ include:
 - `diffusion-perf-opt`: guides diffusion model performance optimization,
   including profiling traces, parallel strategies, stage timing analysis, and
   benchmark-driven tuning
+- `quantization`: guides quantization method selection, model integration,
+  checkpoint loading, and quality/performance validation for vLLM-Omni
 - `add-omni-model`: covers addition of new omni-modality model support
 - `add-tts-model`: covers integration of new TTS models and related serving
   workflows
 - `generate-release-note`: helps prepare release notes for repository changes
+- `precheck-pr`: self-check a branch before creating a PR — validates PR title
+  format, catches dead code, verifies accuracy/perf claims, and confirms merge
+  readiness
+- `vllm-omni-test`: guides generation and execution of CI-aligned tests (L1–L4),
+  pytest marker selection (`core_model` / `advanced_model` / `full_model`,
+  `omni` / `tts` / `diffusion`), Buildkite wiring (`test-ready.yml`,
+  `test-merge.yml`, `test-nightly.yml`, `test-weekly.yml`), and copy-paste
+  local plus CI-like `pytest` commands; see `references/test-routing.md` for
+  level-to-command mapping
 - `review-pr`: provides a structured workflow for reviewing pull requests
+- `vllm-omni-npu-model-runner-upgrade`: upgrades NPU model runners to align with the
+  latest vllm-ascend NPUModelRunner
 
 ## Maintenance Guidelines
 

.claude/skills/review-pr/SKILL.md

@@ -0,0 +1,231 @@
+---
+name: review-pr
+description: Review pull requests and local branches for vllm-project/vllm-omni with a frozen snapshot, module-design ownership, feature-design overlays, targeted validation, and concise evidence-backed findings. Use for default, detailed, or repeat maintainer reviews; checking correctness, compatibility, tests, benchmarks, model additions, distributed changes, or breaking behavior; and identifying or explicitly requesting the most relevant code-owner reviewers. Use precheck-pr instead for an author's pre-submit self-check.
+---
+
+# Review vLLM-Omni Pull Requests
+
+Review like a maintainer: direct, selective, and focused on issues that CI does
+not prove. Prefer a few high-confidence findings over exhaustive commentary.
+Zero findings is a valid result.
+
+## Quality contract
+
+Make every finding:
+
+- **Correct:** prove a reachable failure, not a suspicion.
+- **Prioritized:** lead with merge blockers and high-impact defects.
+- **Actionable:** identify the smallest safe fix direction.
+- **Evidence-based:** cite code, tests, docs, CI, or measurements.
+- **Concise:** avoid review templates and repeated summaries.
+- **Calibrated:** match severity to user and maintainer impact.
+
+Do not report unrelated backlog, style already enforced by pre-commit, or a
+missing test that would not protect changed behavior.
+
+## Select the input and depth
+
+Use `vllm-project/vllm-omni` as the base repository. Accept its forks and local
+checkouts; use another skill for unrelated repositories.
+
+| Input | Review surface |
+| --- | --- |
+| PR number or URL | Frozen PR metadata, full diff, and relevant threads. |
+| Local branch/worktree | Frozen target-base SHA through committed, staged, unstaged, and in-scope untracked changes. |
+| Pre-filled context | Reuse supplied metadata; fetch only missing facts and the full diff. |
+
+Default to maintainer brevity. A detailed or audit request expands coverage and
+lists `path:line` findings, but keeps the same confidence and severity bar.
+
+## Reference guide
+
+Load references in review order: process, one primary module contract, matching
+feature designs, evidence checks, then delivery. Every file is linked directly
+below; do not load unrelated module references.
+
+Each concise reference links to the maintained
+[vLLM-Omni documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/).
+For branch-specific behavior, inspect the matching `docs/` file in the reviewed
+checkout first; use the published latest docs for current guidance and discovery.
+If docs and live code disagree, verify the code/tests and report the drift.
+
+### Review process
+
+| Reference | Read when |
+| --- | --- |
+| [review-execution.md](references/process/review-execution.md) | Every review; freeze inputs, inspect safely, and deliver against the same snapshot. |
+| [general-checks.md](references/process/general-checks.md) | Every review; apply repository-wide correctness and evidence rules. |
+| [design-contracts.md](references/process/design-contracts.md) | Every production review; resolve branch-local module and feature design status. |
+| [review-routing.md](references/process/review-routing.md) | After the diff census; select one primary module and conditional overlays. |
+
+### Primary module contract
+
+| Reference | Read when |
+| --- | --- |
+| [entrypoints.md](references/modules/entrypoints.md) | Offline/CLI/API ingress, validation, rendering, streaming, or sessions change. |
+| [configuration.md](references/modules/configuration.md) | Config construction, deploy/stage schema, defaults, registry, or topology changes. |
+| [input-output-modality.md](references/modules/input-output-modality.md) | Requests, messages, serialization, output types, accumulation, or completion change. |
+| [error-contracts.md](references/modules/error-contracts.md) | Error classification, fatality, propagation, sanitization, or rendering changes. |
+| [engine-orchestration.md](references/modules/engine-orchestration.md) | Cross-stage routing, request state, output ordering, RPC correlation, or terminal convergence changes. |
+| [stage-runtime.md](references/modules/stage-runtime.md) | Placement, startup, readiness, replica identity, affinity, membership, or shutdown changes. |
+| [omni-connector.md](references/modules/omni-connector.md) | Cross-stage/process/device/node transport or synchronization changes. |
+| [model-integration.md](references/modules/model-integration.md) | Registration, preprocessing, loading, runners, or model-specific execution changes. |
+| [ar-runtime.md](references/modules/ar-runtime.md) | AR scheduling, request/cache state, adapters, workers, or upstream vLLM semantics change. |
+| [diffusion.md](references/modules/diffusion.md) | Diffusion runtime, models, batching, parallelism, or offload changes. |
+| [execution-platforms.md](references/modules/execution-platforms.md) | Hardware selection, capabilities, vendor workers, kernels, or patches change. |
+| [cache-management.md](references/modules/cache-management.md) | Cache identity, reuse, validity, reset, eviction, or teardown changes. |
+| [quantization.md](references/modules/quantization.md) | Quantization selection, checkpoint metadata, layer mapping, precision, or constraints change. |
+| [observability.md](references/modules/observability.md) | Metrics, logs, units, labels, correlation, or lifecycle changes. |
+| [profiling.md](references/modules/profiling.md) | Profiling instrumentation, traces, start/stop lifecycle, or overhead changes. |
+| [benchmarking.md](references/modules/benchmarking.md) | Benchmark workload, metric calculation, CLI, or result metadata changes. |
+
+### Feature-design overlays
+
+| Reference | Read when |
+| --- | --- |
+| [runtime-stage-execution.md](references/features/runtime-stage-execution.md) | Disaggregated inference, async chunk/output/materialization, or prefix caching changes. |
+| [communication.md](references/features/communication.md) | A concrete OmniConnector backend or its deployment contract changes. |
+| [diffusion-acceleration.md](references/features/diffusion-acceleration.md) | Diffusion parallelism, attention, quantization, cache, batching, or offload changes. |
+| [infrastructure-performance.md](references/features/infrastructure-performance.md) | Metrics infrastructure or documented speech optimization stacks change. |
+
+### Evidence and quality checks
+
+| Reference | Read when |
+| --- | --- |
+| [model-addition-checklist.md](references/checks/model-addition-checklist.md) | A model, architecture, loader, processor, registry, or stage config is added. |
+| [perf-verification.md](references/checks/perf-verification.md) | The PR makes a latency, throughput, memory, or quality claim. |
+| [test-quality-evaluation.md](references/checks/test-quality-evaluation.md) | Tests change, are absent for risky code, or may not exercise production behavior. |
+| [tests-docs-checklist.md](references/checks/tests-docs-checklist.md) | Coverage, CI markers, examples, user docs, or PR evidence need review. |
+| [verification.md](references/checks/verification.md) | Hardware, a server, or a runnable affected path is available for active verification. |
+
+### Delivery and reviewer coordination
+
+| Reference | Read when |
+| --- | --- |
+| [maintainer-style-study.md](references/delivery/maintainer-style-study.md) | Findings are ready for concise maintainer-style delivery. |
+| [review-requests.md](references/delivery/review-requests.md) | The user asks to identify, suggest, request, or ping code-owner reviewers. |
+
+## Workflow
+
+### 1. Freeze and report the snapshot
+
+Pin the base and head before reading source or running validation. Within 60 seconds,
+report the pinned head, CI, mergeability, and preliminary findings in the host
+conversation. Do not wait for CI or post this update to GitHub.
+
+If the target changes while fetching, discard the evidence and retry once. If
+it changes again, report the churn and wait for a stable target.
+
+For a trusted PR head, materialize the pinned head in an isolated detached
+worktree. A worktree freezes identity but is not a security sandbox. Treat fork
+heads as untrusted unless the user and environment policy explicitly establish
+otherwise: execute them only in a disposable, secret-free sandbox with restricted
+filesystem, network, and resources; without one, use static SHA-addressed reads
+and CI evidence only. For a local review, freeze the committed, index, worktree,
+and NUL-safe in-scope untracked contents. Follow
+[review-execution.md](references/process/review-execution.md) for trust gates,
+state fingerprints, and byte-for-byte staleness checks.
+
+### 2. Build the diff census
+
+Group files into production code, tests, docs, configuration, build/CI, and
+generated artifacts. Map each changed production file and test group to the PR
+goal. Compare the title/body claims with the actual diff; use linked issues only
+when they define the contract or reproduction.
+
+Mark unrelated scope and unexplained generated artifacts. Do not infer behavior
+from the PR description without tracing the live code.
+
+### 3. Route from the live behavior
+
+Trace each claimed behavior through the changed producer to its live consumer,
+then use [design-contracts.md](references/process/design-contracts.md) and
+[review-routing.md](references/process/review-routing.md) to select one primary
+module contract, a second only for a real documented cross-boundary call path,
+and every matching feature-design and evidence overlay. Treat titles and paths
+as hints; live behavior and the frozen head's current design metadata are
+authoritative. For docs-, tests-, or CI-only changes, route to the production
+contract they protect or use only the applicable evidence checks.
+
+### 4. Run the blocker scan
+
+Apply every category in [general-checks.md](references/process/general-checks.md) before
+lower-priority comments.
+
+For each changed value or behavior, trace:
+
+```text
+public ingress -> validation/defaulting -> producer -> transformations
+  -> stage/worker/connector boundary -> final consumer -> terminal cleanup
+```
+
+Cover every applicable offline/online, streaming/non-streaming, sync/async,
+feature-on/off, topology, and compatibility path. Search bounded callers and
+sibling implementations rather than assuming the changed hunk is the only path.
+
+### 5. Apply module and feature contracts
+
+Apply the reference set selected in step 3 and any matching repo-local skill.
+Read the exact module and feature pages in the frozen head, including status,
+ownership boundary, dependencies, candidate invariants, safe-change guide, and
+promotion gate. Candidate or draft rules are questions, not blockers, unless
+current code, tests, or policy enforce them. Inspect both sides of any config,
+registry, serialization, connector, cache, or stage boundary.
+
+When a diff adds or expands a helper, class, fallback, compatibility branch, or
+public behavior, run a subtraction pass: remove out-of-scope behavior and check
+whether each new abstraction can be deleted, merged, moved, or inlined.
+
+### 6. Verify the changed path
+
+Before each validation group, verify the frozen SHA plus the tracked, index,
+untracked, and ignored-file fingerprint, or recreate a pristine snapshot. On a
+trusted head or inside the required sandbox, run an import/version preflight,
+then the narrowest relevant tests and low-cost static checks. Bind every result
+to the head SHA, snapshot fingerprint, and environment fingerprint. Never run
+imports, tests, builds, hooks, or repo-configurable tooling from an untrusted
+head on the reviewer host.
+
+- Treat CI as status evidence; inspect only the first overlapping failure.
+- For docs-only changes, use diff hygiene, links/build checks, and bounded live
+  contract verification instead of dependency setup or pytest.
+- For hardware-dependent paths, run available static/CPU checks and name the
+  exact GPU/NPU gap. Never simulate device evidence.
+- For performance or accuracy claims, require comparable base/head runs with
+  the same environment, workload, warmup, repetitions, and quality criteria.
+
+Stop when each changed semantic path has a supported finding or an explicit
+no-issue conclusion. Do not search further only to increase confidence.
+
+### 7. Consolidate and deliver
+
+Verify each finding against the current diff, deduplicate by root cause, and
+order by severity.
+
+Re-read the remote head and reverify or recreate the pristine validation
+snapshot immediately before delivery. If either changed, mark the review stale
+and restart from the new snapshot.
+
+Return findings first. Use
+[maintainer-style-study.md](references/delivery/maintainer-style-study.md) to keep them
+direct and brief. Each finding must include an exact `path:line`, trigger or
+call path, current behavior, impact, and smallest fix direction. If there are
+no findings, say so briefly and name material validation gaps.
+
+Keep the review read-only unless the user explicitly authorizes posting. Do not
+submit `APPROVE`, `COMMENT`, or `REQUEST_CHANGES`, add labels, edit code, or push
+commits as an implied part of review.
+
+### 8. Optionally request focused owner reviews
+
+Only when the user asks to identify or request reviewers, read
+[review-requests.md](references/delivery/review-requests.md). Rank path-matched
+CODEOWNERS with the frozen module page's owners or required reviewers and
+documented governance expertise; propose one to three focused reviewers with an
+explicit contract rationale.
+
+Identifying or suggesting reviewers is read-only. Requesting reviewers or
+posting `@mention` comments changes external state and requires explicit user
+authorization. When authorized, recheck the head, deduplicate existing
+requests, and post at most one consolidated comment. Do not infer this
+permission from a request to review the code.

.claude/skills/review-pr/agents/openai.yaml

@@ -0,0 +1,4 @@
+interface:
+  display_name: "vLLM-Omni PR Review"
+  short_description: "Review vLLM-Omni PRs and route owner requests"
+  default_prompt: "Use $review-pr to review this vLLM-Omni pull request with owner-specific rules and evidence-backed findings."

.claude/skills/review-pr/references/checks/model-addition-checklist.md

@@ -0,0 +1,121 @@
+# Model Addition Checklist
+
+Read this reference when a PR adds or expands a model architecture, pipeline,
+loader, processor, registry entry, or stage configuration.
+
+Official docs: [model contribution guides](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/model/),
+[adding an omni model](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/model/adding_omni_model/),
+[adding a diffusion model](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/model/adding_diffusion_model/),
+[adding a TTS model](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/model/adding_tts_model/),
+[supported models](https://docs.vllm.ai/projects/vllm-omni/en/latest/models/supported_models/),
+[diffusion features](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion_features/),
+and [feature compatibility](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/feature_compatibility/).
+
+## Match claims to the diff
+
+- Reconcile every claimed architecture, modality, variant, feature, and example
+  with changed files and registry entries.
+- Resolve config model types, architecture names, callable paths, exports, and
+  dependency extras against live symbols.
+- Check sibling generations sharing an architecture name are disambiguated by
+  a stable config predicate rather than import order or an accidental default.
+
+## Close the integration chain
+
+Trace:
+
+```text
+public model id -> config/pipeline selection -> registry -> loader/processor
+  -> stage inputs -> model execution -> stage/public output
+```
+
+Verify optional dependencies fail with an actionable message, weight names and
+dtypes map correctly, and each advertised serving mode reaches the production
+dispatcher. Confirm outputs are non-empty and valid for their modality, shape,
+sample rate, or response schema.
+
+## Keep support documentation truthful
+
+- Build an evidence-backed inventory of the new model's exact architecture and
+  checkpoint identifiers, modalities and tasks, hardware backends, serving
+  modes, and supported acceleration, parallelism, cache, offload, and
+  quantization features. Treat untested support as unknown, not supported.
+- Update `docs/models/supported_models.md` with the architecture, model family,
+  example checkpoints, and only the backends validated by the PR.
+- For a diffusion model, add it to the applicable ImageGen, VideoGen, or
+  AudioGen table in `docs/user_guide/diffusion_features.md`. Fill every feature
+  column according to its legend and document partial support or constraints.
+- Update `docs/user_guide/feature_compatibility.md` when the model changes valid
+  feature combinations, configuration constraints, examples, or limitations,
+  and update any other branch-local support table affected by the claims.
+- Cross-check every documented support mark against the registry, configuration,
+  production-path tests, and PR evidence; do not copy a sibling model's support
+  claims without validation.
+
+## Ship the model recipe
+
+- Require every new model PR to add or update one model-family recipe under
+  `recipes/<vendor>/` and its row in the `recipes/README.md` Available Recipes
+  table. Follow [`recipes/TEMPLATE.md`](../../../../../recipes/TEMPLATE.md) and
+  the layout rules in
+  [`recipes/README.md`](../../../../../recipes/README.md); the
+  external recipes repository is a structural reference, not a substitute.
+- Include only tested task, serving-mode, and hardware configurations. Keep the
+  exact model identifiers, flags, commands, verification, feature support,
+  constraints, and limitations consistent with code, tests, examples, and the
+  support documentation.
+- Cover each validated platform in its own hardware section. Use one recipe per
+  model family by default; justify a split when configurations cannot be kept
+  clear in one file, and do not add placeholder sections for untested platforms.
+
+## Compare the reference implementation
+
+Require every new model PR to include a head-versus-reference accuracy or
+quality and performance comparison:
+
+- Name the canonical implementation and pin its repository revision, package
+  version, configuration, and checkpoint. If it cannot run, explain why and use
+  the closest runnable reference; do not silently omit the comparison.
+- Match inputs or dataset, seed, output settings, precision or quantization,
+  hardware, software, warmup, repetitions, synchronization, and timing and
+  memory scope. Include exact commands and environment details for both sides.
+- Compare accuracy or output quality with a task-appropriate metric and explicit
+  tolerance. When no reliable metric exists, provide paired fixed-input outputs,
+  artifacts, and evaluation criteria, and explain expected numerical or
+  qualitative differences.
+- Report end-to-end latency, task-appropriate throughput, peak device memory,
+  run-to-run variability, and head/reference ratios at equivalent output quality.
+- Prefer a timing split for preprocessing or encoding, each model stage,
+  inter-stage transfer or communication, decoding or postprocessing, and output
+  assembly. Define the boundaries, account for overlap, reconcile the split with
+  end-to-end time, and explain any unavailable stage measurement.
+
+Use [perf-verification.md](perf-verification.md) for the comparison protocol and
+repository benchmark and profiling links.
+
+## Remove accidental surface
+
+Search bounded call sites for:
+
+- inference-dead training `forward()` paths, unused factories/wrappers, never-set
+  branch keys, and immediately discarded parameters;
+- duplicate payload strings, validation, or shape coercion across stages;
+- compatibility aliases in brand-new code without an existing caller;
+- private symbols re-exported as public API or module-level side effects.
+
+Keep code only when it has a distinct live caller, invariant, or compatibility
+contract. Prefer one typed producer-consumer schema to repeated string keys.
+
+## Require proportionate evidence
+
+- Run a representative production-path inference and assert output content, not
+  only process survival.
+- Require profiling or A/B tables when the PR makes performance, memory,
+  precision, or quality claims, or when a suspected hot-path/device bug needs
+  that evidence. Do not impose fixed utilization or regression thresholds that
+  the repository does not define.
+- Require focused registry/config/loading tests, a representative E2E path, and
+  user-facing model/usage documentation.
+
+Use [diffusion.md](../modules/diffusion.md) for diffusion models and
+[perf-verification.md](perf-verification.md) for quantitative claims.

.claude/skills/review-pr/references/checks/perf-verification.md

@@ -0,0 +1,51 @@
+# Performance and Accuracy Verification
+
+Activate this reference when the PR adds a model or claims or intentionally
+changes latency, throughput, memory, scaling, precision, or output quality.
+
+Official docs: [profiling](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/profiling/),
+[serving benchmarks](https://docs.vllm.ai/projects/vllm-omni/en/latest/cli/bench/serve/),
+and [metrics](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/metrics/).
+
+## Comparable A/B contract
+
+Compare frozen base and head for an existing path. For a new model absent from
+base, compare head with a pinned canonical reference implementation. Use the same:
+
+- hardware, driver, software versions, model/checkpoint, and dependencies;
+- input or dataset, seed, precision, batch/concurrency, topology, and feature
+  flags;
+- warmup, measured repetitions, synchronization, timing scope, and memory
+  collection method.
+
+Keep exact commands and report variability rather than a cherry-picked run.
+Use the repository's benchmark when it covers the claim; otherwise use the
+smallest reproducible workload that reaches the changed production path.
+
+## Evidence table
+
+| Dimension | Typical evidence |
+| --- | --- |
+| Latency | End-to-end latency and, when available, TTFT plus a defined per-stage and transfer-time split. |
+| Throughput/scaling | Requests, tokens, frames, or audio duration per second across the claimed concurrency/topology. |
+| Memory | Peak allocated/reserved device memory and OOM boundary if relevant. |
+| Quality/accuracy | Repository metric or known-good output comparison with an explicit tolerance; paired samples when no metric exists. |
+
+Every optimization needs correctness or quality evidence. A faster result on a
+different workload, precision, seed, or topology is not a valid comparison.
+
+## Reviewer verification ladder
+
+1. Run base/head A/B on suitable hardware when affordable.
+2. If only one side can run, verify the script and request a comparable pair.
+3. If hardware or assets are unavailable, audit methodology, code path, and
+   contributor evidence; name the exact unverified claim.
+
+Classify discrepancies before reporting: implementation regression, benchmark
+bug, environmental drift, noise, or unsupported claim. Explain material
+regressions against the PR's stated goal or repository contract; do not invent
+universal percentage thresholds.
+
+Report claimed and measured values together, bind them to the frozen SHAs and
+environment, and keep unavailable hardware as a validation gap rather than a
+fabricated pass.

.claude/skills/review-pr/references/checks/test-quality-evaluation.md

@@ -0,0 +1,39 @@
+# Test Quality Evaluation
+
+Read this reference when tests change, a risky production path has no test, or
+a test-only PR may not prove the intended behavior.
+
+Official docs: [test system overview](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/test_system_overview/)
+and [test writing guide](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/test_writing_guide/).
+
+## Static proof check
+
+For each changed semantic path, ask whether the test would fail if that behavior
+were reverted or broken. Check:
+
+- assertions pin the contract, not only non-null output, logs, or “no crash”;
+- the production dispatcher, registry, connector, or scheduler is reached
+  instead of the changed behavior being mocked away;
+- fakes preserve relevant types, MRO, shapes, devices, async behavior, and
+  lifecycle transitions;
+- seeds, ordering, timing, synchronization, external services, and numeric
+  tolerances are deterministic or explicitly controlled;
+- normal, invalid, boundary, feature-off, failure/cancellation, and regression
+  paths are covered where the diff changes them;
+- run-level and domain markers place hardware/model tests in the intended CI
+  lane without silently skipping the contract.
+
+Map source symbols to tests with bounded `rg` searches; do not assume the test
+tree mirrors production paths.
+
+## Runtime check
+
+Run the narrowest affected tests supported by the environment. Record skipped
+hardware/model cases as gaps. Classify failures as code, test, infrastructure,
+or flaky before turning them into findings; one passing rerun does not erase a
+flakiness signal.
+
+Keep grades or matrices internal. Report only concrete code bugs or the one or
+two test defects that leave changed behavior materially unprotected. Use
+[tests-docs-checklist.md](tests-docs-checklist.md) for CI and documentation
+coverage.

.claude/skills/review-pr/references/checks/tests-docs-checklist.md

@@ -0,0 +1,48 @@
+# Tests and Documentation Checklist
+
+Use this reference to connect changed behavior to coverage, CI selection,
+examples, documentation, and reproducible PR evidence.
+
+Official docs: [test writing guide](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/test_writing_guide/),
+[test execution guide](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/test_execution_guide/),
+and [documentation guide](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/DOCS_GUIDE/).
+
+## Coverage packet
+
+For each high-risk change, record a compact internal packet:
+
+```text
+change and failure risk -> existing unit/E2E coverage -> uncovered boundary
+  -> smallest test and stable assertion that closes the gap
+```
+
+Prefer a regression test that fails on the frozen base for bug fixes. Check
+numeric tolerances, deterministic inputs, realistic mocks, hardware/domain
+markers, and Buildkite/CI wiring using the documentation present on the reviewed
+branch. A skipped test is not a pass.
+
+## Documentation sync
+
+Require user-facing updates when the diff changes a model, feature, CLI/API,
+config key, default, compatibility behavior, or supported platform. Verify:
+
+- the actual docs navigation and support tables used by the branch;
+- exact identifiers, defaults, constraints, known limitations, and migration;
+- a minimal runnable example for newly exposed behavior;
+- examples and docs agree on commands, parameters, output schema, and supported
+  modes;
+- links and the bounded live contract build or resolve correctly.
+
+Do not request docs for internal behavior with no user-visible contract.
+
+## PR evidence
+
+For contributor-run tests or benchmarks, require enough provenance to evaluate
+the claim: hardware, relevant software versions, exact commands, inputs/model,
+result, and approximate runtime. Performance or accuracy evidence must compare
+base and head under equivalent conditions; use
+[perf-verification.md](perf-verification.md).
+
+Turn a missing test or document into a finding only when it protects or explains
+changed behavior. Name the smallest useful addition rather than requesting a
+broad coverage expansion.

.claude/skills/review-pr/references/checks/verification.md

@@ -0,0 +1,36 @@
+# Active Verification
+
+Read this reference when a runnable affected path and suitable local or remote
+hardware are available. Never claim device evidence from static analysis or a
+simulated backend.
+
+Official docs: [installation](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/)
+and [test execution guide](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/test_execution_guide/).
+
+## Choose the narrowest level
+
+| Available environment | Verification |
+| --- | --- |
+| Matching accelerator, model/assets, and server | Targeted unit/integration test plus one representative E2E request. |
+| Matching accelerator without serving setup | Targeted model/runtime test and output or metric inspection. |
+| CPU/static environment only | Import/version preflight, focused CPU tests, config/registry resolution, and static contract checks. |
+| Missing dependencies or inaccessible hardware | Record the exact gap and evaluate contributor/CI evidence. |
+
+## Execute safely
+
+1. Bind the run to the frozen head SHA and record Python, platform, dependency,
+   model, device, and relevant environment fingerprints.
+2. Run the smallest affected unit test before an E2E workload.
+3. Exercise the changed public or production-dispatch path with bounded inputs.
+4. Compare actual output, metrics, cleanup, and exit behavior with the PR claim.
+5. For base/head claims, use the equivalent A/B protocol in
+   [perf-verification.md](perf-verification.md).
+
+Avoid disturbing shared servers, ports, models, or accelerators without user
+authorization. Do not install large dependencies, download gated assets, or
+start remote workloads merely because hardware might exist.
+
+Report the command, environment, result, and any skipped path. Classify failures
+as change-induced, pre-existing, infrastructure, environment, or flaky before
+making a finding. A hardware gap lowers confidence; it is not automatic proof
+of a defect.

.claude/skills/review-pr/references/delivery/maintainer-style-study.md

@@ -0,0 +1,105 @@
+# Maintainer Review Style
+
+Use this reference only when turning proved findings into comments. The baseline
+maintainer samples favor short, direct comments and reserve long explanations
+for architecture or non-obvious failure paths.
+
+Official docs: [contributing and review process](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/).
+
+## Evidence basis
+
+This calibration used 928 PRs merged from 2026-05-10 through 2026-08-08.
+Candidates were ranked by substantive human inline-review threads, then by
+distinct reviewers and review rounds. Author self-comments, bots and AI
+reviewers, and bodyless approvals were excluded. Counts selected the sample;
+the practices below came from reading root comments and follow-up replies.
+
+The main set contains the six leaders after filtering; three other high-ranked
+PRs add core-runtime and benchmark coverage. All were merged.
+
+| PR | Review boundary |
+| --- | --- |
+| [#2906](https://github.com/vllm-project/vllm-omni/pull/2906) | New TTS model, shared utilities, recipes, and request state. |
+| [#3907](https://github.com/vllm-project/vllm-omni/pull/3907) | Full-duplex API, session lifecycle, concurrency, and realtime evidence. |
+| [#2162](https://github.com/vllm-project/vllm-omni/pull/2162) | Diffusion world model, OpenPI serving, tests, examples, and docs. |
+| [#3454](https://github.com/vllm-project/vllm-omni/pull/3454) | Diffusion model, generic CLI surface, loaders, and configuration scope. |
+| [#4425](https://github.com/vllm-project/vllm-omni/pull/4425) | Structured configuration, typing, defaults, and compatibility. |
+| [#5397](https://github.com/vllm-project/vllm-omni/pull/5397) | Distributed offload, collectives, failure handling, and load semantics. |
+| [#4281](https://github.com/vllm-project/vllm-omni/pull/4281) | Composable parallelism, configuration contracts, and examples. |
+| [#3628](https://github.com/vllm-project/vllm-omni/pull/3628) | Stage metrics, benchmark definitions, opt-in API behavior, and cleanup. |
+| [#3855](https://github.com/vllm-project/vllm-omni/pull/3855) | Stage runtime, thread safety, startup, and compatibility. |
+
+Historical comments calibrate delivery; they do not create merge policy. Live
+code, tests, repository docs, and the current `CODEOWNERS` file remain
+authoritative. When refreshing this study, promote a practice only when it
+recurs across at least three PRs or is supported by current repository guidance.
+
+## Recurring review behavior
+
+- Start from a changed contract, not a style preference. State the exact
+  request, configuration, topology, or failure path that triggers the issue.
+- Protect repository boundaries. Keep model-specific behavior in its domain,
+  move genuinely reusable logic to a shared module, and reuse existing loaders,
+  protocols, definitions, and config sources instead of bypassing them.
+- Challenge unrelated scope and abstractions. Ask why a new flag, config, class,
+  helper, fallback, or file is needed when an existing path appears sufficient.
+- Trace concurrency and terminal paths: overlapping requests, disconnect,
+  cancellation, partial failure, stale replies, collective schedule mismatch,
+  cleanup, and retry. Prefer an explicit startup error for unsupported
+  combinations over silently running with incomplete state or wrong weights.
+- Preserve user-visible behavior across variants. An accepted CLI/API/config
+  field must be applied, explicitly rejected, or visibly warned about; compare
+  normal and headless, offline and online, and streaming and non-streaming paths.
+- Ask for evidence at the affected boundary: a regression test using the real
+  tensor or protocol shape, correct pytest markers so CI runs it, comparable A/B
+  performance results, or an SLO-oriented latency/quality measurement.
+- Keep interfaces explicit: avoid `Any` and opaque strings where a precise
+  union, enum, or typed envelope exists; remove dead code and duplicated
+  constants; validate requests before expensive execution.
+- Document module scope and non-obvious invariants. Update examples, recipes,
+  supported-model/feature docs, and hardware assumptions when users rely on
+  them.
+- When expertise crosses an ownership boundary, ask a named owner to inspect a
+  specific file or contract and explain why. Do not send a generic `PTAL` list.
+- Keep non-blocking cleanup in a linked follow-up only after the current PR is
+  safe. If a revision fixes only part of a failure chain, continue the existing
+  thread with the remaining trigger instead of opening a duplicate finding.
+
+## Write the finding
+
+- Lead with the issue or decisive question; omit a review preamble.
+- Name the concrete trigger and impact, then the smallest fix direction.
+- Keep obvious fixes to one sentence. Use more detail only when the call path or
+  ownership boundary is otherwise unclear.
+- Prefer one root-cause comment over repeated symptom comments.
+- Hedge only when evidence is genuinely incomplete; mark that item as a
+  question or validation gap rather than overstating severity.
+- Say briefly that no findings were found when the review is clean; zero
+  comments is a valid outcome when posting is authorized.
+
+Useful forms:
+
+```text
+This leaves <state> allocated when <failure/cancellation path>. Release it in
+the shared terminal cleanup path.
+```
+
+```text
+Does this producer also update <consumer>? It still reads the old field here.
+```
+
+```text
+With <configuration>, this bypasses <existing loader/validator>, so <impact>.
+Reuse that path or reject this combination before startup.
+```
+
+```text
+@<owner>, could you check <specific boundary>? This changes <contract> used by
+<consumer>.
+```
+
+Use a priority label only for a proved high-impact defect. Avoid generic praise,
+routine “Nit:” prefixes, decorative severity labels, audit templates, rule IDs,
+comment-count narration, and “I left a few comments.” Keep approvals, review
+events, reviewer requests, mentions, and GitHub replies subject to the user's
+explicit authorization.

.claude/skills/review-pr/references/delivery/review-requests.md

@@ -0,0 +1,99 @@
+# Focused Code-Owner Review Requests
+
+Read only when the user asks to identify, suggest, request, add, or ping
+reviewers. Finding candidates is read-only; GitHub review requests and
+`@mention` comments are external writes.
+
+Owner sources, in order:
+
+1. the reviewed head's
+   [CODEOWNERS](https://github.com/vllm-project/vllm-omni/blob/main/.github/CODEOWNERS)
+   for path ownership;
+2. the active module design page's `owners` and `required_reviewers` metadata
+   for contract ownership and mandatory design review; and
+3. documented
+   [governance expertise](https://docs.vllm.ai/projects/vllm-omni/en/latest/community/governance/)
+   for specialization and tie-breaking.
+
+Resolve all three from the reviewed head when available because ownership
+changes over time. Use published docs only to discover a page absent from an
+older checkout, never to override the frozen head, and do not copy a static
+owner list into review output. CODEOWNERS remains the path authority; module
+metadata identifies contract experts and required design reviewers.
+
+## Rank candidates
+
+1. Freeze the PR head and collect changed files, author, requested reviewers,
+   submitted reviews, and existing comments.
+2. Resolve the **last matching CODEOWNERS rule** for each changed file. Do not
+   union an earlier broad rule with a later specific rule.
+3. Resolve the primary module and feature overlays through
+   [design-contracts.md](../process/design-contracts.md). Read the active module
+   page's status, `owners`, and `required_reviewers`; do not treat owners on an
+   archived, superseded, or unrelated page as current authority.
+4. Group files by the selected primary owner and the exact module or feature
+   contract needing review. Weight production owners and live consumers above
+   tests/docs/CI owners and the repository-wide fallback. Include an applicable
+   `required_reviewers` entry even when it is not the path owner, and label why.
+5. Use documented governance specialties only to choose among matching owners:
+   for example serving versus configuration, diffusion model versus cache or
+   parallelism, TTS versus generic model execution, or CI versus tests.
+6. Exclude the PR author, bots, users who already completed the needed review,
+   and duplicate pending requests. Do not infer availability or invent
+   expertise from a username.
+7. Select one reviewer for a single-owner change and at most three for a real
+   cross-module change. If more are plausible, rank them but do not ping a
+   broad group; ask the user before exceeding three.
+
+If every specific owner is excluded, consider one repository-wide fallback
+owner and label that choice as a fallback rather than a path match. Never fan
+out to the full fallback list. Prefer a qualified owner who covers multiple
+critical path groups over separate reviewers for each file. If CODEOWNERS is
+missing or invalid, state that gap and use governance only as a labeled fallback.
+
+Recent file history may break a tie only when CODEOWNERS and governance remain
+ambiguous. Never use commit count alone to override an explicit current owner.
+
+## Preview before writing
+
+Show the user the exact candidate set and rationale:
+
+```text
+@owner — <owned paths or module role>; please verify <specific module or feature contract>.
+```
+
+Draft one consolidated comment tied to the frozen head:
+
+```text
+Requesting focused review for `<short-head-sha>`:
+
+- @owner1: could you check the <module contract> in `<path/group>`?
+- @owner2: could you check the <feature or cross-boundary contract> in `<path/group>`?
+
+Relevant validation: <one concise result or exact remaining gap>.
+```
+
+Mention only owners being requested. Keep the reason specific enough that each
+person knows what decision or invariant needs attention; do not post a generic
+“please review” or a generated ownership audit.
+
+## External-write gate
+
+- “Who should review?”, “suggest reviewers”, or “draft a request” authorizes
+  only the preview.
+- “Request/add these reviewers” authorizes the GitHub review-request mutation.
+- “Ping/comment/tag the owners” authorizes the consolidated comment.
+- When the user explicitly asks to request reviews **with an owner comment**,
+  both writes are authorized. Obey narrower wording when only one action is
+  requested.
+
+Immediately before an authorized write, verify the PR is still open, the head
+SHA is unchanged, and the candidate is not the author or already satisfied.
+Report draft/WIP state and avoid notifications unless the user explicitly wants
+reviewers requested before ready-for-review.
+
+Use the GitHub connector for reviewer requests and the single top-level comment;
+use `gh` only when connector coverage is unavailable. Do not submit an approval,
+request-changes event, label, assignment, or multiple comments as part of this
+action. After writing, reread the PR and report the reviewers requested and the
+comment URL. If the head changed, discard the draft and rerank before posting.

.claude/skills/review-pr/references/features/communication.md

@@ -0,0 +1,24 @@
+# Communication Features
+
+Use with the OmniConnector, stage-runtime, configuration, and I/O module
+contracts. Load the concrete backend design from the reviewed head:
+
+| Backend | Design |
+| --- | --- |
+| Mooncake store | [MooncakeStoreConnector](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/omni_connectors/mooncake_store_connector/) |
+| Mooncake transfer engine | [MooncakeTransferEngineConnector](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/omni_connectors/mooncake_transfer_engine_connector/) |
+| Mori transfer engine | [MoriTransferEngineConnector](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/omni_connectors/mori_transfer_engine_connector/) |
+| Shared memory | [SharedMemoryConnector](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/omni_connectors/shared_memory_connector/) |
+| Yuanrong store | [YuanrongConnector](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/omni_connectors/yuanrong_connector/) |
+| Yuanrong transfer engine | [YuanrongTransferEngineConnector](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/omni_connectors/yuanrong_transfer_engine_connector/) |
+
+## Feature checks
+
+- Verify dependency, service, device/network, port, memory-pool, and environment
+  prerequisites with actionable startup errors.
+- Match key/metadata construction, sender and receiver roles, allocation,
+  ownership, completion, and supported fast/serialized paths.
+- Exercise timeout, retry, stale sender/buffer reclamation, partial transfer,
+  cancellation, and connector close semantics.
+- Validate the smallest supported same-host or cross-node topology and document
+  backend limitations and operational diagnostics.

.claude/skills/review-pr/references/features/diffusion-acceleration.md

@@ -0,0 +1,26 @@
+# Diffusion Acceleration Features
+
+Use with the exact diffusion submodule plus platform, cache, quantization, or
+benchmarking contracts selected by the live path.
+
+| Feature family | Designs |
+| --- | --- |
+| Parallelism | [CFG](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/cfg_parallel/), [Expert](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/expert_parallel/), [HSDP](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/hsdp/), [Pipeline](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/pipeline_parallel/), [Sequence](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/sequence_parallel/), [Tensor](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/tensor_parallel/), [VAE patch](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/vae_parallel/) |
+| Attention | [Skip-Softmax](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/skip_softmax/) |
+| Quantization | [Quantization](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/quantization/) |
+| Cache | [Cache-DiT](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/cache_dit/), [TeaCache](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/teacache/) |
+| Batching | [Diffusion Continuous Batching](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/diffusion_continuous_batching/) |
+| Offload | [Distributed Layerwise Offload](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/distributed_layerwise_offload/) |
+
+## Feature checks
+
+- Verify feature selection, supported models/platforms/topologies, defaults,
+  dependencies, and every touched feature combination.
+- Preserve feature-off and single-rank correctness; reject unsupported
+  combinations before collectives, loading, or execution begin.
+- Trace shapes, shards, ranks, collectives, precision, cache validity, residency,
+  and per-request state through the selected runtime hooks.
+- Compare output quality or accuracy against the disabled/reference path before
+  accepting latency or memory improvements.
+- Require exact commands, topology, environment, warmup, repetitions, and
+  measurements for performance claims; update feature compatibility docs.

.claude/skills/review-pr/references/features/infrastructure-performance.md

@@ -0,0 +1,22 @@
+# Infrastructure and Performance Features
+
+Load the matching design from the reviewed head:
+
+- [Prometheus Metrics](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/metrics/)
+  for logger wrapping, gating, lifecycle, stage transfer, and metric definitions.
+- [Speech Generation Performance Optimizations](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/qwen3_omni_tts_performance_optimization/)
+  for stage batching, CUDA graphs, async chunk, streaming, re-prefill, compile,
+  and deployment evidence.
+
+## Feature checks
+
+- Route instrumentation changes through observability or profiling and result
+  methodology through benchmarking and the performance evidence check.
+- Preserve opt-in/default behavior, units, labels, aggregation, timing scope,
+  lifecycle resets, and bounded overhead.
+- For stacked optimizations, measure each increment against the same baseline
+  and explain interactions, quality guards, regressions, and limitations.
+- Require exact model, checkpoint, hardware, software, workload, topology,
+  concurrency, warmup, repetitions, and production commands.
+- Update the design page when metric meaning, supported optimization stack, or
+  deployment guidance changes.

.claude/skills/review-pr/references/features/runtime-stage-execution.md

@@ -0,0 +1,24 @@
+# Runtime and Stage Execution Features
+
+Load the matching design from the reviewed head:
+
+| Feature | Design | Primary module intersections |
+| --- | --- | --- |
+| Disaggregated inference | [Design](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/disaggregated_inference/) | Configuration, stage runtime, OmniConnector, I/O contracts |
+| Async chunk | [Design](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/async_chunk/) | Model integration, orchestration, stage runtime, connector, I/O |
+| Async diffusion output | [Design](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/async_diffusion_output/) | Diffusion runtime, I/O contracts, entrypoints |
+| Async Omni output materialization | [Design](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/omni_async_output_materialization/) | AR runtime, model integration, I/O contracts, orchestration |
+| Automatic prefix caching | [Design](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/feature/prefix_caching/) | AR runtime, cache management, I/O contracts |
+
+## Feature checks
+
+- Confirm the feature's supported models, entrypoints, stage types, topology,
+  configuration, default, and compatibility limitations.
+- Trace enablement from public/config ingress through every participating module
+  and verify the advertised production dispatch actually selects it.
+- Preserve a correct disabled or fallback path and make unsupported combinations
+  fail or warn as documented.
+- Test overlap, ordering, identity, backpressure, cancellation, failure, final
+  partial output, and cleanup at each asynchronous boundary.
+- Update the feature page, support matrices, examples, and recipes when behavior
+  or coverage changes.

.claude/skills/review-pr/references/modules/ar-runtime.md

@@ -0,0 +1,22 @@
+# Autoregressive Runtime
+
+Primary design: [Autoregressive Runtime](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/ar_runtime/).
+
+Use for AR request queues, scheduling state, token and memory budgets, KV/input
+readiness, workers, runners, and upstream vLLM request/cache transitions.
+
+## Contract checks
+
+- Preserve upstream request-state, scheduling, cache, and preemption semantics
+  unless an Omni-specific difference is documented and tested.
+- Convert modality-specific data through explicit input or output adapters; do
+  not inject it through unrelated scheduler state.
+- Keep workers and model runners focused on assigned execution rather than
+  admission or cross-stage routing.
+- Synchronize token, sequence, batch, and memory budgets with worker-visible
+  state and prevent starvation, double counting, or stale readiness.
+- Preserve legal transitions and one terminal cleanup path across completion,
+  abort, timeout, preemption, worker failure, and shutdown.
+
+Test request lifecycle, scheduling decisions, abort, cache state, adapters, and
+every affected worker execution mode against the pinned upstream contract.

.claude/skills/review-pr/references/modules/benchmarking.md

@@ -0,0 +1,24 @@
+# Benchmarking
+
+Primary design: [Benchmarking](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/benchmarking/).
+
+Use when benchmark code, workloads, metric calculations, CLI benchmark
+entrypoints, or result metadata change. Use the performance check separately
+when a PR uses benchmark results as evidence.
+
+## Contract checks
+
+- Record model and revision, code SHA, hardware, software, configuration,
+  workload, precision, topology, concurrency, warmup, repetitions, and
+  measurement window.
+- Validate successful and semantically acceptable outputs before accepting
+  timing, memory, or scaling results.
+- Define every metric's unit, population, aggregation, synchronization, and
+  timing scope.
+- Keep workload parsing and metric computation deterministic and reusable
+  across base and head.
+- Reject comparisons that change material workload or runtime settings without
+  isolating and explaining that difference.
+
+Test workload parsing and metric calculations with deterministic fixtures and
+publish exact commands plus variability for quantitative claims.

.claude/skills/review-pr/references/modules/cache-management.md

@@ -0,0 +1,22 @@
+# Cache Management
+
+Primary design: [Cache Management](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/cache_management/).
+
+Use for AR or diffusion cache identity, reusable state, validity, lifecycle,
+reset, eviction, and teardown.
+
+## Contract checks
+
+- Include every model, request, input, timestep, precision, device, topology,
+  and feature property that can change a reused value in cache identity.
+- Preserve a correct disabled and miss path; never reuse after validity inputs
+  change.
+- Isolate request-scoped state across concurrent requests and define ownership
+  for model-scoped state.
+- Make reset, invalidation, eviction, capacity, and teardown explicit on
+  completion, cancellation, failure, model reload, and shutdown.
+- Keep cache optimizations behind the owning runtime's hooks rather than adding
+  a second request lifecycle.
+
+Test hit, miss, invalidation, disabled behavior, concurrency, capacity, reset,
+and teardown, with quality comparison when reuse is approximate.

.claude/skills/review-pr/references/modules/configuration.md

@@ -0,0 +1,24 @@
+# vLLM-Omni Configuration
+
+Primary design: [vLLM-Omni Configuration](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/vllm_omni_config/).
+
+Use for config construction, deploy YAML, registries, schema, defaults,
+aliases, CLI projection, stage config, and topology. Inspect the reviewed
+page's status: while its contract remains deferred or draft, current code and
+tests—not proposed precedence rules—are authoritative.
+
+## Contract checks
+
+- Resolve each supported structured, legacy, CLI, environment, default, alias,
+  and stage-override producer through its actual runtime consumer.
+- Reject unknown or owner-mismatched fields explicitly; keep process-local
+  runtime objects out of transportable configuration.
+- Verify default, explicit, and feature-off values reach every live factory and
+  consumer without silent reinterpretation.
+- Treat stage count, placement, connector, parallelism, device, and memory
+  fields as one topology contract and reject impossible combinations early.
+- Preserve supported construction-path parity until a documented migration
+  removes a path.
+
+Require normalization and schema tests, a live-consumer assertion, and docs for
+public keys, defaults, constraints, precedence, or migration behavior.

.claude/skills/review-pr/references/modules/diffusion.md

@@ -0,0 +1,33 @@
+# Diffusion Module Family
+
+Primary design: [Diffusion Modules](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/diffusion/).
+
+Use for diffusion-stage behavior, then load the exact submodule page from the
+reviewed head:
+
+| Submodule | Signals | Design |
+| --- | --- | --- |
+| Runtime | Admission, scheduling, execution, progress, outputs, cancellation, hooks, cleanup | [Diffusion Runtime](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/diffusion/diffusion_runtime/) |
+| Model Integration | Pipelines, registry, loader, adapters, shared layers, model-specific processing | [Diffusion Model Integration](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/diffusion/diffusion_model_integration/) |
+| Continuous Batching | Compatibility, request/step batches, per-request progress, capacity | [Continuous Batching](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/diffusion/continuous_batching/) |
+| Parallelism | Rank topology, process groups, sharding, collectives, distributed execution | [Parallelism](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/diffusion/parallelism/) |
+| Offloader | Residency, transfers, memory accounting, prefetch, teardown | [Offloader](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/diffusion/offloader/) |
+
+## Family-wide checks
+
+- Keep one scheduler-owned lifecycle from admission through completion,
+  cancellation, failure, and cleanup; optional features must use defined hooks.
+- Select pipelines through registry and loader contracts instead of scattered
+  model-name conditionals; keep genuine model differences in model directories.
+- Trace latent, conditioning, timestep, generator, shape, layout, dtype, device,
+  batch expansion, and output conversion through the actual consumer.
+- Make batch compatibility and per-request isolation explicit; never associate
+  output by unstable batch position.
+- Derive topology from validated configuration, define every sharding boundary,
+  keep collectives symmetric, and preserve a supported single-rank path.
+- Track one residency owner per offloaded component, wait for transfer readiness,
+  preserve model state, bound retained copies, and clean up deterministically.
+
+Load matching feature designs for cache, quantization, parallelism, continuous
+batching, or offload. Require feature-off correctness, representative inference,
+terminal-path tests, and quality evidence for optimizations.

.claude/skills/review-pr/references/modules/engine-orchestration.md

@@ -0,0 +1,23 @@
+# Engine Orchestration
+
+Primary design: [Engine Orchestration](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/engine_orchestration/).
+
+Use for request-state creation, cross-stage routing, output ordering, companion
+tracking, RPC correlation, cancellation propagation, and terminal convergence.
+Orchestration does not own replica placement, stage-process startup, public
+rendering, connector transport, or payload and error schemas.
+
+## Contract checks
+
+- Keep one owner for cross-stage routing; entrypoints, models, and stage clients
+  must not independently select the next logical stage.
+- Preserve request identity and output order across concurrent stages and
+  correlate every control or RPC result to the originating request and wave.
+- Make terminal state monotonic: no new forwarding after success, abort,
+  request failure, or fatal engine failure.
+- Propagate cancellation and failure to every active stage and release
+  orchestrator-owned tasks, queues, companions, and request state.
+- Preserve shutdown ordering across the orchestrator and stage runtime.
+
+Test representative multi-stage routing, ordering, correlation, cancellation,
+failure, and repeated shutdown.

.claude/skills/review-pr/references/modules/entrypoints.md

@@ -0,0 +1,25 @@
+# Entrypoints and Serving Boundaries
+
+Primary design: [Entrypoints and Serving Boundaries](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/entrypoints/).
+
+Use for offline APIs, CLI and server composition, OpenAI-compatible routes,
+request validation, response conversion, streaming, sessions, and engine
+handoff. Entrypoints adapt public contracts; they do not own cross-stage
+routing, stage lifecycle, configuration precedence, payload schemas, or error
+classification.
+
+## Contract checks
+
+- Normalize and validate public values once before engine submission.
+- Preserve request identity, modality, ordering, and exactly one terminal event
+  across streaming, disconnect, cancellation, and failure.
+- Translate internal outputs and errors explicitly for each advertised offline,
+  HTTP, SSE, WebSocket, or asynchronous-job path.
+- Keep model-specific behavior behind a common adapter or processor instead of
+  adding model-name policy to generic entrypoints.
+- Require API or CLI changes to explain necessity, alternatives, compatibility,
+  migration, and the supported route or transport matrix.
+
+Test the affected public protocol, normalization, engine handoff, rendering,
+and terminal outcomes. Route orchestration, I/O schema, and semantic error
+changes to their module contracts.

.claude/skills/review-pr/references/modules/error-contracts.md

@@ -0,0 +1,24 @@
+# Error Classification, Propagation, and Rendering
+
+Primary design: [Error Classification, Propagation, and Rendering](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/error_contracts/).
+
+Use when a change classifies failures, crosses process boundaries, decides
+fatal versus request-scoped impact, sanitizes context, or renders offline,
+HTTP, SSE, WebSocket, or asynchronous-job errors. The I/O module owns the wire
+schema; this module owns the meaning of its error fields.
+
+## Contract checks
+
+- Keep request-scoped failure isolated unless the underlying engine is no
+  longer usable.
+- Preserve stable classification and request context across process and
+  protocol boundaries without exposing prompts, outputs, secrets, or internal
+  stack details.
+- Keep cancellation distinguishable from internal failure.
+- Terminate each public transport exactly once with consistent status, body,
+  event, finish reason, and metrics classification.
+- Maintain compatibility across mixed or older producers only when the
+  reviewed contract defines that behavior.
+
+Test table-driven classification, wire round trips, fatal/request-scoped
+propagation, cancellation, sanitization, and every affected public renderer.

.claude/skills/review-pr/references/modules/execution-platforms.md

@@ -0,0 +1,23 @@
+# Execution Platforms
+
+Primary design: [Execution Platforms](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/execution_platforms/).
+
+Use for hardware capability detection, worker selection, patches, kernels,
+attention backends, and vendor-specific configuration or connectors.
+
+## Contract checks
+
+- Guard hardware behavior through platform selection or capability detection
+  and fail clearly when unsupported.
+- Keep portable modules free of direct vendor imports when a common selector
+  exists; verify portable imports without the target accelerator.
+- Override only behavior that differs from common or upstream implementations,
+  and compare sibling worker hooks when inheritance does not provide parity.
+- Preserve dtype, layout, rank, stream, synchronization, workspace,
+  graph-capture, and resource-lifetime contracts.
+- Do not let simplified CPU doubles hide platform MRO, initialization, or
+  dispatch failures.
+
+Run portable checks first, then the smallest matching-device test in a fresh
+environment. Report unavailable hardware as a validation gap, never simulated
+device evidence.

.claude/skills/review-pr/references/modules/input-output-modality.md

@@ -0,0 +1,23 @@
+# Input, Output, and Modality Contracts
+
+Primary design: [Input, Output, and Modality Contracts](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/input_output_modality_contracts/).
+
+Use for requests, prompt and modality types, message schemas, serialization,
+output types, accumulation, completion, and compatibility shims crossing
+entrypoint, orchestration, stage, connector, or model boundaries.
+
+## Contract checks
+
+- Give every boundary value an explicit modality and validated typed schema.
+- Preserve request and stage identity across conversion, serialization,
+  streaming updates, batching, cancellation, and errors.
+- Distinguish partial, final, empty, and failed output states for every modality.
+- Verify producer and consumer agreement on shape, layout, dtype, device,
+  ownership, metadata, ordering, and optional or unknown fields.
+- Keep internal objects out of public protocols; translate them at the owning
+  entrypoint.
+- Preserve wire compatibility deliberately and give deprecated shims a bounded
+  migration path.
+
+Require construction, round-trip, accumulation, completion, rejection, and
+compatibility tests at each changed producer-consumer boundary.

.claude/skills/review-pr/references/modules/model-integration.md

@@ -0,0 +1,25 @@
+# Model Integration
+
+Primary design: [Model Integration](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/model_integration/).
+
+Use for model registration, configuration adapters, preprocessing, checkpoint
+loading, stage inputs, model-specific hooks, shared layers, runners, and model
+execution. Use the diffusion family reference for diffusion models.
+
+## Contract checks
+
+- Declare explicitly how the model class, loader, input processor, stage
+  configuration, and runtime capabilities are selected.
+- Preserve weight-name mapping, dtype/device placement, optional-dependency
+  errors, and checkpoint-specific loading semantics.
+- Trace each typed input and output through preprocessing, runner state, model
+  call, and its actual consumer.
+- Reuse upstream vLLM behavior when sufficient; document and test the precise
+  difference for every override or patch.
+- Keep model code out of admission, cross-stage routing, public policy, and
+  terminal lifecycle ownership.
+- Place reusable behavior in shared modules and test it with more than one
+  integration.
+
+Require focused registry, loader, processor, runner, and representative model
+execution tests. Apply the model-addition check for new or expanded support.

.claude/skills/review-pr/references/modules/observability.md

@@ -0,0 +1,21 @@
+# Observability
+
+Primary design: [Observability](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/observability/).
+
+Use for metrics, logs, correlation fields, and cross-stage request visibility.
+
+## Contract checks
+
+- Preserve stable request and stage identity across entrypoints, orchestration,
+  stages, workers, errors, and terminal cleanup.
+- Treat metric names, units, labels, aggregation, and lifecycle as public
+  contracts; document any semantic change.
+- Keep labels bounded: never use request IDs, prompts, outputs, paths, or other
+  unbounded user values.
+- Register and reset metrics exactly once across repeated startup and shutdown,
+  and keep disabled instrumentation inexpensive.
+- Align emitted values with their actual timing and lifecycle owner rather than
+  reconstructing them from ambiguous client strings.
+
+Test registration, label sets, units, resets, error classification, and
+representative multi-stage emission.

.claude/skills/review-pr/references/modules/omni-connector.md

@@ -0,0 +1,24 @@
+# OmniConnector
+
+Primary design: [OmniConnector](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/omni_connector/).
+
+Use for connector contracts and model-agnostic data transfer across stages,
+processes, devices, or nodes. Connector implementations transport and
+synchronize data; they do not choose the next stage or implement model policy.
+
+## Contract checks
+
+- Prove producer and consumer agreement on request/stage identity, shape,
+  layout, dtype, placement, ownership, ordering, and completion.
+- Keep backend selection and metadata/key construction explicit; do not silently
+  reinterpret or drop payloads across SHM, store, network, or vendor paths.
+- Bound connect, put/get, retry, synchronization, and readiness waits with
+  backpressure, cancellation, and actionable errors.
+- Prevent route, endpoint, port, key, and buffer collisions across repeated
+  startup, concurrent requests, and replica replacement.
+- Release connections, handles, buffers, listener work, and stale sender state
+  on success, cancellation, failure, and shutdown.
+
+Load the matching communication feature design for a concrete backend. Require
+focused producer-consumer tests and the smallest runnable multi-process or
+multi-node path.

.claude/skills/review-pr/references/modules/profiling.md

@@ -0,0 +1,21 @@
+# Profiling
+
+Primary design: [Profiling](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/profiling/).
+
+Use for opt-in instrumentation and trace collection across supported profiler
+backends.
+
+## Contract checks
+
+- Keep expensive collection disabled by default and bounded by explicit start,
+  stop, timeout, and cleanup behavior.
+- Preserve scheduling, generated outputs, resource ownership, and concurrency
+  semantics apart from documented synchronization overhead.
+- Include stage, rank, worker, operation, and request correlation where useful
+  without embedding user payloads or secrets.
+- Make repeated start/stop and partial failure safe, including multi-rank trace
+  naming and artifact ownership.
+- Measure disabled and enabled overhead before making performance claims.
+
+Test the disabled path, repeated lifecycle, failure cleanup, trace context, and
+each affected backend.

.claude/skills/review-pr/references/modules/quantization.md

@@ -0,0 +1,22 @@
+# Quantization
+
+Primary design: [Quantization](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/quantization/).
+
+Use for method registration, checkpoint interpretation, layer replacement,
+reduced precision, platform capability, and runtime constraints.
+
+## Contract checks
+
+- Validate checkpoint metadata, selected method, supported layers, device
+  capability, required dependencies, and parallel layout before loading.
+- Preserve loader and weight-name mapping semantics, dtype, scale/granularity,
+  zero point, padding, and buffer handling through the real kernel consumer.
+- Fail clearly for unsupported combinations; do not silently choose a different
+  method or precision.
+- Keep online and pre-quantized checkpoint paths explicit and ensure patches or
+  fallbacks do not bypass model-owned loaders.
+- Require numerical comparison against a defined reference with explicit
+  tolerance before treating performance results as valid.
+
+Test registration, loading, representative layers, metadata, platform guards,
+parallel combinations, and numerical correctness separately from performance.

.claude/skills/review-pr/references/modules/stage-runtime.md

@@ -0,0 +1,23 @@
+# Stage Runtime and Replica Lifecycle
+
+Primary design: [Stage Runtime and Replica Lifecycle](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/module/stage_runtime/).
+
+Use for local or distributed placement, startup, readiness, stage pools,
+replica identity and membership, selection, affinity, liveness, draining, and
+stage-client or process lifecycle. Cross-stage request policy belongs to engine
+orchestration; model scheduling belongs to the owning runtime.
+
+## Contract checks
+
+- Acquire, replace, and retire replicas through one stage-runtime lifecycle.
+- Preserve stable replica identity and request affinity until termination or a
+  declared replica loss.
+- Make readiness explicit before admission; surface partial-startup and
+  membership failures without leaving half-initialized capacity.
+- Keep local and distributed semantics aligned for supported modes.
+- Make drain, shutdown, and cleanup idempotent across repeated signals and
+  process failure.
+
+Test startup, readiness, selection, affinity, membership changes, abort,
+process loss, draining, and repeated shutdown on affected local/distributed
+paths.

.claude/skills/review-pr/references/process/design-contracts.md

@@ -0,0 +1,34 @@
+# Resolve Design Contracts
+
+Use after freezing the review head and before selecting module or feature
+references. The maintained [design index](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/)
+defines two routing axes:
+
+- **Module designs** own code boundaries and invariants. Select one primary
+  module and a second only when the live producer-consumer path crosses it.
+- **Feature designs** describe behavior spanning modules. Load every feature
+  page whose implementation, default, support, or contract changes.
+
+## Resolution order
+
+1. Open `docs/design/index.md` in the reviewed head, not the reviewer's checkout
+   or published latest tree.
+2. Match changed behavior against each module page's `primary_code_paths`,
+   `primary_path_exceptions`, ownership boundary, and `depends_on` metadata.
+3. Read `status`, `architecture_state`, `last_verified_commit`, and any linked
+   in-flight work before treating prose as current behavior.
+4. Use current code and tests when a page is draft, deferred, stale, absent, or
+   conflicts with the frozen implementation. Report material documentation
+   drift separately.
+5. Treat candidate invariants and promotion gates as review questions until
+   repository policy, current code/tests, or owner-approved normative text
+   makes them enforceable. Do not turn a draft identifier into an automatic
+   blocker.
+6. Use published latest docs only for discovery when the reviewed head lacks a
+   page; never import a newer contract into an older branch silently.
+
+When behavior changes an active module or feature contract, require the same PR
+to update that page. Require a new page only when repository policy, an accepted
+RFC/issue, or explicit maintainer acceptance criteria do so; otherwise treat it
+as a non-blocking design suggestion. Do not demand a second page when an existing
+design already owns the exact contract.

.claude/skills/review-pr/references/process/general-checks.md

@@ -0,0 +1,129 @@
+# General Review Checks
+
+Apply these checks to every PR after freezing the review surface. A pattern is
+not a finding until the diff introduces or exposes a reachable trigger, impact,
+and smallest safe fix. Do not report unrelated backlog.
+
+Official docs: [contributing guide](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/)
+[feature compatibility](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/feature_compatibility/),
+and [design documents](https://docs.vllm.ai/projects/vllm-omni/en/latest/design/).
+
+## Contract and scope
+
+- Match the title/body claims to every changed production path, test, config,
+  dependency, generated artifact, and user-visible document.
+- Trace each changed value through public ingress, validation/defaulting,
+  producer, transformations, process or device boundaries, final consumer, and
+  terminal cleanup.
+- Check every applicable sync/async, offline/online, streaming/non-streaming,
+  feature-off, topology, and compatibility path without demanding unsupported
+  modes merely for symmetry.
+
+## Module, API, and class design
+
+Apply these rules to new or materially changed production code; do not require
+unrelated cleanup from untouched files.
+
+1. Start each source module with a concise module docstring that defines its
+   responsibility and boundaries and identifies its primary classes and key
+   functions. Place it after any required license header and before imports.
+2. Give every added or materially changed class and function a clear docstring
+   covering its responsibility, parameters, return value, raised exceptions,
+   and non-obvious invariants. Use explicit typed parameters at owned API
+   boundaries; avoid opaque `**kwargs` and `Any`. Allow them only when an
+   upstream or compatibility interface requires them, and document and validate
+   the accepted keys and concrete types.
+3. Move stable helpers shared by multiple features into an appropriately scoped
+   utility module. Do not let feature modules accumulate large collections of
+   unrelated private utility functions; keep a private helper local only when
+   it is single-purpose and tightly coupled to that feature.
+4. Keep every method on a key class within that class's stated responsibility,
+   and document non-obvious design decisions and invariants. Before adding a
+   method, prefer reuse, consolidation, or extension of an existing abstraction;
+   split the class when new behavior creates a separate responsibility or makes
+   the class unnecessarily long.
+5. Require changes to user-facing behavior to update `docs/`. When an existing
+   active module or feature design page owns the changed contract, update that
+   page. Require a new design page only when the reviewed branch's repository
+   policy, an accepted RFC/issue, or explicit maintainer acceptance criteria do
+   so; otherwise suggest it as non-blocking. For a major architectural change,
+   apply the branch's contributing policy for a design RFC or issue; the current
+   policy uses more than 500 changed production-code lines, excluding kernel,
+   data, config, and test changes. Resolve page status and authority through
+   [design-contracts.md](design-contracts.md), and do not demand a duplicate page
+   when an existing design owns the exact contract. When required, cover scope,
+   data flow, public contracts, failure behavior, alternatives, validation,
+   limitations, and rollout.
+6. Require every performance PR to provide reproducible base-versus-head A/B
+   results under the same hardware, software, model, workload, precision,
+   topology, warmup, repetitions, and measurement scope. Include exact commands,
+   variability, and a correctness or quality guard; one-sided or non-comparable
+   numbers are incomplete evidence. For a new model absent from the frozen base,
+   compare head with its pinned canonical reference implementation under rule 12
+   instead. Follow
+   [perf-verification.md](../checks/perf-verification.md) for the full protocol.
+7. Require every addition, removal, default change, or semantic change to a
+   user-facing inference API or CLI to explain why it is necessary, which
+   alternatives were considered, and why the chosen interface is preferable.
+   Record compatibility, deprecation or migration behavior, and the rationale
+   in the PR. Update the corresponding module and feature design documents when
+   they define the changed interface or workflow, or when policy requires one.
+8. Require every bug-fix PR to add an automated regression test that reproduces
+   the defect on the frozen base and passes on the fixed head. Exercise the
+   production path or nearest stable boundary and assert the corrected behavior,
+   not only process survival. Record the exact base/head command and result, and
+   place hardware- or model-specific coverage in the correct CI lane. A manual
+   reproduction supplements but does not replace the regression test.
+9. Require every new or materially changed recipe to follow
+   [`recipes/TEMPLATE.md`](../../../../../recipes/TEMPLATE.md) and the naming and
+   layout conventions in
+   [`recipes/README.md`](../../../../../recipes/README.md).
+   Complete every applicable summary, use-case, reference, hardware,
+   environment, exact-command, verification, and notes section with tested,
+   reproducible details; explain any intentional deviation in the PR.
+10. Require every new model to verify its supported modalities, platforms,
+    execution modes, and features with production-path evidence. Update
+    `docs/models/supported_models.md` and every applicable feature support or
+    compatibility table; do not claim support that the PR does not validate.
+    Follow
+    [model-addition-checklist.md](../checks/model-addition-checklist.md) for the
+    model-specific documentation checks.
+11. Require every new model PR to add or update its in-repository model-family
+    recipe under `recipes/` and its `recipes/README.md` index entry. Apply rule
+    9, and keep the recipe's model identifiers, tasks, serving modes, hardware,
+    commands, supported features, and limitations aligned with validated PR
+    evidence. An external recipe link does not replace the in-repository recipe.
+12. Require every new model to compare accuracy or output quality and performance
+    against a pinned baseline or canonical reference implementation. Match the
+    checkpoint, inputs, precision, hardware, workload, and measurement method;
+    provide exact commands, results, and explanations for material differences.
+    Prefer a detailed timing split for each stage and explain when it is
+    unavailable. Follow
+    [model-addition-checklist.md](../checks/model-addition-checklist.md) and
+    [perf-verification.md](../checks/perf-verification.md) for the evidence
+    contract.
+
+## Blocking risk scan
+
+| Risk | Prove before reporting |
+| --- | --- |
+| Correctness | A live input reaches a wrong output, exception, silent drop, shape/dtype/device mismatch, or partially initialized state. |
+| Compatibility | A supported API, config key, default, serialization schema, model identifier, or caller breaks without validation, migration, or deprecation. |
+| Lifecycle | Allocation or startup lacks cleanup on partial failure, timeout, cancellation, shutdown, or repeated requests. |
+| Concurrency/distribution | Ordering, identity, rank, world size, completion, timeout, backpressure, or collective assumptions fail on a supported path. |
+| Cache/shared state | Identity omits a correctness-affecting input, or isolation, invalidation, disabled behavior, eviction, or cleanup is incomplete. |
+| Async behavior | Blocking I/O, sleep, a lock across `await`, or unnecessary serialization harms a reachable async path. |
+| Security/data handling | Secrets, user payloads, unsafe deserialization/eval/shell, unvalidated paths, or unbounded metric labels are reachable. |
+| Validation evidence | Tests bypass the production dispatcher, mock away the change, use unrealistic types/MRO, or cannot fail when the defect returns. |
+| User contract | User-visible model, feature, API, CLI, config, default, or compatibility behavior changes without accurate docs or examples. |
+
+Search bounded callers and sibling implementations before reporting dead code,
+duplication, or a missed consumer. Keep new abstractions only when they have a
+distinct live caller, invariant, or compatibility need.
+
+## Finding bar
+
+Anchor each finding to a changed `path:line`; name the trigger or call path,
+current behavior, impact, and smallest fix direction. Treat pending CI, missing
+hardware, or unsupported measurements as validation gaps unless the repository
+contract makes that evidence a merge requirement.

.claude/skills/review-pr/references/process/review-execution.md

@@ -0,0 +1,191 @@
+# Review Execution and Delivery
+
+Use this reference for every review. It defines snapshot collection, gates,
+validation records, delivery permissions, maintainer tone, and re-review.
+
+Official docs: [contributing guide](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/)
+and [CI failure triage](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/failures/).
+
+## Contents
+
+- [Freeze the review surface](#freeze-the-review-surface)
+- [Report status before analysis](#report-status-before-analysis)
+- [Apply review gates](#apply-review-gates)
+- [Run bounded validation](#run-bounded-validation)
+- [Deliver maintainer-style findings](#deliver-maintainer-style-findings)
+- [Re-review safely](#re-review-safely)
+
+## Freeze the review surface
+
+For a GitHub PR, read the base/head SHA before and after fetching metadata and
+the diff:
+
+```bash
+gh api "repos/vllm-project/vllm-omni/pulls/<PR>" \
+  --jq '{base_sha: .base.sha, head_sha: .head.sha}'
+REVIEW_FIELDS="number,url,title,body,isDraft,baseRefName,headRefName,mergeable,mergeStateStatus,statusCheckRollup,files"
+gh pr view <PR> --repo vllm-project/vllm-omni \
+  --json "${REVIEW_FIELDS}"
+gh pr diff <PR> --repo vllm-project/vllm-omni
+gh api "repos/vllm-project/vllm-omni/pulls/<PR>" \
+  --jq '{base_sha: .base.sha, head_sha: .head.sha}'
+```
+
+Discard the snapshot if either SHA changed. Do not mix comments or validation
+from different heads.
+
+After the SHAs stabilize, fetch and materialize a trusted `head_sha` in an
+isolated detached worktree. Run every source read, `rg` search, import, and test
+from that snapshot, not the caller's checkout. A detached worktree provides
+snapshot isolation, not a security boundary.
+
+Treat a fork head as untrusted unless the user and environment policy explicitly
+establish trust. Never run its imports, tests, builds, hooks, package setup, or
+repo-configurable linters/plugins on the reviewer host. Execute them only in a
+disposable sandbox or VM with no credentials or inherited secrets, no host agent
+or service sockets, minimal read-only host mounts, disabled or explicitly
+allowlisted network, resource/time limits, and destruction after the run. If
+that boundary is unavailable, limit the review to the remote diff, SHA-addressed
+reads such as `git show <head_sha>:<path>`, and existing CI evidence; report all
+executable validation as a gap.
+
+Before the first source read, record a pristine snapshot fingerprint outside
+the reviewed worktree. Include `HEAD`, the NUL-delimited index entries and blob
+IDs, and a NUL-safe manifest of every worktree entry except Git metadata,
+including tracked, untracked, and ignored paths with file type, mode, symlink
+target, and content hash. Before each validation group and delivery, recompute
+and byte-compare the fingerprint as well as asserting `HEAD == head_sha`. If a
+tool changes or creates any entry, discard affected evidence and recreate the
+snapshot before another group; do not rely on `HEAD` alone or clean an unknown
+worktree in place. If an exact snapshot cannot be created, use SHA-addressed
+reads and report filesystem-dependent validation as a gap.
+
+For a local branch/worktree, determine the target ref from the user, current PR,
+or configured upstream; never infer it from the branch name. Resolve the target
+once and include all task-owned worktree state:
+
+```bash
+git status --porcelain=v2 -z
+git rev-parse HEAD
+git rev-parse <target-ref>
+git merge-base <target-base-sha> HEAD
+git diff --stat <comparison-commit>
+git diff --name-status <comparison-commit>
+git diff --binary <comparison-commit>
+git diff --cached --binary <comparison-commit>
+git diff --binary
+git ls-files --others --exclude-standard -z
+```
+
+Freeze the exact bytes of every in-scope untracked file from the NUL-delimited
+list, plus a NUL-safe path, file-type, mode, and content-hash manifest. Also
+fingerprint `HEAD`, the target and merge base, porcelain status, and both binary
+index and worktree patches. Recording only untracked names is insufficient.
+Keep these snapshots in the evidence packet and do not mutate the reviewed
+checkout during a read-only review.
+
+For local changes without PR metadata, mark the title/body as synthetic. Base
+them on the user request and commit messages without inventing validation claims.
+
+If no PR, branch, or usable worktree is identifiable, ask for a PR URL/number or
+explicit base and head rather than guessing.
+
+## Report status before analysis
+
+Within 60 seconds of starting, and before source searches or tests, send this
+host update:
+
+```text
+Pinned head: <SHA>
+Base/comparison: <ref and SHA>
+CI: <pass/fail/pending/not applicable>
+Mergeability: <state/not applicable>
+Preliminary findings: <brief finding or none yet>
+```
+
+Mark early findings as preliminary and continue. This is not a GitHub comment.
+
+## Apply review gates
+
+- Report draft/WIP state. Continue a local review when explicitly requested,
+  but do not publish review events without separate authorization.
+- Record DCO, pre-commit, required CI, and mergeability. Pending or unknown gates
+  do not block source review.
+- Treat a failed gate as its own evidence. Do not restate its formatting/lint
+  output as a new code-review finding.
+- Open CI logs only when the first failing step overlaps the frozen diff or
+  blocks the verdict. Start with the first error, not the last cascade.
+- Treat the PR body as navigation. Commands, benchmark tables, and claimed tests
+  become evidence only after their provenance and relevance are checked.
+
+## Run bounded validation
+
+Keep one evidence packet for files read, bounded searches, callers, tests, CI,
+hardware, routes, and findings. Reuse it rather than refetching facts.
+
+Before pytest, run a short import/version compatibility check. Record each
+validation result with:
+
+```text
+repo, head SHA, command, result, Python/platform, dependency or lock fingerprint
+```
+
+After preflight, run targeted tests and low-cost static checks alongside source
+inspection when possible. Imports, tests, builds, and linters may create caches
+or rewrite files, so isolate each potentially mutating group in a disposable
+snapshot or recreate the pinned snapshot before continuing. Map source symbols
+to tests with bounded `rg` searches rather than assuming the test directory
+mirrors production paths.
+
+Classify failures as code, test, infrastructure, or flaky before reporting.
+Skipped hardware tests are gaps, not passes. For available hardware, verify the
+smallest representative unit/E2E path and compare actual output with PR claims.
+
+## Deliver maintainer-style findings
+
+Default to roughly 1-5 short comments, but treat that as calibration rather than
+a quota: report every real blocker and report no finding when there is no issue.
+
+Write each finding as:
+
+```text
+[P1] Short imperative title — path/to/file.py:<line>
+<Trigger or call path>. <Current behavior and impact>. <Smallest fix direction>.
+```
+
+Use [maintainer-style-study.md](../delivery/maintainer-style-study.md) for
+comment tone.
+Keep rule IDs, grades, and audit matrices internal unless the user asks for the
+complete audit.
+
+Verify every inline `path:line` against the frozen diff before delivery. For a
+PR, re-read the remote head and byte-compare the detached snapshot fingerprint,
+or recreate it from `head_sha`, before asserting current lines. For a local
+review, recompute and byte-compare the frozen `HEAD`, target, merge base, status,
+index patch, worktree patch, and untracked-content manifest. Any mismatch makes
+the review stale; discard the affected evidence and restart. Prefer one
+root-cause comment to several symptom comments.
+
+Local presentation is the default. Only explicit authorization permits GitHub
+posting. When authorized, publish one consolidated final review after
+validation; do not post preliminary or incremental comments. Submit a review
+event (`APPROVE`, `COMMENT`, or `REQUEST_CHANGES`) only when the user explicitly
+chooses that event.
+
+Reviewer requests and owner `@mention` comments are separate external writes;
+follow [review-requests.md](../delivery/review-requests.md) when the user asks
+for them.
+
+## Re-review safely
+
+Freeze the new head and compare it with the previously reviewed SHA. Then:
+
+1. inspect the delta and every file affected by conflict resolution or rebase;
+2. revalidate earlier findings against current lines and behavior;
+3. read unresolved, non-outdated threads and the author's evidence;
+4. rerun only checks invalidated by the delta;
+5. look for new regressions introduced by the fix;
+6. do not repeat resolved or outdated comments.
+
+If the target head changes during re-review, discard stale validation and start
+again from the new snapshot.

.claude/skills/review-pr/references/process/review-routing.md

@@ -0,0 +1,76 @@
+# Module and Feature Review Routing
+
+Read [design-contracts.md](design-contracts.md) first. Route from the frozen
+head's live producer-consumer behavior, then confirm the result against its
+`docs/design/` metadata. Titles and paths are evidence, not the deciding rule.
+
+## Primary module contract
+
+Choose one primary module and a second only when the live call path crosses a
+documented dependency or primary-path exception.
+
+| Module | Live behavior signals | Read |
+| --- | --- | --- |
+| Entrypoints | Offline/CLI/API ingress, validation, rendering, streaming, sessions | [entrypoints.md](../modules/entrypoints.md) |
+| Configuration | Schema, defaults, deploy/stage construction, registry, topology | [configuration.md](../modules/configuration.md) |
+| I/O and Modality | Requests, messages, serialization, output types, accumulation, completion | [input-output-modality.md](../modules/input-output-modality.md) |
+| Error Contracts | Classification, fatality, propagation, sanitization, public rendering | [error-contracts.md](../modules/error-contracts.md) |
+| Engine Orchestration | Cross-stage routing, request state, ordering, RPC correlation, terminal convergence | [engine-orchestration.md](../modules/engine-orchestration.md) |
+| Stage Runtime | Placement, startup, readiness, replicas, affinity, membership, shutdown | [stage-runtime.md](../modules/stage-runtime.md) |
+| OmniConnector | Cross-stage/process/device/node transport and synchronization | [omni-connector.md](../modules/omni-connector.md) |
+| Model Integration | Registration, preprocessing, loading, runners, model-specific execution | [model-integration.md](../modules/model-integration.md) |
+| AR Runtime | Scheduling, request/cache state, adapters, workers, upstream vLLM semantics | [ar-runtime.md](../modules/ar-runtime.md) |
+| Diffusion Family | Diffusion runtime, models, batching, parallelism, or offload | [diffusion.md](../modules/diffusion.md) |
+| Execution Platforms | Hardware selection, capabilities, vendor workers, kernels, patches | [execution-platforms.md](../modules/execution-platforms.md) |
+| Cache Management | Cache identity, validity, reuse, eviction, reset, teardown | [cache-management.md](../modules/cache-management.md) |
+| Quantization | Method/metadata selection, layer mapping, reduced precision, constraints | [quantization.md](../modules/quantization.md) |
+| Observability | Metrics, logs, labels, units, correlation, lifecycle | [observability.md](../modules/observability.md) |
+| Profiling | Opt-in instrumentation, traces, start/stop lifecycle, overhead | [profiling.md](../modules/profiling.md) |
+| Benchmarking | Workloads, metric computation, benchmark CLI, result metadata | [benchmarking.md](../modules/benchmarking.md) |
+
+For tests-, docs-, CI-, or recipe-only changes, route to the production module
+whose contract they protect. When none exists, use the applicable evidence
+check without inventing a production owner.
+
+## Feature-design overlays
+
+Load every category whose documented behavior, enablement, default, support, or
+compatibility changes.
+
+| Feature-design section | Read |
+| --- | --- |
+| Runtime and stage execution | [runtime-stage-execution.md](../features/runtime-stage-execution.md) |
+| Communication and concrete connectors | [communication.md](../features/communication.md) |
+| Diffusion acceleration | [diffusion-acceleration.md](../features/diffusion-acceleration.md) |
+| Infrastructure and performance | [infrastructure-performance.md](../features/infrastructure-performance.md) |
+
+## Evidence and change overlays
+
+| Signal | Read | Optional repo-local skill |
+| --- | --- | --- |
+| New or expanded model, loader, processor, registry, or stage config | [model-addition-checklist.md](../checks/model-addition-checklist.md) | [`add-tts-model`](../../../add-tts-model/SKILL.md) or [`add-diffusion-model`](../../../add-diffusion-model/SKILL.md) |
+| Latency, throughput, memory, scaling, precision, or quality claim | [perf-verification.md](../checks/perf-verification.md) | [`diffusion-perf-opt`](../../../diffusion-perf-opt/SKILL.md) for diffusion |
+| Tests changed, absent for risky behavior, or test-only | [test-quality-evaluation.md](../checks/test-quality-evaluation.md) | [`vllm-omni-test`](../../../vllm-omni-test/SKILL.md) |
+| CI, examples, docs, public behavior, or contributor evidence | [tests-docs-checklist.md](../checks/tests-docs-checklist.md) | None |
+| Suitable hardware/server and runnable affected path | [verification.md](../checks/verification.md) | None |
+| User asks who should review or requests owner notification | [review-requests.md](../delivery/review-requests.md) | None |
+
+Use [`quantization`](../../../quantization/SKILL.md) for a quantization module
+change and [`$vllm-omni-npu-model-runner-upgrade`](../../../vllm-omni-npu-upgrade/SKILL.md)
+for an NPU execution-platform change when those skills are available.
+
+For a bug fix, require a reachable reproduction and regression test. For a
+refactor, prove parity and remove obsolete paths. For a feature, verify the
+public/config contract, module integration, compatibility/default behavior,
+production dispatch, feature design, and docs.
+
+## Calibrate findings
+
+- **P0:** security exposure, data corruption, or broad project unusability.
+- **P1:** reachable runtime failure, wrong output, compatibility break, or
+  unsafe lifecycle in the changed behavior.
+- **P2:** real non-blocking defect with a concrete future failure mode.
+
+Treat draft candidate invariants, missing hardware, and unsupported claims as
+questions or validation gaps unless current code, tests, or policy makes them a
+merge requirement.

.claude/skills/vllm-omni-test/SKILL.md

@@ -0,0 +1,894 @@
+---
+name: vllm-omni-test
+description: Generate and run tests for vllm-project/vllm-omni with CI-aligned levels and markers; wire new tests into Buildkite (test-ready.yml for L1/L2, test-merge.yml for L3, test-nightly.yml for L4). On completion, always provide copy-paste local and CI-like pytest commands plus prerequisites. Use when creating regression tests, adding L1-L4 coverage, selecting pytest markers, or validating fixes from issues/PRs.
+---
+
+# vLLM-Omni Test Generator & Runner
+
+## Purpose
+
+Use this skill to generate minimal, stable test cases and run them with the correct marker/level strategy for [vllm-project/vllm-omni](https://github.com/vllm-project/vllm-omni).
+
+**Link convention:** Paths such as `.buildkite/` and `docs/contributing/` live at the **vllm-omni repo root**. Markdown links use repo-relative paths from this skill file (e.g. `../../../.buildkite/cuda/test-ready.yml`, `../../../docs/contributing/ci/test_system_overview.md`).
+
+Default priorities:
+
+1. Reproducible regression coverage for bug fixes
+2. Correct test level and marker selection
+3. Low flake, low dependency tests first
+4. CI-compatible run commands
+5. **Actionable run commands for the human**: whenever you add or change tests, always finish with **copy-paste-ready** `pytest` lines (local: single file and/or single test; CI-like: markers + `--run-level`), plus short **prerequisites** (GPU tier, HF cache, optional `model_prefix`). Do not assume the reader will infer commands from `test-routing.md` alone.
+
+## Inputs
+
+- Issue/PR link and summary
+- Changed files or suspected code path
+- Whether the user wants local quick validation or CI-equivalent validation
+- Hardware constraints (CPU only / CUDA / ROCm / NPU)
+
+## Workflow
+
+### Step 1: Classify Test Goal
+
+- **Bugfix regression**: start from a minimal failing scenario and add assertions that prevent recurrence. Before writing tests, output **`required` / `recommended` / `not_needed`**: **`required`** — stable logic/contract bug that should have been caught; **`recommended`** — environment-sensitive but a small regression still helps; **`not_needed`** — one-off external/config failure or existing tests already cover the path. Prefer the narrowest stable L1 (CPU) case; escalate to L2/L3 only when the bug needs real weights or serving.
+- **Feature coverage**: verify new behavior and one negative/boundary case.
+- **Perf/benchmark claim**: require benchmark-oriented tests and explicit metrics.
+
+### Step 2: Select Test Level
+
+- **L1**: unit/logic, deterministic, CPU-friendly, fastest feedback.
+- **L2**: basic e2e/integration and platform-dependent checks.
+- **L3/L4**: advanced model/integration/perf validation.
+
+Use [references/test-routing.md](references/test-routing.md) for level-to-marker and command mapping.
+
+### Step 3: Pick Markers
+
+Always attach markers deliberately:
+
+- **Level**: `core_model` (L1/L2) and/or `advanced_model` (L3) and/or `full_model` (L4 nightly)
+- **Model type** (required on model-centric e2e — pick exactly one):
+  - `omni` — end-to-end multimodal LLM pipelines (thinker/talker/stages; Qwen-Omni family)
+  - `tts` — speech synthesis / TTS-only models (`/v1/audio/speech`, voice clone, etc.)
+  - `diffusion` — generative diffusion models (image / audio / text / video from noise)
+- **Cross-cutting area** (when relevant): `parallel`, `cache`, `example`, `benchmark`
+- **Hardware**: `cpu`, `gpu`, `cuda`, `rocm`, `npu`, `L4`, `H100`, `distributed_cuda`, …
+- Optional: `slow`, distributed markers when multi-card is required
+
+**Baseline smoke (L2 + L3):** The simplest e2e case per model — default deploy, minimal request — should usually carry **both** `@pytest.mark.core_model` and `@pytest.mark.advanced_model` on the **same** test function so `test-ready.yml` and `test-merge.yml` share one test. `send_*_request` picks validation depth from `--run-level`. References: `test_voxcpm2_tts.py::test_text_to_audio_001`, `test_qwen3_tts_customvoice.py::test_text_to_audio_001`. Heavier scenarios use **`advanced_model` only**; L4 expansion uses **`full_model`**.
+
+For hardware-aware tests, prefer `@hardware_test(...)` or `hardware_marks(...)` in `tests/helpers/mark.py`.
+
+**Diffusion nightly split** (`test-nightly.yml`): all diffusion tests use `pytest.mark.diffusion`, but CI groups them by **output modality**, not by separate pytest markers:
+
+| Nightly group | Scope | Typical models / paths |
+|---------------|--------|-------------------------|
+| **Diffusion X2I(&A&T) Model Test** | x2**i** (image), x2**a** (audio), x2**t** (text) and other **non-video** diffusion | Qwen-Image*, BAGEL, FLUX, SD3, Z-Image, LongCat, DreamZero, … — `test_*_expansion.py` under X2I shards; L4 sweep uses `-k "not test_wan and not test_bagel_expansion and not hunyuan"` for L4 |
+| **Diffusion X2V Model Test** | x2**v** (video) only | Wan2.2, HunyuanVideo 1.5, Wan VACE, … — e.g. `test_wan22_expansion.py`, `test_hunyuan_video_15_expansion.py` |
+
+Wire new **video** diffusion expansion tests into the **X2V** group; wire **image/audio/text** diffusion expansion tests into **X2I(&A&T)**. Do not place x2v modules in X2I shards (see comment in `test-nightly.yml` above the X2I group).
+
+### Naming: generated test module files (L2–L4, model-centric e2e)
+
+When adding **new** pytest modules whose primary scope is a **specific model** (typical under `tests/e2e/offline_inference/` or `tests/e2e/online_serving/`), use this filename pattern:
+
+| Level | Filename pattern | Example (model `Qwen/Qwen2.5-Omni-7B`) |
+|-------|------------------|----------------------------------------|
+| **L2**, **L3** | `test_{lowercase_model_slug}.py` | `test_qwen2_5_omni.py` |
+| **L4** | `test_{lowercase_model_slug}_expansion.py` | `test_qwen2_5_omni_expansion.py` |
+
+**Slug rules for `{lowercase_model_slug}`:**
+
+1. Start from the HuggingFace-style id (e.g. `Qwen/Qwen2.5-Omni-7B`), but **do not** put the org into the filename: use the **repo segment only** (`Qwen2.5-Omni-7B`), not `Qwen_...` / `qwen_qwen2_5_...`.
+2. **Lowercase**; replace `.`, `-`, and whitespace with a single `_` (e.g. `Qwen2.5-Omni-7B` → `qwen2_5_omni`). **Omit** trailing size tokens such as `7b` / `30b` in the basename when a single file covers that model line in the directory (matches `test_qwen2_5_omni.py` in-tree).
+3. If two checkpoints in the same folder need separate modules, add a **minimal** disambiguator (e.g. `_7b` vs `_3b`) only then.
+4. **L1** unit tests are **not** bound to this pattern; use `tests/<area>/test_<feature>.py` as today.
+
+Routing tables and commands: [references/test-routing.md](references/test-routing.md) § *Model-centric e2e filename convention*.
+
+Existing references: `tests/e2e/offline_inference/test_qwen2_5_omni.py` (L2-style omni), `tests/e2e/offline_inference/test_qwen3_5_9b.py` (L2-style omni, single-stage VL), `tests/e2e/online_serving/test_qwen3_omni_expansion.py` (L4-style omni), `tests/e2e/online_serving/test_qwen_image_edit_expansion.py` / `test_qwen_image_expansion.py` (L4-style diffusion).
+
+### Step 4: Generate Test Case Skeleton
+
+**1. Pick the functional scenario** (then choose directory, fixtures, and markers):
+
+| Scenario | Typical location | Fixtures / runner pattern | Baseline markers & level |
+|----------|------------------|---------------------------|---------------------------|
+| **Offline inference e2e** | `tests/e2e/offline_inference/` | **Module (default):** `omni_runner` + `omni_runner_handler`. **Function (isolation only):** `omni_runner_function` + `omni_runner_handler_function`. Diffusion/TTS may use `Omni(...).generate` directly | L2: `core_model` + **one of** `omni` / `tts` / `diffusion`; `@hardware_test(...)` when GPU/NPU is required |
+| **Online serving e2e** | `tests/e2e/online_serving/` | **Module (default):** `omni_server` + `openai_client`. **Function (isolation only):** `omni_server_function` + `openai_client_function`. Clients: `send_omni_request` (omni), `send_audio_speech_request` (tts), `send_diffusion_request` / `send_video_diffusion_request` / `send_images_generations_request` (diffusion) | Baseline smoke: **`core_model` + `advanced_model`**; heavier paths: `advanced_model` only; L4 expansion: `full_model` |
+| **Documentation / runnable examples** | `tests/examples/offline_inference/`, `tests/examples/online_serving/` | **Offline docs (preferred):** extract Python/Bash blocks from the doc README (e.g. `ReadmeSnippet.extract_readme_snippets`), `pytest.mark.parametrize` each snippet, run via `example_runner.run` with a stable `output_subfolder`. **Online docs:** copy client/request scripts into dedicated tests and keep them in sync with the doc page. | Usually **L4**: `advanced_model`, often `example` plus hardware marks matching the nightly docs-example job (see `.buildkite/cuda/test-nightly.yml`). Full conventions: [docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md](../../../docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md) (introduced in [PR #1910](https://github.com/vllm-project/vllm-omni/pull/1910): naming, output directory layout, skip rules, avoid trimming `num_inference_steps` without a strong CI reason). |
+| **Performance / benchmark** | `tests/dfx/perf/tests/*.json` + `run_*_benchmark.py` | JSON or script-driven server + load config; assert explicit metrics / baselines | L4 Perf: JSON `mark` with `full_model` + `omni`/`tts`/`diffusion`; wire `test-nightly.yml` Perf steps |
+| **Invalid parameter / negative HTTP validation** | `tests/dfx/reliability/invalid_param_test/` | Live `omni_server` + low-level `send_*_http_request` with `err_code` / `err_message` | `pytest.mark.slow` + `omni` / `tts` / `diffusion` + `@hardware_marks` (`H100` or `L4`); CI in **`test-weekly.yml`** (not ready/merge/nightly) |
+
+**If the user’s test plan includes invalid parameter validation / invalid params / negative HTTP / 400 validation:** do **not** add those `test_*` functions to `tests/e2e/online_serving/test_*.py` or `*_expansion.py`. **Move or author them** under `tests/dfx/reliability/invalid_param_test/` in the **endpoint-matching script** (see **Invalid parameter validation** below). Success-path e2e and invalid-param dfx tests must stay in separate modules.
+
+**1b. Model type — `omni` vs `tts` vs `diffusion`**
+
+After choosing offline/online/docs/perf, classify the **product under test** and attach **exactly one** model-type marker. All three can live under the same `tests/e2e/` trees; conventions diverge:
+
+| Dimension | **Omni** (`pytest.mark.omni`) | **TTS** (`pytest.mark.tts`) | **Diffusion** (`pytest.mark.diffusion`) |
+|-----------|------------------------------|----------------------------|----------------------------------------|
+| **What it is** | Multimodal LLM pipeline (thinker/talker/stages; text + vision + audio I/O) | Speech synthesis / voice models | Generative diffusion (noise → image, audio, text, or video) |
+| **Examples** | Qwen2.5-Omni, Qwen3-Omni | Qwen3-TTS, VoxCPM2, Higgs-Audio, Step-Audio2 | Qwen-Image, BAGEL, Wan2.2, HunyuanVideo |
+| **Offline runner** | `omni_runner` + `omni_runner_handler`, `generate_multimodal` | `Omni(...).generate` or stage YAML + TTS params | `Omni(...).generate` + `OmniDiffusionSamplingParams` |
+| **Online client** | `openai_client.send_omni_request` (chat completions, modalities) | `openai_client.send_audio_speech_request` (`/v1/audio/speech`) | `send_diffusion_request` (chat/T2I), `send_video_diffusion_request` (`/v1/videos`, X2V), or `send_images_generations_http_request` / `send_images_edits_http_request` (DALL-E routes) |
+| **Typical assertions** | Stage outputs, text/audio keywords via `OmniRunnerHandler` / response handler | WAV bytes, stream chunks, speech endpoint contract | Image/video dimensions, `final_output_type`, `assert_diffusion_response` |
+| **Stage / deploy YAML** | Per-model omni stage configs (`ci/qwen3_omni_moe.yaml`, …) | `qwen3_tts.yaml`, `voxcpm2.yaml`, … | Often default serve; parallel/offload YAML for heavy DiT |
+| **Nightly group** (`test-nightly.yml`) | **Omni Model Test** — `-m "full_model and omni"` | **TTS Model Test** — `-m "full_model and tts"` | **Diffusion X2I(&A&T)** *or* **Diffusion X2V** (see Step 3 table; same `diffusion` marker, different YAML group / file shard) |
+| **L4 pressure** | Expansion per modality/model as needed | Expansion + accuracy/perf in TTS group | X2I: merge feature combos per [#1832](https://github.com/vllm-project/vllm-omni/issues/1832); X2V: separate nightly group |
+
+Do **not** mix fixtures across types (e.g. do not use `omni_runner` layout for a pure diffusion or TTS model without mirroring an in-tree test in that family).
+
+**Diffusion only — X2I(&A&T) vs X2V (nightly routing, not extra markers):**
+
+- **X2I(&A&T)**: image / audio / text generation — Qwen-Image*, FLUX, SD3, Z-Image, BAGEL (expansion), LongCat, audio diffusion, etc.
+- **X2V**: video generation only — Wan2.2, HunyuanVideo 1.5, Wan VACE, LTX video similarity paths.
+
+When adding a new `test_*_expansion.py`, place it in the matching **nightly group** step (explicit file list in `test-nightly.yml`), not only by marker expression.
+
+**2. Use the narrowest deterministic skeleton for the scenario**
+
+*L1 unit / logic (CPU-first):*
+
+**Mocking rule (L1 only):** use **pytest** integration — **`mocker`** (`pytest-mock`) or **`monkeypatch`** (built-in). **Do not** import or call **`unittest.mock`** (`patch`, `MagicMock`, `@patch`, `with patch(...)`, etc.) in L1 tests; patches must auto-revert with the test lifecycle.
+
+```python
+import pytest
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+
+def test_<scenario_name>(mocker):
+    # Prefer mocker.patch / mocker.spy / mocker.Mock — not unittest.mock.patch
+    fake_fn = mocker.patch("vllm_omni.some.module.expensive_call", return_value=...)
+    # Act
+    # Assert
+    fake_fn.assert_called_once()
+
+
+def test_<env_or_attr>(monkeypatch):
+    # For simple env / attribute substitution without a Mock object
+    monkeypatch.setenv("SOME_FLAG", "1")
+    monkeypatch.setattr("vllm_omni.some.module.CONST", 42)
+    ...
+```
+
+**Avoid in L1:**
+
+```python
+# BAD — do not use in L1 unit tests
+from unittest.mock import patch, MagicMock
+
+@patch("vllm_omni.some.module.fn")
+def test_bad(mock_fn): ...
+
+def test_also_bad():
+    with patch("...") as m: ...
+```
+
+See **L1 unit test constraints (mocking)** below for the full do/don't list.
+
+*Offline multimodal e2e — **Omni** (representative):*
+
+```python
+@pytest.mark.core_model
+@pytest.mark.omni
+@hardware_test(...)
+@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
+def test_<scenario>(omni_runner, omni_runner_handler) -> None:
+    request_config = {"prompts": ..., "modalities": [...]}  # optional: images, videos, audios
+    omni_runner_handler.send_omni_request(request_config)
+```
+
+*Offline generative e2e — **Diffusion** (representative):*
+
+```python
+@pytest.mark.core_model
+@pytest.mark.diffusion
+@hardware_test(...)
+@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
+def test_text_to_image_001(omni_runner_handler) -> None:
+    omni_runner_handler.send_diffusion_request({"prompt": "...", "extra_body": {"num_inference_steps": 4, ...}})
+```
+
+*Offline TTS e2e — **Qwen3-TTS** (two-stage; representative):*
+
+```python
+@pytest.mark.advanced_model
+@pytest.mark.tts
+@hardware_test(...)
+@pytest.mark.parametrize("omni_runner", tts_server_params, indirect=True)
+def test_text_to_audio_001(omni_runner, omni_runner_handler) -> None:
+    omni_runner_handler.send_audio_speech_request({
+        "input": "...",
+        "task_type": "Base",
+        "ref_audio": REF_AUDIO_URL,
+        "ref_text": REF_TEXT,
+    })
+```
+
+*Offline TTS e2e — **single-stage** (Coqui XTTS, MOSS-TTS-Nano; representative):*
+
+```python
+@pytest.mark.advanced_model
+@pytest.mark.tts
+@hardware_test(...)
+@pytest.mark.parametrize("omni_runner", tts_server_params, indirect=True)
+def test_voice_clone_001(omni_runner_handler) -> None:
+    omni_runner_handler.send_single_stage_tts_request({
+        "input": "...",
+        "language": "en",
+        "prompt_audio_path": REF_AUDIO_PATH,
+        "response_format": "wav",
+        "run_level": "advanced_model",
+    })
+```
+
+*Online serving e2e — **Omni** (representative):*
+
+```python
+@pytest.mark.core_model
+@pytest.mark.omni
+@hardware_test(...)
+@pytest.mark.parametrize("omni_server", test_params, indirect=True)
+def test_text_to_text_001(omni_server, openai_client) -> None:
+    request_config = {"model": omni_server.model, "messages": ..., "modalities": ["text"]}
+    openai_client.send_omni_request(request_config)
+```
+
+*Online serving e2e — **TTS** (representative):*
+
+```python
+@pytest.mark.core_model
+@pytest.mark.advanced_model
+@pytest.mark.tts
+@hardware_test(...)
+@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
+def test_text_to_audio_001(omni_server, openai_client) -> None:
+    request_config = {"model": omni_server.model, "input": "...", "response_format": "wav", ...}
+    openai_client.send_audio_speech_request(request_config)
+```
+
+*Online serving e2e — **Diffusion** X2I (representative):*
+
+```python
+@pytest.mark.core_model
+@pytest.mark.diffusion
+@pytest.mark.parametrize("omni_server", _get_default_case(MODEL), indirect=True)
+def test_text_to_image_001(omni_server, openai_client) -> None:
+    openai_client.send_diffusion_request({...})  # chat completions + extra_body
+```
+
+*Online serving e2e — **Diffusion** X2V (representative):*
+
+```python
+@pytest.mark.core_model
+@pytest.mark.diffusion
+def test_text_to_video_001(omni_server, openai_client) -> None:
+    openai_client.send_video_diffusion_request({"model": ..., "form_data": {...}})  # /v1/videos
+```
+
+*Documentation example tests:* follow the **Preferred Test Strategy** in [l4_doc_example_tests.inc.md](../../../docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md): dynamic extraction for offline READMEs; explicit copied client code for online pages until extraction is justified; use the documented **naming**, **output directory** (page folder + case id), and **skipping** rules (e.g. Gradio-only scripts).
+
+*Performance tests:* add or extend entries under `tests/dfx/perf/tests/` (and JSON configs where the project uses them), with **explicit baselines**, **`mark`** on each case (`hardware_marks` + `full_model` + type marker), and the same nightly Perf step pattern as in-tree configs.
+
+**3. Cross-cutting rules**
+
+- Reuse existing fixtures for the chosen scenario; do not mix “online client” assumptions into offline `OmniRunner` tests without a clear reason.
+- Avoid external network dependency in assertions unless the scenario is explicitly “online serving” or doc examples that require a model hub (then align with CI secrets/cache).
+- Keep **one test function = one intent** (one modality combo, one endpoint contract, or one acceleration combo).
+- **E2E test function layout**: **one case → one `test_<scenario>` function** with a name that states what is validated (endpoint, size/`n`, server flag, or route). **Do not** merge multiple cases into a single test that branches on `request.node.callspec.id`, `param.id`, or `if case_id == ...`. Use `@pytest.mark.parametrize("omni_server", [...], indirect=True)` per function (usually **one** `OmniServerParams` per test). A loop inside one test is OK only when it serves **that function’s single intent** (e.g. three standard sizes in `test_*_sizes_256_512_1024`).
+- **Runtime fixture scope** (`tests/helpers/fixtures/runtime.py`): default **`omni_server` / `omni_runner`** (module) + matching client/handler; use **`omni_server_function` / `omni_runner_function`** only when each `test_*` must start a fresh instance (see below).
+- **L1 mocks**: never `unittest.mock`; use `mocker` or `monkeypatch` only (see below).
+- **API calls (L2+ e2e, online and offline)**: **reuse** `send_*_request` in `tests/helpers/runtime.py` when it exists; **otherwise add** the helper in `runtime.py` first, then call it from the test. General `assert_*` inside `send_*_request`; special `assert_*` only in the test. See **Runtime send helpers** below — do not call `omni.generate`, raw HTTP, or SDK clients from `test_*.py`.
+- **Response assertions**: reusable checks on API bodies / decoded media belong in **`tests/helpers/assertions.py`** — not as private `_assert_*` helpers inside `test_*.py` (see below).
+- **Model-specific payloads stay in test modules** — per-model `MODEL`, deploy path, `REF_AUDIO_URL`, `get_prompt()`, `_build_request()`, and inline `request_config` dicts live in `test_{slug}.py` / `test_{slug}_expansion.py` (and offline/L1 siblings). **Do not** create `tests/helpers/{slug}.py` to deduplicate them; a little copy across files is preferred (see `test_glm_tts.py`, `test_cosyvoice3_tts_expansion.py`). `tests/helpers/` is repo-wide harness only (`mark`, `media`, `runtime`, `stage_config`, `assertions`, `fixtures/`).
+
+#### L1 unit test constraints (mocking)
+
+L1 tests (`core_model and cpu`, under `tests/diffusion/`, `tests/engine/`, `tests/model_executor/`, etc.) must follow the repo’s **pytest-mock** convention (`pytest-mock>=3.10.0` in `pyproject.toml` `[project.optional-dependencies] dev`).
+
+| Do | Don't |
+|----|-------|
+| `def test_foo(mocker):` + `mocker.patch(...)`, `mocker.spy(...)`, `mocker.Mock()`, `mocker.MagicMock()`, `mocker.AsyncMock()` | `from unittest.mock import patch, MagicMock, Mock, AsyncMock` |
+| `def test_bar(monkeypatch):` + `monkeypatch.setattr` / `setenv` / `delenv` / `setitem` | `@patch(...)` decorator |
+| Let `mocker` auto-stop patches after the test | `with patch(...):` / `patch.object(...)` context managers |
+| Mirror neighboring L1 tests in the same directory | `unittest.mock.create_autospec` unless an existing file already documents an exception |
+
+**Rationale:** `mocker` ties patch lifecycle to pytest fixtures (no leaked patches across tests). `unittest.mock` decorators/context managers are easy to compose incorrectly with parametrized or async tests and are inconsistent with in-tree L1 style.
+
+**Minimal patterns:**
+
+```python
+def test_returns_cached_config(mocker):
+    loader = mocker.patch(
+        "vllm_omni.foo.load_yaml",
+        return_value={"stages": []},
+    )
+    result = get_deploy_config("ci/foo.yaml")
+    assert result["stages"] == []
+    loader.assert_called_once_with("ci/foo.yaml")
+
+
+def test_skips_when_env_unset(monkeypatch):
+    monkeypatch.delenv("VLLM_OMNI_FEATURE", raising=False)
+    assert should_enable_feature() is False
+```
+
+E2E levels (L2+) generally **avoid mocks**; if a rare L2 stub is unavoidable, still prefer `mocker` over `unittest.mock` for consistency.
+
+#### Runtime fixtures — scope (`tests/helpers/fixtures/runtime.py`)
+
+vLLM-Omni e2e tests start a real **OmniServer** (online) or **OmniRunner** (offline). Pick scope by **how often the process must be recreated**, not by test level alone.
+
+| Scope | Online fixtures | Offline fixtures | When to use |
+|-------|-----------------|------------------|-------------|
+| **Module** (default) | `omni_server` → `openai_client` | `omni_runner` → `omni_runner_handler` | **Default for L2/L3/L4 expansion** — amortize model/server init across `test_*` in the same module. Same `OmniServerParams` / runner config can be reused by multiple tests. |
+| **Function** | `omni_server_function` → `openai_client_function` | `omni_runner_function` → `omni_runner_handler_function` | **Only when required** — each `test_*` must get a **clean** server/runner (no shared engine/GPU state). Typical: `tests/dfx/reliability/`, sleep/wakeup, crash/restart, tests that mutate global server state. |
+
+**Rules:**
+
+1. **Default to module scope** (`omni_server` / `omni_runner`) unless the scenario **explicitly** needs a fresh instance per test function.
+2. **Indirect parametrize name must match the fixture name**: `@pytest.mark.parametrize("omni_server", ...)` with `omni_server` + `openai_client`; `@pytest.mark.parametrize("omni_server_function", ...)` with `omni_server_function` + `openai_client_function`. Do not mix module fixture with function client (or vice versa).
+3. Different `OmniServerParams` per `test_*` (e.g. default vs `--enable-cpu-offload`) is still OK with **module** `omni_server` — pytest parametrizes per test node; only switch to `_function` when isolation between tests matters, not merely because `server_args` differ.
+4. One `test_*` with **many** server configs (expansion matrix) → single function + `@pytest.mark.parametrize("omni_server", [...], indirect=True)` + module `omni_server` (see in-tree `test_qwen_image_expansion.py`).
+
+```python
+# Default — module-scoped server (L4 expansion)
+@pytest.mark.parametrize("omni_server", [pytest.param(OmniServerParams(model=MODEL), marks=H100)], indirect=True)
+def test_foo_images_generations_default_1024(omni_server, openai_client) -> None:
+    openai_client.send_images_generations_request({...})
+
+# Function-scoped — reliability / per-test clean state only
+@pytest.mark.parametrize(
+    "omni_server_function",
+    [pytest.param(OmniServerParams(model=MODEL), marks=H100)],
+    indirect=True,
+)
+def test_foo_sleep_wakeup_cycle(omni_server_function, openai_client_function) -> None:
+    openai_client_function.send_omni_sleep_http_request({...})
+    openai_client_function.send_omni_wakeup_http_request({...})
+```
+
+#### Runtime send helpers — online **and** offline (`tests/helpers/runtime.py`)
+
+**L2+ e2e (online serving and offline inference) must call APIs through `tests/helpers/runtime.py`.** Fixtures live in `tests/helpers/fixtures/runtime.py`; **send/assert implementation** lives in `tests/helpers/runtime.py` + `tests/helpers/assertions.py`.
+
+| Principle | Action |
+|-----------|--------|
+| **Reuse first** | Grep `runtime.py` for an existing **`send_*_request`** on `OpenAIClientHandler` (online) or `OmniRunnerHandler` (offline) that matches the endpoint / pipeline shape. |
+| **Extend when close** | If an existing `send_*_request` almost fits (e.g. missing one optional field), extend it in `runtime.py` — do not fork logic in the test file. |
+| **Add when missing** | No suitable helper → add **`send_<feature>_request`** (high-level: call + general `assert_*`) or **`send_<route>_<verb>_http_request`** (low-level HTTP for negative/dfx) in **`runtime.py` first**, then call it from tests. |
+| **Test module owns payload only** | In `test_*.py`: `MODEL`, deploy path, vendored media, `get_prompt()`, and inline **`request_config` dicts** only. **No** `omni.generate(...)`, raw `requests.post`, OpenAI SDK calls, or `_collect_audio()` / `_process_output()` in e2e tests. |
+
+**Online** (`openai_client` from `omni_server`): `OpenAIClientHandler.send_*_request`.
+
+**Offline** (`omni_runner_handler` from `omni_runner`): `OmniRunnerHandler.send_*_request`.
+
+| Do | Don't |
+|----|-------|
+| Online: `openai_client.send_omni_request`, `send_diffusion_request`, `send_audio_speech_request`, `send_video_diffusion_request`, `send_images_generations_request`, … | `requests.post(f"{base_url}/v1/...", json=…)` or `client.chat.completions.create(...)` inside a test |
+| Offline: `omni_runner_handler.send_omni_request`, `send_diffusion_request`, `send_audio_speech_request`, `send_single_stage_tts_request`, `send_single_stage_tts_batch_request`, … | `omni_runner.omni.generate(...)` + hand-rolled tensor/WAV extraction in `test_*.py` |
+| Add missing `send_*` to **`runtime.py`** first; bundle general `assert_*` inside the send helper | A one-off `def _post_*` / `def _collect_audio` at the bottom of a test module |
+| Mirror naming/style of neighboring `send_*` (docstring, `request_config` dict, optional `run_level`, `err_code` / `err_message` for negative cases) | Different parameter shapes per test file for the same endpoint |
+
+**Workflow when generating online/offline e2e tests:**
+
+1. Decide whether the needed check is **general** (every success call of this `send_*`) or **special** (one case / one parameter combo only). See **General vs special assert placement** below.
+2. **Search `runtime.py`** for a matching **`send_*_request`**; if missing, add it (with general `assert_*` inside) before writing the test body.
+3. In the test module: build `request_config` → call **`send_*_request` only** for the general contract; call **`assert_*` in the test file only** for special, case-specific checks (import from `assertions.py`).
+4. Reserve **low-level** `send_*_http_request` for negative/dfx tests (`err_code` / `err_message`) — not for ordinary L2+ success-path e2e.
+5. When wiring Buildkite `source_file_dependencies`, include **`tests/helpers/runtime.py`** and/or **`tests/helpers/assertions.py`** when new helpers were added.
+
+**Exceptions (document in the test docstring why):** models whose offline prompt path cannot go through existing handlers yet (e.g. `test_higgs_audio_v2.py`, `test_voxtral_tts.py` with custom tokenizer compose) may call `omni.generate` directly until a `send_*_request` is added to `runtime.py` — treat as **debt**, not the default for new TTS/diffusion/omni e2e.
+
+**General vs special assert placement:**
+
+| Kind | Definition | Where it lives | Test file calls |
+|------|------------|----------------|-----------------|
+| **General** | Default success contract for this endpoint/client method — e.g. HTTP 200, `data[]` shape, `n` count, `size` dimensions, decodable image/audio, bundled omni/diffusion fields | Implement in `assertions.py`, invoke **inside** `send_*_request` in `runtime.py` (after low-level send / SDK call, when `err_code` is not set) | **`send_*_request` only** — do **not** repeat the same `assert_*` |
+| **Special** | Extra check tied to **this test case only** — e.g. seed byte-identical replay, accuracy/CLIP threshold, perf ceiling, model-specific optional field | Add or reuse `assert_*` in `assertions.py` (never inline in test) | `send_*_request` **then** `assert_<special>(...)` **once** for that case |
+
+Changing `request_config` fields (`size`, `n`, `seed`, server flags) is **not** special validation — the general `assert_*` should read those from `request_config` inside `send_*`.
+
+**`send_*` ↔ `assert_*` pairing (online `OpenAIClientHandler`):**
+
+| High-level `send_*` (prefer in L2+ e2e) | Assert already invoked inside `runtime.py` | Low-level HTTP-only sibling |
+|----------------------------------------|--------------------------------------------|-----------------------------|
+| `send_omni_request` | `assert_omni_response` | `send_chat_completions_http_request` → `assert_http_error` only |
+| `send_diffusion_request` | `assert_diffusion_response` | — |
+| `send_audio_speech_request` | `assert_audio_speech_response` | `send_audio_speech_http_request` → `assert_http_error` only |
+| `send_video_diffusion_request` | `assert_diffusion_response` | `send_videos_*_http_request` → `assert_http_error` only |
+| `send_images_generations_request` *(add when needed)* | **`assert_images_generations_response`** *(general — inside send)* | `send_images_generations_http_request` → `assert_http_error` only |
+| `send_images_edits_request` *(add when needed)* | **`assert_images_edits_response`** *(general — inside send)* | `send_images_edits_http_request` → `assert_http_error` only |
+
+**Rule:** Tests call high-level `send_*_request` for the general contract. **Never** call the same bundled `assert_*` again. Call an extra `assert_*` in the test **only** for special, case-specific validation.
+
+**Common `OpenAIClientHandler` entry points** (non-exhaustive — grep `runtime.py` before adding):
+
+| Area | High-level (SDK + assert) | Low-level HTTP (`*_http_request`) |
+|------|---------------------------|-----------------------------------|
+| Omni chat | `send_omni_request` | `send_chat_completions_http_request` |
+| Diffusion T2I (chat route) | `send_diffusion_request` | — |
+| Diffusion X2V | `send_video_diffusion_request` | `send_videos_create_http_request`, `send_video_content_http_request`, … |
+| DALL-E T2I / edit | `send_images_generations_request`, `send_images_edits_request` | `send_images_generations_http_request`, `send_images_edits_http_request` |
+| TTS (online) | `send_audio_speech_request` | `send_audio_speech_http_request`, `send_audio_generate_http_request`, … |
+| Ops / meta | — | `send_health_http_request`, `send_models_http_request`, `send_omni_sleep_http_request`, … |
+
+**Common `OmniRunnerHandler` entry points** (offline — grep `runtime.py` before adding):
+
+| Area | High-level (`send_*_request` + assert) | Notes |
+|------|----------------------------------------|-------|
+| Omni multimodal | `send_omni_request` | `generate_multimodal` path |
+| Diffusion offline | `send_diffusion_request` | chat-route / `OmniTextPrompt` offline |
+| Qwen-style TTS | `send_audio_speech_request` | two-stage, `generate_multimodal` + `mm_processor_kwargs` |
+| Single-stage TTS (Coqui XTTS, MOSS-TTS-Nano, …) | `send_single_stage_tts_request`, `send_single_stage_tts_batch_request` | `prompt` + `additional_information` + `omni.generate` — **do not** duplicate in tests |
+| New model family | **Add** `send_<family>_request` here first | Then call from `tests/e2e/offline_inference/test_*.py` |
+
+L1 tests under `tests/entrypoints/` may use **FastAPI `TestClient`** or direct handler calls with mocks; they do **not** need `OpenAIClientHandler` / `OmniRunnerHandler`. **L2+ online and offline e2e** must use `runtime.py` `send_*` helpers.
+
+#### Invalid parameter validation (`tests/dfx/reliability/invalid_param_test/`)
+
+When the user asks for **invalid parameter validation**, **invalid request bodies**, **HTTP 4xx contract tests**, or any case that sends **malformed / out-of-range / mismatched** API payloads against a **live** server:
+
+1. **Do not** place these in `tests/e2e/online_serving/` or `*_expansion.py`. If already drafted there, **move** the `test_*` into the correct `invalid_param_test` script and delete the duplicate from e2e.
+2. **Pick the script by HTTP route** (extend an existing file; add a new `test_invalid_<area>.py` only when no in-tree script covers that route family):
+
+| API / area | Script |
+|------------|--------|
+| Omni chat completions, WebSocket video/realtime paths | `test_invalid_omni_chat.py` |
+| `POST /v1/audio/speech`, stream, batch, voices | `test_invalid_audio_speech.py` |
+| Audio diffusion endpoints | `test_invalid_audio_diffusion.py` |
+| `POST /v1/images/generations` | `test_invalid_image_generation.py` |
+| `POST /v1/images/edits` | `test_invalid_image_editing.py` |
+| `POST/GET/DELETE /v1/videos*` | `test_invalid_video_generation.py` |
+| Sleep / wakeup / server control | `test_invalid_server_control.py` |
+
+3. **Match in-tree style** in the chosen script:
+
+| Element | Convention |
+|---------|------------|
+| **Module markers** | `pytestmark = [pytest.mark.slow, pytest.mark.<omni or tts or diffusion>]` |
+| **Server fixture** | `_PARAMS` / `_QWEN3_TTS_SPEECH`-style list of `pytest.param(OmniServerParams(...), id="...", marks=hardware_marks(...))`; `@pytest.mark.parametrize("omni_server", _PARAMS, indirect=True)` |
+| **Hardware** | `hardware_marks(res={"cuda": "H100"})` for heavy diffusion/omni/video; `hardware_marks(res={"cuda": "L4"})` for smaller TTS models (must match weekly `-m "slow and L4"` step) |
+| **HTTP client** | **Low-level** `openai_client.send_*_http_request({..., "err_code": 400, "err_message": (...)} )` — **not** `send_*_request` (success path) |
+| **Case shape** | Prefer **one `test_*` per route family** + `@pytest.mark.parametrize("body_spec, err_message", [...])` with stable `id=` per case; or dedicated `test_<route>_malformed_json` when not parametrized |
+| **Body helpers** | `_minimal_<endpoint>_json(omni_server)` / `_minimal_*_form_data()`; merge overrides with `body.update(body_spec)` |
+| **Known gaps** | `pytest.mark.skip(reason="…#3649")` as `_SKIP_ISSUE_3649` when server validation is not yet strict (mirror neighboring cases) |
+| **Sections** | Route banner comments (`# ─── POST /v1/images/generations ───`) like existing files |
+| **Shared fixtures** | Reuse `tests/dfx/reliability/invalid_param_test/conftest.py` (`tiny_png_bytes`, env defaults) |
+
+4. **Adding a new model** to invalid-param coverage: append a `pytest.param(OmniServerParams(model="...", stage_config_path=..., server_args=...), ...)` entry to the script’s `_PARAMS` list — do **not** create `test_invalid_<model>.py` unless the route is new.
+
+5. **CI — `.buildkite/cuda/test-weekly.yml` only** (not `test-ready.yml` / `test-merge.yml` / `test-nightly.yml`):
+
+| Weekly step | Command | When your cases run |
+|-------------|---------|---------------------|
+| **Invalid parameters Test · H100** | `pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100"` | Diffusion / omni / video invalid-param tests with `H100` hardware mark |
+| **Invalid parameters Test · L4** | `pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and L4"` | TTS / lighter models marked `L4` |
+
+- **Trigger:** `build.env("WEEKLY") == "1"` or PR label `weekly-test`.
+- **Default:** extending an existing `invalid_param_test` script needs **no YAML edit** — the weekly steps already sweep the whole directory.
+- **Edit YAML only** when adding a **new hardware queue**, a **new top-level script** that must run in isolation, or a **model-specific weekly shard** (mirror neighboring reliability steps).
+- Weekly steps **do not** use `source_file_dependencies`.
+
+**Example — append to `test_invalid_image_generation.py`:**
+
+```python
+@pytest.mark.parametrize(
+    "body_spec, err_message",
+    [
+        pytest.param({"seed": -1}, ("seed", "greater_than_equal", "0"), id="seed_negative"),
+    ],
+)
+@pytest.mark.parametrize("omni_server", _PARAMS, indirect=True)
+def test_images_generations_invalid_requests(
+    omni_server: OmniServer,
+    openai_client: OpenAIClientHandler,
+    body_spec: dict[str, object],
+    err_message: str | tuple[str, ...],
+) -> None:
+    body = _minimal_images_gen_json(omni_server)
+    body.update(body_spec)
+    openai_client.send_images_generations_http_request(
+        {"json": body, "timeout": 300, "err_code": 400, "err_message": err_message}
+    )
+```
+
+See [references/test-routing.md](references/test-routing.md) **Invalid parameter / weekly CI**.
+
+#### Assertion helpers (`tests/helpers/assertions.py`)
+
+**Do not** add module-local helpers such as `_assert_images_generations_payload` or `_send_and_assert_*` in e2e test files. Response/media validation belongs in **`tests/helpers/assertions.py`**, grouped by **category**:
+
+| Category | Existing anchors | When to extend vs add |
+|----------|------------------|------------------------|
+| **Image** (chat diffusion + DALL-E JSON) | `assert_image_diffusion_response`, `assert_image_valid` | DALL-E `/v1/images/generations` JSON → add **`assert_images_generations_response`** beside image helpers (reuses `assert_image_valid`); do not fork decode logic into tests |
+| **Video** | `assert_video_diffusion_response`, `assert_video_valid` | Extend these for new video contracts |
+| **Audio** | `assert_audio_diffusion_response`, `assert_audio_speech_response`, `assert_audio_valid` | Extend for new TTS/audio endpoints |
+| **Omni multimodal** | `assert_omni_response` | Extend for new modality combos |
+| **HTTP errors** | `assert_http_error`, `assert_err_message_in_text` | Used inside low-level `send_*_http_request` and negative tests |
+
+| Do | Don't |
+|----|-------|
+| Put **general** `assert_*` inside `send_*_request` in `runtime.py`; tests call `send_*_request` only | Call `assert_diffusion_response` after `send_diffusion_request` (general assert already bundled) |
+| Put **special** `assert_*` in `assertions.py` and call it **in the test** after `send_*_request` when that case needs extra checks | Put general decode/count/size logic in the test because “this case uses `n=4`” (that belongs in general assert reading `request_config`) |
+| Extend the matching category function, or add `assert_<endpoint>_response` next to its category | `_send_and_assert_*` or PIL/base64 loops in `test_*.py` |
+| Use low-level `send_*_http_request` in dfx/negative tests (`err_code`) | Use low-level HTTP send + manual general assert in every L2+ expansion file |
+
+**Workflow when generating tests:**
+
+1. Implement or reuse **general** `assert_*` in `assertions.py` (by category).
+2. Wire it into **`send_*_request`** in `runtime.py` if not already bundled.
+3. In the test: `request_config` → `send_*_request`. Add a separate `assert_*` import/call **only** for special case validation.
+4. Update `source_file_dependencies` when `runtime.py` or `assertions.py` changes.
+
+**Example — DALL-E `/v1/images/generations`:**
+
+```python
+# tests/helpers/assertions.py — general contract (image category)
+def assert_images_generations_response(resp_body: dict, request_config: dict, *, run_level: str | None = None) -> None:
+    ...  # data[], n, size → assert_image_valid
+
+# tests/helpers/runtime.py — general assert INSIDE send
+def send_images_generations_request(self, request_config: dict[str, Any], ...) -> list[HttpResponse]:
+    responses = self.send_images_generations_http_request(request_config, ...)
+    if request_config.get("err_code") is None:
+        body = responses[0].json_body
+        assert isinstance(body, dict)
+        assert_images_generations_response(body, request_config, run_level=self.run_level)
+    return responses
+
+# tests/e2e/.../test_foo_expansion.py — one case per test_* (no case_id branching)
+@pytest.mark.parametrize("omni_server", [pytest.param(OmniServerParams(model=MODEL), marks=SINGLE_L4)], indirect=True)
+def test_foo_images_generations_default_1024(omni_server, openai_client) -> None:
+    openai_client.send_images_generations_request({"json": body, "timeout": 300})
+
+@pytest.mark.parametrize("omni_server", [pytest.param(OmniServerParams(model=MODEL), marks=SINGLE_L4)], indirect=True)
+def test_foo_images_generations_sizes_256_512_1024(omni_server, openai_client) -> None:
+    for size in ("256x256", "512x512", "1024x1024"):
+        openai_client.send_images_generations_request({"json": {**body, "size": size}, "timeout": 300})
+
+# chat diffusion — separate test; send_diffusion_request bundles assert_diffusion_response
+@pytest.mark.parametrize("omni_server", [pytest.param(OmniServerParams(model=MODEL), marks=SINGLE_L4)], indirect=True)
+def test_foo_chat_completions_t2i_fallback(omni_server, openai_client) -> None:
+    openai_client.send_diffusion_request({...})
+```
+
+L1 tests under `tests/entrypoints/` may keep **minimal** asserts next to the handler under test when validating a single branch; still prefer `assertions.py` when the same JSON/media check appears in more than one file or level (e.g. L1 protocol test + L4 e2e).
+
+#### Omni Test Writing Guidance (L1-L4 Layering)
+
+When the goal is a general Omni/multimodal test case, prioritize mapping the test to the correct purpose, directory, and resource assumptions aligned with the layers (see [test_system_overview.md](../../../docs/contributing/ci/test_system_overview.md) and [test_writing_guide.md](../../../docs/contributing/ci/test_writing_guide.md)):
+
+- **L1**: Unit/logic validation on CPU (`core_model and cpu`). Cover input validation, branches, and exception paths (`tests/<component>/test_*.py`). **Mock with `mocker` / `monkeypatch` only — not `unittest.mock`.**
+- **L2**: Basic e2e (online/offline basic scenarios). Prefer dummy/lightweight models to validate the end-to-end request-to-output-structure/streaming chain (typically `tests/e2e/online_serving/` and `tests/e2e/offline_inference/`).
+- **L3/L4**: Important integration, performance, and accuracy validation. L4 emphasizes “full functional scenarios + performance/stress + runnable doc examples” (typically `*_expansion.py` plus related expansion cases).
+
+Also keep markers consistent with the run level: use `core_model` for L1/L2 and `advanced_model` for L3/L4, and pair with `--run-level` to select the intended CI strategy.
+
+#### Diffusion Test Writing Guidance (L4 Coverage Combinations)
+
+When the task involves diffusion models/features, organize L4 test cases following [`#1832`](https://github.com/vllm-project/vllm-omni/issues/1832): combine multiple diffusion features into as few test cases as possible to fit limited CI GPU resources.
+
+Implementation strategy:
+
+1. If “full L4 is too heavy”: first provide a **reduced local validation case in L1/L2** (so the key assertion and the contract fix point are covered deterministically).
+2. Then provide **CI/nightly-ready L3/L4 high-marked cases** (e.g. `advanced_model`) to broaden coverage under resource constraints.
+
+#### L4 nightly sub-pillars — Function vs Accuracy vs Perf
+
+**L4 is not only `test_*_expansion.py`.** In `test-nightly.yml`, each model-type group typically runs **separate jobs**:
+
+| Sub-pillar | Nightly step label pattern | What you add | Typical paths |
+|------------|----------------------------|--------------|---------------|
+| **Function** | `· Function Test with …` | E2e expansion / feature matrix | `tests/e2e/online_serving/test_<model>_expansion.py` (or offline) |
+| **Accuracy** | `· Accuracy Test` | Quality / similarity vs baseline | `tests/e2e/accuracy/test_<model>*.py` |
+| **Perf** | `· Perf Test · <Model>` | Throughput / latency / memory benchmarks | `tests/dfx/perf/tests/test_<model>_vllm_omni.json` + runner script |
+| **Doc** (optional) | `· Doc Test` | Runnable doc examples | `tests/examples/*/test_text_to_image.py`, … |
+
+**Default when the user asks for “L4 functional cases”:** deliver **Function** pillar only (`*_expansion.py` + Function Test shard in `test-nightly.yml`). **Do not** silently add Perf or Accuracy unless the user also asks for **performance** / **benchmark** / **accuracy** / **full L4** / **full L4 coverage**.
+
+**When the user explicitly asks for L4 perf (or full L4 including perf):**
+
+1. **Do not** put throughput/latency assertions inside `test_*_expansion.py` — perf uses the **dfx benchmark harness**, not `omni_server` e2e fixtures.
+2. Add or extend a **JSON config** under `tests/dfx/perf/tests/` (mirror in-tree names: `test_qwen_image_vllm_omni.json`, `test_cosmos3_vllm_omni.json`, `test_qwen3_omni_async_chunk.json`).
+3. Each JSON **case**: `test_name`, optional **`mark`** (`hardware_marks` required when present; `marks`: `full_model` + `omni`/`tts`/`diffusion`), `server_params` (`model`, `serve_args` or `stage_config_name`), `benchmark_params[]` with **`name`** and explicit **`baseline`** (`throughput_qps`, `latency_mean`, `peak_memory_mb_mean`, …). Diffusion cases also set `server_type` (e.g. `"vllm-omni"`) and usually `benchmark_endpoint`.
+4. Run locally via the matching script (model type → runner):
+
+| Model type | Runner script | Example config |
+|------------|---------------|----------------|
+| **Diffusion X2I/X2V** | `tests/dfx/perf/scripts/run_diffusion_benchmark.py` | `tests/dfx/perf/tests/test_qwen_image_vllm_omni.json` |
+| **TTS** | `tests/dfx/perf/scripts/run_benchmark.py` | `tests/dfx/perf/tests/test_tts.json` (shared) or `test_{slug}.json` (dedicated; use when the model must not join the shared nightly matrix before integration — e.g. `test_voxcpm2.json`, `test_coqui_tts.json`) |
+| **Omni** | `tests/dfx/perf/scripts/run_benchmark.py` | `test_qwen3_omni_no_async_chunk.json`, `test_qwen3_omni_async_chunk.json`, `test_qwen3_omni_vllm_text.json`, `test_qwen3_omni_multi_replicas.json` |
+
+5. Wire **`test-nightly.yml` · Perf Test** step for that model (separate from Function Test): export `DIFFUSION_BENCHMARK_DIR` / `BENCHMARK_DIR`, run pytest on the script with **`--test-config-file`** (nightly perf steps do not use `-m`), **upload artifacts** (`buildkite-agent artifact upload`), often **multi-GPU H100** for diffusion perf.
+6. **One benchmark scenario → one `test_name` block** in JSON (same spirit as one `test_*` per function case). Combine server `serve_args` + workload in one entry; use multiple `benchmark_params` rows for size/step sweeps under the same server config.
+
+**Perf local commands:**
+
+```bash
+cd tests
+export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
+# Single file (CI-like)
+pytest -s -v dfx/perf/scripts/run_diffusion_benchmark.py \
+  --test-config-file dfx/perf/tests/test_<model>_vllm_omni.json
+# Bulk load + filter by JSON mark
+pytest -sv dfx/perf/scripts/run_diffusion_benchmark.py -m "full_model and diffusion and H100"
+pytest -sv dfx/perf/scripts/run_benchmark.py -m "full_model and omni and H100"
+```
+
+**Clarify in the test plan** which L4 pillars you deliver: `Function only` | `Function + Perf` | `Function + Accuracy + Perf`.
+
+See [references/test-routing.md](references/test-routing.md) **L4 nightly pillars** for Buildkite step patterns.
+
+### Step 5: Wire Buildkite (when CI must run the new test)
+
+If the test is not already collected by an existing pipeline command (for example, L1 tests marked `core_model and cpu` are already covered by the **Simple Unit Test** step in ready/merge), update the appropriate pipeline under `.buildkite/`:
+
+| Test level | Edit this file | Typical trigger / intent |
+|------------|----------------|---------------------------|
+| **L1** and **L2** | [`.buildkite/cuda/test-ready.yml`](../../../.buildkite/cuda/test-ready.yml) | PR **ready** label; L1 CPU + L2 GPU/basic e2e (steps labeled **Omni ·**, **TTS ·**, **Diffusion ·**) |
+| **L3** | [`.buildkite/cuda/test-merge.yml`](../../../.buildkite/cuda/test-merge.yml) | Post-merge; `advanced_model` integration per model type |
+| **L4** | [`.buildkite/cuda/test-nightly.yml`](../../../.buildkite/cuda/test-nightly.yml) | Nightly; grouped by model type (see below) |
+| **Invalid param / reliability (weekly)** | [`.buildkite/cuda/test-weekly.yml`](../../../.buildkite/cuda/test-weekly.yml) | Weekly; `tests/dfx/reliability/invalid_param_test/` — **not** L1–L4 e2e pipelines |
+
+**Level-specific Buildkite delivery (follow the user’s requested level):**
+
+| User asks for | Document / edit **only** | Do **not** add by default |
+|---------------|---------------------------|---------------------------|
+| **L4** (`*_expansion.py`, `full_model`) | `test-nightly.yml` — append file to the matching nightly shard (X2I / X2V / Omni / TTS) **or** note it is already collected by an existing `-m` / `-k` sweep | `test-merge.yml` (L3 / `advanced_model`) or `test-ready.yml` (L2) |
+| **L3** | `test-merge.yml` with `advanced_model` + `source_file_dependencies` | `test-nightly.yml` unless user also wants nightly |
+| **L2** | `test-ready.yml` with `core_model` + `source_file_dependencies` | merge / nightly pipelines |
+| **Invalid param** | `test-weekly.yml` — **Invalid parameters Test** group (`-m "slow and H100"` / `-m "slow and L4"`). Usually **no YAML edit** when appending cases to existing `invalid_param_test` scripts | `test-ready.yml`, `test-merge.yml`, `test-nightly.yml` |
+
+`test-nightly.yml` shards use **explicit pytest file paths** in `commands` (and PR labels like `diffusion-x2iat-test`); they generally **do not** use `source_file_dependencies`. Only suggest merge/ready YAML when the requested level is L3/L2 (or the user explicitly asks for multi-level CI).
+
+**`test-nightly.yml` top-level groups** (each uses `full_model` + type marker in pytest `-m`):
+
+| Group | Type marker | Notes |
+|-------|-------------|--------|
+| **Omni Model Test** | `omni` | Function / doc / accuracy / **perf** for Qwen-Omni family |
+| **TTS Model Test** | `tts` | `-m "full_model and L4 and tts"` (and H100 variants when enabled); **Perf Test** uses `run_benchmark.py` + `test_tts.json` |
+| **Diffusion X2I(&A&T) Model Test** | `diffusion` | Image/audio/text diffusion; explicit file shards for **Function**; **Perf Test · &lt;Model&gt;** uses `run_diffusion_benchmark.py` + JSON config |
+| **Diffusion X2V Model Test** | `diffusion` | Video-only (Wan, HunyuanVideo, …); PR label `diffusion-x2v-test`; separate **Perf** steps when in-tree |
+
+Guidelines when editing YAML:
+
+- **Match markers and `--run-level`** to the test: L1/L2 in `test-ready.yml` use `core_model` + `omni` / `tts` / `diffusion` as appropriate; L3 merge uses `advanced_model`; L4 nightly uses `full_model` with the same type marker.
+- **Diffusion L4**: add x2i/x2a/x2t expansion files to an **X2I(&A&T)** step; add x2v expansion files to **X2V** — never rely on a broad `tests/e2e/` sweep alone (shards exist to save GPU and avoid wrong queue).
+- **`source_file_dependencies` (required for new E2E jobs in `test-ready.yml` and `test-merge.yml`)**: every step under the **`:card_index_dividers: E2E Test`** group **must** declare `source_file_dependencies` so Buildkite only schedules the job when relevant paths change. Omitting it breaks path-based CI skipping for GPU e2e jobs.
+- **Reuse or extend an existing step** when the new test shares the same marker expression, queue, and timeout; otherwise add a new `steps:` entry with the correct `agents.queue`, `timeout_in_minutes`, and docker/kubernetes plugin blocks consistent with neighboring jobs.
+- **Job timeout — use `timeout_in_minutes` on the step, not `timeout` before pytest**: when adding or documenting a new E2E job in `test-ready.yml` / `test-merge.yml`, set `timeout_in_minutes` on the step and run `pytest` directly in `commands`. **Do not** wrap pytest in `timeout 40m bash -c "..."` / `timeout 20m bash -c "..."`; Buildkite already enforces the step deadline via `timeout_in_minutes`.
+- **Never omit `plugins` in skill examples or PR YAML snippets**: H100 E2E steps on `mithril-h100-pool` use the full `kubernetes` block (`resources`, `volumeMounts`, `env` with `HF_TOKEN` secret, `nodeSelector: gpu-h100-sxm`, `devshm` + `hf-cache` volumes). L4/docker steps on `gpu_1_queue` use the full `docker#v5.2.0` block (`shm-size`, `HF_HOME`, volumes). Do not write `# ... plugins unchanged` — paste the complete block from a sibling step.
+- **Platform forks** (e.g. AMD-ready / AMD-merge) live alongside these files; apply the same level → file mapping for those pipelines when the test is platform-specific.
+
+#### `source_file_dependencies` for E2E Test jobs (`test-ready.yml`, `test-merge.yml`)
+
+When adding or splitting a step inside the **E2E Test** group, **always** include `source_file_dependencies` on that step. List paths the job actually depends on — at minimum:
+
+1. **Test module(s)** — every `tests/e2e/...` file the `pytest` command runs (online and offline if both are in one step).
+2. **Model implementation** — `vllm_omni/model_executor/models/<family>/` and/or `vllm_omni/diffusion/models/<family>/` as applicable.
+3. **Stage / input plumbing** (omni & multi-stage TTS) — `vllm_omni/model_executor/stage_input_processors/<family>.py` when present.
+4. **Deploy / stage YAML** — `vllm_omni/deploy/<config>.yaml` or `vllm_omni/deploy/ci/<config>.yaml` referenced by the test’s `OmniServerParams` / `get_deploy_config_path(...)`.
+5. **Test helpers** (when changed in the same PR) — `tests/helpers/runtime.py` for new `send_*` client methods; `tests/helpers/assertions.py` for new shared `assert_*` helpers.
+
+Use **repo-relative paths** (no leading `./`). Prefer **directory** entries (`.../models/bagel/`) when the whole package matters; use **file** entries for single test modules and YAML configs. Mirror a neighboring step for the same model family (e.g. **Diffusion · Bagel Test** in `test-ready.yml`).
+
+**Example** (L2 ready — diffusion online smoke, H100 kubernetes):
+
+```yaml
+- label: "Diffusion · Bagel Test"
+  source_file_dependencies:
+    - tests/e2e/online_serving/test_bagel.py
+    - vllm_omni/model_executor/models/bagel/
+    - vllm_omni/diffusion/models/bagel/
+    - vllm_omni/model_executor/stage_input_processors/bagel.py
+    - vllm_omni/deploy/bagel.yaml
+  timeout_in_minutes: 40
+  commands:
+    - |
+      export VLLM_IMAGE_FETCH_TIMEOUT=60
+      pytest -s -v tests/e2e/online_serving/test_bagel.py -m 'core_model' --run-level 'core_model'
+  agents:
+    queue: "mithril-h100-pool"
+  plugins:
+    - kubernetes:
+        podSpec:
+          containers:
+            - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+              resources:
+                limits:
+                  nvidia.com/gpu: 1
+              volumeMounts:
+                - name: devshm
+                  mountPath: /dev/shm
+                - name: hf-cache
+                  mountPath: /root/.cache/huggingface
+              env:
+                - name: HF_HOME
+                  value: /root/.cache/huggingface
+                - name: HF_TOKEN
+                  valueFrom:
+                    secretKeyRef:
+                      name: hf-token-secret
+                      key: token
+          nodeSelector:
+            node.kubernetes.io/instance-type: gpu-h100-sxm
+          volumes:
+            - name: devshm
+              emptyDir:
+                medium: Memory
+            - name: hf-cache
+              hostPath:
+                path: /mnt/hf-cache
+                type: DirectoryOrCreate
+```
+
+**Example** (L3 merge — TTS online + offline, L4 docker):
+
+```yaml
+- label: "TTS · Qwen3-TTS Base Test"
+  source_file_dependencies:
+    - tests/e2e/online_serving/test_qwen3_tts_base.py
+    - tests/e2e/offline_inference/test_qwen3_tts_base.py
+    - vllm_omni/model_executor/models/qwen3_tts/
+    - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
+    - vllm_omni/deploy/qwen3_tts.yaml
+  timeout_in_minutes: 20
+  commands:
+    - |
+      export VLLM_LOGGING_LEVEL=DEBUG
+      export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
+      pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+  agents:
+    queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
+  plugins:
+    - docker#v5.2.0:
+        image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+        always-pull: true
+        propagate-environment: true
+        shm-size: "8gb"
+        environment:
+          - "HF_HOME=/fsx/hf_cache"
+        volumes:
+          - "/fsx/hf_cache:/fsx/hf_cache"
+```
+
+**Example** (L3 merge — diffusion online + offline, H100 kubernetes):
+
+```yaml
+- label: "Diffusion · Qwen Image Test"
+  source_file_dependencies:
+    - tests/e2e/online_serving/test_qwen_image.py
+    - tests/e2e/offline_inference/test_qwen_image.py
+    - vllm_omni/diffusion/models/qwen_image/
+  timeout_in_minutes: 40
+  commands:
+    - |
+      export VLLM_IMAGE_FETCH_TIMEOUT=60
+      pytest -s -v \
+        tests/e2e/online_serving/test_qwen_image.py \
+        tests/e2e/offline_inference/test_qwen_image.py \
+        -m 'advanced_model and cuda' --run-level 'advanced_model'
+  agents:
+    queue: "mithril-h100-pool"
+  plugins:
+    - kubernetes:
+        podSpec:
+          containers:
+            - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+              resources:
+                limits:
+                  nvidia.com/gpu: 1
+              volumeMounts:
+                - name: devshm
+                  mountPath: /dev/shm
+                - name: hf-cache
+                  mountPath: /root/.cache/huggingface
+              env:
+                - name: HF_HOME
+                  value: /root/.cache/huggingface
+                - name: HF_TOKEN
+                  valueFrom:
+                    secretKeyRef:
+                      name: hf-token-secret
+                      key: token
+          nodeSelector:
+            node.kubernetes.io/instance-type: gpu-h100-sxm
+          volumes:
+            - name: devshm
+              emptyDir:
+                medium: Memory
+            - name: hf-cache
+              hostPath:
+                path: /mnt/hf-cache
+                type: DirectoryOrCreate
+```
+
+When extending an **existing** E2E step to run an additional test file, **append** that file (and any new model/deploy paths) to `source_file_dependencies` on the same step. When authoring a PR, mention the dependency list in the **Buildkite change** section of your test plan.
+
+#### L3 / L4: validating Buildkite from a feature branch (`cuda/pipeline.yml`)
+
+Production behavior is driven by [`.buildkite/cuda/pipeline.yml`](../../../.buildkite/cuda/pipeline.yml): it builds the CI image, then uploads **L2** (`test-ready.yml` on non-`main`), **L3** (`test-merge.yml` when `build.branch == "main"`), and **L4** (`test-nightly.yml` when `build.env("NIGHTLY") == "1"`).
+
+When you add or debug **L3** or **L4** tests and need those child pipelines to run **off `main`** or **without** `NIGHTLY=1`, apply **temporary** edits (revert before merge):
+
+1. **Point the upload at the right file (optional shortcut)**  
+   In the **Upload Ready Pipeline** step, the command is `buildkite-agent pipeline upload .buildkite/cuda/test-ready.yml`. For a one-off run you may change that path to `.buildkite/cuda/test-merge.yml` (L3) or `.buildkite/cuda/test-nightly.yml` (L4), and adjust the step’s `if:` so it does not conflict with the other upload steps (avoid double-uploading unless intentional).
+
+2. **L3 — merge pipeline**  
+   The gate `if: build.branch == "main"` lives on the **Upload Merge Pipeline** step in `cuda/pipeline.yml` (not inside `test-merge.yml`). **Comment out that `if` line** so `test-merge.yml` is uploaded on your feature branch. Alternatively rely on step (1) instead of the dedicated merge upload step.
+
+3. **L4 — nightly pipeline**  
+   **Comment out** `if: build.env("NIGHTLY") == "1"` on the **Upload Nightly Pipeline** step in `cuda/pipeline.yml` so the nightly definition is uploaded without setting the env var.  
+   Child steps in [`test-nightly.yml`](../../../.buildkite/cuda/test-nightly.yml) often repeat `if: build.env("NIGHTLY") == "1"`; **comment out those lines on the steps you need to run**, otherwise they will still be skipped after upload.
+
+### Step 6: Run Tests
+
+Pick one command path:
+
+- **Quick local regression (preferred first)**: single file or `file.py::test_name`
+- **CI-like level run**: marker expression + `--run-level`
+
+Full templates live in [references/test-routing.md](references/test-routing.md). **After authoring tests, always emit concrete commands** (see **Output Format**).
+
+**Typical copy-paste examples** (run from repo `tests/` directory; requires matching vLLM/vllm-omni install and hardware):
+
+| Area | Example |
+|------|---------|
+| **Omni offline L2** | `pytest -s -v e2e/offline_inference/test_qwen2_5_omni.py -m "core_model and omni and not cpu" --run-level=core_model` |
+| **Omni online L2** | `pytest -s -v e2e/online_serving/test_qwen3_omni.py -m "core_model and omni" --run-level=core_model` |
+| **TTS online L2** | `pytest -s -v e2e/online_serving/test_qwen3_tts_base.py -m "core_model and tts" --run-level=core_model` |
+| **Diffusion X2I L2 online** | `pytest -s -v e2e/online_serving/test_qwen_image.py -m "core_model and diffusion" --run-level=core_model` |
+| **Diffusion X2V L2 online** | `pytest -s -v e2e/online_serving/test_wan22_t2v.py -m "core_model and diffusion" --run-level=core_model` |
+| **Diffusion X2I L4 nightly** | `pytest -s -v e2e/online_serving/test_qwen_image_expansion.py -m "full_model and diffusion and H100" --run-level=full_model` |
+| **Diffusion X2I L4 perf (nightly)** | `pytest -s -v dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file dfx/perf/tests/test_qwen_image_vllm_omni.json` |
+| **Diffusion X2V L4 nightly** | `pytest -s -v e2e/online_serving/test_wan22_expansion.py -m "full_model and cuda" --run-level=full_model` |
+| **Invalid param (weekly H100)** | `pytest -s -v dfx/reliability/invalid_param_test/ -m "slow and H100"` |
+| **Invalid param (weekly L4)** | `pytest -s -v dfx/reliability/invalid_param_test/ -m "slow and L4"` |
+| **L1 CPU** | `pytest -s -v -m "core_model and cpu"` |
+
+**Prerequisites to mention when relevant**: GPU model (e.g. L4 vs H100), `HF_HOME` / token for hub weights, module-level `skipif` (NPU/XPU-only gaps), and whether CI already collects the path (e.g. `test_*_expansion.py` glob in `test-nightly.yml`).
+
+### Step 7: Validate Result Quality
+
+Before finishing:
+
+- Is the new assertion directly tied to the bug/feature contract?
+- Are API calls made via **`runtime.py` `send_*_request`** (not inline HTTP/SDK in the test file)?
+- Is the **general** `assert_*` bundled inside `send_*_request` (not duplicated in the test)?
+- Is each scenario a **separate `test_*` function** (no `case_id` / `if param.id` branching across cases)?
+- **Fixture scope**: default **`omni_server` / `omni_runner`** (module); **`_function`** variants only when each test must restart the instance?
+- Is the test deterministic (no fragile timing/network coupling)?
+- Is runtime appropriate for the selected level?
+- Are markers and `--run-level` consistent?
+- **Invalid-param cases** in `tests/dfx/reliability/invalid_param_test/` (not e2e), using `send_*_http_request` + `err_code`, with `pytest.mark.slow` and correct `H100` / `L4` hardware mark?
+
+## Output Format
+
+When completing a request, return:
+
+1. **Test plan** (level, markers, file target, **L4 pillar(s)** if applicable: Function / Accuracy / Perf / Doc; **or** **Invalid param** pillar with target `invalid_param_test/test_invalid_<area>.py`; module basename for e2e: `test_{slug}.py` / `test_{slug}_expansion.py`)
+2. **Generated/updated test file(s)** — model-specific constants and request payloads stay **inside** those test modules (**never** `tests/helpers/{slug}.py`). Extend **`tests/helpers/runtime.py`** / **`tests/helpers/assertions.py`** only when new **repo-wide** `send_*` or shared assert helpers are required (do not leave ad-hoc HTTP or `_assert_*` in test modules). **If L4 Perf was requested**, also list **`tests/dfx/perf/tests/*.json`**. **If invalid-param cases were requested**, list the **`invalid_param_test/` script** and parametrized `test_*` / new `id=` rows — **not** e2e paths.
+3. **Buildkite change** — **match the requested level and pillar**:
+   - **L4 Function**: `test-nightly.yml` **Function Test** shard (file list / note existing sweep). **No** `test-merge.yml` unless user also asked for L3.
+   - **L4 Perf**: `test-nightly.yml` **Perf Test · &lt;Model&gt;** step (new step or extend commands); include env exports + artifact upload pattern from a sibling perf job.
+   - **L4 Accuracy**: `test-nightly.yml` **Accuracy Test** step under the same model-type group.
+   - **L3**: `test-merge.yml` + full `source_file_dependencies` + `agents` + `plugins`.
+   - **L2**: `test-ready.yml` + same E2E block requirements.
+   - **Invalid param**: `test-weekly.yml` — note **Invalid parameters Test · H100/L4** group; usually **no YAML change** when only extending existing scripts.
+   If the user asked only for **L4 functional cases**, state explicitly that **Perf / Accuracy / Invalid param** were not included (offer to add if needed). If the plan mixes **success e2e** and **invalid param**, split deliverables across e2e vs `invalid_param_test/` and call out both CI files.
+4. **Run commands (required)** — always include, in fenced `bash` blocks:
+   - **Local — whole file**: `cd tests` then `pytest -s -v <path> …`
+   - **Local — single test** (optional but preferred when the change is one function): `pytest -s -v path::test_func …`
+   - **CI-like** (when not L1 CPU): the same **marker + `--run-level`** pairing the level uses (see [references/test-routing.md](references/test-routing.md) and **Step 6** table above)
+   - **Prerequisites** (one line): e.g. “Requires CUDA L4 + weights cached locally”; “Requires H100 + `HF_TOKEN`”; “Syntax-only validation when vllm is not installed locally”
+5. **Result summary** (pass/fail if you executed tests; if not executed, state that explicitly and what the user should run)
+
+## Additional Resources
+
+- Marker and command routing: [references/test-routing.md](references/test-routing.md)
+- CI pipelines (vllm-omni): [test-ready.yml](../../../.buildkite/cuda/test-ready.yml) (L1/L2), [test-merge.yml](../../../.buildkite/cuda/test-merge.yml) (L3), [test-nightly.yml](../../../.buildkite/cuda/test-nightly.yml) (L4), [test-weekly.yml](../../../.buildkite/cuda/test-weekly.yml) (invalid param / reliability)
+- L4 documentation example tests (naming, extraction vs copied scripts, output dirs, skips): [docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md](../../../docs/contributing/ci/test_examples/l4_doc_example_tests.inc.md) — see also [PR #1910](https://github.com/vllm-project/vllm-omni/pull/1910)

.claude/skills/vllm-omni-test/references/test-routing.md

@@ -0,0 +1,395 @@
+# Test Routing Reference
+
+Use this reference to map testing goals to levels, markers, and runnable commands.
+
+**Repo paths** (`.buildkite/`, `docs/contributing/…`): link with repo-relative paths from this file (e.g. `../../../../.buildkite/cuda/test-ready.yml`, `../../../../docs/contributing/ci/test_system_overview.md`).
+
+## Model-centric e2e filename convention (L2–L4)
+
+When generating a **new** test module tied to a **specific model** under `tests/e2e/offline_inference/` or `tests/e2e/online_serving/`:
+
+| Level | Pattern | Example (`Qwen/Qwen2.5-Omni-7B`) |
+|-------|---------|----------------------------------|
+| **L2**, **L3** | `test_{lowercase_model_slug}.py` | `test_qwen2_5_omni.py` |
+| **L4** | `test_{lowercase_model_slug}_expansion.py` | `test_qwen2_5_omni_expansion.py` |
+
+**Slug rules for `{lowercase_model_slug}`** (canonical: [SKILL.md](../SKILL.md) § *Naming: generated test module files*):
+
+1. Start from the HuggingFace-style id (e.g. `Qwen/Qwen2.5-Omni-7B`), but **do not** put the org into the filename: use the **repo segment only** (`Qwen2.5-Omni-7B`), not `Qwen_...` / `qwen_qwen2_5_...`.
+2. **Lowercase**; replace `.`, `-`, and whitespace with a single `_` (e.g. `Qwen2.5-Omni-7B` → `qwen2_5_omni`). **Omit** trailing size tokens such as `7b` / `30b` in the basename when a single file covers that model line in the directory (matches `test_qwen2_5_omni.py` in-tree).
+3. If two checkpoints in the same folder need separate modules, add a **minimal** disambiguator (e.g. `_7b` vs `_3b`) **only then**.
+4. **L1** unit tests are **not** bound to this pattern; use `tests/<area>/test_<feature>.py` as today.
+
+## Model type markers (`omni` / `tts` / `diffusion`)
+
+Every model-centric e2e test must declare **exactly one** type marker:
+
+| Marker | Model family | Typical APIs |
+|--------|--------------|--------------|
+| `pytest.mark.omni` | Multimodal LLM (Qwen-Omni, …) | `send_omni_request`, `omni_runner`, stage YAML |
+| `pytest.mark.tts` | TTS / speech synthesis | `send_audio_speech_request`, `/v1/audio/speech` |
+| `pytest.mark.diffusion` | Diffusion generative models | `send_diffusion_request`, `send_video_diffusion_request`, `send_images_generations_http_request`, `OmniDiffusionSamplingParams` |
+
+**Diffusion** uses one marker for all output modalities. Nightly CI splits diffusion by **group**, not by extra markers:
+
+| Nightly group (`test-nightly.yml`) | Diffusion scope | Examples |
+|-----------------------------------|-----------------|----------|
+| **Diffusion X2I(&A&T) Model Test** | x2**i** / x2**a** / x2**t** — image, audio, text (non-video) | Qwen-Image*, BAGEL, FLUX, SD3, Z-Image, LongCat, DreamZero |
+| **Diffusion X2V Model Test** | x2**v** — video only | Wan2.2, HunyuanVideo 1.5, Wan VACE |
+
+PR labels for selective nightly runs: `diffusion-x2iat-test`, `diffusion-x2v-test` (plus `nightly-test` / `NIGHTLY=1`).
+
+## Level and Marker Mapping
+
+| Goal | Suggested Level | Marker baseline | Typical location |
+|------|------------------|-----------------|------------------|
+| Unit logic, regression on pure code path | L1 | `core_model and cpu` | `tests/<component>/test_*.py` |
+
+### L1 unit tests — mocking (`mocker`, not `unittest.mock`)
+
+L1 modules run in **Simple Test** (`test-ready.yml` / `test-merge.yml`: `-m 'core_model and cpu'`). When isolation requires doubles:
+
+- **Use** the **`mocker`** fixture (`pytest-mock`): `mocker.patch`, `mocker.spy`, `mocker.Mock`, `mocker.MagicMock`, `mocker.AsyncMock`.
+- **Use** **`monkeypatch`** for env vars and simple `setattr` / `delattr` without mock objects.
+- **Do not use** `unittest.mock` — no `from unittest.mock import ...`, no `@patch`, no `with patch(...)`.
+
+```python
+# Good (L1)
+def test_handler(mocker):
+    mocker.patch("vllm_omni.pkg.fn", return_value=0)
+
+# Bad (L1)
+from unittest.mock import patch
+
+@patch("vllm_omni.pkg.fn")
+def test_handler(mock_fn): ...
+```
+
+E2E (L2+) should not rely on mocks unless documenting a rare exception; prefer real (or lightweight) execution paths.
+| Basic integration/e2e | L2 | `core_model` + **one of** `omni` / `tts` / `diffusion` | `tests/e2e/...` |
+| Advanced integration | L3 | `advanced_model` + type marker | `tests/e2e/...` |
+| Full function / nightly | L4 | `full_model` (nightly) + type marker | **Function:** `tests/e2e/*_expansion.py`; **Perf:** `tests/dfx/perf/tests/*.json`; **Accuracy:** `tests/e2e/accuracy/` |
+| Invalid HTTP / param validation | Weekly (dfx) | `pytest.mark.slow` + type marker + `H100` or `L4` | `tests/dfx/reliability/invalid_param_test/test_invalid_*.py` |
+
+## Marker Selection Rules
+
+1. **Level** (pick one):
+   - `core_model` — L1/L2 (`test-ready.yml`)
+   - `advanced_model` — L3 (`test-merge.yml`)
+   - `full_model` — L4 nightly (`test-nightly.yml`); some expansion tests still carry both `advanced_model` and `full_model` during migration
+2. **Model type** (pick one): `omni`, `tts`, or `diffusion`
+3. **Cross-cutting** when relevant: `parallel`, `cache`, `example`, `benchmark`
+4. **Hardware**: `cpu`, `cuda`, `rocm`, `npu`, `L4`, `H100`, `distributed_cuda`, …
+5. Multi-card: `@hardware_test(...)` in `tests/helpers/mark.py`
+
+## Command Templates
+
+### Quick local checks
+
+```bash
+cd tests
+pytest -s -v test_xxxx.py
+```
+
+### L1
+
+```bash
+cd tests
+pytest -s -v -m "core_model and cpu"
+```
+
+### L2
+
+```bash
+cd tests
+pytest -s -v -m "core_model and not cpu" --run-level=core_model
+```
+
+### L3 (merge)
+
+```bash
+cd tests
+pytest -s -v -m "advanced_model" --run-level=advanced_model
+```
+
+### L4 (nightly)
+
+**Function** (e2e expansion — default when user asks for “L4 functional cases”):
+
+```bash
+cd tests
+pytest -s -v e2e/online_serving/test_qwen_image_expansion.py -m "full_model and diffusion and H100" --run-level=full_model
+```
+
+**Perf** (dfx benchmark harness — separate pillar; not `omni_server` fixtures):
+
+```bash
+cd tests
+export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
+# CI-like: single JSON file (nightly Perf Test steps)
+pytest -s -v dfx/perf/scripts/run_diffusion_benchmark.py \
+  --test-config-file dfx/perf/tests/test_qwen_image_vllm_omni.json
+# Local bulk load: all *.json under tests/dfx/perf/tests/, filter by JSON mark
+pytest -sv dfx/perf/scripts/run_diffusion_benchmark.py -m "full_model and diffusion and H100"
+pytest -sv dfx/perf/scripts/run_benchmark.py -m "full_model and omni and H100"
+```
+
+Broad marker sweep (when no explicit file shard):
+
+```bash
+cd tests
+pytest -s -v -m "full_model" --run-level=full_model
+```
+
+## L4 nightly pillars (Function / Accuracy / Perf)
+
+Nightly **L4** for a model is often **multiple jobs** in `test-nightly.yml`, not a single pytest module:
+
+| Pillar | User says | Deliver | Nightly step |
+|--------|-----------|---------|--------------|
+| **Function** | “L4 functional cases” / “functional” / `*_expansion.py` | `tests/e2e/.../test_<model>_expansion.py` | `· Function Test with H100/L4` |
+| **Perf** | “performance” / “perf” / “benchmark” / “full L4” | `tests/dfx/perf/tests/test_<model>_vllm_omni.json` + runner | `· Perf Test · <Model>` |
+| **Accuracy** | “accuracy” / “similarity” | `tests/e2e/accuracy/test_<model>*.py` | `· Accuracy Test` |
+
+**Rule:** “L4 functional cases” → **Function only** by default. State that Perf/Accuracy are separate pillars; add them only when requested.
+
+**Diffusion perf JSON** (`tests/dfx/perf/tests/test_<slug>_vllm_omni.json`): array of **case objects**, each with:
+
+| Field | Required | Notes |
+|-------|----------|-------|
+| `test_name` | Yes | Unique server/workload id |
+| `mark` | Recommended | When present: **`hardware_marks` required** + optional `marks` (`full_model`, `diffusion`, `local_model`, …). Parsed by `resolve_pytest_marks`; enables local `-m` filtering. |
+| `server_type` | Diffusion | e.g. `"vllm-omni"` — routes case to `run_diffusion_benchmark.py` |
+| `benchmark_endpoint` | Diffusion | e.g. `/v1/videos`, `/v1/images/generations` |
+| `server_params` | Yes | `model`, `serve_args`, … |
+| `benchmark_params[]` | Yes | Each row: **`name`** (pytest id suffix), workload fields, **`baseline`** |
+
+Copy structure from `test_qwen_image_vllm_omni.json` or `test_cosmos3_vllm_omni.json` (2-GPU mark example).
+
+**Omni perf JSON**: `test_qwen3_omni_*.json` — same `mark` shape with `"marks": ["full_model", "omni"]`; run via `run_benchmark.py`.
+
+**Runner split**: `is_diffusion_perf_config()` → diffusion when `server_type` is set or `"diffusion"` ∈ `mark.marks`; otherwise omni/tts → `run_benchmark.py`.
+
+**Param ids**: `{test_name}-{benchmark_params.name}` (e.g. `test_tts-p0`, not `test_tts-0`).
+
+**Result filenames**: runtime hardware from `get_runtime_resource_label()`; `H100` omitted on default CI pool.
+
+**Diffusion perf nightly step** (under **Diffusion X2I(&A&T)**; copy full `kubernetes` plugins from `· Perf Test · Qwen-Image`):
+
+```yaml
+      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · <Model-Display-Name>"
+        key: nightly-diffusion-x2iat-performance-<slug>
+        timeout_in_minutes: 180
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
+          - export CACHE_DIT_VERSION=1.3.0
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_<slug>_vllm_omni.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          # … full kubernetes block (often multi-GPU) — copy from Perf Test · Qwen-Image
+```
+
+| Model type | Perf runner | Config example |
+|------------|-------------|----------------|
+| Diffusion X2I/X2V | `dfx/perf/scripts/run_diffusion_benchmark.py` | `test_qwen_image_vllm_omni.json`, `test_cosmos3_vllm_omni.json` |
+| TTS | `dfx/perf/scripts/run_benchmark.py` | `test_tts.json`, `test_voxcpm2.json`, `test_higgs_audio_v3.json` |
+| Omni | `dfx/perf/scripts/run_benchmark.py` | `test_qwen3_omni_async_chunk.json`, `test_qwen3_omni_no_async_chunk.json`, … |
+
+Do **not** put throughput/latency baselines inside `test_*_expansion.py` — that belongs in the dfx perf JSON + nightly Perf job.
+
+## Invalid parameter validation (weekly / dfx)
+
+When the user’s test plan includes **invalid parameter validation**, **invalid request bodies**, or **HTTP 4xx** validation against a live server:
+
+1. **Do not** author these in `tests/e2e/online_serving/test_*.py` or `*_expansion.py`. **Move** any drafted `test_*` into `tests/dfx/reliability/invalid_param_test/`.
+2. **Pick script by route** (extend in-tree file):
+
+| Route family | Script |
+|--------------|--------|
+| Omni chat / WS video·realtime | `test_invalid_omni_chat.py` |
+| `/v1/audio/speech` (+ stream / batch / voices) | `test_invalid_audio_speech.py` |
+| Audio diffusion | `test_invalid_audio_diffusion.py` |
+| `/v1/images/generations` | `test_invalid_image_generation.py` |
+| `/v1/images/edits` | `test_invalid_image_editing.py` |
+| `/v1/videos*` | `test_invalid_video_generation.py` |
+| Sleep / wakeup / server control | `test_invalid_server_control.py` |
+
+3. **Style:** `pytestmark = [pytest.mark.slow, pytest.mark.<type>]`; `_PARAMS` + `hardware_marks`; `send_*_http_request` with `err_code` + `err_message`; parametrized `body_spec` rows with `id=`; `_minimal_*_json()` helpers; `_SKIP_ISSUE_3649` when tracked in [#3649](https://github.com/vllm-project/vllm-omni/issues/3649).
+4. **CI:** [`.buildkite/cuda/test-weekly.yml`](../../../../.buildkite/cuda/test-weekly.yml) group **Reliability Test - Invalid parameters Test** — **not** ready/merge/nightly.
+
+```bash
+# Weekly H100 (diffusion / omni / video invalid-param)
+cd tests
+pytest -s -v dfx/reliability/invalid_param_test/ -m "slow and H100"
+
+# Weekly L4 (e.g. Qwen3-TTS speech invalid-param)
+pytest -s -v dfx/reliability/invalid_param_test/ -m "slow and L4"
+
+# Single script / case
+pytest -s -v dfx/reliability/invalid_param_test/test_invalid_image_generation.py::test_images_generations_invalid_requests -m "slow and H100"
+```
+
+**Trigger:** `WEEKLY=1` or PR label `weekly-test`. Appending cases to existing scripts usually needs **no YAML edit** (directory sweep). No `source_file_dependencies` on weekly steps.
+
+### Platform-targeted examples
+
+```bash
+cd tests
+pytest -s -v -m "core_model and distributed_cuda and L4" --run-level=core_model
+```
+
+### Concrete e2e paths (common in-tree)
+
+Paths are relative to `tests/` after `cd tests`.
+
+| Scenario | Example command |
+|----------|------------------|
+| **Omni** offline L2 | `pytest -s -v e2e/offline_inference/test_qwen2_5_omni.py -m "core_model and omni and not cpu" --run-level=core_model` |
+| **Omni** offline L2 — one test | `pytest -s -v e2e/offline_inference/test_qwen2_5_omni.py::test_text_to_text -m "core_model and omni and not cpu" --run-level=core_model` |
+| **Omni** online L2 | `pytest -s -v e2e/online_serving/test_qwen3_omni.py -m "core_model and omni" --run-level=core_model` |
+| **Omni** offline L2 (Qwen3.5-9B VL) | `pytest -s -v e2e/offline_inference/test_qwen3_5_9b.py -m "core_model and omni and not cpu" --run-level=core_model` |
+| **TTS** online L2 | `pytest -s -v e2e/online_serving/test_qwen3_tts_base.py -m "core_model and tts" --run-level=core_model` |
+| **TTS** online L2 — one test | `pytest -s -v e2e/online_serving/test_qwen3_tts_base.py::test_text_to_audio_001 -m "core_model and tts" --run-level=core_model` |
+| **Diffusion X2I** offline L2 | `pytest -s -v e2e/offline_inference/test_t2i_model.py -m "core_model and diffusion and not cpu" --run-level=core_model` |
+| **Diffusion X2I** L2 online smoke | `pytest -s -v e2e/online_serving/test_qwen_image.py e2e/online_serving/test_bagel.py -m "core_model and diffusion" --run-level=core_model` |
+| **Diffusion X2V** L2 online smoke | `pytest -s -v e2e/online_serving/test_wan22_t2v.py -m "core_model and diffusion" --run-level=core_model` |
+| **Diffusion X2I** L4 online expansion (H100) | `pytest -s -v e2e/online_serving/test_qwen_image_expansion.py -m "full_model and diffusion and H100" --run-level=full_model` |
+| **Diffusion X2I** L4 perf (nightly) | `pytest -s -v dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file dfx/perf/tests/test_qwen_image_vllm_omni.json` |
+| **Diffusion X2I** L4 online expansion (Edit) | `pytest -s -v e2e/online_serving/test_qwen_image_edit_expansion.py -m "full_model and diffusion and H100" --run-level=full_model` |
+| **Diffusion X2V** L4 nightly | `pytest -s -v e2e/online_serving/test_wan22_expansion.py -m "full_model and cuda" --run-level=full_model` |
+| **Invalid param** weekly H100 | `pytest -s -v dfx/reliability/invalid_param_test/ -m "slow and H100"` |
+| **Invalid param** weekly L4 | `pytest -s -v dfx/reliability/invalid_param_test/ -m "slow and L4"` |
+
+### Agent / author completion checklist
+
+When adding or modifying tests, do not stop at “where the file lives” — also deliver:
+
+1. **Local**: `cd tests` + `pytest -s -v <path>` (and `path::test_func` when a single case is enough).
+2. **CI-like**: marker string + `--run-level` matching the pipeline (`core_model` / `advanced_model` / `full_model`).
+3. **L1 mocks**: `mocker` / `monkeypatch` only; no `unittest.mock`.
+4. **API client + assert placement**: **General** validation → implement in `assertions.py`, call **inside** `send_*_request` in `runtime.py`; tests call **`send_*_request` only**. **Special** case validation → `assert_*` in `assertions.py`, called **in the test** after `send_*_request`. Low-level `send_*_http_request` is for negative/dfx tests (`err_code`), not ordinary L2+ success e2e.
+5. **Shared assertions**: logic in **`tests/helpers/assertions.py`**; general checks inside `send_*_request`; special checks only in the test. No `_assert_*` in `test_*.py`.
+6. **One case → one `test_*`**: function name reflects what is validated; no `if case_id == ...` mega-test merging multiple scenarios.
+7. **Fixture scope**: default **`omni_server` + `openai_client`** / **`omni_runner` + `omni_runner_handler`** (module). Use **`omni_server_function` + `openai_client_function`** / **`omni_runner_function` + `omni_runner_handler_function`** only when each `test_*` must spawn a fresh instance. Parametrize name must match fixture (`omni_server` vs `omni_server_function`).
+8. **Type marker**: `omni`, `tts`, or `diffusion` on every model e2e module.
+9. **Diffusion L4 Function**: wire `*_expansion.py` into **X2I(&A&T)** or **X2V** **Function Test** in **`test-nightly.yml` only** — do not add `test-merge.yml` unless the user also requested L3.
+10. **Diffusion L4 Perf** (only when requested): add `tests/dfx/perf/tests/test_<slug>_vllm_omni.json` + **Perf Test · &lt;Model&gt;** step (artifact upload); not part of “L4 functional cases” by default.
+11. **E2E Buildkite (L2/L3 only)**: `test-ready.yml` / `test-merge.yml` steps need **`source_file_dependencies`** + full **`agents` + `plugins`**. **L4 nightly** uses explicit file lists or perf scripts in `test-nightly.yml` (no merge job).
+12. **Invalid param**: cases in **`tests/dfx/reliability/invalid_param_test/`** (route-matching script), `send_*_http_request` + `err_code`, `pytest.mark.slow` + `H100`/`L4`; CI = **`test-weekly.yml`** only — do not put in e2e or nightly.
+13. **Prerequisites**: GPU tier, HF cache/token, and any module `skipif` / platform-only YAML.
+
+## Buildkite pipeline mapping
+
+| Level | Repo file | Model-type grouping |
+|-------|-----------|---------------------|
+| L1, L2 | [`.buildkite/cuda/test-ready.yml`](../../../../.buildkite/cuda/test-ready.yml) | Steps prefixed **Omni ·**, **TTS ·**, **Diffusion ·** under **E2E Test** — **`source_file_dependencies` required** |
+| L3 | [`.buildkite/cuda/test-merge.yml`](../../../../.buildkite/cuda/test-merge.yml) | Per-model E2E steps; `-m "advanced_model and …"` — **`source_file_dependencies` required** |
+| L4 | [`.buildkite/cuda/test-nightly.yml`](../../../../.buildkite/cuda/test-nightly.yml) | **Omni / TTS / Diffusion X2I(&A&T) / X2V** — each group may have **Function**, **Accuracy**, **Perf**, **Doc** steps |
+| Invalid param (weekly) | [`.buildkite/cuda/test-weekly.yml`](../../../../.buildkite/cuda/test-weekly.yml) | **Invalid parameters Test · H100** / **· L4** — sweeps `tests/dfx/reliability/invalid_param_test/` |
+
+### `source_file_dependencies` (E2E Test only — ready & merge)
+
+Required on each step inside the **E2E Test** group in `test-ready.yml` and `test-merge.yml`. Typical entries:
+
+| Category | Path pattern |
+|----------|----------------|
+| Test module | `tests/e2e/online_serving/test_<slug>.py`, `tests/e2e/offline_inference/test_<slug>.py` |
+| API client helpers | `tests/helpers/runtime.py` (when the e2e job uses newly added `send_*` methods) |
+| Shared assert helpers | `tests/helpers/assertions.py` (when the e2e job uses newly added `assert_*` helpers) |
+| AR / omni model | `vllm_omni/model_executor/models/<family>/` |
+| Diffusion model | `vllm_omni/diffusion/models/<family>/` |
+| Stage processor | `vllm_omni/model_executor/stage_input_processors/<family>.py` |
+| Deploy config | `vllm_omni/deploy/<name>.yaml` or `vllm_omni/deploy/ci/<name>.yaml` |
+
+Copy the dependency block from the closest in-tree E2E step for the same model family. Update the list whenever the pytest command or server YAML changes.
+
+### E2E `agents` + `plugins` blocks (required — do not truncate)
+
+When generating or documenting E2E steps, **always include the complete `plugins` section**. Set **`timeout_in_minutes`** on the step for the job deadline; run **`pytest` directly** in `commands` (no `timeout 40m bash -c` wrapper around pytest).
+
+Common patterns in `test-ready.yml` / `test-merge.yml`:
+
+| Queue | Plugin | Required fields |
+|-------|--------|-----------------|
+| `mithril-h100-pool` | `kubernetes` | `resources.limits.nvidia.com/gpu`, `volumeMounts` (`devshm`, `hf-cache`), `env` (`HF_HOME`, `HF_TOKEN` via `secretKeyRef`), `nodeSelector: gpu-h100-sxm`, `volumes` (`devshm` emptyDir, `hf-cache` hostPath `/mnt/hf-cache`) |
+| `gpu_1_queue` / `gpu_4_queue` | `docker#v5.2.0` | `image`, `always-pull`, `propagate-environment`, `shm-size: "8gb"`, `environment: HF_HOME`, `volumes: /fsx/hf_cache` (add `HF_TOKEN` when the step needs hub access) |
+
+**H100 kubernetes template** (copy from **Diffusion · Bagel Test** / **Diffusion · Qwen Image Test** in-tree):
+
+```yaml
+  agents:
+    queue: "mithril-h100-pool"
+  plugins:
+    - kubernetes:
+        podSpec:
+          containers:
+            - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+              resources:
+                limits:
+                  nvidia.com/gpu: 1
+              volumeMounts:
+                - name: devshm
+                  mountPath: /dev/shm
+                - name: hf-cache
+                  mountPath: /root/.cache/huggingface
+              env:
+                - name: HF_HOME
+                  value: /root/.cache/huggingface
+                - name: HF_TOKEN
+                  valueFrom:
+                    secretKeyRef:
+                      name: hf-token-secret
+                      key: token
+          nodeSelector:
+            node.kubernetes.io/instance-type: gpu-h100-sxm
+          volumes:
+            - name: devshm
+              emptyDir:
+                medium: Memory
+            - name: hf-cache
+              hostPath:
+                path: /mnt/hf-cache
+                type: DirectoryOrCreate
+```
+
+**L4 docker template** (copy from **TTS · Qwen3-TTS Base Test** in `test-merge.yml`):
+
+```yaml
+  agents:
+    queue: "gpu_1_queue"
+  plugins:
+    - docker#v5.2.0:
+        image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+        always-pull: true
+        propagate-environment: true
+        shm-size: "8gb"
+        environment:
+          - "HF_HOME=/fsx/hf_cache"
+        volumes:
+          - "/fsx/hf_cache:/fsx/hf_cache"
+```
+
+When extending an existing step (e.g. add `tests/e2e/offline_inference/test_qwen_image.py` to **Diffusion · Qwen Image Test**), update `source_file_dependencies` and `commands` only; **keep the existing `agents` + `plugins` block unchanged** unless the hardware tier changes.
+
+[`.buildkite/cuda/pipeline.yml`](../../../../.buildkite/cuda/pipeline.yml) decides **which** child file is uploaded. To run **L3** on a feature branch, comment out `if: build.branch == "main"` on the merge upload step. To run **L4** without `NIGHTLY=1`, comment out the nightly upload step’s `if: build.env("NIGHTLY") == "1"` and the same `if` on relevant steps inside `test-nightly.yml`. Revert such edits before merging.
+
+Platform-specific pipelines (e.g. AMD) follow the same level → file pairing under `.buildkite/`.
+
+## Diffusion RFC (#1832) Alignment Tips
+
+For **X2I(&A&T)** coverage planning:
+
+- Prioritize high-value feature combinations with minimal case count.
+- Split into lightweight L1/L2 validation plus L4 `*_expansion.py` under the **X2I** nightly shards.
+- **X2V** models (Wan, HunyuanVideo) stay in the **X2V** group — do not merge into X2I sweeps.
+
+If hardware is insufficient, provide an executable reduced case plus a deferred full CI/nightly plan.

.github/CODEOWNERS

@@ -1,6 +1,11 @@
 # CODEOWNERS is used to automatically assign PR reviewers
 # Format: path pattern @username or @team
 # More info: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners
+#
+# The vllm_omni/** source-tree rules below are derived from the `owners` and
+# `primary_code_paths` front matter in docs/design/module/*.md. When updating
+# ownership, update the design doc first, then regenerate the matching rule here.
+# Last-match-wins: more specific rules must come AFTER broader ones.
 
 # Default owners (fallback reviewers for all files)
 * @Gaohan123 @hsliuustc0106 @ywang96
@@ -16,51 +21,128 @@
 /.claude/ @hsliuustc0106
 
 # Tests
-/tests/ @yenuo26
+/tests/ @yenuo26 @NickCao
 
 # CI
-/.buildkite/ @yenuo26 @congw729
+/.buildkite/ @yenuo26 @congw729 @NickCao
 
-# Engine and model_executor
-/vllm_omni/core/ @tzhouam @yinpeiqi @fake0fan
-/vllm_omni/engine/ @tzhouam @yinpeiqi @fake0fan
-/vllm_omni/entrypoints/ @tzhouam @yinpeiqi @fake0fan
-/vllm_omni/model_executor/ @tzhouam @gcanlin
-/vllm_omni/worker/ @tzhouam @gcanlin
+# ComfyUI / external apps
+/apps/ @fhfuih @alex-jw-brooks
 
-# Diffusion
-/vllm_omni/diffusion/ @Isotr0py @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa @david6666666
-/vllm_omni/lora/ @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa @yuanheng-zhao
-
-# Diffusion CPU offloading
-/vllm_omni/diffusion/offloader/ @Isotr0py @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa @yuanheng-zhao
-
-# Omni and TTS support
-/vllm_omni/transformers_utils/ @tzhouam @yuanheng-zhao
-/vllm_omni/model_executor/models/ @linyueqian @ZeldaHuang @princepride @yuanheng-zhao @amy-why-3459 @Bounty-hunter
-
-# Benchmarks
+# Benchmarks (out-of-tree)
 /benchmarks/accuracy/ @david6666666
 /benchmarks/build_dataset/ @linyueqian @yuanheng-zhao
-/benchmarks/diffusion/ @Isotr0py @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa
+/benchmarks/diffusion/ @Isotr0py @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa @xuechendi
 /benchmarks/distributed/ @princepride
 /benchmarks/glm_image/ @princepride @JaredforReal
-/benchmarks/tts/ @linyueqian @ZeldaHuang @princepride
+/benchmarks/tts/ @linyueqian @ZeldaHuang @princepride @Sy0307 @gcanlin
 
-# Usability and Observability
-/vllm_omni/deploy/ @tzhouam @lishunyang12
-/vllm_omni/metrics/ @lishunyang12
-/vllm_omni/profiler/ @gcanlin
+# =====================================================================
+# Module design ownership (docs/design/module/*.md)
+# Each section header cites the source design doc.
+# =====================================================================
 
-# ComfyUI
-/apps/ @fhfuih
+# Autoregressive runtime — docs/design/module/ar_runtime.md
+/vllm_omni/core/ @tzhouam @yinpeiqi @fake0fan @Sy0307 @Gaohan123
+/vllm_omni/worker/ @tzhouam @yinpeiqi @fake0fan @Sy0307 @Gaohan123
 
-# Large Scale Serving
+# Engine orchestration & stage runtime — docs/design/module/engine_orchestration.md, stage_runtime.md
+/vllm_omni/engine/ @tzhouam @fake0fan
+# File-level overrides: input/output modality contracts own these engine files
+# — docs/design/module/input_output_modality_contracts.md
+/vllm_omni/engine/messages.py @Sy0307 @amy-why-3459 @Gaohan123 @alex-jw-brooks
+/vllm_omni/engine/serialization.py @Sy0307 @amy-why-3459 @Gaohan123 @alex-jw-brooks
+/vllm_omni/engine/mm_outputs.py @Sy0307 @amy-why-3459 @Gaohan123 @alex-jw-brooks
+/vllm_omni/engine/output_modality.py @Sy0307 @amy-why-3459 @Gaohan123 @alex-jw-brooks
+/vllm_omni/engine/output_processor.py @Sy0307 @amy-why-3459 @Gaohan123 @alex-jw-brooks
+
+# Input/output modality contracts — docs/design/module/input_output_modality_contracts.md
+/vllm_omni/inputs/ @Sy0307 @amy-why-3459 @Gaohan123 @alex-jw-brooks
+/vllm_omni/outputs/ @Sy0307 @amy-why-3459 @Gaohan123 @alex-jw-brooks
+/vllm_omni/request.py @Sy0307 @amy-why-3459 @Gaohan123 @alex-jw-brooks
+/vllm_omni/data_entry_keys.py @Sy0307 @amy-why-3459 @Gaohan123 @alex-jw-brooks
+
+# Entrypoints and serving boundaries — docs/design/module/entrypoints.md
+/vllm_omni/entrypoints/ @alex-jw-brooks @linyueqian @NickCao
+# File-level overrides within entrypoints/
+/vllm_omni/entrypoints/openai/errors.py @alex-jw-brooks @NickCao
+/vllm_omni/entrypoints/cli/benchmark/ @alex-jw-brooks @Bounty-hunter
+
+# Error contracts — docs/design/module/error_contracts.md
+/vllm_omni/errors.py @alex-jw-brooks @NickCao
+
+# OmniConnector (inter-stage transport) — docs/design/module/omni_connector.md
 /vllm_omni/distributed/ @princepride @yuanheng-zhao @xuechendi @natureofnature @fake0fan
 
-# Quantization and configuration
-/vllm_omni/config/ @lishunyang12
+# Model integration — docs/design/module/model_integration.md
+/vllm_omni/model_executor/ @tzhouam @gcanlin
+/vllm_omni/model_executor/models/ @tzhouam @gcanlin @Sy0307 @amy-why-3459 @linyueqian
+/vllm_omni/model_extras/ @tzhouam @gcanlin @princepride
+/vllm_omni/plugins/ @gcanlin @tjtanaa @xuechendi
+# Config override within model_executor — docs/design/module/vllm_omni_config.md
+/vllm_omni/model_executor/stage_configs/ @lishunyang12 @alex-jw-brooks
+
+# Execution platforms (hardware backends) — docs/design/module/execution_platforms.md
+/vllm_omni/platforms/ @gcanlin @tjtanaa @xuechendi
+/vllm_omni/attention/ @gcanlin @tjtanaa @xuechendi @david6666666 @lishunyang12
+
+# Configuration & deploy — docs/design/module/vllm_omni_config.md
+/vllm_omni/config/ @lishunyang12 @alex-jw-brooks
+/vllm_omni/deploy/ @lishunyang12 @alex-jw-brooks
+
+# Quantization — docs/design/module/quantization.md
 /vllm_omni/quantization/ @david6666666 @Isotr0py @lishunyang12
 
-# Hardware plugins and accelerators
-/vllm_omni/platforms/ @gcanlin @tjtanaa @xuechendi
+# Observability — docs/design/module/observability.md
+/vllm_omni/metrics/ @lishunyang12 @vraiti
+
+# Profiling — docs/design/module/profiling.md
+/vllm_omni/profiler/ @gcanlin
+
+# Benchmarking (in-tree) — docs/design/module/benchmarking.md
+/vllm_omni/benchmarks/ @alex-jw-brooks @Bounty-hunter
+
+# LoRA (no dedicated design doc; ownership intentionally narrowed)
+/vllm_omni/lora/ @SamitHuang @wtomin
+
+# Transformers utils (related path only in design docs; preserves prior ownership)
+/vllm_omni/transformers_utils/ @tzhouam @yuanheng-zhao
+
+# =====================================================================
+# Diffusion module family — docs/design/module/diffusion/
+# Umbrella rule first; sub-module rules after it win via last-match.
+# =====================================================================
+
+# Diffusion umbrella (top-level files: data.py, forward_context.py, etc.)
+# — docs/design/module/diffusion/index.md
+/vllm_omni/diffusion/ @Isotr0py @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa @david6666666 @xuechendi
+
+# Cache management — docs/design/module/cache_management.md
+/vllm_omni/diffusion/cache/ @Isotr0py @princepride @SamitHuang
+
+# Diffusion quantization — docs/design/module/quantization.md
+/vllm_omni/diffusion/quantization/ @david6666666 @Isotr0py @lishunyang12
+
+# Diffusion profiling — docs/design/module/profiling.md
+/vllm_omni/diffusion/profiler/ @gcanlin
+
+# Diffusion runtime (scheduler, executor, worker)
+# — docs/design/module/diffusion/diffusion_runtime.md, continuous_batching.md
+/vllm_omni/diffusion/sched/ @Isotr0py @princepride @SamitHuang @fhfuih
+/vllm_omni/diffusion/executor/ @Isotr0py @princepride @SamitHuang @fhfuih
+/vllm_omni/diffusion/worker/ @Isotr0py @princepride @SamitHuang @fhfuih
+
+# Diffusion model integration (pipelines, registration, checkpoint loading)
+# — docs/design/module/diffusion/diffusion_model_integration.md
+/vllm_omni/diffusion/models/ @fhfuih @Bounty-hunter @wtomin
+/vllm_omni/diffusion/model_loader/ @fhfuih @Bounty-hunter @wtomin @RuixiangMa
+/vllm_omni/diffusion/lora/ @fhfuih @Bounty-hunter @wtomin @RuixiangMa
+
+# Diffusion parallelism (rank topology, TP/SP/PP/VAE patch)
+# — docs/design/module/diffusion/parallelism.md
+/vllm_omni/diffusion/distributed/ @princepride @yuanheng-zhao @xuechendi @Bounty-hunter
+/vllm_omni/diffusion/attention/parallel/ @princepride @yuanheng-zhao @xuechendi @Bounty-hunter
+
+# Diffusion offloader (CPU offloading, DLO)
+# — docs/design/module/diffusion/offloader.md
+/vllm_omni/diffusion/offloader/ @yuanheng-zhao @david6666666 @lishunyang12

.github/ISSUE_TEMPLATE/400-bug-report.yml

@@ -19,6 +19,7 @@ body:
 
       Consider redacting or replacing sensitive values with placeholders like `<YOUR_TOKEN_HERE>` when sharing configuration or code examples.
 - type: textarea
+  id: current-environment
   attributes:
     label: Your current environment
     description: |
@@ -42,6 +43,7 @@ body:
     required: true
 
 - type: textarea
+  id: code-version
   attributes:
     label: Your code version
     description: |
@@ -66,6 +68,7 @@ body:
     required: true
 
 - type: textarea
+  id: bug-description
   attributes:
     label: 🐛 Describe the bug
     description: |

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,21 +1,15 @@
 <!-- markdownlint-disable -->
-PLEASE FILL IN THE PR DESCRIPTION HERE ENSURING ALL CHECKLIST ITEMS (AT THE BOTTOM) HAVE BEEN CONSIDERED.
+PLEASE FILL IN THE PR DESCRIPTION HERE.
 
 ## Purpose
 
 ## Test Plan
 
+**vLLM Version:**
+
+**vLLM-Omni Commit:**
+
 ## Test Result
 
----
-<details>
-<summary> Essential Elements of an Effective PR Description Checklist </summary>
-
-- [ ] The purpose of the PR, such as "Fix some issue (link existing issues this PR will resolve)".
-- [ ] The test plan. Please provide the test scripts & test commands. Please state the reasons if your codes don't require additional test scripts. For test file guidelines, please check the [test style doc](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/tests_style/)
-- [ ] The test results. Please paste the results comparison before and after, or the e2e results.
-- [ ] (Optional) The necessary documentation update, such as updating `supported_models.md` and `examples` for a new model. **Please run `mkdocs serve` to sync the documentation editions to `./docs`.**
-- [ ] (Optional) Release notes update. If your change is user-facing, please update the release notes draft.
-</details>
-
-**BEFORE SUBMITTING, PLEASE READ <https://github.com/vllm-project/vllm-omni/blob/main/CONTRIBUTING.md>** (anything written below this line will be removed by GitHub Actions)
+**BEFORE SUBMITTING:** read [CONTRIBUTING.md](https://github.com/vllm-project/vllm-omni/blob/main/CONTRIBUTING.md) and run the [precheck-pr skill](https://github.com/vllm-project/vllm-omni/blob/main/.claude/skills/precheck-pr/SKILL.md) with the code agent for a self-check against project conventions.
+(anything written below this line will be removed by GitHub Actions)