Merge branch 'main' into codex-split-attention-offload-guides

[Bugfix][XPU][Tests]Scope XPU pytest to explicit paths to fix collection crash (#6175 )
Signed-off-by: Joshna Medisetty <joshna.medisetty@intel.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Chendi.Xue <chendi.xue@intel.com>
2026-08-15 07:00:22 +08:00 · 2026-08-14 15:23:41 -05:00 · 2026-08-14 19:13:00 +00:00 · 2026-08-14 11:47:17 +00:00 · 2026-08-14 09:49:26 +00:00 · 2026-08-14 15:51:23 +08:00
55 changed files with 2439 additions and 1119 deletions
@@ -1,153 +1,103 @@
 #!/bin/bash
-# vllm-omni customized version
-# Based on: vllm/.buildkite/scripts/hardware_ci/run-amd-test.sh
-# Last synced: 2025-12-15
-# Modifications: docker image name for vllm-omni

-# This script runs test inside the corresponding ROCm docker container.
-set -o pipefail
+# Run vLLM-Omni ROCm tests directly in the MI300 Kubernetes pod. The pod's
+# container image is selected by test-template-amd-omni.j2; MI300 has no DinD.
+set -euo pipefail

-# Export Python path
-export PYTHONPATH=".."
+: "${PYTHONFAULTHANDLER:=1}"
+: "${HF_HOME:=/home/buildkite-agent/huggingface}"
+: "${HF_HUB_DOWNLOAD_TIMEOUT:=300}"
+: "${HF_HUB_ETAG_TIMEOUT:=60}"
+: "${MIOPEN_DEBUG_CONV_DIRECT:=0}"
+: "${MIOPEN_DEBUG_CONV_GEMM:=0}"
+: "${VLLM_ROCM_USE_AITER:=0}"
+export PYTHONFAULTHANDLER HF_HOME HF_HUB_DOWNLOAD_TIMEOUT HF_HUB_ETAG_TIMEOUT
+export MIOPEN_DEBUG_CONV_DIRECT MIOPEN_DEBUG_CONV_GEMM VLLM_ROCM_USE_AITER
+export PYTORCH_ROCM_ARCH=""
+export PYTHONPATH="${PYTHONPATH:-..}"
+
+if [[ "${VLLM_CI_DOCKER_DISABLED:-0}" != "1" ]]; then
+    echo "Error: MI300 CI must run natively with Docker disabled." >&2
+    exit 1
+fi
+
+if [[ -n "${TEST_COMMANDS:-}" ]]; then
+    commands="${TEST_COMMANDS}"
+else
+    commands="$*"
+fi
+if [[ -z "${commands}" ]]; then
+    echo "Error: No test commands provided." >&2
+    exit 1
+fi
+
+echo "--- Native in-pod ROCm CI"
+
+job_id="${BUILDKITE_JOB_ID:-${BUILDKITE_PARALLEL_JOB:-local}}"
+job_id="${job_id//[^A-Za-z0-9_.-]/_}"
+native_cache_root="/tmp/vllm-omni-native-${job_id}"
+TMPDIR="${native_cache_root}/tmp"
+TORCHINDUCTOR_CACHE_DIR="${native_cache_root}/torchinductor"
+TRITON_CACHE_DIR="${native_cache_root}/triton"
+VLLM_CACHE_ROOT="${native_cache_root}/vllm"
+XDG_CACHE_HOME="${native_cache_root}/xdg"
+HF_DATASETS_CACHE="${native_cache_root}/huggingface/datasets"
+export TMPDIR TORCHINDUCTOR_CACHE_DIR TRITON_CACHE_DIR VLLM_CACHE_ROOT
+export XDG_CACHE_HOME HF_DATASETS_CACHE
+mkdir -p "${TMPDIR}" "${TORCHINDUCTOR_CACHE_DIR}" "${TRITON_CACHE_DIR}" \
+    "${VLLM_CACHE_ROOT}" "${XDG_CACHE_HOME}" "${HF_HOME}" \
+    "${HF_DATASETS_CACHE}"
+
+if [[ "${VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE:-0}" == "1" ]]; then
+    if ! command -v findmnt >/dev/null 2>&1; then
+        echo "Error: findmnt is required to verify the Hugging Face cache mount." >&2
+        exit 1
+    fi
+    hf_mount=$(findmnt -n -T "${HF_HOME}" -o TARGET 2>/dev/null || true)
+    if [[ -z "${hf_mount}" || "${hf_mount}" == "/" ]]; then
+        echo "Error: MI300 CI requires a persistent volume at or above ${HF_HOME}." >&2
+        exit 1
+    fi
+fi

-# Print ROCm version
-echo "--- ROCm info"
 rocminfo

-# cleanup older docker images
-cleanup_docker() {
-  # Get Docker's root directory
-  docker_root=$(docker info -f '{{.DockerRootDir}}')
-  if [ -z "$docker_root" ]; then
-    echo "Failed to determine Docker root directory."
-    exit 1
-  fi
-  echo "Docker root directory: $docker_root"
-  # Check disk usage of the filesystem where Docker's root directory is located
-  disk_usage=$(df "$docker_root" | tail -1 | awk '{print $5}' | sed 's/%//')
-  # Define the threshold
-  threshold=70
-  if [ "$disk_usage" -gt "$threshold" ]; then
-    echo "Disk usage is above $threshold%. Cleaning up Docker images and volumes..."
-    # Remove dangling images (those that are not tagged and not used by any container)
-    docker image prune -f
-    # Remove unused volumes / force the system prune for old images as well.
-    docker volume prune -f && docker system prune --force --filter "until=72h" --all
-    echo "Docker images and volumes cleanup completed."
-  else
-    echo "Disk usage is below $threshold%. No cleanup needed."
-  fi
-}
+expected_gpus="${VLLM_CI_EXPECTED_GPU_COUNT:-1}"
+python3 - <<'PY'
+import os

-# Call the cleanup docker function
-cleanup_docker
+import torch

-echo "--- Pulling container"
-## Temporary change to use AMD Docker Hub to store the vllm-omni image
-# to bypass the rate limit issue with ECR Public Gallery.
-# Images are now stored in a separate repository for vllm-omni, instead of vllm-ci.
-# TODO: @tjtanaa point back to ECR Public Gallery
-# once the amd agents are configured to use ECR Public Gallery.
-# image_name="public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:${BUILDKITE_COMMIT}-rocm-omni"
-image_name="rocm/vllm-omni:${BUILDKITE_COMMIT}"
-container_name="rocm_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)"
+expected = int(os.environ.get("VLLM_CI_EXPECTED_GPU_COUNT", "1"))
+assert torch.version.hip, "PyTorch is not a ROCm build"
+assert torch.cuda.is_available(), "ROCm GPU is not available to PyTorch"
+actual = torch.cuda.device_count()
+assert actual == expected, f"Expected {expected} ROCm GPU(s), found {actual}"
+PY

-# TODO: @tjtanaa uncomment this once the amd agents are configured to use ECR Public Gallery.
-# # Install AWS CLI to authenticate to ECR Public Gallery to get higher rate limit for pulling images
-# sudo apt-get update && sudo apt-get install -y awscli
-##  Use safe docker login helper to prevent race conditions
-# source "$(dirname "${BASH_SOURCE[0]}")/../../common/scripts/docker_login_ecr_public.sh"
-# safe_docker_login_ecr_public
-
-## Pull the container from AMD Docker Hub
-
-docker pull "${image_name}"
-
-remove_docker_container() {
-   docker ps -aq --filter "name=${container_name}" | xargs -r docker rm -f || true
-   docker image rm -f "${image_name}" || true
-}
-trap remove_docker_container EXIT
-
-echo "--- Running container"
-
-HF_CACHE="$(realpath ~)/huggingface"
-mkdir -p "${HF_CACHE}"
-HF_MOUNT="/root/.cache/huggingface"
-
-if [[ -n "${TEST_COMMAND:-}" ]]; then
-    commands="$TEST_COMMAND"
-else
-    commands="$@"
-fi
-echo "Commands:$commands"
-
-PARALLEL_JOB_COUNT=8
-MYPYTHONPATH=".."
-
-# Test that we're launching on the machine that has
-# proper access to GPUs
-render_gid=$(getent group render | cut -d: -f3)
-if [[ -z "$render_gid" ]]; then
-  echo "Error: 'render' group not found. This is required for GPU access." >&2
-  exit 1
-fi
-
-# check if the command contains shard flag, we will run all shards in parallel because the host have 8 GPUs.
-# TODO: @tjtanaa reenable to run VLLM_ROCM_USE_AITER=1 when AITER is shipped with prebuilt kernels.
-if [[ $commands == *"--shard-id="* ]]; then
-  # assign job count as the number of shards used
-  commands=$(echo "$commands" | sed -E "s/--num-shards[[:blank:]]*=[[:blank:]]*[0-9]*/--num-shards=${PARALLEL_JOB_COUNT} /g" | sed 's/ \\ / /g')
-  for GPU in $(seq 0 $(($PARALLEL_JOB_COUNT-1))); do
-    # assign shard-id for each shard
-    commands_gpu=$(echo "$commands" | sed -E "s/--shard-id[[:blank:]]*=[[:blank:]]*[0-9]*/--shard-id=${GPU} /g" | sed 's/ \\ / /g')
-    echo "Shard ${GPU} commands:$commands_gpu"
-    echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES"
-    docker run \
-        --device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \
-        --network=host \
-        --shm-size=16gb \
-        --group-add "$render_gid" \
-        -e MIOPEN_DEBUG_CONV_DIRECT=0 \
-        -e MIOPEN_DEBUG_CONV_GEMM=0 \
-        -e VLLM_ROCM_USE_AITER=0 \
-        -e HIP_VISIBLE_DEVICES="${GPU}" \
-        -e HF_TOKEN \
-        -v "${HF_CACHE}:${HF_MOUNT}" \
-        -e "HF_HOME=${HF_MOUNT}" \
-        -e "PYTHONPATH=${MYPYTHONPATH}" \
-        --name "${container_name}_${GPU}" \
-        "${image_name}" \
-        /bin/bash -c "${commands_gpu}" \
-        |& while read -r line; do echo ">>Shard $GPU: $line"; done &
-    PIDS+=($!)
-  done
-  #wait for all processes to finish and collect exit codes
-  for pid in "${PIDS[@]}"; do
-    wait "${pid}"
-    STATUS+=($?)
-  done
-  for st in "${STATUS[@]}"; do
-    if [[ ${st} -ne 0 ]]; then
-      echo "One of the processes failed with $st"
-      exit "${st}"
+echo "Commands:${commands}"
+if /bin/bash -o pipefail -c '
+set -E
+test_status=0
+trap '\''
+    command_status=$?
+    # Keep running subsequent commands, but retain a failure for the final exit.
+    # Prefer a test failure over pytest "no tests collected" when both occur.
+    if (( test_status == 0 || (test_status == 5 && command_status != 5) )); then
+        test_status=${command_status}
    fi
-  done
+'\'' ERR
+eval "$1"
+exit "${test_status}"
+' _ "${commands}"; then
+    exit 0
 else
-  echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES"
-  docker run \
-          --device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \
-          --network=host \
-          --shm-size=16gb \
-          --group-add "$render_gid" \
-          -e MIOPEN_DEBUG_CONV_DIRECT=0 \
-          -e MIOPEN_DEBUG_CONV_GEMM=0 \
-          -e VLLM_ROCM_USE_AITER=0 \
-          -e HF_TOKEN \
-          -v "${HF_CACHE}:${HF_MOUNT}" \
-          -e "HF_HOME=${HF_MOUNT}" \
-          -e "PYTHONPATH=${MYPYTHONPATH}" \
-          --name "${container_name}" \
-          "${image_name}" \
-          /bin/bash -c "${commands}"
+    exit_code=$?
 fi
+
+if [[ ${exit_code} -eq 5 && "${VLLM_CI_ALLOW_NO_TESTS:-0}" == "1" ]]; then
+    echo "Pytest collected no tests; VLLM_CI_ALLOW_NO_TESTS=1, treating exit code 5 as success."
+    exit 0
+fi
+
+exit "${exit_code}"
@@ -7,54 +7,64 @@ steps:

 - group: ":card_index_dividers: Simple Test"
  steps:
-    - label: "Simple · Diffusion & Model Executor Test (1/2)"
-      agent_pool: mi325_1
+    - label: "Simple · Model Executor Test · Shard %N/%t"
+      agent_pool: mi300_1
+      parallelism: 2
      depends_on: amd-build
-      timeout_in_minutes: 70
      mirror_hardwares: [amdproduction]
      grade: Blocking
      commands:
        - export VLLM_ROCM_USE_AITER=0
        # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
        # that is not supported on CPU
-        # CPU tests on ROCm gpu machines are very slow, so we give them more time.
-        - "timeout 60m pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+        - "pytest -sv tests/model_executor -m 'core_model and cpu' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml:coverage-model-executor-$$BUILDKITE_PARALLEL_JOB.xml"

-    - label: "Simple · Diffusion & Model Executor Test (2/2)"
-      agent_pool: mi325_1
+    - label: "Simple · Diffusion Test · Shard %N/%t"
+      agent_pool: mi300_1
+      parallelism: 2
      depends_on: amd-build
-      timeout_in_minutes: 70
      mirror_hardwares: [amdproduction]
      grade: Blocking
      commands:
        - export VLLM_ROCM_USE_AITER=0
-        # CPU tests on ROCm gpu machines are very slow, so we give them more time.
-        - "timeout 60m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+        # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
+        # that is not supported on CPU
+        - "pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml:coverage-diffusion-$$BUILDKITE_PARALLEL_JOB.xml"
+
+    - label: "Simple · Engine&Entrypoints Test"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        - "pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"

    - label: "Simple · Other Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
      depends_on: amd-build
-      timeout_in_minutes: 50
      mirror_hardwares: [amdproduction]
      grade: Blocking
      commands:
        - export VLLM_ROCM_USE_AITER=0
        # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
        # that is not supported on CPU
-        - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+        - "pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine  --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"

 - group: ":card_index_dividers: Diffusion Test"
  steps:
    - label: "Diffusion · LoRA Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
+      artifact_paths:
+        - lora-server.log
      commands:
-        - timeout 20m pytest -s -v tests/diffusion/lora/test_images_generations_lora.py
+        - pytest -sv tests/diffusion/lora/ -m 'advanced_model and cuda' --run-level "advanced_model" 2>&1 | tee lora-server.log

    - label: "Diffusion Model CPU offloading Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
@@ -62,7 +72,7 @@ steps:
        - export VLLM_LOGGING_LEVEL=DEBUG
        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
        - |
-          timeout 20m bash -c '
+          bash -c '
            set +e
            pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
            EXIT1=\$?
@@ -72,7 +82,7 @@ steps:
          '

    - label: "Diffusion Sequence Parallelism Test (Need 4 GPUs)"
-      agent_pool: mi325_4
+      agent_pool: mi300_4
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
@@ -84,7 +94,7 @@ steps:

    # merge-only tests
    - label: "Diffusion Tensor Parallelism Test"
-      agent_pool: mi325_2
+      agent_pool: mi300_2
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
@@ -92,7 +102,7 @@ steps:
        - pytest -s -v tests/diffusion/distributed/test_tensor_parallel.py

    - label: "Diffusion Model Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
@@ -100,7 +110,7 @@ steps:
        - timeout 30m pytest -s -v tests/e2e/offline_inference/test_z_image.py

    - label: "Diffusion GPU Worker Test"
-      agent_pool: mi325_2
+      agent_pool: mi300_2
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
@@ -108,7 +118,7 @@ steps:
        - timeout 20m pytest -s -v tests/diffusion/test_diffusion_worker.py

    - label: "Diffusion · Wan22 Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
@@ -121,7 +131,7 @@ steps:

 ## ISSUE depends on `diffusers` package: https://github.com/huggingface/diffusers/issues/13274
 # - label: "Audio Generation Model Test"
-#   agent_pool: mi325_1
+#   agent_pool: mi300_1
 #   depends_on: amd-build
 #   mirror_hardwares: [amdproduction]
 #   grade: Blocking
@@ -132,7 +142,7 @@ steps:
 #     - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_expansion.py -m "advanced_model and diffusion and L4" --run-level advanced_model

 - label: "Engine Test"
-  agent_pool: mi325_1
+  agent_pool: mi300_1
  depends_on: amd-build
  mirror_hardwares: [amdproduction]
  grade: Blocking
@@ -147,7 +157,7 @@ steps:
 - group: ":card_index_dividers: E2E Test"
  steps:
    - label: "Omni Model Test Qwen2-5-Omni"
-      agent_pool: mi325_4
+      agent_pool: mi300_4
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      commands:
@@ -155,7 +165,7 @@ steps:
        - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model"

    - label: "Omni Model Test Qwen3-Omni"
-      agent_pool: mi325_2
+      agent_pool: mi300_2
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
@@ -163,22 +173,23 @@ steps:
        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
        - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model'

-    - label: "Qwen3-TTS CustomVoice E2E Test"
-      agent_pool: mi325_1
+    - label: "Qwen3-TTS CustomVoice E2E Test · Shard %N/%t"
+      agent_pool: mi300_1
+      parallelism: 2
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
      commands:
        - |
-          timeout 50m bash -c '
+          bash -c '
            export VLLM_LOGGING_LEVEL=DEBUG
            export VLLM_WORKER_MULTIPROC_METHOD=spawn
            export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-            pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m "advanced_model" --run-level "advanced_model"
+            pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m "advanced_model" --run-level "advanced_model" --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
          '

    - label: "Qwen3-TTS Base E2E Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
@@ -196,7 +207,7 @@ steps:
 # Need to debug before reneable numerical changes across large PRs
 # # split Bagel Model Test with H100 (Real Weights) into three tests
 # - label: "Bagel Text2Img Model Test (1/3)"
-#   agent_pool: mi325_1
+#   agent_pool: mi300_1
 #   depends_on: amd-build
 #   mirror_hardwares: [amdproduction]
 #   grade: Blocking
@@ -208,7 +219,7 @@ steps:
 #     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "advanced_model" --run-level "advanced_model" -k "shared_memory" -k "rocm"

 # - label: "Bagel Img2Img Model Test (2/3)"
-#   agent_pool: mi325_1
+#   agent_pool: mi300_1
 #   depends_on: amd-build
 #   mirror_hardwares: [amdproduction]
 #   grade: Blocking
@@ -220,7 +231,7 @@ steps:
 #     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "advanced_model" --run-level "advanced_model" -k "rocm"

 # - label: "Bagel Online Serving Test (3/3)"
-#   agent_pool: mi325_1
+#   agent_pool: mi300_1
 #   depends_on: amd-build
 #   mirror_hardwares: [amdproduction]
 #   grade: Blocking
@@ -7,8 +7,9 @@ steps:

 - group: ":card_index_dividers: Simple Test"
  steps:
-    - label: "Simple · Diffusion & Model Executor Test"
-      agent_pool: mi325_1
+    - label: "Simple · Model Executor Test · Shard %N/%t"
+      agent_pool: mi300_1
+      parallelism: 2
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
@@ -16,10 +17,30 @@ steps:
        - export VLLM_ROCM_USE_AITER=0
        # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
        # that is not supported on CPU
-        - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+        - "pytest -sv tests/model_executor -m 'core_model and cpu' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml:coverage-model-executor-$$BUILDKITE_PARALLEL_JOB.xml"
+
+    - label: "Simple · Diffusion Test"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
+        # that is not supported on CPU
+        - "pytest -sv tests/diffusion -m 'core_model and cpu'  --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+
+    - label: "Simple · Engine&Entrypoints Test"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        - "pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"

    - label: "Simple · Other Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
@@ -27,19 +48,19 @@ steps:
        - export VLLM_ROCM_USE_AITER=0
        # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
        # that is not supported on CPU
-        - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+        - "pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine  --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"

 - label: "Custom Pipeline Test"
-  agent_pool: mi325_1
+  agent_pool: mi300_1
  depends_on: amd-build
  mirror_hardwares: [amdproduction]
  grade: Blocking
  commands:
-    - timeout 20m pytest -s -v tests/e2e/features/custom_pipeline/ -m "core_model"
+    - pytest -s -v tests/e2e/features/custom_pipeline/ -m "core_model"


 - label: "Voxtral TTS CUDA Unit Test"
-  agent_pool: mi325_1
+  agent_pool: mi300_1
  depends_on: amd-build
  mirror_hardwares: [amdproduction]
  grade: Blocking
@@ -49,33 +70,25 @@ steps:
 - group: ":card_index_dividers: Diffusion Test"
  steps:
    - label: "Diffusion · Batch Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
      commands:
-        - timeout 20m pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model"
+        - pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model"

    - label: "Diffusion · Offloader Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
      commands:
        - export VLLM_LOGGING_LEVEL=DEBUG
        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-        - |
-          timeout 20m bash -c '
-            set +e
-            pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
-            EXIT1=\$?
-            pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
-            EXIT2=\$?
-            exit \$((EXIT1 | EXIT2))
-          '
+        - pytest -s -v tests/diffusion/offloader -m "core_model and cuda" --run-level "core_model"

    - label: "Diffusion · Cache Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
@@ -83,10 +96,10 @@ steps:
        - export GPU_ARCHS=gfx942
        - export VLLM_LOGGING_LEVEL=DEBUG
        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-        - timeout 15m pytest -s -v -m "core_model and cache and diffusion and not distributed_cuda and L4"
+        - pytest -sv tests/diffusion/cache -m 'core_model and cuda' --run-level "core_model"

    - label: "Diffusion Sequence Parallelism Test"
-      agent_pool: mi325_2
+      agent_pool: mi300_2
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
@@ -96,7 +109,7 @@ steps:
        - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py -m core_model

    - label: "Diffusion GPU Worker Test"
-      agent_pool: mi325_2
+      agent_pool: mi300_2
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
@@ -104,7 +117,7 @@ steps:
        - timeout 20m pytest -s -v tests/diffusion/test_diffusion_worker.py

    - label: "Diffusion · Model Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
@@ -112,7 +125,7 @@ steps:
        - timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model"

    - label: "Diffusion · Wan22 Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
@@ -126,7 +139,7 @@ steps:

 ## ISSUE depends on `diffusers` package: https://github.com/huggingface/diffusers/issues/13274
 # - label: "Audio Generation Model Test"
-#   agent_pool: mi325_1
+#   agent_pool: mi300_1
 #   depends_on: amd-build
 #   mirror_hardwares: [amdproduction]
 #   grade: Blocking
@@ -137,7 +150,7 @@ steps:
 #     - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_expansion.py -m "advanced_model and diffusion and L4" --run-level advanced_model

 - label: "Engine Test"
-  agent_pool: mi325_1
+  agent_pool: mi300_1
  depends_on: amd-build
  mirror_hardwares: [amdproduction]
  grade: Blocking
@@ -149,7 +162,7 @@ steps:
      '

 # - label: "Omni Model Test Qwen3-Omni"
-#   agent_pool: mi325_2
+#   agent_pool: mi300_2
 #   depends_on: amd-build
 #   mirror_hardwares: [amdproduction]
 #   grade: Blocking
@@ -162,7 +175,7 @@ steps:
 - group: ":card_index_dividers: E2E Test"
  steps:
    - label: "Qwen3-TTS E2E Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
@@ -173,7 +186,7 @@ steps:
        - timeout 30m pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m "core_model" --run-level "core_model"

    - label: "CosyVoice3-TTS E2E Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
      depends_on: amd-build
      mirror_hardwares: [amdproduction]
      grade: Blocking
@@ -186,7 +199,7 @@ steps:
 # TODO: Bagel test on ROCm is very unstable. @tjtanaa
 # Need to debug before reneable numerical changes across large PRs
 # - label: "Bagel Text2Img Model Test"
-#   agent_pool: mi325_1
+#   agent_pool: mi300_1
 #   depends_on: amd-build
 #   mirror_hardwares: [amdproduction]
 #   grade: Blocking
@@ -198,7 +211,7 @@ steps:
 #     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"

 # - label: "Bagel Img2Img Model Test"
-#   agent_pool: mi325_1
+#   agent_pool: mi300_1
 #   depends_on: amd-build
 #   mirror_hardwares: [amdproduction]
 #   grade: Blocking
@@ -210,7 +223,7 @@ steps:
 #     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"

 # - label: "Bagel Online Serving Test"
-#   agent_pool: mi325_1
+#   agent_pool: mi300_1
 #   depends_on: amd-build
 #   mirror_hardwares: [amdproduction]
 #   grade: Blocking
@@ -1,11 +1,65 @@
 {# vllm-omni customized version
   Based on: https://github.com/vllm-project/ci-infra/blob/main/buildkite/test-template-amd.j2
-   Last synced: 2025-12-15
-   Modifications: Removed unused CUDA/NVIDIA logic, keeping only AMD tests
+   Last synced: 2026-08-04
+   Modifications: Use the shared MI300 queues and Omni command-passing contract
 #}
 {% set docker_image_amd = "rocm/vllm-omni:$BUILDKITE_COMMIT" %}
 {% set default_working_dir = "/app/vllm-omni" %}

+{# MI300 runs tests natively in Kubernetes; Docker-in-Docker is unavailable. #}
+{% macro amd_native_k8s_plugin(indent, step) -%}
+{{ indent }}plugins:
+{{ indent }}  - kubernetes:
+{{ indent }}      podSpecPatch:
+{{ indent }}        automountServiceAccountToken: false
+{{ indent }}        securityContext:
+{{ indent }}          seccompProfile:
+{{ indent }}            type: RuntimeDefault
+{{ indent }}        imagePullSecrets:
+{{ indent }}          - name: docker-config
+{{ indent }}        containers:
+{{ indent }}          - name: container-0
+{{ indent }}            image: {{ docker_image_amd }}
+{{ indent }}            imagePullPolicy: Always
+{{ indent }}            securityContext:
+{{ indent }}              allowPrivilegeEscalation: false
+{{ indent }}              capabilities:
+{{ indent }}                add:
+{{ indent }}                  - IPC_LOCK
+{{ indent }}            resources:
+{{ indent }}              limits:
+{{ indent }}                amd.com/gpu: "{{ step.agent_pool[-1:] }}"
+{{ indent }}              requests:
+{{ indent }}                amd.com/gpu: "{{ step.agent_pool[-1:] }}"
+{{ indent }}            volumeMounts:
+{{ indent }}              - name: devshm
+{{ indent }}                mountPath: /dev/shm
+{{ indent }}            env:
+{{ indent }}              - name: AMD_CI_RUNTIME
+{{ indent }}                value: native
+{{ indent }}              - name: NATIVE_CI
+{{ indent }}                value: "true"
+{{ indent }}              - name: VLLM_CI_DOCKER_DISABLED
+{{ indent }}                value: "1"
+{{ indent }}              - name: VLLM_CI_EXPECTED_GPU_COUNT
+{{ indent }}                value: "{{ step.agent_pool[-1:] }}"
+{{ indent }}              - name: HF_HOME
+{{ indent }}                value: /home/buildkite-agent/huggingface
+{{ indent }}              - name: MIOPEN_DEBUG_CONV_DIRECT
+{{ indent }}                value: "0"
+{{ indent }}              - name: MIOPEN_DEBUG_CONV_GEMM
+{{ indent }}                value: "0"
+{{ indent }}              - name: VLLM_ROCM_USE_AITER
+{{ indent }}                value: "0"
+{{ indent }}              - name: PYTORCH_ROCM_ARCH
+{{ indent }}                value: ""
+{{ indent }}        volumes:
+{{ indent }}          - name: devshm
+{{ indent }}            emptyDir:
+{{ indent }}              medium: Memory
+{{ indent }}              sizeLimit: 16Gi
+{%- endmacro %}
+
 {% if env %}
 env:
 {% for name in env %}
@@ -53,19 +107,31 @@ steps:
          {% if step.agent_pool %}
          queue: amd_{{ step.agent_pool }}
          {% else %}
-          queue: amd_mi325_1
+          queue: amd_mi300_1
          {% endif %}
+{{ amd_native_k8s_plugin("        ", step) }}
 {% set cmd_body = (step.command or (step.commands | join("\n"))) | trim %}
 {% set indented_cmd = cmd_body | replace("\n", "\n            ") %}
        command: bash .buildkite/amd/scripts/run-amd-test.sh
        env:
-          DOCKER_BUILDKIT: "1"
+          AMD_CI_RUNTIME: "native"
+          NATIVE_CI: "true"
+          VLLM_CI_DOCKER_DISABLED: "1"
+          VLLM_CI_EXPECTED_GPU_COUNT: "{{ step.agent_pool[-1:] }}"
+          VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE: "1"
+          HF_HOME: "/home/buildkite-agent/huggingface"
+          HF_HUB_DOWNLOAD_TIMEOUT: "300"
+          HF_HUB_ETAG_TIMEOUT: "60"
+          MIOPEN_DEBUG_CONV_DIRECT: "0"
+          MIOPEN_DEBUG_CONV_GEMM: "0"
+          VLLM_ROCM_USE_AITER: "0"
+          PYTORCH_ROCM_ARCH: ""
 {% if step.env %}
 {% for name in step.env %}
          {{ name }}: "{{ step.env[name] }}"
 {% endfor %}
 {% endif %}
-          TEST_COMMAND: |-
+          TEST_COMMANDS: |-
            (command rocm-smi || true) && cd {{ (step.working_dir or default_working_dir) | safe }}
 {% if env %}
 {% for name in env %}
@@ -82,6 +148,15 @@ steps:
 {% endif %}
            {{ indented_cmd | safe }}
        priority: 100
+{% if step.parallelism %}
+        parallelism: {{ step.parallelism }}
+{% endif %}
+{% if step.artifact_paths %}
+        artifact_paths:
+{% for path in step.artifact_paths %}
+          - "{{ path }}"
+{% endfor %}
+{% endif %}
        {% if step.timeout_in_minutes %}
        timeout_in_minutes: {{ step.timeout_in_minutes }}
        {% endif %}
@@ -100,19 +175,31 @@ steps:
      {% if step.agent_pool %}
      queue: amd_{{ step.agent_pool }}
      {% else %}
-      queue: amd_mi325_1
+      queue: amd_mi300_1
      {% endif %}
+{{ amd_native_k8s_plugin("    ", step) }}
 {% set cmd_body = (step.command or (step.commands | join("\n"))) | trim %}
 {% set indented_cmd = cmd_body | replace("\n", "\n        ") %}
    command: bash .buildkite/amd/scripts/run-amd-test.sh
    env:
-      DOCKER_BUILDKIT: "1"
+      AMD_CI_RUNTIME: "native"
+      NATIVE_CI: "true"
+      VLLM_CI_DOCKER_DISABLED: "1"
+      VLLM_CI_EXPECTED_GPU_COUNT: "{{ step.agent_pool[-1:] }}"
+      VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE: "1"
+      HF_HOME: "/home/buildkite-agent/huggingface"
+      HF_HUB_DOWNLOAD_TIMEOUT: "300"
+      HF_HUB_ETAG_TIMEOUT: "60"
+      MIOPEN_DEBUG_CONV_DIRECT: "0"
+      MIOPEN_DEBUG_CONV_GEMM: "0"
+      VLLM_ROCM_USE_AITER: "0"
+      PYTORCH_ROCM_ARCH: ""
 {% if step.env %}
 {% for name in step.env %}
      {{ name }}: "{{ step.env[name] }}"
 {% endfor %}
 {% endif %}
-      TEST_COMMAND: |-
+      TEST_COMMANDS: |-
        (command rocm-smi || true) && cd {{ (step.working_dir or default_working_dir) | safe }}
 {% if env %}
 {% for name in env %}
@@ -129,6 +216,15 @@ steps:
 {% endif %}
        {{ indented_cmd | safe }}
    priority: 100
+{% if step.parallelism %}
+    parallelism: {{ step.parallelism }}
+{% endif %}
+{% if step.artifact_paths %}
+    artifact_paths:
+{% for path in step.artifact_paths %}
+      - "{{ path }}"
+{% endfor %}
+{% endif %}
    {% if step.timeout_in_minutes %}
    timeout_in_minutes: {{ step.timeout_in_minutes }}
    {% endif %}
@@ -9,6 +9,11 @@
 #   a2b3_npu_1  — ascend-a2b3, resource_class npu-1 (A2/B3 CI image)
 #   a2b3_npu_4  — ascend-a2b3, resource_class npu-4
 #   a3_npu_2    — ascend-a3, resource_class npu-2
+#
+# The ``l4_*`` presets run under the ``docker`` plugin, which does not put
+# ``buildkite-agent`` on PATH inside the container the way the ``kubernetes``
+# presets do, so they set ``mount-buildkite-agent: true``; without it any step
+# calling ``buildkite-agent`` (artifact upload, annotate, meta-data) fails there.

 x-npu-k8s-plugins: &npu_k8s_plugins
  - kubernetes:
@@ -26,6 +31,7 @@ mirror_hardwares:
          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
          always-pull: true
          propagate-environment: true
+          mount-buildkite-agent: true
          shm-size: "8gb"
          environment:
            - HF_HOME=/fsx/hf_cache
@@ -41,6 +47,7 @@ mirror_hardwares:
          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
          always-pull: true
          propagate-environment: true
+          mount-buildkite-agent: true
          shm-size: "8gb"
          environment:
            - HF_HOME=/fsx/hf_cache
@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+# Run a model's E2E tests once per entry mode as separate coverage runs and upload
+# the reports, so per-mode attribution survives (see vllm-project/vllm-omni#5332).
+#
+# Usage: run_cov_split.sh --model-id <id> [--offline <paths>] [--online <paths>] \
+#                         -- <pytest args...>
+#
+#   --model-id  Names the artifacts: coverage-<id>-{offline,online}-<step id>.xml.gz.
+#               Use the model's directory name under vllm_omni/*/models/.
+#   --offline   Pytest path(s) for the offline half. Repeatable.
+#   --online    Pytest path(s) for the online half. Repeatable.
+#   --          Everything after this is forwarded verbatim to every pytest run,
+#               quoting preserved: the job's own -m expression, --run-level, and
+#               anything else it needs (e.g. --test-config-file).
+#
+# At least one of --offline/--online is required, so a job with only one mode runs
+# just that half. Total runtime is bounded by the step's timeout_in_minutes.
+#
+# Exits non-zero if any half or the upload failed.
+set -uo pipefail
+
+MODEL_ID=""
+OFFLINE=()
+ONLINE=()
+PYTEST_ARGS=()
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --model-id) MODEL_ID="$2"; shift 2 ;;
+        --offline)  OFFLINE+=("$2"); shift 2 ;;
+        --online)   ONLINE+=("$2"); shift 2 ;;
+        --)         shift; PYTEST_ARGS=("$@"); break ;;
+        *) echo "run_cov_split.sh: unknown argument: $1" >&2; exit 2 ;;
+    esac
+done
+
+if [[ -z "${MODEL_ID}" ]]; then
+    echo "run_cov_split.sh: missing --model-id" >&2
+    exit 2
+fi
+
+if ((${#OFFLINE[@]} == 0 && ${#ONLINE[@]} == 0)); then
+    echo "run_cov_split.sh: need at least one of --offline / --online" >&2
+    exit 2
+fi
+
+# Two steps can test the same model, and artifacts are scoped to the build (a
+# nightly one carries the ready, merge and nightly tiers at once), so unsuffixed
+# reports would share a path -- a glob download then races to rename onto one file.
+# Not BUILDKITE_JOB_ID: the step id survives retries, so attempts collapse under
+# Buildkite's latest-attempt lookup instead of accumulating. A parallel step would
+# need BUILDKITE_PARALLEL_JOB as well; no pilot is one.
+REPORT_SUFFIX="${BUILDKITE_STEP_ID:-local}"
+OFFLINE_REPORT="coverage-${MODEL_ID}-offline-${REPORT_SUFFIX}.xml"
+ONLINE_REPORT="coverage-${MODEL_ID}-online-${REPORT_SUFFIX}.xml"
+
+run_half() {
+    local mode="$1"
+    shift
+    local report="coverage-${MODEL_ID}-${mode}-${REPORT_SUFFIX}.xml"
+    echo "--- coverage: ${MODEL_ID} ${mode}"
+    pytest -s -v "$@" "${PYTEST_ARGS[@]}" --cov=vllm_omni --cov-report="xml:${report}"
+    local status=$?
+    # Cobertura lists every statement in vllm_omni whether or not it ran, so the
+    # report is ~7MB no matter how little the job covers. The repetition is what
+    # dominates it, and gzip takes that to ~400KB.
+    gzip -9 -f "${report}" || status=1
+    return "${status}"
+}
+
+# Clear both modes, not just the ones being run: the upload glob matches both, so
+# a report left by an earlier run would otherwise be published as if the mode it
+# belongs to had run this time.
+rm -f "${OFFLINE_REPORT}" "${ONLINE_REPORT}" "${OFFLINE_REPORT}.gz" "${ONLINE_REPORT}.gz"
+
+EXIT=0
+((${#OFFLINE[@]})) && { run_half offline "${OFFLINE[@]}" || EXIT=1; }
+((${#ONLINE[@]})) && { run_half online "${ONLINE[@]}" || EXIT=1; }
+
+buildkite-agent artifact upload "coverage-${MODEL_ID}-*-${REPORT_SUFFIX}.xml.gz" || EXIT=1
+
+exit "${EXIT}"
@@ -94,13 +94,19 @@ steps:
          - vllm_omni/model_executor/models/qwen3_tts/
          - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
          - vllm_omni/deploy/qwen3_tts.yaml
+          - .buildkite/common/scripts/run_cov_split.sh
+          - pyproject.toml
+        # 40 -> 60: the coverage split runs the model twice.
+        timeout_in_minutes: 60
        commands:
          - |
-            timeout 40m bash -c "
-              export VLLM_LOGGING_LEVEL=DEBUG
-              export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
-              pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-            "
+            export VLLM_LOGGING_LEVEL=DEBUG
+            export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
+            .buildkite/common/scripts/run_cov_split.sh \
+              --model-id qwen3_tts \
+              --offline tests/e2e/offline_inference/test_qwen3_tts_base.py \
+              --online tests/e2e/online_serving/test_qwen3_tts_base.py \
+              -- -m 'advanced_model and cuda' --run-level 'advanced_model'
        mirror_hardwares: l4_1

      - label: "TTS · Higgs-Audio-V3 Test"
@@ -203,9 +209,17 @@ steps:
          - vllm_omni/diffusion/models/bagel/
          - vllm_omni/model_executor/stage_input_processors/bagel.py
          - vllm_omni/deploy/bagel.yaml
-        timeout_in_minutes: 20
+          - .buildkite/common/scripts/run_cov_split.sh
+          - pyproject.toml
+        # 20 -> 40: the coverage split runs the model twice.
+        timeout_in_minutes: 40
        commands:
-          - pytest -s -v tests/e2e/offline_inference/test_bagel.py tests/e2e/online_serving/test_bagel.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+          - |
+            .buildkite/common/scripts/run_cov_split.sh \
+              --model-id bagel \
+              --offline tests/e2e/offline_inference/test_bagel.py \
+              --online tests/e2e/online_serving/test_bagel.py \
+              -- -m 'advanced_model and cuda' --run-level 'advanced_model'
        mirror_hardwares: h100_1

      - label: "Diffusion · Qwen Image Test"
@@ -87,8 +87,9 @@ time timeout -k 30 30m docker run \
    echo $ZE_AFFINITY_MASK
    pip install tblib==3.1.0
    cd /workspace/vllm-omni
-    pytest -v -s -m "core_model and xpu and B60"
+    XPU_TEST_PATHS="tests/diffusion tests/dfx tests/e2e"
+    pytest -v -s $XPU_TEST_PATHS -m "core_model and xpu and B60"
    pytest -v -s tests/diffusion/quantization/test_mxfp8_config.py
-    pytest -v -s -m "advanced_model and xpu and B60"
-    pytest -v -s -m "omni and xpu and B60"
+    pytest -v -s $XPU_TEST_PATHS -m "advanced_model and xpu and B60"
+    pytest -v -s $XPU_TEST_PATHS -m "omni and xpu and B60"
 '
@@ -46,6 +46,8 @@ htmlcov/
 .cache
 nosetests.xml
 coverage.xml
+coverage-*.xml
+coverage-*.xml.gz
 *.cover
 *.py,cover
 .hypothesis/
@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.26.0
+ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.27.0
 FROM ${BASE_IMAGE} AS base

 # Declare a variable to know if we want to use the nightly build or the stable build.
@@ -10,7 +10,7 @@ FROM ${BASE_IMAGE} AS base
 #    we should swap over to use stable release ASAP.
 #    We should avoid relying on custom commits.
 ARG USE_NIGHTLY_BUILD=0
-ARG VLLM_VERSION_OR_COMMIT_HASH=v0.26.0
+ARG VLLM_VERSION_OR_COMMIT_HASH=v0.27.0
 ARG ARG_PYTORCH_ROCM_ARCH
 ENV PYTORCH_ROCM_ARCH=${ARG_PYTORCH_ROCM_ARCH:-${PYTORCH_ROCM_ARCH}}

@@ -79,9 +79,16 @@ RUN printf '%s\n' \
    '    os._exit(_exit_code)' \
    > ${COMMON_WORKDIR}/vllm-omni/conftest.py

+# Some optional dependencies pull in the CPU ONNX Runtime distribution. The
+# CPU and ROCm wheels own the same import path and cannot coexist, so remove
+# the CPU wheel and reinstall the ROCm wheel after dependency resolution.
 RUN cd ${COMMON_WORKDIR}/vllm-omni && \
    uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" \
        --no-cache-dir ".[dev]" --no-build-isolation && \
+    python3 -m pip uninstall -y onnxruntime && \
+    python3 -m pip install --no-cache-dir --force-reinstall --no-deps \
+        "onnxruntime-rocm>=1.22.2" && \
+    python3 -c "import importlib.metadata as m; import onnxruntime as ort; assert 'ROCMExecutionProvider' in ort.get_available_providers(), ort.get_available_providers(); assert m.version('onnxruntime-rocm'); assert not any(d.metadata['Name'] == 'onnxruntime' for d in m.distributions())" && \
    python3 -c "from torchcodec.decoders import VideoDecoder"

 RUN ln -sf /usr/bin/python3 /usr/bin/python
@@ -60,7 +60,11 @@ nav:
      - Overview and Compatibility:
        - Overview: user_guide/diffusion_features.md
        - Feature Compatibility: user_guide/feature_compatibility.md
-      - CPU Offloading: user_guide/diffusion/cpu_offload.md
+      - CPU Offloading:
+        - Overview: user_guide/diffusion/cpu_offload.md
+        - Model-Level: user_guide/diffusion/offloader/cpu_offload.md
+        - Layerwise: user_guide/diffusion/offloader/layerwise_offload.md
+        - Distributed Layerwise: user_guide/diffusion/offloader/distributed_layerwise_offload.md
      - Cache Acceleration:
        - TeaCache: user_guide/diffusion/cache_acceleration/teacache.md
        - Cache-DiT: user_guide/diffusion/cache_acceleration/cache_dit.md
@@ -72,7 +76,13 @@ nav:
        - Sequence Parallel: user_guide/diffusion/parallelism/sequence_parallel.md
        - Tensor Parallel: user_guide/diffusion/parallelism/tensor_parallel.md
        - VAE Parallelism: user_guide/diffusion/parallelism/vae_parallelism.md
-      - Attention Backends: user_guide/diffusion/attention_backends.md
+      - Attention Backends:
+        - Overview: user_guide/diffusion/attention_backends.md
+        - Dense Backends: user_guide/diffusion/attention_backends/dense_backends.md
+        - TRTLLM: user_guide/diffusion/attention_backends/trtllm.md
+        - SageAttention: user_guide/diffusion/attention_backends/sage.md
+        - Hugging Face Hub: user_guide/diffusion/attention_backends/huggingface_hub.md
+        - RainFusion: user_guide/diffusion/attention_backends/rainfusion.md
      - Regional Compilation: user_guide/diffusion/regional_compilation.md
      - Frame Interpolation: user_guide/diffusion/frame_interpolation.md
      - Startup and Loading: user_guide/diffusion/startup_and_loading.md
@@ -125,11 +135,16 @@ nav:
          - Tensor Parallel: design/feature/tensor_parallel.md
          - VAE Patch Parallelism: design/feature/vae_parallel.md
        - Attention Optimization:
+          - Backend Selection: design/feature/attention_backend_selection.md
          - Skip-Softmax: design/feature/skip_softmax.md
+        - CPU Offloading:
+          - Overview: design/feature/offloader/README.md
+          - Model-Level: design/feature/offloader/cpu_offload.md
+          - Layerwise: design/feature/offloader/layerwise_offload.md
+          - Distributed Layerwise: design/feature/offloader/distributed_layerwise_offload.md
        - Cache-DiT: design/feature/cache_dit.md
        - TeaCache: design/feature/teacache.md
        - Diffusion Continuous Batching: design/feature/diffusion_continuous_batching.md
-        - Distributed Layerwise Offload: design/feature/distributed_layerwise_offload.md
    - Infrastructure and Performance:
      - Prometheus Metrics: design/metrics.md
      - Speech Generation Performance Optimizations: design/qwen3_omni_tts_performance_optimization.md
@@ -327,6 +327,96 @@ On PR builds, `upload_pipeline.py` logs `skip '…' (no changes under …)` for
 - Use **groups** for dashboard readability; keep **E2E Test** as the group name CUDA diff filtering expects for `--e2e` nightly runs on main.
 - Prefix labels by model domain: **Omni ·**, **TTS ·**, **Diffusion ·**, **Simple ·**, etc., matching existing steps.

+### Per-model coverage
+
+Pilot (v1): two Merge-tier (L3) jobs in `.buildkite/cuda/test-merge.yml` upload
+per-model, per-entry-mode coverage as Buildkite artifacts — "Diffusion · Bagel
+Test" and "TTS · Qwen3-TTS Base Test". Both run on a single GPU so the pilot is
+cheap to reproduce. This is a pilot, not a rollout: no dashboard/visualization
+lives in this repo, and nothing is gated on coverage.
+
+**Naming convention**: `coverage-<model_id>-<mode>-<step_id>.xml.gz`, where
+`<model_id>` is the model's directory name under
+`vllm_omni/diffusion/models/<model_id>/` or
+`vllm_omni/model_executor/models/<model_id>/` (e.g. `z_image`, `qwen3_omni`),
+`<mode>` is `online` or `offline`, and `<step_id>` is `BUILDKITE_STEP_ID`
+(`local` outside Buildkite). `gunzip` before feeding a report to a Cobertura
+consumer.
+
+Reports are gzipped because `--cov=vllm_omni` sets coverage's `source`, which
+makes it walk the package and emit one `<line>` element per statement for every
+file — including the ones a single model's tests never import. That inventory,
+not the coverage, is the size: a Bagel report is 7.1 MB, of which 96.5% is
+`<line>` elements and 77% belongs to the 840 of 1169 files that run never
+imported. A run covering zero lines produces the same 7 MB. gzip takes it to
+roughly 410 KB and changes nothing about the data.
+
+`<step_id>` is what keeps two steps that cover the same model apart. Artifacts are
+scoped to the build, and a nightly build carries the ready, merge and nightly tiers
+at once, so several same-model steps land in one namespace — `qwen3_tts` already has
+both a Base and a CustomVoice job, and `minicpmo_4_5` a base and a duplex one. Two
+uploads on one path do not fail loudly: an exact-name download can report the path
+as ambiguous, and a glob download fetches every match concurrently and renames each
+onto the same destination, so whichever finishes last wins. The step id remains the
+logical step identifier across retries, while Buildkite's default artifact lookup
+selects only the latest attempt — `BUILDKITE_JOB_ID` would instead give every
+attempt its own filename, leaving a failed attempt's partial report to be picked up
+alongside the real one. A step using `parallelism`/matrix would need
+`BUILDKITE_PARALLEL_JOB` added, since its jobs share a step id; neither pilot is
+one.
+
+**Opting in a new model**: replace the job's combined `pytest` command with
+[`run_cov_split.sh`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/scripts/run_cov_split.sh),
+which runs each mode, names the reports, uploads them, and fails the step if
+either half or the upload failed:
+
+```yaml
+commands:
+  - |
+    .buildkite/common/scripts/run_cov_split.sh \
+      --model-id <model_id> \
+      --offline <offline test path> \
+      --online <online test path> \
+      -- <the job's existing pytest flags, e.g. -m '...' --run-level '...'>
+```
+
+Everything after `--` is forwarded verbatim to both runs with quoting preserved,
+so an expression like `-m 'advanced_model and cuda'` survives intact and a job can
+pass anything else it needs. `--offline`/`--online` are repeatable and only one is
+required, so a job with tests in a single mode runs just that half. Runtime is
+bounded solely by the step's `timeout_in_minutes` — raise it, since the split
+loads the model once per mode.
+
+Before splitting, confirm each half actually collects at least one test under the job's forwarded filters — pytest exits with code 5 ("no tests collected") if a half is empty, which the script reports as a failure, red-ing a job that used to pass as one combined invocation.
+
+Also check the tests' `num_cards` against the job's `mirror_hardwares` preset.
+`cuda_marks` turns `num_cards > 1` into `skipif(device_count() < num_cards)`, so a
+test asking for more GPUs than the preset provides is silently skipped and its
+coverage artifact comes back empty. Splitting such a job produces a file that
+looks fine and measures nothing — "Diffusion · Z Image Test" is the current
+example (online tests want 4 cards, the job runs on `l4_1`), which is why it is
+not a coverage pilot.
+
+Online-mode coverage depends on the `[tool.coverage.run]` `patch`/`sigterm`
+settings in `pyproject.toml`: online tests launch the server with
+`subprocess.Popen` and stop it with SIGTERM, and without those settings the XML
+reflects only the pytest parent process, not the server's code paths.
+
+The upload depends on `buildkite-agent` being callable inside the container. The
+`kubernetes` presets (`h100_*`, `*_npu_*`) provide it; the `docker` ones (`l4_*`)
+only do because they set `mount-buildkite-agent: true`. A new docker preset that
+runs a coverage job needs the same.
+
+List both `run_cov_split.sh` and `pyproject.toml` in every opted-in job's
+`source_file_dependencies` — both change what the job measures, so without them a
+change there is filtered out of normal PR builds and only surfaces in a later
+nightly. `tests/buildkite/test_upload_pipeline.py` asserts this for the pilots.
+Editing only the surrounding CI YAML still does not schedule them, so a PR that
+touches just the wiring needs a full E2E run (or the commands run on a GPU host)
+to produce artifacts. When checking a new model's
+artifacts, compare `lines-covered` between the online and offline XML rather than
+just confirming both files exist.
+
 ### Validation checklist

 | Check | Command / location |
@@ -406,7 +406,7 @@ the source for configuration, compatibility, and implementation details.
  stage contract.
 * **Quantization and memory efficiency:** [Quantization](feature/quantization.md)
  resolves per-pipeline or per-component quantization configurations, while
-  [Distributed Layerwise Offload](feature/distributed_layerwise_offload.md)
+  [Distributed Layerwise Offload](feature/offloader/distributed_layerwise_offload.md)
  streams diffusion blocks from host memory within the existing parallel
  topology.

@@ -0,0 +1,89 @@
+# Diffusion Attention Backend Selection
+
+This document defines the selection and extension contract for diffusion
+attention backends. User-facing backend choices, installation, and tuning are
+in the [attention backend guides](../../user_guide/diffusion/attention_backends.md).
+
+## Scope
+
+The contract applies to DiT and other diffusion attention layers. It is
+separate from vLLM's autoregressive attention selector.
+
+The selection path has four responsibilities:
+
+1. normalize user configuration into `AttentionConfig` and `AttentionSpec`;
+2. resolve a spec for an attention role;
+3. ask the active platform to validate an explicit backend or choose a
+   hardware default; and
+4. load the selected `AttentionBackend` class from the registry path.
+
+## Resolution contract
+
+`get_attn_backend_for_role()` resolves configuration in this order:
+
+1. exact `per_role[role]`;
+2. category `per_role[role_category]`;
+3. `default`;
+4. platform default.
+
+An explicit resolution returns both the backend class and its
+`AttentionSpec`. A platform-default resolution returns the class and `None`.
+Layers must therefore treat the spec as optional and must not infer that a
+missing spec means SDPA.
+
+Class resolution is cached by backend name, head size, and whether TRTLLM may
+be selected automatically. Logging is cached separately by role and source so
+the startup record identifies why each role received its backend.
+
+## Registry and platform boundary
+
+`DiffusionAttentionBackendEnum` maps stable configuration names to default
+qualified class paths. `register_diffusion_backend()` may replace a path at
+runtime without changing the public enum value.
+
+The active `OmniPlatform` owns compatibility policy through
+`get_diffusion_attn_backend_cls()`. It must:
+
+- validate explicit selections and fail with an actionable error when the
+  requested kernel cannot run;
+- choose only an available, compatible default when no backend is explicit;
+- account for head size and other platform-visible constraints; and
+- return a qualified class path implementing `AttentionBackend`.
+
+The selector must not duplicate device capability or package-availability
+policy that belongs to the platform.
+
+## Typed backend options
+
+Backend-specific settings remain typed fields on `AttentionSpec` rather than
+unstructured keyword dictionaries:
+
+- `quant` is consumed by FlashInfer and TRTLLM with backend-specific value
+  validation;
+- `skip_softmax` is consumed by TRTLLM; and
+- `block_sparse` is consumed by block-sparse backends such as RainFusion.
+
+A backend reads only the fields it owns and rejects incompatible values. New
+options should be added to a shared typed spec only when more than one backend
+shares their semantics; otherwise add a dedicated typed block.
+
+## Model integration contract
+
+Each diffusion attention call declares a stable role and, when appropriate, a
+broader category. Model-specific roles permit precise overrides; categories
+preserve common `self` or `cross` policies. A model also decides whether its
+path is compatible with automatic TRTLLM selection, because only the model
+knows whether masking and packing satisfy the kernel contract.
+
+## Adding or changing a backend
+
+An implementation change is complete only when it:
+
+1. implements the `AttentionBackend` interface and exposes a stable enum name;
+2. adds platform validation and default routing where appropriate;
+3. defines and validates any typed configuration;
+4. tests explicit selection, default selection, and incompatible paths;
+5. documents installation, fallback behavior, and quality implications in the
+   corresponding user guide; and
+6. updates the overview matrix without moving algorithm details back into the
+   landing page.
@@ -0,0 +1,80 @@
+# CPU Offloading
+
+This document defines the shared architecture for diffusion CPU offloading.
+The three strategies have separate user and design pages:
+
+| Strategy | User guide | Design contract |
+| --- | --- | --- |
+| Model-level | [Guide](../../../user_guide/diffusion/offloader/cpu_offload.md) | [Design](cpu_offload.md) |
+| Layerwise | [Guide](../../../user_guide/diffusion/offloader/layerwise_offload.md) | [Design](layerwise_offload.md) |
+| Distributed layerwise | [Guide](../../../user_guide/diffusion/offloader/distributed_layerwise_offload.md) | [Design](distributed_layerwise_offload.md) |
+
+## Strategy selection
+
+`OffloadConfig.from_od_config()` converts diffusion configuration into one
+`OffloadStrategy`. If callers set multiple flags, selection is deterministic:
+
+1. `DISTRIBUTED_LAYERWISE`;
+2. `LAYERWISE`;
+3. `MODEL_LEVEL`; and
+4. `NONE`.
+
+The factory logs when a higher-priority strategy disables another. It derives
+parallel and HSDP state from `DiffusionParallelConfig`; callers do not provide
+a separate offload group size.
+
+`get_offload_backend()` then validates platform offload support, resolves the
+device, and creates exactly one backend. Returning `None` means offloading is
+disabled or unsupported and must not leave partially installed hooks.
+
+## Shared lifecycle
+
+Every backend implements `OffloadBackend`:
+
+- `enable(pipeline)` discovers modules, establishes initial residency, and
+  installs hooks;
+- `disable()` removes owned hooks and resources; and
+- `is_enabled()` reports lifecycle state.
+
+`disable()` does not promise to restore the pipeline's original device
+placement. The caller owns any subsequent rematerialization.
+
+Hooks are registered through `HookRegistry` and `ModelHook`; offload backends
+must use distinct hook names and remove only hooks they own.
+
+## Discovery and topology
+
+Pipeline component discovery prefers `SupportsComponentDiscovery` declarations:
+
+- `_dit_modules`;
+- `_encoder_modules`;
+- `_vae_modules`; and
+- `_resident_modules`.
+
+Dotted paths are supported. Legacy pipelines may use the fallback scan of
+well-known attribute names, but new integrations should declare components
+explicitly.
+
+Ordinary layerwise offload obtains topology from declared
+`_layerwise_offload_blocks_attrs` (with a deprecated singular-name
+compatibility path). Distributed layerwise offload may instead use a pipeline
+`OffloadPlan`, then falls back to declared block attributes and its documented
+compatibility discovery. Discovery metadata describes structure only; the
+backend remains responsible for transfer, synchronization, and storage
+ownership.
+
+## Cross-strategy invariants
+
+- At most one strategy owns offload hooks for a pipeline.
+- A parameter has one authoritative host representation while offloaded.
+- Device storage is not freed until dependent compute or transfer work is
+  complete.
+- Non-persistent and model-specific buffers remain correct after movement or
+  checkpoint rematerialization.
+- Unsupported parallel or loading combinations fail before hooks mutate the
+  model.
+- Platform streams, events, synchronization, and cache management go through
+  the vLLM-Omni platform abstraction.
+
+The diffusion [Offloader module design](../../module/diffusion/offloader.md)
+describes how these feature contracts fit into the larger diffusion runtime.
@@ -0,0 +1,56 @@
+# Model-Level Offload
+
+Model-level offload enforces mutual exclusion between pipeline component
+groups. For commands and model support, see the
+[model-level user guide](../../../user_guide/diffusion/offloader/cpu_offload.md).
+
+## Generic hook path
+
+`ModelLevelOffloadBackend.enable()` discovers DiTs, encoders, and VAEs. It
+makes encoders and VAEs device resident for initialization, then installs
+`SequentialOffloadHook` instances:
+
+- each DiT hook offloads all encoders and other DiTs before loading itself;
+- each encoder hook offloads every DiT before loading itself; and
+- VAEs are not part of the mutual-exclusion hook set.
+
+The hook moves parameters and buffers without recursively calling
+`module.to()`. This avoids recursion through modules that retain references to
+other transformer blocks. Host tensors are pinned when configured; DTensors
+are not pinned through the ordinary tensor path.
+
+Before forward returns control to the model, the hook synchronizes the active
+platform so compute cannot race the transfer. XPU uses blocking device-to-host
+copies because its allocator cannot safely evict storage while an asynchronous
+copy is pending.
+
+## Split-model delegation
+
+If a pipeline provides callable `enable_omni_model_cpu_offload`, the backend
+delegates instead of installing generic encoder/DiT hooks. This supports models
+whose mutually exclusive components are nested inside one transformer, such
+as a reasoner and generator.
+
+The delegated pipeline owns its internal contexts, but must preserve the same
+contract:
+
+- only the executing component is device resident;
+- enable and disable are idempotent;
+- transfers use the supplied device, pinning, and HSDP settings; and
+- disabling removes all model-local offload hooks or contexts.
+
+The delegation is currently duck typed. If another independent implementation
+is added, formalize the enable/disable pair as a protocol rather than expanding
+attribute-name heuristics.
+
+## Extension invariants
+
+- New ordinary pipelines declare both DiT and encoder groups through
+  `SupportsComponentDiscovery`.
+- Models with multiple DiTs offload the inactive DiTs as well as encoders.
+- Empty or parameterless modules do not trigger transfers.
+- Hook removal does not move modules; callers must not assume original
+  residency is restored.
+
+Shared selection and lifecycle behavior is defined in the
+[CPU Offloading design](README.md).
@@ -7,7 +7,7 @@ either shard those host-side weights across an existing parallel group or keep
 the standard loader's rank-local weights and avoid an additional collective.

 For user-facing commands, see the
-[distributed layerwise offloading guide](../../user_guide/diffusion/cpu_offload.md)
+[distributed layerwise offloading guide](../../../user_guide/diffusion/offloader/distributed_layerwise_offload.md)
 and the [Cosmos3 recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/cosmos3/Cosmos3-DistOffload.md).

 ## Status
@@ -56,7 +56,7 @@ AllGather:                [full N+1]             [full N+2]
 Buffers:    [current slot]       [prefetch slot]       [current slot]
 ```

-![DLO double-buffer prefetch pipeline](../figures/dlo/dlo_pipeline.gif)
+![DLO double-buffer prefetch pipeline](../../figures/dlo/dlo_pipeline.gif)

 The backend uses two shared device buffers, so accelerator weight residency is
 bounded by the largest streamed blocks rather than the complete model.
@@ -0,0 +1,57 @@
+# Layerwise Offload
+
+Layerwise offload streams transformer blocks through one accelerator. For
+commands and model support, see the
+[layerwise user guide](../../../user_guide/diffusion/offloader/layerwise_offload.md).
+
+## Hook ring
+
+`LayerWiseOffloadBackend` discovers every streamable block and registers one
+`LayerwiseOffloadHook` per block. Hooks form a ring: each hook owns the pinned
+host representation of its registered block and a reference to the next block
+it must prefetch; the final hook prefetches the first block for the next
+denoising iteration.
+
+During initialization, parameters and buffers are grouped by dtype and copied
+into contiguous pinned CPU tensors. Original tensor storage is replaced with
+an empty or meta placeholder while identity and module structure are retained.
+Metadata records names, offsets, shapes, and dtypes for rematerialization.
+
+## Forward lifecycle
+
+For each block:
+
+1. the previous hook's completion event ensures this block is materialized;
+2. the copy stream asynchronously allocates and fills the next block's device
+   tensors;
+3. parameter and buffer storage is rebound to views of those tensors;
+4. the current block computes on the normal stream; and
+5. its post-forward hook replaces device storage with placeholders after all
+   dependent work is safe.
+
+The copy stream waits on the compute stream before reusing storage, and the
+compute stream waits on the per-block prefetch event before forward. These
+dependencies, not incidental global synchronization, define correctness.
+
+## Topology contract
+
+A DiT declares ordered block-container attributes using
+`_layerwise_offload_blocks_attrs` or an `OffloadPlan`. Each resolved item must
+be an executable `nn.Module`; ordering must match forward execution. Multiple
+containers are concatenated in execution order.
+
+Non-block DiT modules, encoders, VAEs, and declared resident modules are not
+streamed by this backend.
+
+## Invariants and limitations
+
+- Each offloaded tensor has one pinned host backing store.
+- Parameter identity is preserved when storage is rebound.
+- Blocks with heterogeneous shapes and dtypes use their recorded metadata;
+  implementations must not assume identical blocks.
+- The ring is initialized before the first forward so block zero is available.
+- This backend is single-device; distributed weight reconstruction belongs to
+  [Distributed Layerwise Offload](distributed_layerwise_offload.md).
+
+Shared selection and lifecycle behavior is defined in the
+[CPU Offloading design](README.md).
@@ -186,7 +186,7 @@ Distributed layerwise offload has an additional boundary: its sharded mmap
 AllGather path is not compatible with online quantization because the path
 expects already materialized checkpoint weights. Use the rank-local
 `--dlo-no-use-allgather` path or a compatible pre-quantized checkpoint when
-combining these features. See the [DLO design document](distributed_layerwise_offload.md)
+combining these features. See the [DLO design document](offloader/distributed_layerwise_offload.md)
 for the full compatibility matrix.

 ## Adding a quantization backend
@@ -2,8 +2,10 @@

 Skip-Softmax is the sparse-attention mode of the `TRTLLM_ATTN` backend. Usage — the config keys
 and how to pick an operating point — is in
-[Diffusion Attention Backends](../../user_guide/diffusion/attention_backends.md). This page explains
-the algorithm.
+[TRTLLM Attention](../../user_guide/diffusion/attention_backends/trtllm.md#skip-softmax).
+The shared selector contract is documented in
+[Diffusion Attention Backend Selection](attention_backend_selection.md). This
+page explains the algorithm.

 ## Motivation

@@ -53,15 +53,22 @@ implementation contract; it is not, by itself, a general support claim.
 #### Attention optimization

 The [Diffusion Attention Backends](../user_guide/diffusion/attention_backends.md)
-guide lists all selectable backends and their platform defaults. Skip-Softmax
-is the backend-related optimization with a standalone feature-design contract:
+guides list selectable backends, platform defaults, installation, and tuning.
+The design contracts separate selection mechanics from backend algorithms:

+- [Attention Backend Selection](feature/attention_backend_selection.md)
 - [Skip-Softmax](feature/skip_softmax.md)

+#### CPU offloading
+
+- [Overview and Shared Contracts](feature/offloader/README.md)
+- [Model-Level Offload](feature/offloader/cpu_offload.md)
+- [Layerwise Offload](feature/offloader/layerwise_offload.md)
+- [Distributed Layerwise Offload](feature/offloader/distributed_layerwise_offload.md)
+
 - [Cache-DiT](feature/cache_dit.md)
 - [TeaCache](feature/teacache.md)
 - [Diffusion Continuous Batching](feature/diffusion_continuous_batching.md)
- [Distributed Layerwise Offload](feature/distributed_layerwise_offload.md)

 ## Infrastructure and Performance

@@ -48,10 +48,10 @@ defines the shared configuration and backend extension points.
 | Goal | User guide | Related design contract |
 | --- | --- | --- |
 | Compare acceleration methods and supported combinations | [Overview](../user_guide/diffusion_features.md), [Feature Compatibility](../user_guide/feature_compatibility.md) | [Diffusion acceleration designs](../design/index.md#diffusion-acceleration) |
-| Move weights between host and device memory | [CPU Offloading](../user_guide/diffusion/cpu_offload.md) | [Distributed Layerwise Offload](../design/feature/distributed_layerwise_offload.md) |
+| Move weights between host and device memory | [CPU Offloading](../user_guide/diffusion/cpu_offload.md) | [CPU Offloading](../design/feature/offloader/README.md) |
 | Reuse denoising computation | [Cache Acceleration](../user_guide/diffusion/cache_acceleration/cache_dit.md) | [Cache-DiT](../design/feature/cache_dit.md), [TeaCache](../design/feature/teacache.md) |
 | Distribute diffusion work across devices | [Parallelism](../user_guide/diffusion/parallelism/overview.md) | [Parallelism designs](../design/index.md#parallelism) |
-| Select dense, sparse, or quantized attention paths | [Attention Backends](../user_guide/diffusion/attention_backends.md) | [Skip-Softmax](../design/feature/skip_softmax.md) |
+| Select dense, sparse, or quantized attention paths | [Attention Backends](../user_guide/diffusion/attention_backends.md) | [Attention Backend Selection](../design/feature/attention_backend_selection.md) |
 | Compile repeated diffusion regions | [Regional Compilation](../user_guide/diffusion/regional_compilation.md) | User-facing optimization guide |
 | Add generated video frames | [Frame Interpolation](../user_guide/diffusion/frame_interpolation.md) | User-facing extension guide |
 | Reduce diffusion model startup time | [Startup and Loading](../user_guide/diffusion/startup_and_loading.md) | User-facing loading guide |
@@ -1,494 +1,192 @@
 # Diffusion Attention Backends

-This document describes the diffusion attention backends available in vLLM-Omni, how to select them globally and per-role, the per-platform defaults, and how to use SageAttention.
+Use this page to select and configure a diffusion attention backend. Backend-
+specific installation, tuning, and compatibility details live in separate
+guides so that the selection contract stays easy to scan.

-## Overview
+Diffusion attention backend selection applies to DiT attention in image and
+video generation models. It does **not** change autoregressive LLM attention,
+which uses vLLM's attention selector.

-Diffusion attention backend selection is resolved in `vllm_omni.diffusion.attention.selector`. It looks up the backend from a structured `AttentionConfig` carried on `OmniDiffusionConfig` and falls back to the platform default when nothing is configured.
+For the internal selector, registry, and platform contract, see
+[Attention Backend Selection](../../design/feature/attention_backend_selection.md).

-This backend is used by diffusion attention layers such as the DiT attention in video and image generation models. It does **not** affect autoregressive (LLM) attention paths — those go through vLLM's own attention backend selector.
+## Choose a guide

-The full set of backends and their platform defaults is in the **Backend Options** and **Platform Defaults** sections below. If no attention backend is configured, vLLM-Omni asks the current platform to choose the default.
+| Need | Guide |
+| --- | --- |
+| Select a conservative or platform-native dense kernel | [Dense Backends](attention_backends/dense_backends.md) |
+| Use FlashInfer trtllm-gen FMHA, Skip-Softmax, or TRTLLM SAGE quantization | [TRTLLM Attention](attention_backends/trtllm.md) |
+| Install and use SageAttention 2.2 or SageAttention3 | [SageAttention](attention_backends/sage.md) |
+| Match training or rollout kernels loaded from Hugging Face | [Hugging Face Hub Backends](attention_backends/huggingface_hub.md) |
+| Use block-sparse video attention on Ascend NPU | [RainFusion](attention_backends/rainfusion.md) |

-## Backend Options
-
-| Value | Notes |
-|---|---|
-| `TRTLLM_ATTN` | FlashInfer's trtllm-gen FMHA (TensorRT-LLM's generated kernels, vendored by FlashInfer). Dense BF16, GQA native, `head_dim=128`. Datacenter Blackwell only (sm_100 / sm_103). Packed paths can provide `cu_seqlens` directly. Supports optional **Skip-Softmax** sparse attention — see below. Requires `flashinfer`. |
-| `FLASH_ATTN` | Wraps FlashAttention 4 on Blackwell when `flash-attn-4` is installed, then falls back to FlashAttention 3/2. Default on Hopper / Ada / Ampere when a compatible FlashAttention package is installed. |
-| `CUDNN_ATTN` | Pins `sdpa_kernel([CUDNN_ATTENTION])`. Default on Blackwell (sm_10x / sm_12x) with cuDNN ≥ 9.5. Wins on mask-heavy DiTs (HunyuanVideo-1.5: 2× e2e vs SDPA). |
-| `FLASHINFER_ATTN` | Uses FlashInfer's batch-prefill wrapper and supports mixed Q/K and V input dtypes through backend-specific configuration. Used as Blackwell fallback when cuDNN is unavailable. Requires `flashinfer` >= 0.6.16rc1 for mixed-dtype configurations. |
-| `TORCH_SDPA` | PyTorch `scaled_dot_product_attention` with the default backend dispatcher. Most conservative; always available. |
-| `SAGE_ATTN` | SageAttention 2.2 — INT8-quantized attention with FP16 accumulation. Lossy but typically visually indistinguishable on diffusion outputs. Requires `sageattention`. |
-| `SAGE_ATTN_3` | Requires `sageattn3` from `SageAttention/sageattention3_blackwell`. CUDA only, intended for Blackwell GPUs, with GQA/MQA requests falling back to PyTorch SDPA. |
-| `FLASH_ATTN_HUB` | FlashAttention 2 from HuggingFace `kernels` library. Useful for train/rollout alignment. |
-| `FLASH_ATTN_3_HUB` | FlashAttention 3 from HuggingFace `kernels` library. CUDA Hopper (sm_90+) only; falls back to `FLASH_ATTN_HUB` on older GPUs. |
-| `RAINFUSION_ATTN` | MindIE-SD **RainFusion** block-sparse video attention — see [below](#rainfusion_attn-backend-and-block-sparse-video-attention). Ascend NPU only; requires `mindiesd`. Delegates to `FLASH_ATTN` for anything that is not a packed video sequence. |
+## Backend options

+| Value | Family | Primary use | Detail |
+| --- | --- | --- | --- |
+| `TORCH_SDPA` | Dense | Conservative reference; always available | [Dense Backends](attention_backends/dense_backends.md#torch_sdpa) |
+| `FLASH_ATTN` | Dense | FlashAttention 4/3/2 depending on the installed package and GPU | [Dense Backends](attention_backends/dense_backends.md#flash_attn) |
+| `CUDNN_ATTN` | Dense | Mask-heavy DiTs on Blackwell with cuDNN 9.5 or newer | [Dense Backends](attention_backends/dense_backends.md#cudnn_attn) |
+| `FLASHINFER_ATTN` | Dense or quantized | FlashInfer batch prefill; optional mixed Q/K and V dtypes | [Dense Backends](attention_backends/dense_backends.md#flashinfer_attn) |
+| `TRTLLM_ATTN` | Dense, sparse, or quantized | Datacenter Blackwell with `head_dim=128` and compatible packed paths | [TRTLLM Attention](attention_backends/trtllm.md) |
+| `SAGE_ATTN` | Quantized | SageAttention 2.2 INT8 attention | [SageAttention](attention_backends/sage.md#sage_attn) |
+| `SAGE_ATTN_3` | Quantized | SageAttention3 on Blackwell | [SageAttention](attention_backends/sage.md#sage_attn_3) |
+| `FLASH_ATTN_HUB` | Hub kernel | FlashAttention 2 from Hugging Face `kernels` | [Hugging Face Hub Backends](attention_backends/huggingface_hub.md) |
+| `FLASH_ATTN_3_HUB` | Hub kernel | FlashAttention 3 from Hugging Face `kernels` on Hopper or newer | [Hugging Face Hub Backends](attention_backends/huggingface_hub.md) |
+| `RAINFUSION_ATTN` | Block sparse | MindIE-SD RainFusion video attention on Ascend NPU | [RainFusion](attention_backends/rainfusion.md) |

 ## Configuration

-Diffusion attention backends can be configured three ways, in priority order:
+Backend selection is resolved in this order:

-1. **`--diffusion-attention-config`** — structured per-role config (highest priority).
-2. **`--diffusion-attention-backend` / `DIFFUSION_ATTENTION_BACKEND` env var** — global shorthand that sets the default backend.
-3. **Platform default** — used when nothing is configured.
+1. `--diffusion-attention-config` per-role configuration.
+2. `--diffusion-attention-backend` or `DIFFUSION_ATTENTION_BACKEND` as a
+   global default.
+3. The current platform's default.

-`--diffusion-attention-backend` is shorthand for `--diffusion-attention-config.default.backend`. It may be combined with `--diffusion-attention-config.per_role.*` overrides, but is mutually exclusive with `--diffusion-attention-config.default.backend`.
+`--diffusion-attention-backend` is shorthand for
+`--diffusion-attention-config.default.backend`. Do not pass it together with an
+explicit `default.backend` in the structured configuration.

 ### Global default

-Set the default backend for every diffusion attention layer:
-
 ```bash
-# CLI flag
-vllm-omni serve <model> --diffusion-attention-backend SAGE_ATTN
+vllm-omni serve <model> --diffusion-attention-backend FLASH_ATTN

-# Environment variable (also recognized for backwards compatibility)
-export DIFFUSION_ATTENTION_BACKEND=SAGE_ATTN
+# Backwards-compatible environment variable
+export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
 ```

 ### Per-role configuration

-Roles are free-form strings declared by each diffusion model. The two common categories are `"self"` and `"cross"`; model-specific roles (e.g. `"ltx2.audio_to_video"`) may also be declared. A role string is matched in this order:
+Roles are declared by each diffusion model. Common categories are `self` and
+`cross`; a model may also use a more specific role such as
+`ltx2.audio_to_video`. Resolution order is:

-1. Exact `per_role[role]` match
-2. `per_role[role_category]` fallback (e.g. `"ltx2.audio_to_video"` → `"cross"`)
-3. `default`
-4. Platform default
-
-Use vLLM-style dotted flags or one JSON blob:
+1. Exact `per_role[role]` match.
+2. Category `per_role[role_category]` match.
+3. `default`.
+4. Platform default.

 ```bash
 # Dotted flags
 vllm-omni serve <model> \
-    --diffusion-attention-config.default.backend FLASH_ATTN \
-    --diffusion-attention-config.per_role.cross.backend TORCH_SDPA
+  --diffusion-attention-config.default.backend FLASH_ATTN \
+  --diffusion-attention-config.per_role.cross.backend TORCH_SDPA

-# JSON
+# Equivalent JSON
 vllm-omni serve <model> \
-    --diffusion-attention-config '{"default":{"backend":"FLASH_ATTN"},"per_role":{"cross":{"backend":"TORCH_SDPA"}}}'
+  --diffusion-attention-config \
+  '{"default":{"backend":"FLASH_ATTN"},"per_role":{"cross":{"backend":"TORCH_SDPA"}}}'
 ```

-A backend that needs configuration exposes it as a typed field on the spec:
-`TRTLLM_ATTN`'s Skip-Softmax (see [below](#trtllm_attn-backend-and-skip-softmax)) and
-`RAINFUSION_ATTN`'s RainFusion block (see [below](#rainfusion_attn-backend-and-block-sparse-video-attention)):
-
-```bash
--diffusion-attention-config.default.backend TRTLLM_ATTN \
--diffusion-attention-config.default.skip_softmax.target_sparsity 0.5
-```
-
-### Programmatic API
-
-When constructing `OmniDiffusionConfig` directly:
+### Python API

 ```python
-from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, AttnQuantSpec, OmniDiffusionConfig
+from vllm_omni.diffusion.data import (
+    AttentionConfig,
+    AttentionSpec,
+    OmniDiffusionConfig,
+)

 config = OmniDiffusionConfig(
    diffusion_attention_config=AttentionConfig(
        default=AttentionSpec(backend="FLASH_ATTN"),
-        per_role={
-            "cross": AttentionSpec(backend="TORCH_SDPA"),
-        },
+        per_role={"cross": AttentionSpec(backend="TORCH_SDPA")},
    ),
    ...,
 )
 ```

-A plain dict is also accepted and normalized to `AttentionConfig`.
+Backend-specific typed blocks are documented with their consumers:

-#### FlashInfer QK16/V8 quantized attention
+- `quant`: [FlashInfer](attention_backends/dense_backends.md#flashinfer-quantized-attention)
+  and [TRTLLM SAGE](attention_backends/trtllm.md#sage-quantization).
+- `skip_softmax`: [TRTLLM Skip-Softmax](attention_backends/trtllm.md#skip-softmax).
+- `block_sparse`: [RainFusion](attention_backends/rainfusion.md#configuration).

-`FLASHINFER_ATTN` accepts quantization options through `AttentionSpec.quant`. For QK16/V8:
-
-```python
-config = OmniDiffusionConfig(
-    diffusion_attention_config=AttentionConfig(
-        default=AttentionSpec(
-            backend="FLASHINFER_ATTN",
-            quant=AttnQuantSpec(
-                dtype_qk="bfloat16",
-                dtype_vo="fp8_e4m3",
-            ),
-        ),
-    ),
-    ...,
-)
-```
-
-`dtype_qk` configures the Q and K input dtypes. `dtype_vo` configures the V input dtype.
-
-## Platform Defaults
+## Platform defaults

 ### Blackwell (sm_100 / sm_103 / sm_120 / sm_121)

-Auto-route preference, in order:
+The CUDA auto-route preference is:

-1. `TRTLLM_ATTN` — on **datacenter** Blackwell (sm_100 / sm_103) when `flashinfer` is installed, the model's `head_dim` is 128, **and the model declares a compatible packed/mask-free path**
-2. `CUDNN_ATTN` — when cuDNN ≥ 9.5 is available (ships in PyTorch 2.5+ wheels)
-3. `FLASHINFER_ATTN` — when `flashinfer` is installed but cuDNN < 9.5
-4. `FLASH_ATTN` — when `flash-attn` is installed with the Blackwell CUTE kernel
-5. `TORCH_SDPA` — last resort
+1. `TRTLLM_ATTN` on datacenter Blackwell (sm_100/sm_103) when FlashInfer is
+   available, `head_dim=128`, and the model declares a compatible packed or
+   mask-free path.
+2. `CUDNN_ATTN` when cuDNN 9.5 or newer is available.
+3. `FLASHINFER_ATTN` when FlashInfer is available but cuDNN is too old.
+4. `FLASH_ATTN` when a compatible package is installed.
+5. `TORCH_SDPA`.

-`TRTLLM_ATTN` is skipped on workstation Blackwell (sm_120 / sm_121) and for any `head_dim != 128`, so those GPUs keep the `CUDNN_ATTN` route described below.
+`TRTLLM_ATTN` is not auto-selected on workstation Blackwell
+(sm_120/sm_121), for other head dimensions, or for paths that require an
+unsupported mask.

-`TRTLLM_ATTN` outranks `CUDNN_ATTN` on datacenter Blackwell for compatible packed/mask-free pipelines. Workstation Blackwell (sm_120 / sm_121) and pipelines that require attention masks retain their normal fallback.
+### Hopper, Ada, and Ampere

-The startup log line `Defaulting to diffusion attention backend CUDNN_ATTN (Blackwell sm_120, cuDNN 91002)` confirms the route.
+The CUDA auto-route uses `FLASH_ATTN` when available and otherwise falls back
+to `TORCH_SDPA`. `CUDNN_ATTN` and `FLASHINFER_ATTN` remain explicit options.

-**Why CUDNN_ATTN by default**: on mask-heavy diffusion models (HunyuanVideo-1.5, Qwen-Image), cuDNN's pinned FMHA kernel sidesteps a PyTorch SDPA dispatch quirk where the unpinned dispatcher picks `EFFICIENT_ATTENTION` (~25 ms) for masked calls instead of cuDNN (~11 ms). The pin gives 2× e2e on HV-1.5 with no regression on lighter models.
+Other platforms validate an explicit backend and choose their own default
+through the platform implementation. Check the startup log to confirm the
+resolved backend.

-### Hopper (sm_90) / Ada (sm_89) / Ampere (sm_80–sm_86)
+## Choosing a backend manually

-Auto-route preference:
+Override the platform default when you need:

-1. `FLASH_ATTN` — when `flash-attn` is installed
-2. `TORCH_SDPA` — fallback
+- a correctness reference (`TORCH_SDPA`);
+- a backend-specific workaround;
+- training/rollout kernel alignment (Hub backends); or
+- an explicitly validated sparse or quantized speedup.

-`CUDNN_ATTN` and `FLASHINFER_ATTN` are still selectable via env var on these GPUs but are not in the auto-route — FlashAttention 2 is the well-tuned path on pre-Blackwell hardware.
+The startup log prints the resolved backend and whether it came from explicit
+configuration or platform defaulting. If no resolution message appears, check
+earlier logs for diffusion-stage initialization failures.

-## TRTLLM_ATTN Backend and Skip-Softmax
+## Reference benchmark

-`TRTLLM_ATTN` runs FlashInfer's trtllm-gen FMHA. Dense it is a BF16 backend like the others; on top of
-that it can enable **Skip-Softmax**, a sparse-attention mode that trades a little fidelity for speed
-(algorithm: [design doc](../../design/feature/skip_softmax.md)).
+The following BF16 results were measured on an sm_120 RTX Pro 6000 Blackwell
+with the same prompt and seed across runs. Treat them as reference results, not
+portable guarantees.

-Enable it through the typed `skip_softmax` block on the attention spec:
-
-| Key | Valid values | Meaning |
-|---|---|---|
-| `target_sparsity` | finite, `[0, 1]` | Operating point on a calibrated curve. Needs a calibration for the model. |
-| `threshold` | finite, `≥ 0` | Direct threshold, no calibration needed. Mutually exclusive with `target_sparsity`. |
-| `disabled_until_timestep` | finite, `[0, 1]` | Holds early, high-noise steps dense; skip turns on once the normalized timestep `t ≤ D`. |
-
-```bash
-vllm-omni serve Wan-AI/Wan2.2-T2V-A14B-Diffusers \
-  --diffusion-attention-config '{"default": {"backend": "TRTLLM_ATTN",
-      "skip_softmax": {"target_sparsity": 0.65, "disabled_until_timestep": 0.86}}}'
-```
-
-Programmatically the same block is a typed `SkipSoftmaxSpec` (values validated at construction):
-
-```python
-from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, SkipSoftmaxSpec
-
-AttentionConfig(
-    default=AttentionSpec(
-        backend="TRTLLM_ATTN",
-        skip_softmax=SkipSoftmaxSpec(target_sparsity=0.65, disabled_until_timestep=0.86),
-    ),
-)
-```
-
-**Start at `target_sparsity=0.65, disabled_until_timestep=0.86`** and raise `target_sparsity` only
-as far as your quality bar allows — the output diverges from dense as it climbs. The gain grows with
-sequence length, since Skip-Softmax only accelerates attention. Dense BF16 is `TRTLLM_ATTN` with no
-`skip_softmax` block.
-
-Requires datacenter Blackwell with `head_dim == 128`; elsewhere, or without FlashInfer, selecting
-`TRTLLM_ATTN` raises rather than silently degrading.
-
-## TRTLLM_ATTN SAGE Quantization
-
-On top of dense/Skip-Softmax, `TRTLLM_ATTN` can run **SAGE** (SageAttention) — the trtllm-gen
-FMHA kernel with per-block quantized Q/K and per-channel fp8 V, so the two attention matmuls run
-in low precision. Enable it through the typed `quant` block on the attention spec:
-
-| Key | Valid values | Meaning |
-|---|---|---|
-| `dtype_qk` | `int8`, `fp8_e4m3` | Quantization dtype for Q and K. Absent ⇒ dense (SAGE off). |
-| `q_block_size` | `1`, `4`, `16` | Per-token block size for Q scales (default `1`). |
-| `k_block_size` | `1`, `4`, `16` | Per-token block size for K scales (default `16`). |
-
-V is always quantized per-channel to fp8_e4m3 and K-smoothing is applied inside the routine, so
-neither is a user knob. Only block sizes `1`, `4`, `16` have compiled kernels.
-
-The `quant` block is a shared `AttnQuantSpec` consumed by more than one backend; each reads the
-fields that apply and rejects the rest. `TRTLLM_ATTN` (SAGE) reads `dtype_qk` (`int8` / `fp8_e4m3`),
-`q_block_size`, and `k_block_size`; `FLASHINFER_ATTN` reads `dtype_qk` (`float16` / `bfloat16`),
-`dtype_vo`, and `flashinfer_backend` (see its section). A `dtype_qk` from the wrong set raises.
-
-```bash
-vllm-omni serve Wan-AI/Wan2.2-T2V-A14B-Diffusers \
-  --diffusion-attention-config '{"default": {"backend": "TRTLLM_ATTN",
-      "quant": {"dtype_qk": "fp8_e4m3", "q_block_size": 1, "k_block_size": 16}}}'
-```
-
-```python
-from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, AttnQuantSpec
-
-AttentionConfig(
-    default=AttentionSpec(
-        backend="TRTLLM_ATTN",
-        quant=AttnQuantSpec(dtype_qk="fp8_e4m3", q_block_size=1, k_block_size=16),
-    ),
-)
-```
-
-**Requirements.** Needs FlashInfer ≥ 0.6.16rc1. Kernel
-availability is arch-dependent: `fp8_e4m3` QK has kernels on both **SM100** (B200) and **SM103**
-(B300); `int8` QK kernels are compiled for **SM100 only**.
-
-## RAINFUSION_ATTN Backend and Block-Sparse Video Attention
-
-`RAINFUSION_ATTN` runs MindIE-SD's RainFusion (`rf_v2`) kernel on Ascend NPU. It pools each
-128-token block of keys, ranks them per query block, and computes attention against only the
-top-scoring ones. Video tokens are rearranged into `(t, h, w)` order first, so the kept blocks are
-spatiotemporal neighbours rather than arbitrary rows of the packed sequence.
-
-Only the video segment is sparsified. The prefix rows (text, visual conditions, audio) and the
-first-frame blocks are always kept dense, which is why the realized sparsity is lower than the
-nominal `sparsity` you configure. Anything the kernel cannot handle — a warmup denoise step, an
-exempt layer, a sequence with no published video geometry, a video segment under 32 blocks —
-delegates to `FLASH_ATTN`, so a model can select this backend unconditionally.
-
-Enable it through the typed `block_sparse` block on the attention spec, shared by every
-block-sparse backend:
-
-| Key | Valid values | Meaning |
-|---|---|---|
-| `sparsity` | finite, `[0, 1]` | Nominal fraction of key blocks dropped per query block. `0` disables sparsity. Defaults to `0.8`. |
-| `start_step` | `≥ 0` | Keep the first N denoise steps dense. Layout is decided early, so these steps dominate structural fidelity. |
-| `skip_layers` | index selector, e.g. `"0-3,38"` | DiT blocks that always stay dense. |
-
-```bash
-vllm-omni serve MiniMaxAI/MiniMax-H3 \
-  --diffusion-attention-config '{"default": {"backend": "RAINFUSION_ATTN",
-      "block_sparse": {"sparsity": 0.8, "start_step": 0}}}'
-```
-
-Programmatically the same block is a typed `BlockSparseSpec` (values validated at construction):
-
-```python
-from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, BlockSparseSpec
-
-AttentionConfig(
-    default=AttentionSpec(
-        backend="RAINFUSION_ATTN",
-        block_sparse=BlockSparseSpec(sparsity=0.8, start_step=0, skip_layers="0-1"),
-    ),
-)
-```
-
-**Tune in the order `start_step` → `sparsity` → `skip_layers`.** Raise `start_step` first: it is the
-cheapest way to recover structure, because the early high-noise steps decide global layout while the
-later steps only refine texture. Then walk `sparsity` up to your quality bar. Reach for
-`skip_layers` last, once an A/B against dense at the same seed points at specific blocks.
-
-Requires Ascend NPU with `mindiesd`; selecting it on any other platform raises. It is also
-incompatible with ring sequence parallelism, since `rf_v2` needs the whole key sequence to rank
-blocks — use Ulysses SP (`ring_degree=1`).
-
-### Which geometries run sparse
-
-MindIE-SD rf_v2 handles arbitrary video grids by spatially rearranging video first and, when
-necessary, promoting a real-video suffix to its always-kept prefix segment. It never pads: rf_v2
-does not consume an attention mask for padded keys. The remaining video rows stay sparse, while the
-promoted rows remain visible to every query and key.
-
-For example, MiniMax-H3 at 1344x768 has grid `(62, 24, 42)`: its 62496 video rows leave a 32-row
-128-block residue, and its two non-8-wide latent columns form a 2928-row rearranged suffix.
-MindIE-SD promotes 2976 rows, leaving 59520 sparse rows (`465 × 128`). This keeps the block mask
-and kernel tiling aligned, at the cost of retaining more blocks.
-
-8x8-aligned spatial grids remain preferable for performance. A latent `h` or `w` not divisible by 8
-creates an always-kept residual suffix and lowers realized sparsity. "Multiple of 8" on the latent
-grid means **width and height that are multiples of 256**. This path requires a MindIE-SD release
-that implements protected video tails for rf_v2.
-
-## End-to-End Benchmark (BF16, sm_120 RTX Pro 6000 Blackwell)
-
-Same prompt and seed across runs. `Total generation time` from `text_to_video.py` / `text_to_image.py`.
-
-| Model | Shape | TORCH_SDPA | CUDNN_ATTN | FLASHINFER_ATTN |
-|---|---|---|---|---|
+| Model | Shape | `TORCH_SDPA` | `CUDNN_ATTN` | `FLASHINFER_ATTN` |
+| --- | --- | ---: | ---: | ---: |
 | HunyuanVideo-1.5 (T2V) | 480p / 33f / 50 steps | 147.05 s | **73.02 s** | 127.84 s |
 | Wan 2.2 14B (T2V) | 480p / 33f / 40 steps | 117.75 s | 117.17 s | **115.07 s** |
 | Qwen-Image (T2I) | 1024² / 50 steps | 17.41 s | **15.14 s** | 16.02 s |
 | FLUX.2-dev (T2I) | 1024² / 50 steps, TP=2 | 53.62 s | **53.30 s** | 54.94 s |

-Pattern: mask-heavy DiTs (HV-1.5, Qwen-Image) favor `CUDNN_ATTN`; lighter-mask DiTs and TP-saturated configs (Wan 2.2, FLUX.2 TP=2) tie within noise.
+Mask-heavy DiTs favored `CUDNN_ATTN`; lighter-mask or TP-saturated workloads
+were close enough that users should benchmark their exact model and shape.

-## Known Limitations
+## Compatibility anchors

-### LTX-2.0: `CUDNN_ATTN` crashes under torch.compile
+The following headings preserve links to sections that moved into dedicated
+guides.

-LTX-2's audio attention has a symbolic head_dim under torch.compile tracing. cuDNN's SDPA backend selector rejects symbolic dims and Dynamo aborts compilation. Tracked in [#3121](https://github.com/vllm-project/vllm-omni/issues/3121).
+## TRTLLM_ATTN Backend and Skip-Softmax

-**Workaround**: explicitly select `FLASHINFER_ATTN` or `TORCH_SDPA` for LTX-2.0:
+See [TRTLLM Attention](attention_backends/trtllm.md#skip-softmax).

-```bash
-DIFFUSION_ATTENTION_BACKEND=FLASHINFER_ATTN python examples/offline_inference/text_to_video/text_to_video.py \
-    --model Lightricks/LTX-2 ...
-```
+## TRTLLM_ATTN SAGE Quantization

-### FlashAttention-4 on Blackwell
+See [TRTLLM Attention](attention_backends/trtllm.md#sage-quantization).

-Install the optional CUDA 13 extra to use the CuTe-based FA4 path:
+## RAINFUSION_ATTN Backend and Block-Sparse Video Attention

-```bash
-pip install 'vllm-omni[fa4]'
-```
-
-`FLASH_ATTN` prefers `flash_attn.cute` on Blackwell and falls back to FA3/FA2
-when it is unavailable. Version `4.0.0b18` is required; earlier beta
-wheels had known JIT failures on Blackwell.
-
-## Choosing a Backend Manually
-
-### When to override the default
-
- **Quality validation**: compare a new backend against `TORCH_SDPA` as the reference, since SDPA's default dispatcher is the most extensively tested.
- **Lossy speedup hunting**: try `SAGE_ATTN` (INT8 quantized) on diffusion outputs — typically indistinguishable visually but always validate.
- **Workaround for known issues**: see Known Limitations above.
-
-### Verifying which backend is in use
-
-The startup log prints one of:
-
-```
-Using diffusion attention backend 'CUDNN_ATTN'           # explicit override
-Defaulting to diffusion attention backend CUDNN_ATTN ... # auto-route
-Defaulting to diffusion attention backend SDPA           # nothing else available
-```
-
-If you don't see one of these, the model didn't reach diffusion stage init — check earlier logs for failures.
+See [RainFusion](attention_backends/rainfusion.md).

 ## SageAttention Installation

-vLLM-Omni expects SageAttention to be installed into the same Python environment as vLLM-Omni.
-
-Build from source:
-
-```bash
-git clone https://github.com/thu-ml/SageAttention.git
-cd SageAttention
-
-export EXT_PARALLEL=4 NVCC_APPEND_FLAGS="--threads 8" MAX_JOBS=32
-pip install . --no-build-isolation
-```
-
-Quick check:
-
-```bash
-python -c "import sageattention; print(sageattention.__file__)"
-```
+See [SageAttention](attention_backends/sage.md#installation).

 ## SageAttention3 Installation

-vLLM-Omni expects SageAttention3 to be installed into the same Python environment as vLLM-Omni.
-
-Build from source:
-
-```bash
-git clone https://github.com/thu-ml/SageAttention.git
-cd SageAttention/sageattention3_blackwell
-python setup.py install
-```
-
-Quick check:
-
-```bash
-python -c "import sageattn3; print(sageattn3.__file__)"
-```
-
-Notes:
-
- `SAGE_ATTN_3` is only selected on CUDA when `sageattn3` is importable and the GPU is Blackwell-class.
- SageAttention3's Blackwell kernel assumes `Hq == Hkv`. In vLLM-Omni, GQA/MQA diffusion requests fall back to PyTorch SDPA for correctness.
+See [SageAttention](attention_backends/sage.md#sageattention3-installation).

 ## HuggingFace Kernels Hub Backends

-To achieve perfect numerical consistency between **training** (typically using HuggingFace Diffusers with Hub kernels) and **serving/rollout** (using vLLM-Omni), you can use the Hub-based attention backends. This eliminates numerical drift / sampling divergence caused by executing different local kernel versions during rollout.
-
-The following backend options are supported:
- `FLASH_ATTN_HUB` (HuggingFace `kernels-community/flash-attn2`)
- `FLASH_ATTN_3_HUB` (HuggingFace `kernels-community/flash-attn3`, Hopper sm_90+ only)
-
-### Installation
-
-To use these backends, you must install the `kernels` library:
-
-```bash
-pip install kernels==0.14.1
-```
-
-If the `kernels` library is not available in the environment, vLLM-Omni will log a warning and fall back gracefully to the corresponding local backend implementations (`FLASH_ATTN`). On CUDA GPUs below Hopper (compute capability < 9.0), `FLASH_ATTN_3_HUB` falls back to `FLASH_ATTN_HUB`.
-
-### Usage
-
-Select a Hub backend using the global CLI flag or environment variables:
-
-```bash
-# Environment variable
-export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN_HUB
-
-# CLI flag
-vllm-omni serve <model> --diffusion-attention-backend FLASH_ATTN_HUB
-```
-
-## Usage Examples
-
-### Default (auto-route)
-
-```bash
-python examples/offline_inference/text_to_video/text_to_video.py \
-    --model hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v \
-    --prompt "A dog running across a field of golden wheat." \
-    --height 480 --width 832 --num-frames 33 \
-    --num-inference-steps 50 --seed 42 --guidance-scale 6.0 \
-    --output hv15.mp4
-```
-
-On Blackwell this picks `CUDNN_ATTN` automatically. Check the log for the `Defaulting to ...` line.
-
-### Explicit backend selection
-
-```bash
-DIFFUSION_ATTENTION_BACKEND=FLASHINFER_ATTN python examples/offline_inference/text_to_video/text_to_video.py \
-    --model Lightricks/LTX-2 \
-    --prompt "A dog running across a field of golden wheat." \
-    --height 480 --width 832 --num-frames 33 \
-    --num-inference-steps 40 --seed 42 --guidance-scale 4.0 \
-    --output ltx2.mp4
-```
-
-### SageAttention (lossy)
-
-```bash
-DIFFUSION_ATTENTION_BACKEND=SAGE_ATTN python examples/offline_inference/text_to_video/text_to_video.py \
-    --model hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v \
-    --prompt "A dog running across a field of golden wheat." \
-    --height 480 --width 832 --num-frames 33 \
-    --num-inference-steps 30 --seed 42 --guidance-scale 6.0 \
-    --tensor-parallel-size 2 \
-    --output hv15_sage.mp4
-```
-
-Example: Wan2.2 TI2V 5B
-
-```bash
-DIFFUSION_ATTENTION_BACKEND=SAGE_ATTN python examples/offline_inference/text_to_video/text_to_video.py \
-    --model Wan-AI/Wan2.2-TI2V-5B-Diffusers \
-    --prompt "A dog running across a field of golden wheat." \
-    --height 704 --width 1280 --num-frames 49 \
-    --num-inference-steps 30 --seed 42 --guidance-scale 5.0 \
-    --tensor-parallel-size 2 \
-    --output outputs/wan22_sage.mp4
-```
-
-### Enable SageAttention3
-
-Example:
-
-```bash
-DIFFUSION_ATTENTION_BACKEND=SAGE_ATTN_3 python examples/offline_inference/text_to_video/text_to_video.py \
-    --model hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v \
-    --prompt "A dog running across a field of golden wheat." \
-    --height 480 --width 832 --num-frames 33 \
-    --num-inference-steps 30 --seed 42 --guidance-scale 6.0 \
-    --tensor-parallel-size 2 \
-    --output outputs/hv15_sage3.mp4
-```
+See [Hugging Face Hub Backends](attention_backends/huggingface_hub.md).
@@ -0,0 +1,110 @@
+# Dense Attention Backends
+
+Use these backends when you want dense attention without backend-specific
+sparsity. Start with `TORCH_SDPA` for correctness comparisons, then benchmark
+the fastest compatible kernel for your model, shape, and hardware.
+
+For selection precedence and per-role configuration, see the
+[attention backend overview](../attention_backends.md).
+
+## `TORCH_SDPA`
+
+`TORCH_SDPA` calls PyTorch `scaled_dot_product_attention` and lets PyTorch's
+dispatcher choose the implementation. It is always available and is the most
+conservative reference when validating another backend.
+
+```bash
+vllm-omni serve <model> --diffusion-attention-backend TORCH_SDPA
+```
+
+## `FLASH_ATTN`
+
+`FLASH_ATTN` uses the installed FlashAttention implementation. On Blackwell it
+prefers the CuTe-based FlashAttention 4 path and falls back to
+FlashAttention 3 or 2 when that path is unavailable. On Hopper, Ada, and
+Ampere it is the preferred automatic route when a compatible package is
+installed.
+
+```bash
+vllm-omni serve <model> --diffusion-attention-backend FLASH_ATTN
+```
+
+### FlashAttention 4 on Blackwell
+
+Install the optional CUDA 13 extra:
+
+```bash
+pip install 'vllm-omni[fa4]'
+```
+
+Version `4.0.0b18` is required; earlier beta wheels had known JIT failures on
+Blackwell. If the CuTe path is unavailable, the backend falls back to the
+compatible FlashAttention 3 or 2 path.
+
+## `CUDNN_ATTN`
+
+`CUDNN_ATTN` pins PyTorch SDPA to `CUDNN_ATTENTION`. It is particularly useful
+for mask-heavy DiTs and is automatically preferred on Blackwell when cuDNN
+9.5 or newer is available and the higher-priority TRTLLM route is not
+compatible.
+
+```bash
+vllm-omni serve <model> --diffusion-attention-backend CUDNN_ATTN
+```
+
+### LTX-2.0 limitation
+
+LTX-2 audio attention has a symbolic head dimension during `torch.compile`
+tracing. The cuDNN SDPA selector rejects that symbolic dimension and Dynamo
+aborts compilation. This is tracked in
+[issue #3121](https://github.com/vllm-project/vllm-omni/issues/3121).
+
+Use `FLASHINFER_ATTN` or `TORCH_SDPA` as a workaround:
+
+```bash
+DIFFUSION_ATTENTION_BACKEND=FLASHINFER_ATTN \
+  python examples/offline_inference/text_to_video/text_to_video.py \
+  --model Lightricks/LTX-2 ...
+```
+
+## `FLASHINFER_ATTN`
+
+`FLASHINFER_ATTN` uses FlashInfer's batch-prefill wrapper. It is an explicit
+option on CUDA platforms and an automatic Blackwell fallback when FlashInfer
+is installed but cuDNN is too old for `CUDNN_ATTN`.
+
+```bash
+vllm-omni serve <model> --diffusion-attention-backend FLASHINFER_ATTN
+```
+
+### FlashInfer quantized attention
+
+The backend accepts an `AttentionSpec.quant` block. For QK16/V8, keep Q and K
+in FP16 or BF16 and use FP8 E4M3 for V:
+
+```python
+from vllm_omni.diffusion.data import (
+    AttentionConfig,
+    AttentionSpec,
+    AttnQuantSpec,
+    OmniDiffusionConfig,
+)
+
+config = OmniDiffusionConfig(
+    diffusion_attention_config=AttentionConfig(
+        default=AttentionSpec(
+            backend="FLASHINFER_ATTN",
+            quant=AttnQuantSpec(
+                dtype_qk="bfloat16",
+                dtype_vo="fp8_e4m3",
+            ),
+        ),
+    ),
+    ...,
+)
+```
+
+`dtype_qk` controls Q and K; `dtype_vo` controls V. Mixed-dtype
+configurations require FlashInfer 0.6.16rc1 or newer. The shared quantization
+schema is also consumed by TRTLLM, but each backend validates its own allowed
+fields and values; see [TRTLLM SAGE quantization](trtllm.md#sage-quantization).
@@ -0,0 +1,32 @@
+# Hugging Face Hub Attention Backends
+
+Hub backends load the same published kernels commonly used with Hugging Face
+Diffusers. They are useful when training and serving must use matching kernel
+implementations to reduce numerical drift and sampling divergence.
+
+| Backend | Kernel | Platform |
+| --- | --- | --- |
+| `FLASH_ATTN_HUB` | `kernels-community/flash-attn2` | Compatible CUDA GPUs |
+| `FLASH_ATTN_3_HUB` | `kernels-community/flash-attn3` | Hopper sm_90 or newer |
+
+## Installation
+
+```bash
+pip install kernels==0.14.1
+```
+
+If `kernels` is unavailable, vLLM-Omni warns and falls back to the
+corresponding local FlashAttention implementation. On CUDA devices older than
+Hopper, `FLASH_ATTN_3_HUB` falls back to `FLASH_ATTN_HUB`.
+
+## Usage
+
+```bash
+export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN_HUB
+
+# Equivalent CLI selection
+vllm-omni serve <model> --diffusion-attention-backend FLASH_ATTN_HUB
+```
+
+For per-role kernel alignment, configure the Hub backend on only the relevant
+role; see the [attention backend overview](../attention_backends.md#per-role-configuration).
@@ -0,0 +1,72 @@
+# RainFusion Attention
+
+`RAINFUSION_ATTN` runs MindIE-SD RainFusion (`rf_v2`) block-sparse video
+attention on Ascend NPU. It pools 128-token key blocks, ranks them per query
+block, and attends to the highest-scoring blocks after arranging video tokens
+in `(t, h, w)` order.
+
+Only the video segment is sparse. Prefix rows such as text, visual conditions,
+and audio, plus first-frame blocks, remain dense. Unsupported calls—including
+warmup steps, skipped layers, missing video geometry, and video segments below
+32 blocks—delegate to `FLASH_ATTN`, so compatible models can select RainFusion
+globally.
+
+## Configuration
+
+| Key | Valid values | Meaning |
+| --- | --- | --- |
+| `sparsity` | finite, `[0, 1]` | Nominal dropped-key-block fraction; default `0.8`; `0` disables sparsity |
+| `start_step` | integer, `>= 0` | Number of early denoise steps kept dense |
+| `skip_layers` | selector such as `"0-3,38"` | DiT blocks kept dense |
+
+```bash
+vllm-omni serve MiniMaxAI/MiniMax-H3 \
+  --diffusion-attention-config '{"default":{"backend":"RAINFUSION_ATTN",\
+    "block_sparse":{"sparsity":0.8,"start_step":0}}}'
+```
+
+```python
+from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, BlockSparseSpec
+
+config = AttentionConfig(
+    default=AttentionSpec(
+        backend="RAINFUSION_ATTN",
+        block_sparse=BlockSparseSpec(
+            sparsity=0.8,
+            start_step=0,
+            skip_layers="0-1",
+        ),
+    ),
+)
+```
+
+Tune `start_step`, then `sparsity`, then `skip_layers`. Increasing the dense
+early-step window is usually the cheapest way to recover global structure;
+use layer exclusions only after a same-seed dense comparison identifies
+sensitive blocks.
+
+## Requirements and compatibility
+
+RainFusion requires Ascend NPU and `mindiesd`. Selecting it on another
+platform raises. It is incompatible with ring sequence parallelism because
+the kernel needs the complete key sequence for ranking; use Ulysses sequence
+parallelism with `ring_degree=1`.
+
+## Geometry handling
+
+RainFusion handles arbitrary video grids by rearranging video spatially and,
+when necessary, promoting a real-video suffix to the always-kept prefix. It
+does not pad because `rf_v2` does not consume a padding attention mask.
+
+For MiniMax-H3 at 1344x768, grid `(62, 24, 42)` produces 62,496 video rows.
+The implementation promotes 2,976 rows and leaves 59,520 sparse rows
+(`465 x 128`) so the mask and kernel tiling remain aligned.
+
+Spatial grids aligned to 8x8 are still preferable. A latent height or width
+not divisible by 8 creates an always-kept suffix and reduces realized
+sparsity. At the image level, that means width and height divisible by 256.
+Protected video tails require a compatible MindIE-SD release.
+
+For common configuration and selector behavior, see the
+[attention backend overview](../attention_backends.md) and the
+[backend selection design](../../../design/feature/attention_backend_selection.md).
@@ -0,0 +1,63 @@
+# SageAttention
+
+SageAttention backends provide lossy low-precision attention for diffusion
+models. Validate output quality against `TORCH_SDPA` at the same seed before
+using either backend in production.
+
+## `SAGE_ATTN`
+
+`SAGE_ATTN` uses SageAttention 2.2 with INT8-quantized attention and FP16
+accumulation.
+
+### Installation
+
+Install SageAttention into the same environment as vLLM-Omni:
+
+```bash
+git clone https://github.com/thu-ml/SageAttention.git
+cd SageAttention
+export EXT_PARALLEL=4 NVCC_APPEND_FLAGS="--threads 8" MAX_JOBS=32
+pip install . --no-build-isolation
+```
+
+Verify the installation:
+
+```bash
+python -c "import sageattention; print(sageattention.__file__)"
+```
+
+Select it globally:
+
+```bash
+vllm-omni serve <model> --diffusion-attention-backend SAGE_ATTN
+```
+
+## `SAGE_ATTN_3`
+
+`SAGE_ATTN_3` uses the SageAttention3 Blackwell implementation.
+
+### SageAttention3 installation
+
+```bash
+git clone https://github.com/thu-ml/SageAttention.git
+cd SageAttention/sageattention3_blackwell
+python setup.py install
+```
+
+Verify the installation:
+
+```bash
+python -c "import sageattn3; print(sageattn3.__file__)"
+```
+
+```bash
+vllm-omni serve <model> --diffusion-attention-backend SAGE_ATTN_3
+```
+
+`SAGE_ATTN_3` requires CUDA, an importable `sageattn3`, and a Blackwell-class
+GPU. Its kernel assumes the query-head count equals the key/value-head count.
+GQA and MQA diffusion calls therefore fall back to PyTorch SDPA for
+correctness.
+
+For common configuration and platform routing, see the
+[attention backend overview](../attention_backends.md).
@@ -0,0 +1,103 @@
+# TRTLLM Attention
+
+`TRTLLM_ATTN` runs FlashInfer's trtllm-gen FMHA kernels. It supports dense
+BF16 attention plus optional Skip-Softmax sparsity or SAGE quantization.
+
+For common selection and per-role configuration, see the
+[attention backend overview](../attention_backends.md).
+
+## Requirements
+
+Selecting `TRTLLM_ATTN` requires:
+
+- datacenter Blackwell, sm_100 or sm_103;
+- `head_dim=128`;
+- FlashInfer; and
+- a model-declared packed or mask-free compatible path.
+
+It is not supported on workstation Blackwell (sm_120/sm_121). An incompatible
+explicit selection raises instead of silently changing the backend.
+
+## Dense mode
+
+Select `TRTLLM_ATTN` without `skip_softmax` or `quant` to run dense BF16:
+
+```bash
+vllm-omni serve <model> --diffusion-attention-backend TRTLLM_ATTN
+```
+
+## Skip-Softmax
+
+Skip-Softmax trades some fidelity for speed by skipping low-contribution
+attention work. See the [feature design](../../../design/feature/skip_softmax.md)
+for the algorithm and implementation contract.
+
+| Key | Valid values | Meaning |
+| --- | --- | --- |
+| `target_sparsity` | finite, `[0, 1]` | Point on a model-calibrated curve |
+| `threshold` | finite, `>= 0` | Direct threshold; mutually exclusive with `target_sparsity` |
+| `disabled_until_timestep` | finite, `[0, 1]` | Keep early high-noise steps dense until normalized `t <= D` |
+
+```bash
+vllm-omni serve Wan-AI/Wan2.2-T2V-A14B-Diffusers \
+  --diffusion-attention-config '{"default":{"backend":"TRTLLM_ATTN",\
+    "skip_softmax":{"target_sparsity":0.65,"disabled_until_timestep":0.86}}}'
+```
+
+```python
+from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, SkipSoftmaxSpec
+
+config = AttentionConfig(
+    default=AttentionSpec(
+        backend="TRTLLM_ATTN",
+        skip_softmax=SkipSoftmaxSpec(
+            target_sparsity=0.65,
+            disabled_until_timestep=0.86,
+        ),
+    ),
+)
+```
+
+Start with `target_sparsity=0.65` and `disabled_until_timestep=0.86`. Increase
+sparsity only after comparing quality against dense output at the same seed.
+The benefit grows with sequence length because only attention is accelerated.
+
+## SAGE quantization
+
+TRTLLM SAGE quantizes Q/K per block and V per channel so both attention
+matrix multiplications use low precision.
+
+| Key | Valid values | Meaning |
+| --- | --- | --- |
+| `dtype_qk` | `int8`, `fp8_e4m3` | Q/K quantization dtype; absent means dense |
+| `q_block_size` | `1`, `4`, `16` | Q scale block size; default `1` |
+| `k_block_size` | `1`, `4`, `16` | K scale block size; default `16` |
+
+V is always quantized per channel to FP8 E4M3 and K-smoothing is internal.
+Only block sizes with compiled kernels are accepted.
+
+```bash
+vllm-omni serve Wan-AI/Wan2.2-T2V-A14B-Diffusers \
+  --diffusion-attention-config '{"default":{"backend":"TRTLLM_ATTN",\
+    "quant":{"dtype_qk":"fp8_e4m3","q_block_size":1,"k_block_size":16}}}'
+```
+
+```python
+from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, AttnQuantSpec
+
+config = AttentionConfig(
+    default=AttentionSpec(
+        backend="TRTLLM_ATTN",
+        quant=AttnQuantSpec(
+            dtype_qk="fp8_e4m3",
+            q_block_size=1,
+            k_block_size=16,
+        ),
+    ),
+)
+```
+
+This path requires FlashInfer 0.6.16rc1 or newer. FP8 Q/K kernels are
+available on sm_100 and sm_103; INT8 Q/K kernels are compiled for sm_100 only.
+The shared `AttnQuantSpec` is backend typed: values intended for
+`FLASHINFER_ATTN` are rejected here and vice versa.
@@ -1,389 +1,75 @@
 # CPU Offloading for Diffusion Models

-## Overview
+vLLM-Omni provides three CPU-offload strategies for diffusion models. Choose
+the coarsest strategy that meets your memory target: finer-grained streaming
+saves more device memory but adds more transfer and setup overhead.

-vLLM-Omni provides two offloading strategies to reduce GPU memory usage for diffusion models:
+For the shared factory, discovery, and lifecycle contract, see the
+[CPU Offloading design](../../design/feature/offloader/README.md).

-1. **Model-level (Sequential) Offloading**: Mutual exclusion between DiT model and encoder - only one is on GPU at a time.
-2. **Layerwise (Blockwise) Offloading**: Keeps only one transformer block on GPU at a time with compute-memory overlap.
+## Choose a strategy

-Both strategies use pinned memory for faster CPU-GPU transfers. The strategies are **mutually exclusive** for now - if both are enabled, layerwise takes priority.
+| Strategy | Device residency | Parallel scope | Guide |
+| --- | --- | --- | --- |
+| Model-level (sequential) | One pipeline component group at a time | Single device | [Model-Level Offloading](offloader/cpu_offload.md) |
+| Layerwise (blockwise) | One transformer block, with next-block prefetch | Single device | [Layerwise Offloading](offloader/layerwise_offload.md) |
+| Distributed layerwise | Fixed two-block device buffer; optional host-weight sharding and AllGather | Multiple GPU/NPU ranks | [Distributed Layerwise Offloading](offloader/distributed_layerwise_offload.md) |

+All strategies use pinned host memory for faster transfers where applicable.
+Configuration priority is:
+
+1. Distributed layerwise offloading.
+2. Layerwise offloading.
+3. Model-level offloading.
+
+Treat the flags as mutually exclusive. If more than one is enabled, the
+higher-priority strategy is selected.
+
+## Quick selection
+
+- Use [model-level offloading](offloader/cpu_offload.md) when swapping whole
+  encoders and DiTs is enough to fit and phase-boundary transfers are
+  acceptable.
+- Use [layerwise offloading](offloader/layerwise_offload.md) for compute-heavy video
+  DiTs where block transfers can overlap computation.
+- Use [distributed layerwise offloading](offloader/distributed_layerwise_offload.md)
+  when a multi-rank deployment also needs bounded device residency and,
+  optionally, sharded host weights.
+
+## Supported models
+
+| Architecture | Example models | DiT class | Model-level | Layerwise | Distributed layerwise | Layerwise block attributes |
+| --- | --- | --- | :---: | :---: | :---: | --- |
+| Flux2Pipeline | `black-forest-labs/FLUX.2-dev` | `Flux2Transformer2DModel` | yes | yes | — | `transformer_blocks`, `single_transformer_blocks` |
+| LongCatImagePipeline | `meituan-longcat/LongCat-Image` | `LongCatImageTransformer2DModel` | — | yes | — | `transformer_blocks`, `single_transformer_blocks` |
+| NextStep11Pipeline | `stepfun-ai/NextStep-1.1` | `NextStepModel` | — | yes | — | `layers` |
+| OvisImagePipeline | `AIDC-AI/Ovis-Image-7B` | `OvisImageTransformer2DModel` | — | yes | — | `transformer` |
+| QwenImagePipeline | `Qwen/Qwen-Image` | `QwenImageTransformer2DModel` | yes | yes | — | `transformer_blocks` |
+| StableDiffusionXLPipeline | `stabilityai/stable-diffusion-xl-base-1.0` | `SDXLUNet2DConditionModel` | yes | yes | — | `down_blocks`, `up_blocks` |
+| StableDiffusion3Pipeline | `stabilityai/stable-diffusion-3.5-medium` | `SD3Transformer2DModel` | — | yes | — | `transformer_blocks` |
+| Wan22I2VPipeline | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | `WanTransformer3DModel` | yes | yes | — | `blocks` |
+| Wan22Pipeline | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | `WanTransformer3DModel` | yes | yes | — | `blocks` |
+| SoulXSingerPipeline / SoulXSingerSVCPipeline | `Soul-AILab/SoulX-Singer` | `DiffLlama` | yes | yes | — | `layers` |
+| BagelPipeline | `ByteDance-Seed/BAGEL-7B-MoT` | `Qwen2MoTModel` | — | yes | — | `layers`, customized modules |
+| Cosmos3OmniDiffusersPipeline | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | `Cosmos3VFMTransformer`, `Cosmos3LanguageModel` | yes | yes | yes | `layers`, `gen_layers` |
+
+Model-level support requires discoverable DiT and encoder components.
+Layerwise support requires transformer block topology. Distributed support
+reuses that topology but still requires validation for the model, checkpoint,
+and parallel configuration.
+
+## Compatibility anchors
+
+These headings preserve links to sections that moved into dedicated guides.

 ## Model-level (Sequential) Offloading

-### How It Works
-
-Model-level offloading implements mutual exclusion between DiT transformer and encoder modules using pre forward hooks:
-
- **When encoders run**: DiT transformer is offloaded to CPU
- **When DiT runs**: Encoders are offloaded to CPU, if more than one dit models, only one loaded on GPU, others get offloaded to CPU.
- **VAE**: Stays resident on GPU
-
-Before each module's forward pass, the hook automatically moves it to GPU while offloading the other module group to CPU. Transfers use pinned memory for speed.
-
-### Usage
-
-**Python API:**
-```python
-from vllm_omni import Omni
-
-m = Omni(model="Wan-AI/Wan2.2-T2V-A14B-Diffusers", enable_cpu_offload=True)
-```
-
-**CLI:**
-```bash
-vllm serve Wan-AI/Wan2.2-T2V-A14B-Diffusers --omni --enable-cpu-offload
-```
-
-### To Support a Model
-
-Implement the `SupportsComponentDiscovery` protocol to declare which
-submodules serve as pipeline components (used by offloading, HSDP
-sharding, and other framework features):
-
-```python
-from typing import ClassVar
-from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery
-
-class MyPipeline(nn.Module, SupportsComponentDiscovery):
-    _dit_modules: ClassVar[list[str]] = ["transformer"]
-    _encoder_modules: ClassVar[list[str]] = ["text_encoder", "vision_model"]
-    _vae_modules: ClassVar[list[str]] = ["vae"]
-    _resident_modules: ClassVar[list[str]] = []  # optional
-
-    def __init__(self):
-        super().__init__()
-        self.transformer = ...     # DiT — stays on GPU during denoising
-        self.text_encoder = ...    # Encoder — offloaded to CPU during denoising
-        self.vision_model = ...    # Encoder — offloaded to CPU during denoising
-        self.vae = ...             # VAE — always on GPU
-```
-
- `_dit_modules`: attribute names of denoising submodules (kept on GPU
-  during the diffusion loop).
- `_encoder_modules`: attribute names of encoder/vision submodules
-  (offloaded to CPU during the diffusion loop).
- `_vae_modules`: attribute names of VAE(s) (always kept on GPU, not
-  part of the mutual exclusion hooks).
- `_resident_modules`: attribute names of small submodules that must
-  stay on GPU during layerwise offloading (e.g. embedders, connectors).
-  Optional — defaults to `[]`.
-
-All attribute names support dotted paths for nested submodules
-(e.g. `"pipe.transformer"`, `"bagel.time_embedder"`).
-
-Both DiT and encoder lists are needed because the offload hooks use
-mutual exclusion: when one group runs, the other moves to CPU.
-
-### Limitations
- Cold start latency increases
- Adds overhead from CPU-GPU transfers between encoder and denoising phases
- Support single GPU only for now
-
-
-### Component offloading for split models (e.g. Cosmos3)
-
-Some models split their transformer into mutually-exclusive *components* that run
-in different phases of a single forward pass rather than as separate pipeline
-components -- e.g. Cosmos3's understanding (reasoner) component runs once per
-generation while the generation (generator) component runs every denoising step.
-Such models have no separate text encoder to swap against, so the transformer
-owns a small model-local offload path and wraps each phase with
-`with self._offload_context(name):`
-
-```python
-class Cosmos3VFMTransformer(nn.Module):
-    def forward(self, ...):
-        with self._offload_context("reasoner"):
-            ...  # understanding pass, runs once
-        with self._offload_context("generator"):
-            ...  # denoising pass, runs every step
-```
-
-Model-level offloading then keeps exactly one component GPU-resident at a time
-(the other on CPU), reusing the same `SequentialOffloadHook` `.to()` movers. The
-pipeline opts in by exposing `enable_omni_model_cpu_offload` (which drives the
-transformer's `enable_model_cpu_offload` and pins the VAE). Layerwise offloading
-works for these models too -- each component declares its own block container via
-`_layerwise_offload_blocks_attrs`.
-
+See [Model-Level Offloading](offloader/cpu_offload.md).

 ## Layerwise (Blockwise) Offloading

-### How It Works
-
-Layerwise offloading keeps only one transformer block on GPU at a time.
-
-As each block completes, the next block is prefetched to GPU while the current block is freed. The pre and forward hooks utilized by layerwise offloading apply a separate CUDA stream (`copy_stream`) to overlap weight transfer with computation, and retain flattened tensors in pinned CPU memory for block parameters re-materialization. Encoders, VAE, and non-block DiT modules (embeddings, norms) always stay on GPU.
-
-**Execution Flow:**
-
-| Block | Pre-forward Hook | Forward | Post-forward Hook |
-|-------|------------------|---------|-------------------|
-| block-0 | Prefetch block-1 (async) | Compute block-0 | Free block-0 |
-| block-1 | Prefetch block-2 (async) | Compute block-1 | Free block-1 |
-| ... | ... | ... | ... |
-| block-(n-1) | **Prefetch block-0** (async) | Compute block-(n-1) | Free block-(n-1) |
-
-Each transformer block has a `LayerwiseOffloadHook` that prefetches the next block before forward and frees the current block after forward.
-
-Layerwise offloading is primarily recommended for large **video generation models** where the compute cost per block is high enough to effectively overlap with memory prefetch operations. For example, Wan2.2 T2V and I2V pipelines.
-
-### Usage
-
-**Python API:**
-```python
-from vllm_omni import Omni
-
-# Text-to-video
-m = Omni(model="Wan-AI/Wan2.2-T2V-A14B-Diffusers", enable_layerwise_offload=True)
-
-# Or image-to-video
-m = Omni(model="Wan-AI/Wan2.2-I2V-A14B-Diffusers", enable_layerwise_offload=True)
-```
-
-**CLI:**
-```bash
-# Text-to-video
-vllm serve Wan-AI/Wan2.2-T2V-A14B-Diffusers --omni --enable-layerwise-offload
-
-# Or image-to-video
-vllm serve Wan-AI/Wan2.2-I2V-A14B-Diffusers --omni --enable-layerwise-offload
-```
-
-### To Support a Model
-
-Models must define the blocks attribute name for layerwise offloading:
-
-```python
-class WanTransformer3DModel(nn.Module):
-    _layerwise_offload_blocks_attrs = ["blocks"]  # Attribute names containing transformer blocks
-
-    def __init__(self):
-        self.blocks = nn.ModuleList([...])  # Transformer blocks
-```
-
-For models with multiple block types:
-
-```python
-class Flux2Transformer2DModel(nn.Module):
-    _layerwise_offload_blocks_attrs = ["transformer_blocks", "single_transformer_blocks"]
-```
-
-### Limitations
- Cold start latency increases because offloaded components must be moved to CPU
-  during setup; layerwise offload may add extra weight consolidation and pinning
-  work.
- Performance depends on compute cost and H2D bandwidth as well
- Support single GPU only for now
-
+See [Layerwise Offloading](offloader/layerwise_offload.md).

 ## Distributed Layerwise Offloading

-### How It Works
-
-Distributed layerwise offloading extends single-GPU layerwise offloading to
-multi-device deployments. In the default mode, each rank stores only
-**1/dp_size** of the model weights in host memory; full layer weights are
-reconstructed at runtime via **AllGather** on a dedicated communication stream,
-overlapped with computation via a fixed double-buffer scheme. The
-`--dlo-no-use-allgather` mode instead streams standard-loader rank-local
-weights independently and does not shard weights across ranks.
-
-For the implementation design and compatibility matrix, see
-[Distributed Layerwise Offload](../../design/feature/distributed_layerwise_offload.md).
-
-**Key features:**
- **Weight sharding + AllGather**: each rank stores 1/dp_size of weights,
-  reconstructed per-layer via `all_gather_into_tensor`
- **Fixed double-buffer**: exactly 2 transformer blocks on each device at any
-  time, regardless of model size
- **DP multi-concurrency**: N concurrent requests processed in parallel
-  (AllGather only gathers weight shards, which are request-independent)
- **mmap weight loading**: weights loaded as mmap views pointing to shared OS
-  page cache, eliminating O(dp_size × model_size) RSS during model creation
- **Platform-agnostic**: works on NVIDIA GPU (CUDA/NCCL) and Ascend NPU
-  (CANN/HCCL) via vLLM-Omni's platform abstraction
-
-**Execution Flow (per device):**
-
-```
-Compute Stream:  [Layer N]          [Layer N+1]          [Layer N+2]
-H2D Stream:      [H2D Shard N+1]   [H2D Shard N+2]
-AllGather:       [AG N+1]          [AG N+2]
-Slot Usage:      Slot0: Layer N    Slot1: Layer N+1
-```
-
-### Usage
-
-**CLI:**
-```bash
-# 4× GPU/NPU with AllGather (recommended)
-vllm serve /path/to/model --omni \
-  --enable-distributed-layerwise-offload \
-  --data-parallel-size 4
-
-# Without DLO AllGather (each rank streams standard-loader rank-local weights)
-vllm serve /path/to/model --omni \
-  --enable-distributed-layerwise-offload \
-  --data-parallel-size 4 \
-  --dlo-no-use-allgather
-
-# With SP instead of DP (long sequences)
-vllm serve /path/to/model --omni \
-  --enable-distributed-layerwise-offload \
-  --usp 4
-```
-
-**Python API:**
-```python
-from vllm_omni import Omni
-
-m = Omni(
-    model="/path/to/model",
-    enable_distributed_layerwise_offload=True,
-    dlo_use_allgather=True,  # default
-)
-```
-
-### CLI Flags
-
-| Flag | Description | Default |
-|------|-------------|---------|
-| `--enable-distributed-layerwise-offload` | Enable DLO | `false` |
-| `--data-parallel-size N` | Number of DP ranks; DLO weight-sharding group when AllGather is enabled | `1` |
-| `--dlo-use-allgather` | Shard + AllGather (saves CPU, requires concurrent requests) | `true` |
-| `--dlo-no-use-allgather` | Use standard-loader rank-local weights; no DLO sharding or AllGather | `false` |
-
-### How model weights are loaded (mmap path)
-
-When DLO + AllGather is active, the offloader:
-
-1. **Saves non-persistent buffers** (e.g. RoPE `inv_freq`, timestep `freqs`)
-   from the normally-created transformer
-2. **Converts to meta device** via `to_empty(device="meta")`, releasing random
-   initialization weights
-3. **Loads checkpoint weights as mmap views** via `safe_open().get_tensor()`,
-   which return views into the OS page cache (shared across all ranks, 0 RSS)
-4. **Calls `post_load_weights()`** to apply model-specific dtype conversions
-   (e.g. Cosmos3's `time_embedder` → FP32)
-5. **Restores non-persistent buffers** from saved copies
-
-This approach requires **zero model-specific code changes** — no pipeline or
-transformer modifications are needed.
-
-### OffloadPlan (declarative topology metadata)
-
-Models can optionally declare an `OffloadPlan` class variable to provide
-topology metadata (block attribute names, submodules to offload) without
-any offload-specific logic:
-
-```python
-from vllm_omni.diffusion.offloader import OffloadPlan
-
-class MyPipeline(nn.Module):
-    _dit_modules = ["transformer"]
-    _offload_plan = OffloadPlan(
-        block_attrs={"transformer": ("blocks",)},
-        offload_submodules={"context_encoder": "layers"},
-    )
-```
-
-When not declared, the offloader falls back to `_layerwise_offload_blocks_attrs`
-and heuristic attribute search.  This is backward-compatible — existing models
-work without any changes.
-
-### DP Multi-concurrency
-
-When `--data-parallel-size > 1` and AllGather is enabled, the scheduler batches
-up to `dp_size` requests per denoise step.  Each DP rank processes a different
-request while AllGather synchronizes only weight shards (request-independent).
-
-**Requirements for concurrent requests:**
- `num_inference_steps` must be specified explicitly (None is not allowed)
- All concurrent requests must have the same `num_inference_steps` value
- These constraints exist because AllGather is a collective that requires
-  every rank to participate at each denoise step
-
-### Limitations
- Online quantization (FP8) is incompatible with mmap loading — falls back to
-  regular loading only when the model is outside the DLO+AllGather mmap path;
-  online quantization with DLO+AllGather is rejected. Use
-  `--dlo-no-use-allgather` or disable online quantization.
- Tensor Parallel > 1 is rejected in the DLO+AllGather mmap path because TP-aware
-  weight-loader callbacks are bypassed. The no-AllGather path retains the
-  standard TP loader and is experimental.
- HSDP + AllGather is rejected because it would double-shard parameters. HSDP
-  with no-AllGather is accepted at configuration level, but has limited
-  end-to-end validation.
- `num_inference_steps=None` is not allowed in DP multi-concurrency mode
-
-**Module Discovery**
-
-The offloader discovers pipeline components in two ways:
-
-1. **Protocol-based** (preferred): If the pipeline implements
-    `SupportsComponentDiscovery`, its `_dit_modules`, `_encoder_modules`,
-    `_vae_modules`, and `_resident_modules` class variables are used
-    directly.  All attribute names support dotted paths (e.g.
-    `"pipe.transformer"`, `"bagel.time_embedder"`) for nested submodules.
-
-2. **Fallback attribute scan**: Otherwise, the offloader scans for
-    well-known attribute names:
-    - **DiT modules**: `transformer`, `transformer_2`, `dit`, `sr_dit`, `language_model`, `transformer_blocks`, `model`
-    - **Encoders**: `text_encoder`, `text_encoder_2`, `text_encoder_3`, `image_encoder`
-    - **VAE**: `vae`, `audio_vae`
-
-**Hook System**
-
-Both strategies use vLLM-Omni's hook registry system (`HookRegistry` and `ModelHook`) to register pre/post forward callbacks on modules, enabling automatic swapping without modifying model code.
-
-**Backend Architecture**
-
-```
-OffloadBackend (base class)
-├── ModelLevelOffloadBackend → uses SequentialOffloadHook (.to() swap)
-│                              (delegates to a pipeline's enable_omni_model_cpu_offload
-│                               for split models like Cosmos3)
-├── LayerWiseOffloadBackend → uses LayerwiseOffloadHook
-│                          (single-GPU, full weights on host)
-└── DistributedLayerwiseOffloadBackend → uses DistributedLayerwiseOffloadHook
-                                         (multi-GPU, sharded AllGather or rank-local mode)
-```
-
-Factory function `get_offload_backend()` selects the appropriate backend based on
-configuration.
-
-For split models, `ModelLevelOffloadBackend.enable()` detects a pipeline's
-`enable_omni_model_cpu_offload` hook and delegates to it; Cosmos3 then swaps its
-reasoner/generator components inside the model forward pass.
-
-
-## Supported Models
-
-| Architecture | Example Models | DiT Class | Model-Level Offload | Layerwise Offload | Distributed Layerwise Offload | Blocks Attrs (Layerwise specific) |
-|--------------|----------------|-----------|---------------------|-------------------|-------------------------------|-----------------------------------|
-| Flux2Pipeline | `black-forest-labs/FLUX.2-dev` | `Flux2Transformer2DModel` | ✓ | ✓ | - | `"transformer_blocks"`, `"single_transformer_blocks"` |
-| LongCatImagePipeline | `meituan-longcat/LongCat-Image` | `LongCatImageTransformer2DModel` | - | ✓ | - | `"transformer_blocks"`, `"single_transformer_blocks"` |
-| NextStep11Pipeline | `stepfun-ai/NextStep-1.1` | `NextStepModel` | - | ✓ | - | `"layers"` |
-| OvisImagePipeline | `AIDC-AI/Ovis-Image-7B` | `OvisImageTransformer2DModel` | - | ✓ | - | `"transformer"` |
-| QwenImagePipeline | `Qwen/Qwen-Image` | `QwenImageTransformer2DModel` | ✓ | ✓ | - | `"transformer_blocks"` |
-| StableDiffusionXLPipeline | `stabilityai/stable-diffusion-xl-base-1.0` | `SDXLUNet2DConditionModel` | ✓ | ✓ | - | `"down_blocks"`, `"up_blocks"` |
-| StableDiffusion3Pipeline | `stabilityai/stable-diffusion-3.5-medium` | `SD3Transformer2DModel` | - | ✓ | - | `"transformer_blocks"` |
-| Wan22I2VPipeline | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | `WanTransformer3DModel` | ✓ | ✓ | - | `"blocks"` |
-| Wan22Pipeline | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | `WanTransformer3DModel` | ✓ | ✓ | - | `"blocks"` |
-| SoulXSingerPipeline / SoulXSingerSVCPipeline | `Soul-AILab/SoulX-Singer` | `DiffLlama` (`cfm_decoder.model.diff_estimator`) | ✓ | ✓ | - | `"layers"` |
-| BagelPipeline | `ByteDance-Seed/BAGEL-7B-MoT` | `Qwen2MoTModel` | - | ✓ | - | `"layers"`, `"customized modules"` |
-| Cosmos3OmniDiffusersPipeline | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | `Cosmos3VFMTransformer`, `Cosmos3LanguageModel` | ✓ | ✓ | ✓ | `"layers"`, `"gen_layers"` |
-
-**Notes:**
- Model-Level Offloading is expected to be supported by all common diffusion models (DiT and encoders) naturally
- Layerwise Offloading requires DiT class to define `_layerwise_offload_blocks_attrs` pointing to transformer blocks
- Distributed Layerwise Offloading reuses the layerwise block topology, but the
-  model and parallelism combination still needs validation. AllGather mode
-  additionally requires a compatible checkpoint/mmap layout; no-AllGather mode
-  retains the standard loader's rank-local tensors. See the
-  [DLO design](../../design/feature/distributed_layerwise_offload.md) and
-  [Cosmos3 DistOffload recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/cosmos3/Cosmos3-DistOffload.md)
-  for usage examples.
+See [Distributed Layerwise Offloading](offloader/distributed_layerwise_offload.md).
@@ -0,0 +1,81 @@
+# Model-Level Offloading
+
+Model-level, or sequential, offloading keeps only the pipeline component group
+currently executing on the accelerator. It is the simplest offload strategy
+and is selected with `--enable-cpu-offload`.
+
+## How it works
+
+Pre-forward hooks enforce mutual exclusion between DiT and encoder modules:
+
+- before an encoder runs, the DiT moves to CPU;
+- before a DiT runs, encoders and other DiTs move to CPU; and
+- VAE modules remain on the accelerator.
+
+Pinned host memory reduces transfer overhead. Transfers occur at phase
+boundaries, so cold-start and encoder-to-denoiser transitions become slower.
+
+## Usage
+
+```python
+from vllm_omni import Omni
+
+omni = Omni(
+    model="Wan-AI/Wan2.2-T2V-A14B-Diffusers",
+    enable_cpu_offload=True,
+)
+```
+
+```bash
+vllm serve Wan-AI/Wan2.2-T2V-A14B-Diffusers \
+  --omni --enable-cpu-offload
+```
+
+## Model integration
+
+Pipelines should implement `SupportsComponentDiscovery`:
+
+```python
+from typing import ClassVar
+
+from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery
+
+
+class MyPipeline(nn.Module, SupportsComponentDiscovery):
+    _dit_modules: ClassVar[list[str]] = ["transformer"]
+    _encoder_modules: ClassVar[list[str]] = ["text_encoder", "vision_model"]
+    _vae_modules: ClassVar[list[str]] = ["vae"]
+    _resident_modules: ClassVar[list[str]] = []
+```
+
+All entries may be dotted paths. DiT and encoder lists are both required for
+mutual exclusion. VAE modules are pinned but not swapped; resident modules are
+small modules that must stay on the accelerator for layerwise paths.
+
+## Split-model components
+
+Some models, such as Cosmos3, split one transformer into mutually exclusive
+components that run in different phases. The pipeline exposes
+`enable_omni_model_cpu_offload`, and the backend delegates to the model-local
+contexts:
+
+```python
+class Cosmos3VFMTransformer(nn.Module):
+    def forward(self, ...):
+        with self._offload_context("reasoner"):
+            ...
+        with self._offload_context("generator"):
+            ...
+```
+
+This preserves the same invariant—exactly one component is device resident—
+while reusing sequential `.to()` movers.
+
+## Limitations
+
+- Single device only.
+- Higher cold-start latency.
+- Transfers between encoder and denoising phases add latency.
+
+See the [model-level design](../../../design/feature/offloader/cpu_offload.md)
+for lifecycle and extension invariants.
@@ -0,0 +1,118 @@
+# Distributed Layerwise Offloading
+
+Distributed layerwise offloading (DLO) extends block streaming to multi-device
+deployments. With AllGather enabled, each rank stores roughly `1 / dp_size` of
+the host weights and reconstructs each layer at runtime. Without AllGather,
+each rank streams its standard-loader rank-local weights independently.
+
+See the [DLO feature design](../../../design/feature/offloader/distributed_layerwise_offload.md)
+for the implementation contract and compatibility matrix.
+
+## Execution model
+
+DLO overlaps three operations with a fixed two-block device buffer:
+
+```text
+Compute stream:  [Layer N]          [Layer N+1]        [Layer N+2]
+H2D stream:      [H2D shard N+1]    [H2D shard N+2]
+AllGather:       [AG N+1]           [AG N+2]
+Slots:           slot 0: Layer N    slot 1: Layer N+1
+```
+
+AllGather communicates only request-independent weight shards, so data-
+parallel ranks may process different requests concurrently.
+
+## Usage
+
+```bash
+# Four ranks with sharded host weights and AllGather
+vllm serve /path/to/model --omni \
+  --enable-distributed-layerwise-offload \
+  --data-parallel-size 4
+
+# Standard-loader rank-local weights, without DLO AllGather
+vllm serve /path/to/model --omni \
+  --enable-distributed-layerwise-offload \
+  --data-parallel-size 4 \
+  --dlo-no-use-allgather
+
+# Sequence parallel deployment
+vllm serve /path/to/model --omni \
+  --enable-distributed-layerwise-offload \
+  --usp 4
+```
+
+```python
+from vllm_omni import Omni
+
+omni = Omni(
+    model="/path/to/model",
+    enable_distributed_layerwise_offload=True,
+    dlo_use_allgather=True,
+)
+```
+
+## Flags
+
+| Flag | Meaning | Default |
+| --- | --- | --- |
+| `--enable-distributed-layerwise-offload` | Enable DLO | `false` |
+| `--data-parallel-size N` | DP ranks and AllGather weight-sharding group | `1` |
+| `--dlo-use-allgather` | Shard host weights and reconstruct with AllGather | `true` |
+| `--dlo-no-use-allgather` | Keep standard-loader rank-local weights | `false` |
+| `--dlo-resident-layers N` | Keep N leading main-DiT blocks on device; requires no-AllGather and model-declared resident paths | `0` |
+
+## mmap weight loading
+
+The DLO plus AllGather path:
+
+1. saves non-persistent buffers such as RoPE frequencies;
+2. moves the normally created transformer to the meta device;
+3. loads checkpoint tensors as mmap views backed by the shared OS page cache;
+4. calls model-specific `post_load_weights()` conversions; and
+5. restores the saved non-persistent buffers.
+
+This avoids one full checkpoint RSS copy per rank and does not require
+model-specific loading code.
+
+## Declarative topology
+
+Models may declare an `OffloadPlan` instead of embedding offload logic:
+
+```python
+from vllm_omni.diffusion.offloader import OffloadPlan
+
+
+class MyPipeline(nn.Module):
+    _dit_modules = ["transformer"]
+    _offload_plan = OffloadPlan(
+        block_attrs={"transformer": ("blocks",)},
+        offload_submodules={"context_encoder": "layers"},
+    )
+```
+
+When no plan exists, discovery falls back to
+`_layerwise_offload_blocks_attrs` and then heuristic attribute lookup.
+
+## Data-parallel concurrency
+
+With `data_parallel_size > 1` and AllGather enabled, the scheduler can process
+up to `dp_size` requests per denoising step. Every concurrent request must set
+the same explicit `num_inference_steps`; `None` is rejected because every rank
+must enter each collective.
+
+## Limitations
+
+- Online FP8 quantization is rejected with the DLO plus AllGather mmap path.
+  Use `--dlo-no-use-allgather` or disable online quantization.
+- Tensor parallel size greater than one is rejected in the mmap path because
+  it bypasses TP-aware loader callbacks. The no-AllGather path retains the
+  standard loader and remains experimental with TP.
+- HSDP plus AllGather is rejected to avoid double sharding. HSDP without
+  AllGather has limited end-to-end validation.
+- Resident leading layers require `--dlo-no-use-allgather` and a model
+  `OffloadPlan` that declares eligible `resident_dit_paths`.
+- DP concurrency requires an explicit, identical inference-step count.
+
+See the [Cosmos3 DistOffload recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/cosmos3/Cosmos3-DistOffload.md)
+for an end-to-end example.
@@ -0,0 +1,65 @@
+# Layerwise Offloading
+
+Layerwise, or blockwise, offloading keeps one transformer block on the
+accelerator and prefetches the next block while the current block computes.
+It is best suited to compute-heavy video DiTs whose block execution can hide
+host-to-device transfers.
+
+## Execution flow
+
+Each block has a pre-forward and post-forward hook. Parameters are consolidated
+in pinned host tensors and rematerialized for execution on a dedicated copy
+stream.
+
+| Block | Pre-forward hook | Forward | Post-forward hook |
+| --- | --- | --- | --- |
+| block 0 | Prefetch block 1 | Compute block 0 | Free block 0 |
+| block 1 | Prefetch block 2 | Compute block 1 | Free block 1 |
+| ... | ... | ... | ... |
+| last block | Prefetch block 0 | Compute last block | Free last block |
+
+Encoders, VAE modules, and non-block DiT modules such as embeddings and norms
+remain device resident.
+
+## Usage
+
+```python
+from vllm_omni import Omni
+
+omni = Omni(
+    model="Wan-AI/Wan2.2-T2V-A14B-Diffusers",
+    enable_layerwise_offload=True,
+)
+```
+
+```bash
+vllm serve Wan-AI/Wan2.2-T2V-A14B-Diffusers \
+  --omni --enable-layerwise-offload
+```
+
+## Model integration
+
+Transformer classes declare containers of executable blocks:
+
+```python
+class WanTransformer3DModel(nn.Module):
+    _layerwise_offload_blocks_attrs = ["blocks"]
+
+
+class Flux2Transformer2DModel(nn.Module):
+    _layerwise_offload_blocks_attrs = [
+        "transformer_blocks",
+        "single_transformer_blocks",
+    ]
+```
+
+See the [layerwise design](../../../design/feature/offloader/layerwise_offload.md) for
+the discovery and hook invariants. `OffloadPlan` is a separate declarative
+topology path for distributed layerwise offload.
+
+## Limitations
+
+- Single device only.
+- Setup consolidates and pins block parameters, increasing cold-start time.
+- Performance depends on block compute time and host-to-device bandwidth;
+  lightweight blocks may not hide transfers.
@@ -64,6 +64,41 @@ omni = Omni(
 )
 ```

+### Combined with Online FP8 Quantization
+
+HSDP can be combined with online FP8 quantization for diffusion models that
+support both features. HSDP shards the quantized weights across GPUs to reduce
+the per-GPU weight memory footprint.
+
+```python
+from vllm_omni import Omni
+from vllm_omni.diffusion.data import DiffusionParallelConfig
+
+omni = Omni(
+    model="<your-model>",
+    quantization="fp8",
+    parallel_config=DiffusionParallelConfig(
+        use_hsdp=True,
+        hsdp_shard_size=2,
+    ),
+)
+```
+
+For online serving:
+
+```bash
+vllm serve <your-model> \
+    --omni \
+    --quantization fp8 \
+    --use-hsdp \
+    --hsdp-shard-size 2
+```
+
+The model must support both HSDP and online FP8 quantization. HSDP remains
+incompatible with Tensor Parallelism. See the
+[FP8 Quantization Guide](../../quantization/fp8.md) for supported models and
+hardware.
+
 ---

 ## Example Script
@@ -147,3 +182,4 @@ For detailed instructions on adding HSDP support to new models, see the [HSDP Co
 1. ✅ **Enable HSDP** - Set `use_hsdp=True` and `hsdp_shard_size` to reduce per-GPU memory for large models
 2. ✅ **Combine with SP** - Use together with `ulysses_degree` for video models requiring both memory reduction and sequence parallelism
 3. ⚠️ **Incompatible with TP** - `tensor_parallel_size` must be 1 when HSDP is enabled
+4. ✅ **Combine with online FP8** - Use `quantization="fp8"` or `--quantization fp8` to reduce the sharded weight memory footprint
@@ -106,7 +106,10 @@ The following tables show which models support each feature:

 > Notes:

-> 1. CPU Offload has two methods: Module-wise (default for models with DiT + text encoder) and Layerwise. The tables below show **Layerwise support** only. Split models like Cosmos3 (no separate text encoder) swap their reasoner/generator components for module-wise offload; see the [CPU Offload Guide](diffusion/cpu_offload.md).
+> 1. CPU Offload has three strategies: model-level, layerwise, and distributed
+>    layerwise. The tables below show **layerwise support** only. Split models
+>    like Cosmos3 swap their reasoner/generator components for model-level
+>    offload; see the [CPU Offload Guide](diffusion/cpu_offload.md).
 > 2. The **💾Quantization** column is collapsed for readability. See [Quantization](quantization/overview.md) for per-method and per-model support details.

 ### ImageGen
@@ -214,7 +217,7 @@ The following tables show which models support each feature:
    3. CPU Offloading (Layerwise) and CPU Offloading (Module-wise) are not compatible.
    4. The CPU Offloading (Layerwise) row describes local layerwise offload.
       Multi-device Distributed Layerwise Offload has a separate topology and
-       compatibility matrix in the [CPU Offloading guide](diffusion/cpu_offload.md#distributed-layerwise-offloading).
+       compatibility matrix in the [Distributed Layerwise Offloading guide](diffusion/offloader/distributed_layerwise_offload.md).
    5. The compatibility matrix uses FP8 as the representative quantization method.
    6. Step Execution is not compatible with any diffusion cache backend. LoRA is supported, but each scheduled batch must use a single adapter (requests with different `lora_request` or `lora_scale` are kept in separate batches).

@@ -165,7 +165,7 @@ vllm serve Qwen/Qwen-Image --omni --port 8091 \

 - **Tensor Parallelism — Text Encoder Not Sharded**: TP currently only shards the DiT blocks. Each TP rank retains a **full copy of the text encoder weights**, leading to significant GPU memory overhead proportional to TP degree. Tracked in [Issue #771](https://github.com/vllm-project/vllm-omni/issues/771).

- **CPU Offloading — Two Modes Are Mutually Exclusive**: Model-level offload (`enable_cpu_offload`) and layerwise offload (`enable_layerwise_offload`) cannot be used simultaneously. If both are set, layerwise takes priority and model-level is silently ignored.
+- **CPU Offloading — Strategies Are Mutually Exclusive**: Distributed layerwise, layerwise, and model-level offload should not be enabled together. Selection priority is distributed layerwise, then layerwise, then model-level.

 - **CPU Offloading — VAE stays on GPU**: Both offloading strategies keep the VAE on GPU at all times. For high-resolution generation, VAE decode can still cause OOM. Mitigate by combining with `vae_use_tiling=True` or VAE Patch Parallelism.

@@ -81,6 +81,10 @@ plugins:
        user_guide/quantization/index.md: user_guide/quantization/overview.md
        design/feature/diffusion_request_level_batching.md: design/feature/diffusion_continuous_batching.md
        design/feature/diffusion_step_execution.md: design/feature/diffusion_continuous_batching.md
+        design/feature/cpu_offloading.md: design/feature/offloader/README.md
+        design/feature/model_level_offload.md: design/feature/offloader/cpu_offload.md
+        design/feature/layerwise_offload.md: design/feature/offloader/layerwise_offload.md
+        design/feature/distributed_layerwise_offload.md: design/feature/offloader/distributed_layerwise_offload.md
  - glightbox
  - git-revision-date-localized:
      # exclude files
@@ -40,6 +40,7 @@ dev = [
    "pytest-asyncio==1.4.0",
    "pytest-cov==7.1.0",
    "pytest-mock==3.15.1",
+    "pytest-shard==0.1.2",
    "websockets==16.1",
    "pytest-subtests==0.15.0",
    "datasets==3.6.0",
@@ -301,6 +302,21 @@ filterwarnings = [
    "ignore:.*does not have '__bases__' attribute.*:UserWarning",
 ]

+[tool.coverage.run]
+core = "sysmon"
+source = ["vllm_omni"]
+# Online-serving tests launch the server via subprocess.Popen and stop it with
+# SIGTERM. pytest-cov no longer instruments subprocesses implicitly, and a
+# SIGTERM-ed process discards its data, so without patch/sigterm a coverage run
+# over an online test measures only the pytest parent.
+patch = ["subprocess"]
+concurrency = ["multiprocessing", "thread"]
+sigterm = true
+
+[tool.coverage.report]
+show_missing = true
+skip_covered = true
+
 [tool.typos.default]
 extend-ignore-identifiers-re = [
    ".*_thw",
@@ -5,6 +5,7 @@ import sys
 from pathlib import Path

 import pytest
+import yaml

 sys.path.insert(0, str(Path(__file__).resolve().parents[2] / ".buildkite" / "common" / "scripts"))

@@ -139,3 +140,48 @@ def test_mirror_hardwares_a2b3_npu_4_expands_agents_image_and_plugins() -> None:
    assert step["plugins"][0]["kubernetes"]["podSpecPatch"]["imagePullSecrets"] == [
        {"name": "swr-secret"},
    ]
+
+
+COVERAGE_PILOT_LABELS = ("Diffusion · Bagel Test", "TTS · Qwen3-TTS Base Test")
+MERGE_PIPELINE = Path(".buildkite/cuda/test-merge.yml")
+
+
+def _surviving_labels(changed_files: list[str]) -> set[str]:
+    doc = yaml.safe_load(MERGE_PIPELINE.read_text(encoding="utf-8"))
+    rendered = _render_test_pipeline(doc, changed_files=changed_files)
+    labels: set[str] = set()
+
+    def walk(steps: list | None) -> None:
+        for step in steps or []:
+            if not isinstance(step, dict):
+                continue
+            if "label" in step:
+                labels.add(step["label"])
+            walk(step.get("steps"))
+
+    walk(rendered.get("steps"))
+    return labels
+
+
+@pytest.mark.parametrize(
+    "changed_file",
+    [
+        ".buildkite/common/scripts/run_cov_split.sh",
+        "pyproject.toml",
+    ],
+)
+def test_coverage_pilots_selected_by_their_shared_inputs(changed_file: str) -> None:
+    """The split helper and the coverage config both change what these jobs measure.
+
+    Without them in source_file_dependencies the jobs are filtered out of normal
+    PR builds, so a regression would only surface in a later nightly.
+    """
+    labels = _surviving_labels([changed_file])
+    for pilot in COVERAGE_PILOT_LABELS:
+        assert pilot in labels, f"{pilot} dropped when {changed_file} changed"
+
+
+def test_coverage_pilots_not_selected_by_unrelated_change() -> None:
+    labels = _surviving_labels(["vllm_omni/entrypoints/openai/serving_chat.py"])
+    for pilot in COVERAGE_PILOT_LABELS:
+        assert pilot not in labels
@@ -60,9 +60,18 @@ def _dense_attention(query: torch.Tensor, key: torch.Tensor, value: torch.Tensor
    return torch.einsum("bhqk,bkhd->bqhd", probs, value)


-def _cuda_flash_attn_usable() -> bool:
+def _gpu_flash_attn_usable() -> bool:
    if not torch.cuda.is_available():
        return False
+    if torch.version.hip is not None:
+        for module in ("aiter", "flash_attn"):
+            try:
+                imported = __import__(module, fromlist=["flash_attn_varlen_func"])
+                if getattr(imported, "flash_attn_varlen_func", None) is not None:
+                    return True
+            except ImportError:
+                pass
+        return False
    try:
        spec = find_spec("vllm.vllm_flash_attn")
        if spec is None or spec.origin is None:
@@ -227,12 +236,11 @@ def test_paged_attention_matches_dense_reference_cpu(history_chunks, action_len,
    assert st.adapter(POS).completed_chunks == before + (1 if commit_current else 0)


-@pytest.mark.skipif(not _cuda_flash_attn_usable(), reason="usable CUDA FlashAttention is required")
+@pytest.mark.skipif(not _gpu_flash_attn_usable(), reason="usable GPU FlashAttention is required")
@pytest.mark.parametrize("history_chunks", [1, 3])
@pytest.mark.parametrize("action_len", [0, 3])
@pytest.mark.parametrize("commit_current", [False, True])
 def test_paged_attention_matches_dense_reference_gpu(history_chunks, action_len, commit_current):
-    pytest.importorskip("vllm.vllm_flash_attn")
    torch.manual_seed(0)
    device = torch.device("cuda")
    dtype = torch.float16
@@ -265,10 +265,15 @@ def test_fa_vs_sdpa():
    print(f"Max absolute difference (valid region): {max_diff:.6f}")
    print(f"Mean absolute difference (valid region): {mean_diff:.6f}")

-    # Assert that outputs are close
-    # Using higher tolerance for bfloat16 and different implementations
-    assert max_diff < 0.01, f"Max difference {max_diff} exceeds threshold 0.01"
-    assert mean_diff < 0.001, f"Mean difference {mean_diff} exceeds threshold 0.001"
+    # AITER's ROCm BF16 kernel is deterministic, but differs from ROCm SDPA by
+    # about 1.3% relative error. An absolute-only bound misclassifies that
+    # expected rounding difference for larger output values (for example,
+    # -2.65625 vs -2.6875). Keep the tighter historical bounds elsewhere.
+    if current_omni_platform.is_rocm():
+        torch.testing.assert_close(output_fa_valid, output_sdpa_valid, rtol=1e-2, atol=1.5e-2)
+    else:
+        assert max_diff < 0.01, f"Max difference {max_diff} exceeds threshold 0.01"
+        assert mean_diff < 0.001, f"Mean difference {mean_diff} exceeds threshold 0.001"

    print("✓ Case 2 PASSED: FA and SDPA outputs are very close!")

@@ -416,8 +416,8 @@ def test_sp_correctness_advanced(model_name: str):


@pytest.mark.skipif(
-    not (current_omni_platform.is_cuda() or current_omni_platform.is_xpu()),
-    reason="Only tested on CUDA and XPU",
+    not (current_omni_platform.is_cuda() or current_omni_platform.is_rocm() or current_omni_platform.is_xpu()),
+    reason="Only tested on CUDA, ROCM and XPU",
 )
@pytest.mark.diffusion
@pytest.mark.parallel
@@ -447,8 +447,8 @@ def test_local_sp_padding_mask(monkeypatch: pytest.MonkeyPatch) -> None:


@pytest.mark.skipif(
-    not (current_omni_platform.is_cuda() or current_omni_platform.is_xpu()),
-    reason="Only tested on CUDA and XPU",
+    not (current_omni_platform.is_cuda() or current_omni_platform.is_rocm() or current_omni_platform.is_xpu()),
+    reason="Only tested on CUDA, ROCM and XPU",
 )
@pytest.mark.diffusion
@pytest.mark.parallel
@@ -468,8 +468,8 @@ def test_local_sp_padding_mask_no_padding(monkeypatch: pytest.MonkeyPatch) -> No


@pytest.mark.skipif(
-    not (current_omni_platform.is_cuda() or current_omni_platform.is_xpu()),
-    reason="Only tested on CUDA and XPU",
+    not (current_omni_platform.is_cuda() or current_omni_platform.is_rocm() or current_omni_platform.is_xpu()),
+    reason="Only tested on CUDA, ROCM and XPU",
 )
@pytest.mark.diffusion
@pytest.mark.parallel
@@ -24,11 +24,20 @@ from safetensors.torch import save_file

 from tests.helpers.mark import hardware_test
 from tests.helpers.runtime import OmniServer, OmniServerParams
+from vllm_omni.platforms import current_omni_platform

 os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"

 MODEL = "Tongyi-MAI/Z-Image-Turbo"

+# ROCm's default FLASH_ATTN diffusion backend resolves to AITER. On a cold CI
+# worker, JIT-building its MHA kernel can take longer than the server readiness
+# timeout. This test validates LoRA switching rather than attention-backend or
+# compiled-execution performance, so use the eager SDPA path on ROCm only.
+ROCM_SERVER_ARGS = (
+    ["--enforce-eager", "--diffusion-attention-backend", "TORCH_SDPA"] if current_omni_platform.is_rocm() else []
+)
+

 PROMPT = "a photo of a cat sitting on a laptop keyboard"
 SIZE = "256x256"
@@ -42,7 +51,14 @@ server_params = [
            server_args=[
                "--num-gpus",
                "1",
+                *ROCM_SERVER_ARGS,
            ],
+            # A cold-cache Z-Image load can take slightly more than the
+            # orchestrator's 900s default on CI (weights alone have been
+            # observed at ~690s and full orchestrator readiness at ~906s).
+            # Match the large-model offline runner's startup allowance without
+            # weakening timeout detection for every online-serving test.
+            init_timeout=1800,
        ),
        id="zimage_turbo_lora",
    ),
@@ -2,6 +2,7 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

 import json
+import sys
 from multiprocessing.reduction import ForkingPickler
 from types import SimpleNamespace
 from unittest.mock import Mock
@@ -1428,34 +1429,219 @@ def test_r7_r8_ref2va_video_segment_matrix(monkeypatch, tmp_path, case, start_ti
    assert transcode_calls[0][1]["target_frame_count"] == 209


-def test_ref2va_qwen_sampling_decodes_prepared_video_once(monkeypatch):
+def test_ref2va_qwen_sampling_uses_one_selective_decode(monkeypatch):
    from vllm_omni.diffusion.models.minimax_h3 import reference_video as reference_video_module

-    decoded = np.arange(25 * 2 * 2 * 3, dtype=np.uint8).reshape(25, 2, 2, 3)
-    load_calls = []
+    decoded = np.arange(3 * 2 * 2 * 3, dtype=np.uint8).reshape(3, 2, 2, 3)
+    decode_calls = []
    monkeypatch.setattr(
        reference_video_module,
        "_probe_video",
-        lambda _path: {"frame_count": len(decoded)},
+        lambda _path: {
+            "frame_count": 25,
+            "width": 2,
+            "height": 2,
+        },
    )
    monkeypatch.setattr(
        reference_video_module,
-        "load_video_frames",
-        lambda path: load_calls.append(path) or decoded,
+        "_decode_video_frames_ffmpeg",
+        lambda path, **kwargs: decode_calls.append((path, kwargs)) or decoded,
    )

    sampled = reference_video_module.sample_reference_video_frames("prepared.mp4")

-    assert load_calls == ["prepared.mp4"]
-    for frame, expected in zip(
-        sampled["frames"],
-        (decoded[0], decoded[12], decoded[24]),
-        strict=True,
-    ):
+    assert decode_calls == [
+        (
+            "prepared.mp4",
+            {
+                "frame_count": 25,
+                "indices": [0, 12, 24],
+                "width": 2,
+                "height": 2,
+            },
+        )
+    ]
+    for frame, expected in zip(sampled["frames"], decoded, strict=True):
        np.testing.assert_array_equal(frame, expected)
    assert sampled["block_timestamps"] == [0.25, 1.0]


+@pytest.mark.parametrize(
+    ("indices", "frame_count"),
+    [([0, 12], 25), (None, 2)],
+)
+def test_ref2va_ffmpeg_decode_reads_exact_rgb_frames(monkeypatch, indices, frame_count):
+    from vllm_omni.diffusion.models.minimax_h3 import reference_video as reference_video_module
+
+    expected = np.arange(2 * 2 * 3 * 3, dtype=np.uint8).reshape(2, 2, 3, 3)
+    observed = []
+
+    def fake_run(command, **kwargs):
+        observed.append((command, kwargs))
+        return SimpleNamespace(stdout=expected.tobytes())
+
+    monkeypatch.setattr(reference_video_module.subprocess, "run", fake_run)
+
+    actual = reference_video_module._decode_video_frames_ffmpeg(
+        "prepared.mp4",
+        frame_count=frame_count,
+        indices=indices,
+        width=3,
+        height=2,
+    )
+
+    np.testing.assert_array_equal(actual, expected)
+    command, kwargs = observed[0]
+    assert command[0] == "ffmpeg"
+    assert command[command.index("-frames:v") + 1] == "2"
+    if indices is None:
+        assert "-vf" not in command
+    else:
+        assert command[command.index("-vf") + 1] == "select=eq(n\\,0)+eq(n\\,12)"
+    assert command[-1] == "pipe:1"
+    assert kwargs == {"check": True, "capture_output": True}
+
+
+def test_ref2va_load_video_frames_uses_ffmpeg_without_decord(monkeypatch):
+    from vllm_omni.diffusion.models.minimax_h3 import reference_video as reference_video_module
+
+    expected = np.zeros((2, 2, 3, 3), dtype=np.uint8)
+    decode_calls = []
+    monkeypatch.setitem(sys.modules, "decord", None)
+    monkeypatch.setattr(
+        reference_video_module,
+        "_probe_video",
+        lambda _path: {
+            "frame_count": 2,
+            "width": 3,
+            "height": 2,
+        },
+    )
+    monkeypatch.setattr(
+        reference_video_module,
+        "_decode_video_frames_ffmpeg",
+        lambda path, **kwargs: decode_calls.append((path, kwargs)) or expected,
+    )
+
+    actual = reference_video_module.load_video_frames("prepared.mp4")
+
+    assert actual is expected
+    assert decode_calls == [
+        (
+            "prepared.mp4",
+            {
+                "frame_count": 2,
+                "width": 3,
+                "height": 2,
+            },
+        )
+    ]
+
+
+@pytest.mark.parametrize(
+    ("kwargs", "message"),
+    [
+        ({"frame_count": 0, "width": 3, "height": 2}, "video has no frames"),
+        ({"frame_count": 1, "width": 0, "height": 2}, "invalid dimensions"),
+        (
+            {"frame_count": 1, "indices": [], "width": 3, "height": 2},
+            "invalid frame indices",
+        ),
+    ],
+)
+def test_ref2va_ffmpeg_decode_rejects_invalid_metadata(kwargs, message):
+    from vllm_omni.diffusion.models.minimax_h3 import reference_video as reference_video_module
+    from vllm_omni.errors import OmniClientError
+
+    with pytest.raises(OmniClientError, match=message):
+        reference_video_module._decode_video_frames_ffmpeg("prepared.mp4", **kwargs)
+
+
+def test_ref2va_probe_uses_container_frame_count_without_scan(monkeypatch):
+    from vllm_omni.diffusion.models.minimax_h3 import reference_video as reference_video_module
+
+    calls = []
+    probe = {
+        "streams": [
+            {
+                "codec_type": "video",
+                "width": 4,
+                "height": 2,
+                "r_frame_rate": "24/1",
+                "nb_frames": "3",
+                "duration": "0.125",
+                "sample_aspect_ratio": "1:1",
+            }
+        ],
+        "format": {"duration": "0.125", "size": "123"},
+    }
+    monkeypatch.setattr(
+        reference_video_module.subprocess,
+        "run",
+        lambda command, **kwargs: calls.append((command, kwargs)) or SimpleNamespace(stdout=json.dumps(probe)),
+    )
+
+    metadata = reference_video_module._probe_video("prepared.mp4")
+
+    assert metadata["frame_count"] == 3
+    assert len(calls) == 1
+    assert "-count_frames" not in calls[0][0]
+
+
+def test_ref2va_probe_counts_frames_only_when_metadata_is_missing(monkeypatch):
+    from vllm_omni.diffusion.models.minimax_h3 import reference_video as reference_video_module
+
+    calls = []
+    stream = {
+        "codec_type": "video",
+        "width": 4,
+        "height": 2,
+        "r_frame_rate": "24/1",
+        "duration": "0.125",
+        "sample_aspect_ratio": "1:1",
+    }
+    first_probe = {
+        "streams": [stream],
+        "format": {"duration": "0.125", "size": "123"},
+    }
+    second_probe = {
+        "streams": [{**stream, "nb_read_frames": "3"}],
+        "format": {"duration": "0.125", "size": "123"},
+    }
+    monkeypatch.setattr(
+        reference_video_module.subprocess,
+        "run",
+        lambda command, **kwargs: calls.append((command, kwargs))
+        or SimpleNamespace(stdout=json.dumps(first_probe if len(calls) == 1 else second_probe)),
+    )
+
+    metadata = reference_video_module._probe_video("prepared.mp4")
+
+    assert metadata["frame_count"] == 3
+    assert len(calls) == 2
+    assert "-count_frames" in calls[1][0]
+
+
+def test_ref2va_ffmpeg_decode_rejects_incomplete_output(monkeypatch):
+    from vllm_omni.diffusion.models.minimax_h3 import reference_video as reference_video_module
+    from vllm_omni.errors import OmniClientError
+
+    monkeypatch.setattr(
+        reference_video_module.subprocess,
+        "run",
+        lambda *_args, **_kwargs: SimpleNamespace(stdout=b"short"),
+    )
+
+    with pytest.raises(OmniClientError, match="decoded 5 bytes.*expected 18"):
+        reference_video_module._decode_video_frames_ffmpeg(
+            "prepared.mp4",
+            frame_count=1,
+            width=3,
+            height=2,
+        )
+
+
 def test_ref2va_two_video_recipe_tolerates_container_rounding(monkeypatch, tmp_path):
    from vllm_omni.diffusion.models.minimax_h3 import reference_video as reference_video_module

@@ -0,0 +1,104 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import pytest
+import torch
+
+from vllm_omni.platforms import current_omni_platform
+
+pytestmark = [pytest.mark.core_model, pytest.mark.diffusion]
+
+
+@pytest.mark.skipif(not current_omni_platform.is_npu(), reason="requires Ascend NPU")
+@pytest.mark.parametrize("num_q_heads,num_kv_heads,seq_len", [(8, 2, 16), (64, 8, 1)])
+def test_qwen3_vl_npu_rope_matches_bnsd_reference(
+    num_q_heads: int,
+    num_kv_heads: int,
+    seq_len: int,
+) -> None:
+    from vllm_omni.platforms.npu.models.minimax_h3 import _apply_rotary_pos_emb_npu
+
+    device = torch.device("npu")
+    dtype = torch.bfloat16
+    q = torch.randn(1, num_q_heads, seq_len, 128, device=device, dtype=dtype)
+    k = torch.randn(1, num_kv_heads, seq_len, 128, device=device, dtype=dtype)
+    cos = torch.randn(1, seq_len, 128, device=device, dtype=dtype)
+    sin = torch.randn(1, seq_len, 128, device=device, dtype=dtype)
+
+    q_out, k_out = _apply_rotary_pos_emb_npu(q, k, cos, sin)
+    cos_bnsd = cos.unsqueeze(1)
+    sin_bnsd = sin.unsqueeze(1)
+
+    def rotate_half(x: torch.Tensor) -> torch.Tensor:
+        x1, x2 = x.chunk(2, dim=-1)
+        return torch.cat((-x2, x1), dim=-1)
+
+    torch.testing.assert_close(
+        q_out,
+        q * cos_bnsd + rotate_half(q) * sin_bnsd,
+        atol=2e-2,
+        rtol=2e-2,
+    )
+    torch.testing.assert_close(
+        k_out,
+        k * cos_bnsd + rotate_half(k) * sin_bnsd,
+        atol=2e-2,
+        rtol=2e-2,
+    )
+
+
+@pytest.mark.skipif(not current_omni_platform.is_npu(), reason="requires Ascend NPU")
+def test_qwen3_vl_npu_rope_preserves_batched_mrope_positions() -> None:
+    """Keep per-sample Qwen3-VL M-RoPE frequencies on the BNSD NPU path."""
+    from types import SimpleNamespace
+
+    from vllm_omni.diffusion.models.minimax_h3.encoder import (
+        MiniMaxH3Qwen3VLTextRotaryEmbedding,
+    )
+    from vllm_omni.platforms.npu.models.minimax_h3 import _apply_rotary_pos_emb_npu
+
+    device = torch.device("npu")
+    dtype = torch.bfloat16
+    batch_size, seq_len, head_dim = 2, 16, 128
+    config = SimpleNamespace(
+        hidden_size=head_dim,
+        num_attention_heads=1,
+        rope_parameters={
+            "rope_theta": 1_000_000.0,
+            "mrope_section": [24, 20, 20],
+        },
+    )
+    rope = MiniMaxH3Qwen3VLTextRotaryEmbedding(config).to(device)
+    positions = torch.arange(seq_len, device=device)
+    position_ids = torch.stack(
+        (
+            torch.stack((positions, positions + 7)),
+            torch.stack((positions * 2, positions * 2 + 3)),
+            torch.stack((positions * 3, positions * 3 + 5)),
+        )
+    )
+    q = torch.randn(batch_size, 8, seq_len, head_dim, device=device, dtype=dtype)
+    k = torch.randn(batch_size, 2, seq_len, head_dim, device=device, dtype=dtype)
+    cos, sin = rope(q, position_ids)
+
+    q_out, k_out = _apply_rotary_pos_emb_npu(q, k, cos, sin)
+    cos_bnsd = cos.unsqueeze(1)
+    sin_bnsd = sin.unsqueeze(1)
+
+    def rotate_half(x: torch.Tensor) -> torch.Tensor:
+        x1, x2 = x.chunk(2, dim=-1)
+        return torch.cat((-x2, x1), dim=-1)
+
+    assert not torch.equal(cos[0], cos[1])
+    torch.testing.assert_close(
+        q_out,
+        q * cos_bnsd + rotate_half(q) * sin_bnsd,
+        atol=2e-2,
+        rtol=2e-2,
+    )
+    torch.testing.assert_close(
+        k_out,
+        k * cos_bnsd + rotate_half(k) * sin_bnsd,
+        atol=2e-2,
+        rtol=2e-2,
+    )
@@ -0,0 +1,46 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import pytest
+import torch
+import torch.nn.functional as F
+
+from vllm_omni.platforms import current_omni_platform
+
+pytestmark = [pytest.mark.core_model, pytest.mark.diffusion]
+
+
+@pytest.mark.skipif(not current_omni_platform.is_npu(), reason="requires Ascend NPU")
+@pytest.mark.parametrize("shape", [(1, 16, 256), (2, 1, 256)])
+def test_qwen3_vl_npu_swiglu_matches_split_reference(
+    shape: tuple[int, int, int],
+) -> None:
+    from vllm_omni.platforms.npu.models.minimax_h3 import npu_swiglu_from_packed
+
+    packed = torch.randn(*shape, device="npu", dtype=torch.bfloat16)
+    gate, up = packed.chunk(2, dim=-1)
+
+    torch.testing.assert_close(
+        npu_swiglu_from_packed(packed),
+        F.silu(gate) * up,
+        atol=2e-2,
+        rtol=2e-2,
+    )
+
+
+@pytest.mark.skipif(not current_omni_platform.is_npu(), reason="requires Ascend NPU")
+def test_qwen3_vl_npu_swiglu_matches_two_projection_reference() -> None:
+    from vllm_omni.platforms.npu.models.minimax_h3 import npu_swiglu_from_packed
+
+    x = torch.randn(1, 16, 128, device="npu", dtype=torch.bfloat16)
+    weight = torch.randn(256, 128, device="npu", dtype=torch.bfloat16)
+    packed = F.linear(x, weight)
+    gate = F.linear(x, weight[:128])
+    up = F.linear(x, weight[128:])
+
+    torch.testing.assert_close(
+        npu_swiglu_from_packed(packed),
+        F.silu(gate) * up,
+        atol=2e-2,
+        rtol=2e-2,
+    )
@@ -36,15 +36,23 @@ class _ToyKernel:
 # --- helper to build a toy module simulating online-FP8 post-load state ---


-def _make_fp8_toy_module(out_features: int = 16, in_features: int = 32):
+def _make_fp8_toy_module(
+    out_features: int = 16,
+    in_features: int = 32,
+    *,
+    quant_method_cls=None,
+):
    """Return a module whose weight is ``qweight.t()`` (non-contiguous) and whose
    ``quant_method`` is an ``Fp8LinearMethod`` with a ``_ToyKernel``.
    """
    from vllm.model_executor.layers.quantization.fp8 import Fp8LinearMethod

+    if quant_method_cls is None:
+        quant_method_cls = Fp8LinearMethod
+
    # Create the quant-method instance without calling __init__ (avoids
    # heavyweight upstream config dependencies).  isinstance() still works.
-    qm = object.__new__(Fp8LinearMethod)
+    qm = object.__new__(quant_method_cls)
    qm.fp8_linear = _ToyKernel()

    # randn does not support float8 on CPU; zeros does and the test only
@@ -104,6 +112,28 @@ def test_rewrites_non_contiguous_weight():
    assert module.weight.shape == (16, 32)


+def test_rewrites_per_tensor_online_fp8_weight():
+    """Regression test for vLLM's new online FP8 frontend (#45463)."""
+    from vllm.model_executor.layers.quantization.online.fp8 import (
+        Fp8PerTensorOnlineLinearMethod,
+    )
+
+    module = _make_fp8_toy_module(
+        16,
+        32,
+        quant_method_cls=Fp8PerTensorOnlineLinearMethod,
+    )
+
+    n_rewritten_layers = prepare_fp8_layers_for_fsdp(module)
+
+    assert n_rewritten_layers == 1
+    assert module.weight.is_contiguous()
+    assert module.weight.shape == (16, 32)
+    w, *_ = module.quant_method.fp8_linear._get_layer_params(module)
+    assert w.shape == (32, 16)
+    assert w.stride() == (1, 32)
+
+
 def test_patched_get_layer_params_returns_transposed_view():
    module = _make_fp8_toy_module(16, 32)
    prepare_fp8_layers_for_fsdp(module)
@@ -3,6 +3,7 @@

 from __future__ import annotations

+import os
 import subprocess
 import sys
 from pathlib import Path
@@ -14,9 +15,18 @@ REPO_ROOT = Path(__file__).resolve().parents[2]


 def _assert_isolated_import_succeeds(script: str) -> None:
+    # These tests validate Python import/package boundaries and do not use a
+    # GPU. Other tests in the combined entrypoints/engine suite may temporarily
+    # configure only one of the CUDA/ROCm visibility variables. Do not let an
+    # unrelated, inconsistent device-selection environment prevent the child
+    # interpreter from reaching the boundary assertion.
+    env = os.environ.copy()
+    env.pop("CUDA_VISIBLE_DEVICES", None)
+    env.pop("HIP_VISIBLE_DEVICES", None)
    result = subprocess.run(
        [sys.executable, "-c", script],
        cwd=REPO_ROOT,
+        env=env,
        capture_output=True,
        text=True,
        timeout=60,
@@ -25,6 +35,14 @@ def _assert_isolated_import_succeeds(script: str) -> None:
    assert result.returncode == 0, result.stderr


+def test_isolated_import_ignores_conflicting_gpu_visibility(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Import-boundary checks must not depend on GPU state leaked by prior tests."""
+    monkeypatch.setenv("HIP_VISIBLE_DEVICES", "0")
+    monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "0,1")
+
+    _assert_isolated_import_succeeds("import vllm_omni.outputs")
+
+
 def test_stable_engine_imports_do_not_load_experimental_duplex() -> None:
    _assert_isolated_import_succeeds("""
 import sys
@@ -5,6 +5,7 @@
 import asyncio
 import base64
 import os
+import tempfile

 import httpx
 import pytest
@@ -101,7 +102,7 @@ class TestDecodeAudioUrlSuffixSanitization:
        try:
            # ".." contains non-alnum dots, so falls back to .wav
            assert path.endswith(".wav")
-            assert os.path.dirname(path) == "/tmp"
+            assert os.path.dirname(path) == tempfile.gettempdir()
        finally:
            _cleanup(path)

@@ -44,31 +44,33 @@ def _nearest_multiple(value: float, multiple: int) -> int:


 def _probe_video(path: str) -> dict[str, Any]:
-    result = subprocess.run(
-        [
-            "ffprobe",
-            "-v",
-            "error",
-            "-count_frames",
-            "-show_entries",
-            (
-                "stream=codec_type,codec_name,width,height,r_frame_rate,nb_read_frames,nb_frames,duration,"
-                "sample_aspect_ratio:format=format_name,size,duration"
-            ),
-            "-of",
-            "json",
-            path,
-        ],
-        check=True,
-        capture_output=True,
-        text=True,
+    show_entries = (
+        "stream=codec_type,codec_name,width,height,r_frame_rate,nb_read_frames,nb_frames,duration,"
+        "sample_aspect_ratio:format=format_name,size,duration"
    )
+    command = ["ffprobe", "-v", "error", "-show_entries", show_entries, "-of", "json", path]
+    result = subprocess.run(command, check=True, capture_output=True, text=True)
    probe = json.loads(result.stdout)
    streams = probe.get("streams") or []
    video_streams = [stream for stream in streams if stream.get("codec_type") == "video"]
    if not video_streams:
        raise OmniClientError(f"media has no video stream: {path}")
    stream = video_streams[0]
+    raw_count = stream.get("nb_read_frames") or stream.get("nb_frames")
+    if raw_count in (None, "", "N/A"):
+        result = subprocess.run(
+            ["ffprobe", "-v", "error", "-count_frames", *command[3:]],
+            check=True,
+            capture_output=True,
+            text=True,
+        )
+        probe = json.loads(result.stdout)
+        streams = probe.get("streams") or []
+        video_streams = [stream for stream in streams if stream.get("codec_type") == "video"]
+        if not video_streams:
+            raise OmniClientError(f"media has no video stream: {path}")
+        stream = video_streams[0]
+        raw_count = stream.get("nb_read_frames") or stream.get("nb_frames")
    try:
        numerator, denominator = str(stream["r_frame_rate"]).split("/", 1)
        fps = float(numerator) / float(denominator)
@@ -76,7 +78,6 @@ def _probe_video(path: str) -> dict[str, Any]:
        raise OmniClientError(f"cannot determine video FPS: {path}") from exc
    if not math.isfinite(fps) or fps <= 0:
        raise OmniClientError(f"cannot determine video FPS: {path}")
-    raw_count = stream.get("nb_read_frames") or stream.get("nb_frames")
    if raw_count in (None, "", "N/A"):
        raise OmniClientError(f"cannot determine video frame count: {path}")
    raw_duration = stream.get("duration")
@@ -404,13 +405,13 @@ def load_video_frames(path: str) -> np.ndarray:
    try:
        import decord
    except ImportError:
-        import av
-
-        with av.open(path) as container:
-            frames = [frame.to_ndarray(format="rgb24") for frame in container.decode(video=0)]
-        if not frames:
-            raise OmniClientError(f"video has no frames: {path}")
-        return np.stack(frames)
+        meta = _probe_video(path)
+        return _decode_video_frames_ffmpeg(
+            path,
+            frame_count=int(meta["frame_count"]),
+            width=int(meta["width"]),
+            height=int(meta["height"]),
+        )

    reader = decord.VideoReader(path)
    if len(reader) <= 0:
@@ -419,6 +420,70 @@ def load_video_frames(path: str) -> np.ndarray:
    return frames.asnumpy() if hasattr(frames, "asnumpy") else np.asarray(frames)


+def _decode_video_frames_ffmpeg(
+    path: str,
+    *,
+    frame_count: int,
+    width: int,
+    height: int,
+    indices: list[int] | None = None,
+) -> np.ndarray:
+    frame_count = int(frame_count)
+    width = int(width)
+    height = int(height)
+    if frame_count <= 0:
+        raise OmniClientError(f"video has no frames: {path}")
+    if width <= 0 or height <= 0:
+        raise OmniClientError(f"video has invalid dimensions {width}x{height}: {path}")
+    if indices is not None and (not indices or any(index < 0 or index >= frame_count for index in indices)):
+        raise OmniClientError(f"invalid frame indices for {path}: {indices}")
+
+    output_frame_count = frame_count if indices is None else len(indices)
+    command = [
+        "ffmpeg",
+        "-loglevel",
+        "error",
+        "-threads",
+        "0",
+        "-i",
+        path,
+        "-map",
+        "0:v:0",
+        "-an",
+    ]
+    if indices is not None:
+        select = "+".join(f"eq(n\\,{index})" for index in indices)
+        command.extend(["-vf", f"select={select}"])
+    command.extend(
+        [
+            "-vsync",
+            "0",
+            "-frames:v",
+            str(output_frame_count),
+            "-f",
+            "rawvideo",
+            "-pix_fmt",
+            "rgb24",
+            "pipe:1",
+        ]
+    )
+    result = subprocess.run(
+        command,
+        check=True,
+        capture_output=True,
+    )
+    frame_size = width * height * 3
+    expected_size = output_frame_count * frame_size
+    if len(result.stdout) != expected_size:
+        raise OmniClientError(f"decoded {len(result.stdout)} bytes from {path}, expected {expected_size}")
+    return np.frombuffer(result.stdout, dtype=np.uint8).reshape(
+        output_frame_count,
+        height,
+        width,
+        3,
+    )
+
+
 def sample_reference_video_frames(prepared_path: str) -> dict[str, Any]:
    meta = _probe_video(prepared_path)
    ratio = MINIMAX_H3_FPS / MINIMAX_H3_QWEN_VIDEO_SAMPLE_FPS
@@ -434,11 +499,17 @@ def sample_reference_video_frames(prepared_path: str) -> dict[str, Any]:
    if not indices:
        raise OmniClientError(f"no frames sampled from {prepared_path}")

-    # The preparation step emits a lossless RGB stream. Decode it once and
-    # sample from that array so Qwen3VL sees exactly the same pixels as the
-    # video VAE, without starting one full-video ffmpeg decode per sample.
-    decoded_frames = load_video_frames(prepared_path)
-    frames = [np.asarray(decoded_frames[source_index]) for source_index in indices]
+    # The preparation step emits a lossless RGB stream. Decode only the sampled
+    # frames in one ffmpeg process so Qwen3VL sees the exact prepared pixels
+    # without decoding the entire high-bitrate stream into host memory.
+    decoded_frames = _decode_video_frames_ffmpeg(
+        prepared_path,
+        frame_count=int(meta["frame_count"]),
+        indices=indices,
+        width=meta["width"],
+        height=meta["height"],
+    )
+    frames = [np.asarray(frame) for frame in decoded_frames]

    timestamps = [index / MINIMAX_H3_QWEN_VIDEO_SAMPLE_FPS for index in range(len(indices))]
    timestamps += [timestamps[-1]] * ((-len(timestamps)) % MINIMAX_H3_QWEN_TEMPORAL_PATCH)
@@ -642,10 +642,10 @@ class QwenImageCrossAttention(nn.Module):
        txt_query = self.norm_added_q(txt_query)
        txt_key = self.norm_added_k(txt_key)

-        img_cos = vid_freqs.real.to(img_query.dtype)
-        img_sin = vid_freqs.imag.to(img_query.dtype)
-        txt_cos = txt_freqs.real.to(txt_query.dtype)
-        txt_sin = txt_freqs.imag.to(txt_query.dtype)
+        img_cos = torch.real(vid_freqs).to(img_query.dtype)
+        img_sin = torch.imag(vid_freqs).to(img_query.dtype)
+        txt_cos = torch.real(txt_freqs).to(txt_query.dtype)
+        txt_sin = torch.imag(txt_freqs).to(txt_query.dtype)

        img_query = self.rope(img_query, img_cos, img_sin)
        img_key = self.rope(img_key, img_cos, img_sin)
@@ -33,9 +33,17 @@ import types
 from torch import nn
 from vllm.logger import init_logger
 from vllm.model_executor.layers.quantization.fp8 import Fp8LinearMethod
+from vllm.model_executor.layers.quantization.online.fp8 import (
+    Fp8PerTensorOnlineLinearMethod,
+)

 logger = init_logger(__name__)

+_FP8_TRANSPOSED_WEIGHT_METHODS = (
+    Fp8LinearMethod,
+    Fp8PerTensorOnlineLinearMethod,
+)
+

 def _build_transposed_get_layer_params(original_bound_method):
    """Return a ``_get_layer_params`` replacement that transposes ``w``.
@@ -55,9 +63,8 @@ def _build_transposed_get_layer_params(original_bound_method):
 def prepare_fp8_layers_for_fsdp(model: nn.Module) -> int:
    """Make online-FP8 linear layers in ``model`` FSDP2-compatible.

-    For every layer whose ``quant_method`` is an :class:`Fp8LinearMethod`
-    and whose weight is currently a non-contiguous transpose view, this
-    function:
+    For every layer whose quantization method stores FP8 weights as a
+    non-contiguous transpose view, this function:

    1. Replaces ``layer.weight`` with the underlying ``(out, in)`` row-major
       contiguous storage so FSDP2 ``fully_shard`` accepts it.
@@ -76,7 +83,7 @@ def prepare_fp8_layers_for_fsdp(model: nn.Module) -> int:
    patched_kernel_ids: set[int] = set()
    for module in model.modules():
        qm = getattr(module, "quant_method", None)
-        if not isinstance(qm, Fp8LinearMethod):
+        if not isinstance(qm, _FP8_TRANSPOSED_WEIGHT_METHODS):
            continue

        weight = getattr(module, "weight", None)
@@ -43,6 +43,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
    ChatCompletionRequest,
    ChatCompletionResponse,
 )
+from vllm.entrypoints.openai.cli_args import make_arg_parser

 # yapf conflicts with isort for this block
 # yapf: disable
@@ -3826,26 +3827,16 @@ async def omni_wakeup(request: OmniWakeupRequest, raw_request: Request):


 if __name__ == "__main__":
-    import asyncio
-
-    from vllm.entrypoints.openai.cli_args import make_arg_parser
-
    parser = TrackingArgumentParser(description="vLLM-Omni OpenAI-Compatible REST API server")
    parser = make_arg_parser(parser)
-    registered_flags = set()
-    for action in parser._actions:
-        registered_flags.update(action.option_strings)
-
-    # FIXME - this is broken on `main`. `make_arg_parser` does not handle omni engine args properly.
-    if "--omni" not in registered_flags:
-        parser.add_argument("--omni", action="store_true", default=False, help="Enable vLLM-Omni mode.")
-    if "--enable-sleep-mode" not in registered_flags:
-        parser.add_argument(
-            "--enable-sleep-mode", action="store_true", default=False, help="Enable GPU memory pool for sleep mode."
-        )
+    # Ensure that passing --omni won't crash the server.
+    # NOTE: the value here does not matter since we are always running the Omni server
+    # when __main__ is called, i.e., --omni is only used when called through the entrypoints.
+    parser.add_argument("--omni", action="store_true", default=False)
    args = parser.parse_args()
-    if not hasattr(args, "model_tag"):
-        setattr(args, "model_tag", args.model)
-    if hasattr(args, "model_tag") and args.model_tag is None:
-        args.model_tag = args.model
+    # sync args.model to model_tag, because if we pass the model positionally,
+    # args.model will be the default from vLLM's ModelConfig (currently
+    # Qwen/Qwen3-0.6B) and crash cryptically.
+    if args.model_tag is not None:
+        args.model = args.model_tag
    asyncio.run(omni_run_server(args))
@@ -365,6 +365,22 @@ def _resolve_fa_version(head_size: int) -> int:
    return version


+def _rocm_flash_attn_varlen_func():
+    """Resolve a ROCm varlen kernel, preferring AITER when available.
+
+    The caller gathers paged KV into packed tensors before invoking this
+    function, avoiding AITER releases whose ``block_table`` kernel is broken.
+    """
+    try:
+        from aiter import flash_attn_varlen_func
+
+        return flash_attn_varlen_func
+    except ImportError:
+        from flash_attn import flash_attn_varlen_func
+
+        return flash_attn_varlen_func
+
+
 def ar_diffusion_paged_attention(
    query: torch.Tensor,
    key_cache: torch.Tensor,
@@ -401,6 +417,34 @@ def ar_diffusion_paged_attention(
            softmax_scale,
            causal=causal,
        )
+    elif torch.version.hip is not None:
+        # vllm.vllm_flash_attn contains CUDA-only extensions. ROCm's AITER and
+        # upstream flash-attn expose the standard cu_seqlens_k API instead of
+        # vLLM's seqused_k/fa_version API. The ROCm flash-attn paged kernel also
+        # requires 128-token blocks, while AR-Diffusion uses frame-aligned
+        # 16-token blocks, so gather the visible blocks on-device first.
+        flash_attn_varlen_func = _rocm_flash_attn_varlen_func()
+        cu_seqlens_k = torch.cat([seq_lens.new_zeros(1), torch.cumsum(seq_lens, dim=0, dtype=torch.int32)])
+        positions = torch.arange(int(max_seq_len), device=query_flat.device)
+        logical_blocks = torch.div(positions, key_cache.shape[1], rounding_mode="floor")
+        offsets = positions % key_cache.shape[1]
+        physical_blocks = block_table[:, logical_blocks].long()
+        gathered_k = key_cache[physical_blocks, offsets]
+        gathered_v = value_cache[physical_blocks, offsets]
+        valid = positions.unsqueeze(0) < seq_lens.unsqueeze(1)
+        packed_k = gathered_k[valid]
+        packed_v = gathered_v[valid]
+        out = flash_attn_varlen_func(
+            q=query_flat,
+            k=packed_k,
+            v=packed_v,
+            cu_seqlens_q=query_start_loc,
+            cu_seqlens_k=cu_seqlens_k,
+            max_seqlen_q=int(max_query_len),
+            max_seqlen_k=int(max_seq_len),
+            softmax_scale=float(softmax_scale),
+            causal=causal,
+        )
    else:
        from vllm.vllm_flash_attn import flash_attn_varlen_func

@@ -0,0 +1,71 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+"""NPU patches for the MiniMax H3 Qwen3-VL text encoder."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import torch
+import torch.nn.functional as F
+import torch_npu
+from vllm.logger import init_logger
+
+from vllm_omni.platforms.npu.layers.rotary_embedding import (
+    npu_rotary_mul_with_bsnd_fallback,
+)
+
+logger = init_logger(__name__)
+
+_ROPE_PATCHED = False
+_SWIGLU_PATCHED = False
+
+
+def _apply_rotary_pos_emb_npu(
+    q: torch.Tensor,
+    k: torch.Tensor,
+    cos: torch.Tensor,
+    sin: torch.Tensor,
+) -> tuple[torch.Tensor, torch.Tensor]:
+    """Apply Qwen3-VL text RoPE with fused BNSD rotary multiplication."""
+    return (
+        npu_rotary_mul_with_bsnd_fallback(q, cos, sin, unsqueeze_dim=1),
+        npu_rotary_mul_with_bsnd_fallback(k, cos, sin, unsqueeze_dim=1),
+    )
+
+
+def apply_minimax_h3_qwen3vl_patch() -> None:
+    """Route MiniMax H3 Qwen3-VL text RoPE to the Ascend fused operator."""
+    global _ROPE_PATCHED
+    if _ROPE_PATCHED:
+        return
+
+    from vllm_omni.diffusion.models.minimax_h3 import encoder
+
+    encoder._apply_rotary_pos_emb = _apply_rotary_pos_emb_npu
+    _ROPE_PATCHED = True
+    logger.debug("Applied NPU fused RoPE patch for MiniMax H3 Qwen3-VL text encoder")
+
+
+def npu_swiglu_from_packed(gate_up: torch.Tensor) -> torch.Tensor:
+    """Apply fused SwiGLU to a packed gate/up projection tensor."""
+    return torch_npu.npu_swiglu(gate_up, dim=-1)
+
+
+def _forward_minimax_h3_qwen3vl_text_mlp_npu(self: Any, x: torch.Tensor) -> torch.Tensor:
+    """Run the Qwen3-VL MLP with one packed GEMM and fused SwiGLU."""
+    gate_up = F.linear(x, self.gate_up_proj.weight)
+    return self.down_proj(npu_swiglu_from_packed(gate_up))
+
+
+def apply_minimax_h3_qwen3vl_swiglu_patch() -> None:
+    """Route MiniMax H3 Qwen3-VL text MLP to the Ascend fused SwiGLU path."""
+    global _SWIGLU_PATCHED
+    if _SWIGLU_PATCHED:
+        return
+
+    from vllm_omni.diffusion.models.minimax_h3 import encoder
+
+    encoder.MiniMaxH3Qwen3VLTextMLP.forward = _forward_minimax_h3_qwen3vl_text_mlp_npu
+    _SWIGLU_PATCHED = True
@@ -36,6 +36,7 @@ class NPUOmniPlatform(OmniPlatform, NPUPlatform):
        from vllm_ascend.utils import adapt_patch

        from vllm_omni.platforms.npu._310p import apply_patches as apply_310p_patches
+        from vllm_omni.platforms.npu.models.minimax_h3 import apply_minimax_h3_qwen3vl_patch
        from vllm_omni.platforms.npu.models.qwen3_tts_code2wav import (
            apply_qwen3_tts_code2wav_patch,
        )
@@ -44,6 +45,7 @@ class NPUOmniPlatform(OmniPlatform, NPUPlatform):
        )

        adapt_patch(is_global_patch=True)
+        apply_minimax_h3_qwen3vl_patch()
        apply_qwen3_tts_code2wav_patch()
        apply_qwen3_tts_tokenizer_v2_patch()
        apply_310p_patches()
@@ -80,6 +82,14 @@ class NPUOmniPlatform(OmniPlatform, NPUPlatform):
    def init_diffusion_model_runner_runtime(cls, vllm_config: Any, od_config: Any, device: torch.device) -> None:
        from vllm_ascend.ascend_forward_context import set_mc2_mask, set_mc2_tokens_capacity

+        from vllm_omni.platforms.npu.models.minimax_h3 import (
+            apply_minimax_h3_qwen3vl_swiglu_patch,
+        )
+
+        # The patch imports the MiniMax encoder, which depends on
+        # current_omni_platform. Run it only after platform construction has
+        # completed, but before the diffusion pipeline is loaded.
+        apply_minimax_h3_qwen3vl_swiglu_patch()
        set_mc2_tokens_capacity(vllm_config, od_config.max_num_seqs, 1)
        set_mc2_mask(vllm_config, device)
Author	SHA1	Message	Date
Hongsheng Liu	7ec78360bb	Merge branch 'main' into codex-split-attention-offload-guides	2026-08-15 07:00:22 +08:00
Joshna-Medisetty Cursor Chendi.Xue	4b6d6d5cf7	[Bugfix][XPU][Tests]Scope XPU pytest to explicit paths to fix collection crash (#6175 ) Signed-off-by: Joshna Medisetty <joshna.medisetty@intel.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Chendi.Xue <chendi.xue@intel.com>	2026-08-14 15:23:41 -05:00
Alex Brooks	f7a5146093	[Feat/Bugfix] Fix API Server model tag <-> model sync & flag normalization (#3805 ) Signed-off-by: Alex Brooks <albrooks@redhat.com>	2026-08-14 19:13:00 +00:00
Wenjia Li	8ecd1f6d5c	[Perf][NPU] Fuse MiniMax H3 Qwen3-VL SwiGLU (#6167 ) Signed-off-by: Wenjia Li <wjialish@gmail.com>	2026-08-14 11:47:17 +00:00
Wenjia Li	ffd11c181e	[Perf][NPU] Fuse MiniMax H3 Qwen3-VL RoPE (#6061 ) Signed-off-by: Wenjia Li <wjialish@gmail.com>	2026-08-14 09:49:26 +00:00
R0CKSTAR	728640f4d9	[Bugfix][MUSA][Qwen Image] Avoid complex RoPE alias guards (#6110 ) Signed-off-by: Xiaodong Ye <xiaodong.ye@mthreads.com>	2026-08-14 15:51:23 +08:00
R0CKSTAR	022d7a7695	[Perf][Diffusion] Avoid redundant MiniMax-H3 reference video scans (#6064 ) Signed-off-by: Xiaodong Ye <xiaodong.ye@mthreads.com>	2026-08-14 15:49:26 +08:00
baonudesifeizhai	b300f66656	[Bugfix] Fix HSDP compatibility with the new online FP8 linear method (#5677 ) Signed-off-by: roG0d <baonudesifeizhai@gmail.com>	2026-08-14 15:45:55 +08:00
TJianandHongsheng Liu	bc73546c06	[ROCm] [CI] Migrate CI to mi300x for v0.27.x (#5886 ) Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com> Signed-off-by: TJian <tunjian.tan@embeddedllm.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>	2026-08-14 15:38:47 +08:00
Shenglei Fu Claude Sonnet 5 Alicia wangyu Hongsheng Liu	2b08419d8b	[CI] Per-model, per-entry-mode code coverage collection (#5593 ) Signed-off-by: Shenglei Fu <sfu@confluent.io> Signed-off-by: Shenglei Fu <117230642+ShengleiFu@users.noreply.github.com> Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com> Co-authored-by: Alicia <115451386+congw729@users.noreply.github.com> Co-authored-by: wangyu <53896905+yenuo26@users.noreply.github.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>	2026-08-14 15:36:56 +08:00
Hongsheng Liu	5569abf0a2	Update docs/design/feature/offloader/README.md Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com>	2026-08-13 23:37:08 +08:00
Hongsheng Liu	10853fb4f3	Update docs/design/feature/offloader/README.md Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com>	2026-08-13 23:37:01 +08:00
Hongsheng Liu	a52c19ea8b	docs: organize offloader feature designs Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com>	2026-08-12 11:35:07 +08:00
Hongsheng Liu	8b61499662	docs: rename offloader guides Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com>	2026-08-12 09:30:34 +08:00
Hongsheng Liu	7872663bf0	docs: split diffusion feature guides Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com>	2026-08-12 08:55:41 +08:00