Merge branch 'main' into codex-split-attention-offload-guides

Signed-off-by: Joshna Medisetty <joshna.medisetty@intel.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Chendi.Xue <chendi.xue@intel.com>

.buildkite/amd/scripts/run-amd-test.sh

@@ -1,153 +1,103 @@
 #!/bin/bash
-# vllm-omni customized version
-# Based on: vllm/.buildkite/scripts/hardware_ci/run-amd-test.sh
-# Last synced: 2025-12-15
-# Modifications: docker image name for vllm-omni
 
-# This script runs test inside the corresponding ROCm docker container.
+# Run vLLM-Omni ROCm tests directly in the MI300 Kubernetes pod. The pod's
-set -o pipefail
+# container image is selected by test-template-amd-omni.j2; MI300 has no DinD.
+set -euo pipefail
 
-# Export Python path
+: "${PYTHONFAULTHANDLER:=1}"
-export PYTHONPATH=".."
+: "${HF_HOME:=/home/buildkite-agent/huggingface}"
+: "${HF_HUB_DOWNLOAD_TIMEOUT:=300}"
+: "${HF_HUB_ETAG_TIMEOUT:=60}"
+: "${MIOPEN_DEBUG_CONV_DIRECT:=0}"
+: "${MIOPEN_DEBUG_CONV_GEMM:=0}"
+: "${VLLM_ROCM_USE_AITER:=0}"
+export PYTHONFAULTHANDLER HF_HOME HF_HUB_DOWNLOAD_TIMEOUT HF_HUB_ETAG_TIMEOUT
+export MIOPEN_DEBUG_CONV_DIRECT MIOPEN_DEBUG_CONV_GEMM VLLM_ROCM_USE_AITER
+export PYTORCH_ROCM_ARCH=""
+export PYTHONPATH="${PYTHONPATH:-..}"
+
+if [[ "${VLLM_CI_DOCKER_DISABLED:-0}" != "1" ]]; then
+    echo "Error: MI300 CI must run natively with Docker disabled." >&2
+    exit 1
+fi
+
+if [[ -n "${TEST_COMMANDS:-}" ]]; then
+    commands="${TEST_COMMANDS}"
+else
+    commands="$*"
+fi
+if [[ -z "${commands}" ]]; then
+    echo "Error: No test commands provided." >&2
+    exit 1
+fi
+
+echo "--- Native in-pod ROCm CI"
+
+job_id="${BUILDKITE_JOB_ID:-${BUILDKITE_PARALLEL_JOB:-local}}"
+job_id="${job_id//[^A-Za-z0-9_.-]/_}"
+native_cache_root="/tmp/vllm-omni-native-${job_id}"
+TMPDIR="${native_cache_root}/tmp"
+TORCHINDUCTOR_CACHE_DIR="${native_cache_root}/torchinductor"
+TRITON_CACHE_DIR="${native_cache_root}/triton"
+VLLM_CACHE_ROOT="${native_cache_root}/vllm"
+XDG_CACHE_HOME="${native_cache_root}/xdg"
+HF_DATASETS_CACHE="${native_cache_root}/huggingface/datasets"
+export TMPDIR TORCHINDUCTOR_CACHE_DIR TRITON_CACHE_DIR VLLM_CACHE_ROOT
+export XDG_CACHE_HOME HF_DATASETS_CACHE
+mkdir -p "${TMPDIR}" "${TORCHINDUCTOR_CACHE_DIR}" "${TRITON_CACHE_DIR}" \
+    "${VLLM_CACHE_ROOT}" "${XDG_CACHE_HOME}" "${HF_HOME}" \
+    "${HF_DATASETS_CACHE}"
+
+if [[ "${VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE:-0}" == "1" ]]; then
+    if ! command -v findmnt >/dev/null 2>&1; then
+        echo "Error: findmnt is required to verify the Hugging Face cache mount." >&2
+        exit 1
+    fi
+    hf_mount=$(findmnt -n -T "${HF_HOME}" -o TARGET 2>/dev/null || true)
+    if [[ -z "${hf_mount}" || "${hf_mount}" == "/" ]]; then
+        echo "Error: MI300 CI requires a persistent volume at or above ${HF_HOME}." >&2
+        exit 1
+    fi
+fi
 
-# Print ROCm version
-echo "--- ROCm info"
 rocminfo
 
-# cleanup older docker images
+expected_gpus="${VLLM_CI_EXPECTED_GPU_COUNT:-1}"
-cleanup_docker() {
+python3 - <<'PY'
-  # Get Docker's root directory
+import os
-  docker_root=$(docker info -f '{{.DockerRootDir}}')
-  if [ -z "$docker_root" ]; then
-    echo "Failed to determine Docker root directory."
-    exit 1
-  fi
-  echo "Docker root directory: $docker_root"
-  # Check disk usage of the filesystem where Docker's root directory is located
-  disk_usage=$(df "$docker_root" | tail -1 | awk '{print $5}' | sed 's/%//')
-  # Define the threshold
-  threshold=70
-  if [ "$disk_usage" -gt "$threshold" ]; then
-    echo "Disk usage is above $threshold%. Cleaning up Docker images and volumes..."
-    # Remove dangling images (those that are not tagged and not used by any container)
-    docker image prune -f
-    # Remove unused volumes / force the system prune for old images as well.
-    docker volume prune -f && docker system prune --force --filter "until=72h" --all
-    echo "Docker images and volumes cleanup completed."
-  else
-    echo "Disk usage is below $threshold%. No cleanup needed."
-  fi
-}
 
-# Call the cleanup docker function
+import torch
-cleanup_docker
 
-echo "--- Pulling container"
+expected = int(os.environ.get("VLLM_CI_EXPECTED_GPU_COUNT", "1"))
-## Temporary change to use AMD Docker Hub to store the vllm-omni image
+assert torch.version.hip, "PyTorch is not a ROCm build"
-# to bypass the rate limit issue with ECR Public Gallery.
+assert torch.cuda.is_available(), "ROCm GPU is not available to PyTorch"
-# Images are now stored in a separate repository for vllm-omni, instead of vllm-ci.
+actual = torch.cuda.device_count()
-# TODO: @tjtanaa point back to ECR Public Gallery
+assert actual == expected, f"Expected {expected} ROCm GPU(s), found {actual}"
-# once the amd agents are configured to use ECR Public Gallery.
+PY
-# image_name="public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:${BUILDKITE_COMMIT}-rocm-omni"
-image_name="rocm/vllm-omni:${BUILDKITE_COMMIT}"
-container_name="rocm_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)"
 
-# TODO: @tjtanaa uncomment this once the amd agents are configured to use ECR Public Gallery.
+echo "Commands:${commands}"
-# # Install AWS CLI to authenticate to ECR Public Gallery to get higher rate limit for pulling images
+if /bin/bash -o pipefail -c '
-# sudo apt-get update && sudo apt-get install -y awscli
+set -E
-##  Use safe docker login helper to prevent race conditions
+test_status=0
-# source "$(dirname "${BASH_SOURCE[0]}")/../../common/scripts/docker_login_ecr_public.sh"
+trap '\''
-# safe_docker_login_ecr_public
+    command_status=$?
-
+    # Keep running subsequent commands, but retain a failure for the final exit.
-## Pull the container from AMD Docker Hub
+    # Prefer a test failure over pytest "no tests collected" when both occur.
-
+    if (( test_status == 0 || (test_status == 5 && command_status != 5) )); then
-docker pull "${image_name}"
+        test_status=${command_status}
-
-remove_docker_container() {
-   docker ps -aq --filter "name=${container_name}" | xargs -r docker rm -f || true
-   docker image rm -f "${image_name}" || true
-}
-trap remove_docker_container EXIT
-
-echo "--- Running container"
-
-HF_CACHE="$(realpath ~)/huggingface"
-mkdir -p "${HF_CACHE}"
-HF_MOUNT="/root/.cache/huggingface"
-
-if [[ -n "${TEST_COMMAND:-}" ]]; then
-    commands="$TEST_COMMAND"
-else
-    commands="$@"
-fi
-echo "Commands:$commands"
-
-PARALLEL_JOB_COUNT=8
-MYPYTHONPATH=".."
-
-# Test that we're launching on the machine that has
-# proper access to GPUs
-render_gid=$(getent group render | cut -d: -f3)
-if [[ -z "$render_gid" ]]; then
-  echo "Error: 'render' group not found. This is required for GPU access." >&2
-  exit 1
-fi
-
-# check if the command contains shard flag, we will run all shards in parallel because the host have 8 GPUs.
-# TODO: @tjtanaa reenable to run VLLM_ROCM_USE_AITER=1 when AITER is shipped with prebuilt kernels.
-if [[ $commands == *"--shard-id="* ]]; then
-  # assign job count as the number of shards used
-  commands=$(echo "$commands" | sed -E "s/--num-shards[[:blank:]]*=[[:blank:]]*[0-9]*/--num-shards=${PARALLEL_JOB_COUNT} /g" | sed 's/ \\ / /g')
-  for GPU in $(seq 0 $(($PARALLEL_JOB_COUNT-1))); do
-    # assign shard-id for each shard
-    commands_gpu=$(echo "$commands" | sed -E "s/--shard-id[[:blank:]]*=[[:blank:]]*[0-9]*/--shard-id=${GPU} /g" | sed 's/ \\ / /g')
-    echo "Shard ${GPU} commands:$commands_gpu"
-    echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES"
-    docker run \
-        --device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \
-        --network=host \
-        --shm-size=16gb \
-        --group-add "$render_gid" \
-        -e MIOPEN_DEBUG_CONV_DIRECT=0 \
-        -e MIOPEN_DEBUG_CONV_GEMM=0 \
-        -e VLLM_ROCM_USE_AITER=0 \
-        -e HIP_VISIBLE_DEVICES="${GPU}" \
-        -e HF_TOKEN \
-        -v "${HF_CACHE}:${HF_MOUNT}" \
-        -e "HF_HOME=${HF_MOUNT}" \
-        -e "PYTHONPATH=${MYPYTHONPATH}" \
-        --name "${container_name}_${GPU}" \
-        "${image_name}" \
-        /bin/bash -c "${commands_gpu}" \
-        |& while read -r line; do echo ">>Shard $GPU: $line"; done &
-    PIDS+=($!)
-  done
-  #wait for all processes to finish and collect exit codes
-  for pid in "${PIDS[@]}"; do
-    wait "${pid}"
-    STATUS+=($?)
-  done
-  for st in "${STATUS[@]}"; do
-    if [[ ${st} -ne 0 ]]; then
-      echo "One of the processes failed with $st"
-      exit "${st}"
     fi
-  done
+'\'' ERR
+eval "$1"
+exit "${test_status}"
+' _ "${commands}"; then
+    exit 0
 else
-  echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES"
+    exit_code=$?
-  docker run \
-          --device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \
-          --network=host \
-          --shm-size=16gb \
-          --group-add "$render_gid" \
-          -e MIOPEN_DEBUG_CONV_DIRECT=0 \
-          -e MIOPEN_DEBUG_CONV_GEMM=0 \
-          -e VLLM_ROCM_USE_AITER=0 \
-          -e HF_TOKEN \
-          -v "${HF_CACHE}:${HF_MOUNT}" \
-          -e "HF_HOME=${HF_MOUNT}" \
-          -e "PYTHONPATH=${MYPYTHONPATH}" \
-          --name "${container_name}" \
-          "${image_name}" \
-          /bin/bash -c "${commands}"
 fi
+
+if [[ ${exit_code} -eq 5 && "${VLLM_CI_ALLOW_NO_TESTS:-0}" == "1" ]]; then
+    echo "Pytest collected no tests; VLLM_CI_ALLOW_NO_TESTS=1, treating exit code 5 as success."
+    exit 0
+fi
+
+exit "${exit_code}"

.buildkite/amd/test-amd-merge.yml

@@ -7,54 +7,64 @@ steps:
 
 - group: ":card_index_dividers: Simple Test"
   steps:
-    - label: "Simple · Diffusion & Model Executor Test (1/2)"
+    - label: "Simple · Model Executor Test · Shard %N/%t"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
+      parallelism: 2
       depends_on: amd-build
-      timeout_in_minutes: 70
       mirror_hardwares: [amdproduction]
       grade: Blocking
       commands:
         - export VLLM_ROCM_USE_AITER=0
         # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
         # that is not supported on CPU
-        # CPU tests on ROCm gpu machines are very slow, so we give them more time.
+        - "pytest -sv tests/model_executor -m 'core_model and cpu' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml:coverage-model-executor-$$BUILDKITE_PARALLEL_JOB.xml"
-        - "timeout 60m pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
 
-    - label: "Simple · Diffusion & Model Executor Test (2/2)"
+    - label: "Simple · Diffusion Test · Shard %N/%t"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
+      parallelism: 2
       depends_on: amd-build
-      timeout_in_minutes: 70
       mirror_hardwares: [amdproduction]
       grade: Blocking
       commands:
         - export VLLM_ROCM_USE_AITER=0
-        # CPU tests on ROCm gpu machines are very slow, so we give them more time.
+        # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
-        - "timeout 60m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+        # that is not supported on CPU
+        - "pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml:coverage-diffusion-$$BUILDKITE_PARALLEL_JOB.xml"
+
+    - label: "Simple · Engine&Entrypoints Test"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        - "pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
 
     - label: "Simple · Other Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
       depends_on: amd-build
-      timeout_in_minutes: 50
       mirror_hardwares: [amdproduction]
       grade: Blocking
       commands:
         - export VLLM_ROCM_USE_AITER=0
         # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
         # that is not supported on CPU
-        - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+        - "pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine  --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
 
 - group: ":card_index_dividers: Diffusion Test"
   steps:
     - label: "Diffusion · LoRA Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
+      artifact_paths:
+        - lora-server.log
       commands:
-        - timeout 20m pytest -s -v tests/diffusion/lora/test_images_generations_lora.py
+        - pytest -sv tests/diffusion/lora/ -m 'advanced_model and cuda' --run-level "advanced_model" 2>&1 | tee lora-server.log
 
     - label: "Diffusion Model CPU offloading Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
@@ -62,7 +72,7 @@ steps:
         - export VLLM_LOGGING_LEVEL=DEBUG
         - export VLLM_WORKER_MULTIPROC_METHOD=spawn
         - |
-          timeout 20m bash -c '
+          bash -c '
             set +e
             pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
             EXIT1=\$?
@@ -72,7 +82,7 @@ steps:
           '
 
     - label: "Diffusion Sequence Parallelism Test (Need 4 GPUs)"
-      agent_pool: mi325_4
+      agent_pool: mi300_4
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
@@ -84,7 +94,7 @@ steps:
 
     # merge-only tests
     - label: "Diffusion Tensor Parallelism Test"
-      agent_pool: mi325_2
+      agent_pool: mi300_2
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
@@ -92,7 +102,7 @@ steps:
         - pytest -s -v tests/diffusion/distributed/test_tensor_parallel.py
 
     - label: "Diffusion Model Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
@@ -100,7 +110,7 @@ steps:
         - timeout 30m pytest -s -v tests/e2e/offline_inference/test_z_image.py
 
     - label: "Diffusion GPU Worker Test"
-      agent_pool: mi325_2
+      agent_pool: mi300_2
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
@@ -108,7 +118,7 @@ steps:
         - timeout 20m pytest -s -v tests/diffusion/test_diffusion_worker.py
 
     - label: "Diffusion · Wan22 Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
@@ -121,7 +131,7 @@ steps:
 
 ## ISSUE depends on `diffusers` package: https://github.com/huggingface/diffusers/issues/13274
 # - label: "Audio Generation Model Test"
-#   agent_pool: mi325_1
+#   agent_pool: mi300_1
 #   depends_on: amd-build
 #   mirror_hardwares: [amdproduction]
 #   grade: Blocking
@@ -132,7 +142,7 @@ steps:
 #     - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_expansion.py -m "advanced_model and diffusion and L4" --run-level advanced_model
 
 - label: "Engine Test"
-  agent_pool: mi325_1
+  agent_pool: mi300_1
   depends_on: amd-build
   mirror_hardwares: [amdproduction]
   grade: Blocking
@@ -147,7 +157,7 @@ steps:
 - group: ":card_index_dividers: E2E Test"
   steps:
     - label: "Omni Model Test Qwen2-5-Omni"
-      agent_pool: mi325_4
+      agent_pool: mi300_4
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       commands:
@@ -155,7 +165,7 @@ steps:
         - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model"
 
     - label: "Omni Model Test Qwen3-Omni"
-      agent_pool: mi325_2
+      agent_pool: mi300_2
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
@@ -163,22 +173,23 @@ steps:
         - export VLLM_WORKER_MULTIPROC_METHOD=spawn
         - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model'
 
-    - label: "Qwen3-TTS CustomVoice E2E Test"
+    - label: "Qwen3-TTS CustomVoice E2E Test · Shard %N/%t"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
+      parallelism: 2
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
       commands:
         - |
-          timeout 50m bash -c '
+          bash -c '
             export VLLM_LOGGING_LEVEL=DEBUG
             export VLLM_WORKER_MULTIPROC_METHOD=spawn
             export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-            pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m "advanced_model" --run-level "advanced_model"
+            pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m "advanced_model" --run-level "advanced_model" --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
           '
 
     - label: "Qwen3-TTS Base E2E Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
@@ -196,7 +207,7 @@ steps:
 # Need to debug before reneable numerical changes across large PRs
 # # split Bagel Model Test with H100 (Real Weights) into three tests
 # - label: "Bagel Text2Img Model Test (1/3)"
-#   agent_pool: mi325_1
+#   agent_pool: mi300_1
 #   depends_on: amd-build
 #   mirror_hardwares: [amdproduction]
 #   grade: Blocking
@@ -208,7 +219,7 @@ steps:
 #     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "advanced_model" --run-level "advanced_model" -k "shared_memory" -k "rocm"
 
 # - label: "Bagel Img2Img Model Test (2/3)"
-#   agent_pool: mi325_1
+#   agent_pool: mi300_1
 #   depends_on: amd-build
 #   mirror_hardwares: [amdproduction]
 #   grade: Blocking
@@ -220,7 +231,7 @@ steps:
 #     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "advanced_model" --run-level "advanced_model" -k "rocm"
 
 # - label: "Bagel Online Serving Test (3/3)"
-#   agent_pool: mi325_1
+#   agent_pool: mi300_1
 #   depends_on: amd-build
 #   mirror_hardwares: [amdproduction]
 #   grade: Blocking

.buildkite/amd/test-amd-ready.yml

@@ -7,8 +7,9 @@ steps:
 
 - group: ":card_index_dividers: Simple Test"
   steps:
-    - label: "Simple · Diffusion & Model Executor Test"
+    - label: "Simple · Model Executor Test · Shard %N/%t"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
+      parallelism: 2
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
@@ -16,10 +17,30 @@ steps:
         - export VLLM_ROCM_USE_AITER=0
         # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
         # that is not supported on CPU
-        - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+        - "pytest -sv tests/model_executor -m 'core_model and cpu' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml:coverage-model-executor-$$BUILDKITE_PARALLEL_JOB.xml"
+
+    - label: "Simple · Diffusion Test"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
+        # that is not supported on CPU
+        - "pytest -sv tests/diffusion -m 'core_model and cpu'  --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+
+    - label: "Simple · Engine&Entrypoints Test"
+      agent_pool: mi300_1
+      depends_on: amd-build
+      mirror_hardwares: [amdproduction]
+      grade: Blocking
+      commands:
+        - export VLLM_ROCM_USE_AITER=0
+        - "pytest -sv tests/entrypoints tests/engine -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
 
     - label: "Simple · Other Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
@@ -27,19 +48,19 @@ steps:
         - export VLLM_ROCM_USE_AITER=0
         # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
         # that is not supported on CPU
-        - "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+        - "pytest -sv tests/ -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --ignore=tests/entrypoints --ignore=tests/engine  --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
 
 - label: "Custom Pipeline Test"
-  agent_pool: mi325_1
+  agent_pool: mi300_1
   depends_on: amd-build
   mirror_hardwares: [amdproduction]
   grade: Blocking
   commands:
-    - timeout 20m pytest -s -v tests/e2e/features/custom_pipeline/ -m "core_model"
+    - pytest -s -v tests/e2e/features/custom_pipeline/ -m "core_model"
 
 
 - label: "Voxtral TTS CUDA Unit Test"
-  agent_pool: mi325_1
+  agent_pool: mi300_1
   depends_on: amd-build
   mirror_hardwares: [amdproduction]
   grade: Blocking
@@ -49,33 +70,25 @@ steps:
 - group: ":card_index_dividers: Diffusion Test"
   steps:
     - label: "Diffusion · Batch Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
       commands:
-        - timeout 20m pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model"
+        - pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model"
 
     - label: "Diffusion · Offloader Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
       commands:
         - export VLLM_LOGGING_LEVEL=DEBUG
         - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-        - |
+        - pytest -s -v tests/diffusion/offloader -m "core_model and cuda" --run-level "core_model"
-          timeout 20m bash -c '
-            set +e
-            pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
-            EXIT1=\$?
-            pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
-            EXIT2=\$?
-            exit \$((EXIT1 | EXIT2))
-          '
 
     - label: "Diffusion · Cache Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
@@ -83,10 +96,10 @@ steps:
         - export GPU_ARCHS=gfx942
         - export VLLM_LOGGING_LEVEL=DEBUG
         - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-        - timeout 15m pytest -s -v -m "core_model and cache and diffusion and not distributed_cuda and L4"
+        - pytest -sv tests/diffusion/cache -m 'core_model and cuda' --run-level "core_model"
 
     - label: "Diffusion Sequence Parallelism Test"
-      agent_pool: mi325_2
+      agent_pool: mi300_2
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
@@ -96,7 +109,7 @@ steps:
         - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py -m core_model
 
     - label: "Diffusion GPU Worker Test"
-      agent_pool: mi325_2
+      agent_pool: mi300_2
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
@@ -104,7 +117,7 @@ steps:
         - timeout 20m pytest -s -v tests/diffusion/test_diffusion_worker.py
 
     - label: "Diffusion · Model Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
@@ -112,7 +125,7 @@ steps:
         - timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model"
 
     - label: "Diffusion · Wan22 Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
@@ -126,7 +139,7 @@ steps:
 
 ## ISSUE depends on `diffusers` package: https://github.com/huggingface/diffusers/issues/13274
 # - label: "Audio Generation Model Test"
-#   agent_pool: mi325_1
+#   agent_pool: mi300_1
 #   depends_on: amd-build
 #   mirror_hardwares: [amdproduction]
 #   grade: Blocking
@@ -137,7 +150,7 @@ steps:
 #     - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_expansion.py -m "advanced_model and diffusion and L4" --run-level advanced_model
 
 - label: "Engine Test"
-  agent_pool: mi325_1
+  agent_pool: mi300_1
   depends_on: amd-build
   mirror_hardwares: [amdproduction]
   grade: Blocking
@@ -149,7 +162,7 @@ steps:
       '
 
 # - label: "Omni Model Test Qwen3-Omni"
-#   agent_pool: mi325_2
+#   agent_pool: mi300_2
 #   depends_on: amd-build
 #   mirror_hardwares: [amdproduction]
 #   grade: Blocking
@@ -162,7 +175,7 @@ steps:
 - group: ":card_index_dividers: E2E Test"
   steps:
     - label: "Qwen3-TTS E2E Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
@@ -173,7 +186,7 @@ steps:
         - timeout 30m pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m "core_model" --run-level "core_model"
 
     - label: "CosyVoice3-TTS E2E Test"
-      agent_pool: mi325_1
+      agent_pool: mi300_1
       depends_on: amd-build
       mirror_hardwares: [amdproduction]
       grade: Blocking
@@ -186,7 +199,7 @@ steps:
 # TODO: Bagel test on ROCm is very unstable. @tjtanaa
 # Need to debug before reneable numerical changes across large PRs
 # - label: "Bagel Text2Img Model Test"
-#   agent_pool: mi325_1
+#   agent_pool: mi300_1
 #   depends_on: amd-build
 #   mirror_hardwares: [amdproduction]
 #   grade: Blocking
@@ -198,7 +211,7 @@ steps:
 #     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"
 
 # - label: "Bagel Img2Img Model Test"
-#   agent_pool: mi325_1
+#   agent_pool: mi300_1
 #   depends_on: amd-build
 #   mirror_hardwares: [amdproduction]
 #   grade: Blocking
@@ -210,7 +223,7 @@ steps:
 #     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"
 
 # - label: "Bagel Online Serving Test"
-#   agent_pool: mi325_1
+#   agent_pool: mi300_1
 #   depends_on: amd-build
 #   mirror_hardwares: [amdproduction]
 #   grade: Blocking

.buildkite/amd/test-template-amd-omni.j2

@@ -1,11 +1,65 @@
 {# vllm-omni customized version
    Based on: https://github.com/vllm-project/ci-infra/blob/main/buildkite/test-template-amd.j2
-   Last synced: 2025-12-15
+   Last synced: 2026-08-04
-   Modifications: Removed unused CUDA/NVIDIA logic, keeping only AMD tests
+   Modifications: Use the shared MI300 queues and Omni command-passing contract
 #}
 {% set docker_image_amd = "rocm/vllm-omni:$BUILDKITE_COMMIT" %}
 {% set default_working_dir = "/app/vllm-omni" %}
 
+{# MI300 runs tests natively in Kubernetes; Docker-in-Docker is unavailable. #}
+{% macro amd_native_k8s_plugin(indent, step) -%}
+{{ indent }}plugins:
+{{ indent }}  - kubernetes:
+{{ indent }}      podSpecPatch:
+{{ indent }}        automountServiceAccountToken: false
+{{ indent }}        securityContext:
+{{ indent }}          seccompProfile:
+{{ indent }}            type: RuntimeDefault
+{{ indent }}        imagePullSecrets:
+{{ indent }}          - name: docker-config
+{{ indent }}        containers:
+{{ indent }}          - name: container-0
+{{ indent }}            image: {{ docker_image_amd }}
+{{ indent }}            imagePullPolicy: Always
+{{ indent }}            securityContext:
+{{ indent }}              allowPrivilegeEscalation: false
+{{ indent }}              capabilities:
+{{ indent }}                add:
+{{ indent }}                  - IPC_LOCK
+{{ indent }}            resources:
+{{ indent }}              limits:
+{{ indent }}                amd.com/gpu: "{{ step.agent_pool[-1:] }}"
+{{ indent }}              requests:
+{{ indent }}                amd.com/gpu: "{{ step.agent_pool[-1:] }}"
+{{ indent }}            volumeMounts:
+{{ indent }}              - name: devshm
+{{ indent }}                mountPath: /dev/shm
+{{ indent }}            env:
+{{ indent }}              - name: AMD_CI_RUNTIME
+{{ indent }}                value: native
+{{ indent }}              - name: NATIVE_CI
+{{ indent }}                value: "true"
+{{ indent }}              - name: VLLM_CI_DOCKER_DISABLED
+{{ indent }}                value: "1"
+{{ indent }}              - name: VLLM_CI_EXPECTED_GPU_COUNT
+{{ indent }}                value: "{{ step.agent_pool[-1:] }}"
+{{ indent }}              - name: HF_HOME
+{{ indent }}                value: /home/buildkite-agent/huggingface
+{{ indent }}              - name: MIOPEN_DEBUG_CONV_DIRECT
+{{ indent }}                value: "0"
+{{ indent }}              - name: MIOPEN_DEBUG_CONV_GEMM
+{{ indent }}                value: "0"
+{{ indent }}              - name: VLLM_ROCM_USE_AITER
+{{ indent }}                value: "0"
+{{ indent }}              - name: PYTORCH_ROCM_ARCH
+{{ indent }}                value: ""
+{{ indent }}        volumes:
+{{ indent }}          - name: devshm
+{{ indent }}            emptyDir:
+{{ indent }}              medium: Memory
+{{ indent }}              sizeLimit: 16Gi
+{%- endmacro %}
+
 {% if env %}
 env:
 {% for name in env %}
@@ -53,19 +107,31 @@ steps:
           {% if step.agent_pool %}
           queue: amd_{{ step.agent_pool }}
           {% else %}
-          queue: amd_mi325_1
+          queue: amd_mi300_1
           {% endif %}
+{{ amd_native_k8s_plugin("        ", step) }}
 {% set cmd_body = (step.command or (step.commands | join("\n"))) | trim %}
 {% set indented_cmd = cmd_body | replace("\n", "\n            ") %}
         command: bash .buildkite/amd/scripts/run-amd-test.sh
         env:
-          DOCKER_BUILDKIT: "1"
+          AMD_CI_RUNTIME: "native"
+          NATIVE_CI: "true"
+          VLLM_CI_DOCKER_DISABLED: "1"
+          VLLM_CI_EXPECTED_GPU_COUNT: "{{ step.agent_pool[-1:] }}"
+          VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE: "1"
+          HF_HOME: "/home/buildkite-agent/huggingface"
+          HF_HUB_DOWNLOAD_TIMEOUT: "300"
+          HF_HUB_ETAG_TIMEOUT: "60"
+          MIOPEN_DEBUG_CONV_DIRECT: "0"
+          MIOPEN_DEBUG_CONV_GEMM: "0"
+          VLLM_ROCM_USE_AITER: "0"
+          PYTORCH_ROCM_ARCH: ""
 {% if step.env %}
 {% for name in step.env %}
           {{ name }}: "{{ step.env[name] }}"
 {% endfor %}
 {% endif %}
-          TEST_COMMAND: |-
+          TEST_COMMANDS: |-
             (command rocm-smi || true) && cd {{ (step.working_dir or default_working_dir) | safe }}
 {% if env %}
 {% for name in env %}
@@ -82,6 +148,15 @@ steps:
 {% endif %}
             {{ indented_cmd | safe }}
         priority: 100
+{% if step.parallelism %}
+        parallelism: {{ step.parallelism }}
+{% endif %}
+{% if step.artifact_paths %}
+        artifact_paths:
+{% for path in step.artifact_paths %}
+          - "{{ path }}"
+{% endfor %}
+{% endif %}
         {% if step.timeout_in_minutes %}
         timeout_in_minutes: {{ step.timeout_in_minutes }}
         {% endif %}
@@ -100,19 +175,31 @@ steps:
       {% if step.agent_pool %}
       queue: amd_{{ step.agent_pool }}
       {% else %}
-      queue: amd_mi325_1
+      queue: amd_mi300_1
       {% endif %}
+{{ amd_native_k8s_plugin("    ", step) }}
 {% set cmd_body = (step.command or (step.commands | join("\n"))) | trim %}
 {% set indented_cmd = cmd_body | replace("\n", "\n        ") %}
     command: bash .buildkite/amd/scripts/run-amd-test.sh
     env:
-      DOCKER_BUILDKIT: "1"
+      AMD_CI_RUNTIME: "native"
+      NATIVE_CI: "true"
+      VLLM_CI_DOCKER_DISABLED: "1"
+      VLLM_CI_EXPECTED_GPU_COUNT: "{{ step.agent_pool[-1:] }}"
+      VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE: "1"
+      HF_HOME: "/home/buildkite-agent/huggingface"
+      HF_HUB_DOWNLOAD_TIMEOUT: "300"
+      HF_HUB_ETAG_TIMEOUT: "60"
+      MIOPEN_DEBUG_CONV_DIRECT: "0"
+      MIOPEN_DEBUG_CONV_GEMM: "0"
+      VLLM_ROCM_USE_AITER: "0"
+      PYTORCH_ROCM_ARCH: ""
 {% if step.env %}
 {% for name in step.env %}
       {{ name }}: "{{ step.env[name] }}"
 {% endfor %}
 {% endif %}
-      TEST_COMMAND: |-
+      TEST_COMMANDS: |-
         (command rocm-smi || true) && cd {{ (step.working_dir or default_working_dir) | safe }}
 {% if env %}
 {% for name in env %}
@@ -129,6 +216,15 @@ steps:
 {% endif %}
         {{ indented_cmd | safe }}
     priority: 100
+{% if step.parallelism %}
+    parallelism: {{ step.parallelism }}
+{% endif %}
+{% if step.artifact_paths %}
+    artifact_paths:
+{% for path in step.artifact_paths %}
+      - "{{ path }}"
+{% endfor %}
+{% endif %}
     {% if step.timeout_in_minutes %}
     timeout_in_minutes: {{ step.timeout_in_minutes }}
     {% endif %}

.buildkite/common/ci_mirror_hardwares.yml

@@ -9,6 +9,11 @@
 #   a2b3_npu_1  — ascend-a2b3, resource_class npu-1 (A2/B3 CI image)
 #   a2b3_npu_4  — ascend-a2b3, resource_class npu-4
 #   a3_npu_2    — ascend-a3, resource_class npu-2
+#
+# The ``l4_*`` presets run under the ``docker`` plugin, which does not put
+# ``buildkite-agent`` on PATH inside the container the way the ``kubernetes``
+# presets do, so they set ``mount-buildkite-agent: true``; without it any step
+# calling ``buildkite-agent`` (artifact upload, annotate, meta-data) fails there.
 
 x-npu-k8s-plugins: &npu_k8s_plugins
   - kubernetes:
@@ -26,6 +31,7 @@ mirror_hardwares:
           image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
           always-pull: true
           propagate-environment: true
+          mount-buildkite-agent: true
           shm-size: "8gb"
           environment:
             - HF_HOME=/fsx/hf_cache
@@ -41,6 +47,7 @@ mirror_hardwares:
           image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
           always-pull: true
           propagate-environment: true
+          mount-buildkite-agent: true
           shm-size: "8gb"
           environment:
             - HF_HOME=/fsx/hf_cache

.buildkite/common/scripts/run_cov_split.sh

@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+# Run a model's E2E tests once per entry mode as separate coverage runs and upload
+# the reports, so per-mode attribution survives (see vllm-project/vllm-omni#5332).
+#
+# Usage: run_cov_split.sh --model-id <id> [--offline <paths>] [--online <paths>] \
+#                         -- <pytest args...>
+#
+#   --model-id  Names the artifacts: coverage-<id>-{offline,online}-<step id>.xml.gz.
+#               Use the model's directory name under vllm_omni/*/models/.
+#   --offline   Pytest path(s) for the offline half. Repeatable.
+#   --online    Pytest path(s) for the online half. Repeatable.
+#   --          Everything after this is forwarded verbatim to every pytest run,
+#               quoting preserved: the job's own -m expression, --run-level, and
+#               anything else it needs (e.g. --test-config-file).
+#
+# At least one of --offline/--online is required, so a job with only one mode runs
+# just that half. Total runtime is bounded by the step's timeout_in_minutes.
+#
+# Exits non-zero if any half or the upload failed.
+set -uo pipefail
+
+MODEL_ID=""
+OFFLINE=()
+ONLINE=()
+PYTEST_ARGS=()
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --model-id) MODEL_ID="$2"; shift 2 ;;
+        --offline)  OFFLINE+=("$2"); shift 2 ;;
+        --online)   ONLINE+=("$2"); shift 2 ;;
+        --)         shift; PYTEST_ARGS=("$@"); break ;;
+        *) echo "run_cov_split.sh: unknown argument: $1" >&2; exit 2 ;;
+    esac
+done
+
+if [[ -z "${MODEL_ID}" ]]; then
+    echo "run_cov_split.sh: missing --model-id" >&2
+    exit 2
+fi
+
+if ((${#OFFLINE[@]} == 0 && ${#ONLINE[@]} == 0)); then
+    echo "run_cov_split.sh: need at least one of --offline / --online" >&2
+    exit 2
+fi
+
+# Two steps can test the same model, and artifacts are scoped to the build (a
+# nightly one carries the ready, merge and nightly tiers at once), so unsuffixed
+# reports would share a path -- a glob download then races to rename onto one file.
+# Not BUILDKITE_JOB_ID: the step id survives retries, so attempts collapse under
+# Buildkite's latest-attempt lookup instead of accumulating. A parallel step would
+# need BUILDKITE_PARALLEL_JOB as well; no pilot is one.
+REPORT_SUFFIX="${BUILDKITE_STEP_ID:-local}"
+OFFLINE_REPORT="coverage-${MODEL_ID}-offline-${REPORT_SUFFIX}.xml"
+ONLINE_REPORT="coverage-${MODEL_ID}-online-${REPORT_SUFFIX}.xml"
+
+run_half() {
+    local mode="$1"
+    shift
+    local report="coverage-${MODEL_ID}-${mode}-${REPORT_SUFFIX}.xml"
+    echo "--- coverage: ${MODEL_ID} ${mode}"
+    pytest -s -v "$@" "${PYTEST_ARGS[@]}" --cov=vllm_omni --cov-report="xml:${report}"
+    local status=$?
+    # Cobertura lists every statement in vllm_omni whether or not it ran, so the
+    # report is ~7MB no matter how little the job covers. The repetition is what
+    # dominates it, and gzip takes that to ~400KB.
+    gzip -9 -f "${report}" || status=1
+    return "${status}"
+}
+
+# Clear both modes, not just the ones being run: the upload glob matches both, so
+# a report left by an earlier run would otherwise be published as if the mode it
+# belongs to had run this time.
+rm -f "${OFFLINE_REPORT}" "${ONLINE_REPORT}" "${OFFLINE_REPORT}.gz" "${ONLINE_REPORT}.gz"
+
+EXIT=0
+((${#OFFLINE[@]})) && { run_half offline "${OFFLINE[@]}" || EXIT=1; }
+((${#ONLINE[@]})) && { run_half online "${ONLINE[@]}" || EXIT=1; }
+
+buildkite-agent artifact upload "coverage-${MODEL_ID}-*-${REPORT_SUFFIX}.xml.gz" || EXIT=1
+
+exit "${EXIT}"

.buildkite/cuda/test-merge.yml

@@ -94,13 +94,19 @@ steps:
           - vllm_omni/model_executor/models/qwen3_tts/
           - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
           - vllm_omni/deploy/qwen3_tts.yaml
+          - .buildkite/common/scripts/run_cov_split.sh
+          - pyproject.toml
+        # 40 -> 60: the coverage split runs the model twice.
+        timeout_in_minutes: 60
         commands:
           - |
-            timeout 40m bash -c "
+            export VLLM_LOGGING_LEVEL=DEBUG
-              export VLLM_LOGGING_LEVEL=DEBUG
+            export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
-              export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
+            .buildkite/common/scripts/run_cov_split.sh \
-              pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+              --model-id qwen3_tts \
-            "
+              --offline tests/e2e/offline_inference/test_qwen3_tts_base.py \
+              --online tests/e2e/online_serving/test_qwen3_tts_base.py \
+              -- -m 'advanced_model and cuda' --run-level 'advanced_model'
         mirror_hardwares: l4_1
 
       - label: "TTS · Higgs-Audio-V3 Test"
@@ -139,7 +145,12 @@ steps:
         timeout_in_minutes: 50
         commands:
           - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-        mirror_hardwares: h100_1
+        # h100_2, not h100_1: advanced_model asserts transcribe every audio
+        # response with Whisper, and tests/helpers/media.py only uses an
+        # accelerator for Whisper when a spare GPU exists. On a single card it
+        # grades on CPU: ~8-10 min per clip vs ~30 s, which alone overruns the
+        # 50-minute job timeout (build 2956).
+        mirror_hardwares: h100_2
 
       - label: "Omni · MiniCPM-o 4.5 Duplex Test"
         source_file_dependencies:
@@ -198,9 +209,17 @@ steps:
           - vllm_omni/diffusion/models/bagel/
           - vllm_omni/model_executor/stage_input_processors/bagel.py
           - vllm_omni/deploy/bagel.yaml
-        timeout_in_minutes: 20
+          - .buildkite/common/scripts/run_cov_split.sh
+          - pyproject.toml
+        # 20 -> 40: the coverage split runs the model twice.
+        timeout_in_minutes: 40
         commands:
-          - pytest -s -v tests/e2e/offline_inference/test_bagel.py tests/e2e/online_serving/test_bagel.py -m 'advanced_model and cuda' --run-level 'advanced_model'
+          - |
+            .buildkite/common/scripts/run_cov_split.sh \
+              --model-id bagel \
+              --offline tests/e2e/offline_inference/test_bagel.py \
+              --online tests/e2e/online_serving/test_bagel.py \
+              -- -m 'advanced_model and cuda' --run-level 'advanced_model'
         mirror_hardwares: h100_1
 
       - label: "Diffusion · Qwen Image Test"

.buildkite/cuda/test-nightly.yml

@@ -345,7 +345,7 @@ steps:
         commands:
           - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
           - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
-          - export CACHE_DIT_VERSION=1.3.0
+          - export CACHE_DIT_VERSION=1.5.0
           - |
             set +e
             pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
@@ -361,7 +361,7 @@ steps:
         commands:
           - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
           - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
-          - export CACHE_DIT_VERSION=1.3.0
+          - export CACHE_DIT_VERSION=1.5.0
           - |
             set +e
             pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json
@@ -377,7 +377,7 @@ steps:
         commands:
           - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
           - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
-          - export CACHE_DIT_VERSION=1.3.0
+          - export CACHE_DIT_VERSION=1.5.0
           - |
             set +e
             pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json
@@ -393,7 +393,7 @@ steps:
         commands:
           - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
           - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
-          - export CACHE_DIT_VERSION=1.3.0
+          - export CACHE_DIT_VERSION=1.5.0
           - |
             set +e
             pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json
@@ -409,7 +409,7 @@ steps:
         commands:
           - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
           - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
-          - export CACHE_DIT_VERSION=1.3.0
+          - export CACHE_DIT_VERSION=1.5.0
           - |
             set +e
             pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_bagel_vllm_omni.json
@@ -456,7 +456,7 @@ steps:
         timeout_in_minutes: 90
         commands:
           - pytest -s -v tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py -m "full_model and cuda" --run-level "full_model"
-          - pytest -s -v tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_ltx2_expansion.py -m "full_model and cuda" --run-level "full_model"
+          - pytest -s -v tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_ltx.py -m "full_model and cuda" --run-level "full_model"
         mirror_hardwares: h100_2
 
       - label: ":full_moon: Diffusion X2V · Wan2.2 Accuracy Test"
@@ -477,10 +477,16 @@ steps:
         commands:
           - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py -m full_model --run-level full_model
           - pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m full_model --run-level full_model
-          - pytest -s -v 'tests/e2e/accuracy/ltx/test_ltx_official_similarity.py::test_ltx_one_stage_matches_official[ltx2_3_i2v]' -m full_model --run-level full_model
           - pytest -s -v tests/e2e/accuracy/test_video_streaming_output_similarity.py -m full_model --run-level full_model
         mirror_hardwares: h100_2
 
+      - label: ":full_moon: Diffusion X2V · LTX Accuracy Test"
+        timeout_in_minutes: 180
+        commands:
+          - pytest -s -v 'tests/e2e/accuracy/ltx/test_ltx_official_similarity.py::test_ltx_matches_official[ltx2_3_i2v]' -m full_model --run-level full_model
+          - pytest -s -v 'tests/e2e/accuracy/ltx/test_ltx_official_similarity.py::test_ltx_matches_official[ltx23_two_stage_layer_fused_i2v]' -m full_model --run-level full_model
+        mirror_hardwares: h100_1
+
       - label: ":full_moon: Diffusion X2V · Perf Test"
         key: nightly-diffusion-x2v-performance
         timeout_in_minutes: 180

.buildkite/cuda/test-ready.yml

@@ -128,6 +128,21 @@ steps:
               image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
               always-pull: true
               propagate-environment: true
+              # vLLM 0.27 added a pre-flight capacity check in
+              # distributed/device_communicators/shm_broadcast.py. Without this
+              # the container gets Docker's 64 MiB default and every test in
+              # this step dies at engine start with
+              #   RuntimeError: Insufficient space in /dev/shm:
+              #                 240 MiB required, 64 MiB available
+              # (9 failed in 8.7s on build #2949). vLLM 0.26 had no such check
+              # and allocated regardless, so this step passed on main while
+              # being under-provisioned the whole time — 0.27 turned a silent
+              # risk into an explicit error.
+              #
+              # This is the only inline docker plugin block in .buildkite/cuda/;
+              # every other step inherits common/ci_mirror_hardwares.yml, which
+              # already sets 8gb. Matching that rather than inventing a value.
+              shm-size: "8gb"
               environment:
                 - "HF_HOME=/fsx/hf_cache"
               volumes:
@@ -212,6 +227,18 @@ steps:
             "
         mirror_hardwares: h100_1
 
+      - label: "Omni · MiniCPM-o 4.5 Perf Test"
+        timeout_in_minutes: 60
+        source_file_dependencies:
+          - tests/dfx/perf/tests/test_minicpmo_4_5_ready.json
+          - vllm_omni/deploy/minicpmo_4_5.yaml
+          - vllm_omni/model_executor/models/minicpmo_4_5/
+          - vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and cuda'
+        mirror_hardwares: h100_1
+
       - label: "TTS · VoxCPM2 Test"
         source_file_dependencies:
           - tests/e2e/online_serving/test_voxcpm2_tts.py

.buildkite/intel/pipeline-intel.yml

@@ -10,7 +10,7 @@ steps:
           DOCKER_BUILDKIT: "1"
           # Buildkite will automatically replace this with the actual commit hash
           VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
-          VLLM_VERSION: "v0.26.0"
+          VLLM_VERSION: "v0.27.0"
         priority: 100
         timeout_in_minutes: 60
         soft_fail: false

.buildkite/intel/scripts/run-xpu-test.sh

@@ -34,8 +34,23 @@ docker_build() {
     docker build "${out[@]}" "$@" -f docker/Dockerfile.xpu .
 }
 
+build_vllm_base() (
+    local vllm_source_dir
+    vllm_source_dir=$(mktemp -d)
+    trap 'rm -rf "${vllm_source_dir}"' EXIT
+
+    git clone --depth 1 --branch "${VLLM_VERSION}" \
+        https://github.com/vllm-project/vllm "${vllm_source_dir}"
+
+    local out=("${export_args[@]/'{{IMAGE}}'/${base_image_name}}")
+    docker build "${out[@]}" \
+        --target vllm-openai \
+        -f "${vllm_source_dir}/docker/Dockerfile.xpu" \
+        "${vllm_source_dir}"
+)
+
 if [ -z "$(docker images -q "${base_image_name}")" ]; then
-    docker_build "${base_image_name}" --target vllm-base --build-arg "VLLM_VERSION=${VLLM_VERSION}"
+    build_vllm_base
 fi
 
 # Try building the docker image
@@ -72,8 +87,9 @@ time timeout -k 30 30m docker run \
     echo $ZE_AFFINITY_MASK
     pip install tblib==3.1.0
     cd /workspace/vllm-omni
-    pytest -v -s -m "core_model and xpu and B60"
+    XPU_TEST_PATHS="tests/diffusion tests/dfx tests/e2e"
+    pytest -v -s $XPU_TEST_PATHS -m "core_model and xpu and B60"
     pytest -v -s tests/diffusion/quantization/test_mxfp8_config.py
-    pytest -v -s -m "advanced_model and xpu and B60"
+    pytest -v -s $XPU_TEST_PATHS -m "advanced_model and xpu and B60"
-    pytest -v -s -m "omni and xpu and B60"
+    pytest -v -s $XPU_TEST_PATHS -m "omni and xpu and B60"
 '

.buildkite/npu/test-npu-nightly.yml

@@ -115,7 +115,7 @@ steps:
           DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
         commands:
           - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
-          - export CACHE_DIT_VERSION=1.3.0
+          - export CACHE_DIT_VERSION=1.5.0
           - |
             set +e
             pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json

.buildkite/npu/test-npu-ready.yml

@@ -17,6 +17,15 @@ steps:
           VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
         commands:
           - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'core_model and npu and A3 and omni' --run-level 'core_model'
+      - label: "Omni · MiniCPM-o 4.5 Perf Test"
+        timeout_in_minutes: 60
+        mirror_hardwares: a3_npu_2
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and npu and A3 and omni'
 
   - group: ":card_index_dividers: TTS Model Test"
     key: ready-tts-group

.buildkite/release/release-pipeline.yml

@@ -40,9 +40,11 @@ steps:
     commands:
       - "bash .buildkite/release/scripts/publish-release-wheels-pypi.sh"
 
+  # Auto-skip when NIGHTLY=1 (scheduled); otherwise require manual unblock.
   - block: "Unblock to build release Docker images"
     depends_on: ~
     key: block-build-release-images
+    if: build.env("NIGHTLY") != "1"
 
   - group: "Build release Docker images"
     key: "build-release-images"
@@ -63,7 +65,7 @@ steps:
               --progress plain \
               -f docker/Dockerfile.cuda .
           - "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m)"
-          # re-tag to default image tag and push, just in case arm64 build fails
+          # Fallback default tag (no arch suffix) if arm64 build fails.
           - "docker tag public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m) public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT"
           - "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT"
         env:
@@ -86,6 +88,26 @@ steps:
         env:
           DOCKER_BUILDKIT: "1"
 
+  # NIGHTLY=1 only: promote ECR images to DockerHub nightly tags, then prune.
+  - label: "Publish nightly multi-arch image to DockerHub"
+    depends_on:
+      - build-release-image-x86
+      - build-release-image-arm64
+    if: build.env("NIGHTLY") == "1"
+    key: publish-nightly-images-dockerhub
+    agents:
+      queue: small_cpu_queue_release
+    commands:
+      - "bash .buildkite/release/scripts/publish-release-images.sh nightly"
+      - "bash .buildkite/release/scripts/cleanup-nightly-builds.sh"
+    plugins:
+      - docker-login#v3.0.0:
+          username: vllmbot
+          password-env: DOCKERHUB_TOKEN
+    env:
+      DOCKER_BUILDKIT: "1"
+      DOCKERHUB_USERNAME: "vllmbot"
+
   - block: "Publish release images to DockerHub"
     key: block-publish-release-images
     depends_on:

.buildkite/release/scripts/cleanup-nightly-builds.sh

@@ -0,0 +1,147 @@
+#!/bin/bash
+# SPDX-License-Identifier: Apache-2.0
+#
+# Clean up old nightly builds from DockerHub (vllm/vllm-omni), keeping the
+# newest 14 tags with the nightly- prefix.
+
+set -ex
+
+TAG_PREFIX="nightly-"
+REPO="vllm/vllm-omni"
+REPO_API_URL="https://hub.docker.com/v2/repositories/${REPO}/tags"
+
+echo "Cleaning up tags with prefix: $TAG_PREFIX in repository: $REPO"
+
+if [ -z "$DOCKERHUB_TOKEN" ]; then
+  echo "Error: DOCKERHUB_TOKEN environment variable is not set"
+  exit 1
+fi
+
+if [ -z "$DOCKERHUB_USERNAME" ]; then
+  echo "Error: DOCKERHUB_USERNAME environment variable is not set"
+  exit 1
+fi
+
+echo "Getting DockerHub bearer token..."
+set +x
+BEARER_TOKEN=$(curl -s -X POST \
+  -H "Content-Type: application/json" \
+  -d "{\"username\": \"$DOCKERHUB_USERNAME\", \"password\": \"$DOCKERHUB_TOKEN\"}" \
+  "https://hub.docker.com/v2/users/login" | jq -r '.token')
+set -x
+
+if [ -z "$BEARER_TOKEN" ] || [ "$BEARER_TOKEN" = "null" ]; then
+  echo "Error: Failed to get DockerHub bearer token"
+  exit 1
+fi
+
+get_all_tags() {
+  local page=1
+  local all_tags=""
+  # Hub `name` is a partial match (not a strict prefix). Use it to narrow the
+  # list, then enforce startswith() client-side. Exhaust API pages; do not
+  # stop merely because a page has no matching tags after filtering.
+  local name_query
+  name_query=$(printf '%s' "$TAG_PREFIX" | jq -sRr @uri)
+
+  while true; do
+    set +x
+    local response
+    response=$(curl -s -H "Authorization: Bearer $BEARER_TOKEN" \
+      "$REPO_API_URL?page=$page&page_size=100&name=${name_query}")
+    set -x
+
+    local page_count
+    page_count=$(echo "$response" | jq -r '.results | length')
+    if [ -z "$page_count" ] || [ "$page_count" = "0" ] || [ "$page_count" = "null" ]; then
+      break
+    fi
+
+    local tags
+    tags=$(echo "$response" | jq -r --arg prefix "$TAG_PREFIX" \
+      '.results[] | select(.name | startswith($prefix)) | "\(.last_updated)|\(.name)"')
+
+    if [ -n "$tags" ]; then
+      all_tags="$all_tags$tags"$'\n'
+    fi
+    page=$((page + 1))
+  done
+
+  echo "$all_tags" | sort -r | cut -d'|' -f2
+}
+
+delete_tag() {
+  local tag_name="$1"
+  echo "Deleting tag: $tag_name"
+
+  local delete_url="https://hub.docker.com/v2/repositories/${REPO}/tags/$tag_name"
+  set +x
+  local response http_code body
+  response=$(curl -s -w "\n%{http_code}" -X DELETE \
+    -H "Authorization: Bearer $BEARER_TOKEN" "$delete_url")
+  set -x
+  http_code=$(printf '%s\n' "$response" | tail -n1)
+  body=$(printf '%s\n' "$response" | sed '$d')
+
+  if [ "$http_code" -ge 200 ] && [ "$http_code" -lt 300 ]; then
+    echo "Successfully deleted tag: $tag_name (HTTP ${http_code})"
+    return 0
+  fi
+
+  local detail=""
+  if echo "$body" | jq -e '.detail' > /dev/null 2>&1; then
+    detail=$(echo "$body" | jq -r '.detail')
+  elif [ -n "$body" ]; then
+    detail="$body"
+  else
+    detail="(empty response body)"
+  fi
+  echo "Error: Failed to delete tag $tag_name (HTTP ${http_code}): ${detail}"
+  return 1
+}
+
+echo "Fetching all tags from DockerHub..."
+all_tags=$(get_all_tags)
+
+if [ -z "$all_tags" ]; then
+  echo "No tags found to clean up"
+  exit 0
+fi
+
+total_tags=$(echo "$all_tags" | wc -l)
+echo "Found $total_tags tags"
+
+tags_to_keep=14
+tags_to_delete=$((total_tags - tags_to_keep))
+
+if [ $tags_to_delete -le 0 ]; then
+  echo "No tags need to be deleted (only $total_tags tags found, keeping $tags_to_keep)"
+  exit 0
+fi
+
+echo "Will delete $tags_to_delete old tags, keeping the newest $tags_to_keep"
+
+tags_to_delete_list=$(echo "$all_tags" | tail -n +$((tags_to_keep + 1)))
+
+if [ -z "$tags_to_delete_list" ]; then
+  echo "No tags to delete"
+  exit 0
+fi
+
+echo "Deleting old tags..."
+delete_failures=0
+while IFS= read -r tag; do
+  if [ -n "$tag" ]; then
+    if ! delete_tag "$tag"; then
+      delete_failures=$((delete_failures + 1))
+    fi
+    sleep 1
+  fi
+done <<< "$tags_to_delete_list"
+
+if [ "$delete_failures" -gt 0 ]; then
+  echo "Cleanup finished with ${delete_failures} deletion failure(s)"
+  exit 1
+fi
+
+echo "Cleanup completed successfully"

.buildkite/release/scripts/publish-release-images.sh

@@ -1,50 +1,87 @@
 #!/bin/bash
 # SPDX-License-Identifier: Apache-2.0
 #
-# Publish release Docker images from ECR to DockerHub.
+# Publish Docker images from ECR staging to DockerHub.
-# Pulls per-arch images, tags with `latest` and versioned tags, pushes them,
+#
-# then creates and pushes a multi-arch manifest.
+# Usage:
+#   publish-release-images.sh            # release mode (default)
+#   publish-release-images.sh release
+#   publish-release-images.sh nightly
+#
+# Release tags: latest, v<version>
+# Nightly tags: nightly, nightly-<commit>
 
 set -euo pipefail
 
-RELEASE_VERSION=$(buildkite-agent meta-data get release-version --default "" | sed 's/^v//')
+MODE="${1:-release}"
-if [ -z "${RELEASE_VERSION}" ]; then
-  echo "ERROR: release-version metadata not set"
-  exit 1
-fi
 
-COMMIT="$BUILDKITE_COMMIT"
 DOCKERHUB_REPO="vllm/vllm-omni"
 ECR_REPO="public.ecr.aws/q9t5s3a7/vllm-omni-release-repo"
+COMMIT="$BUILDKITE_COMMIT"
+PRIMARY_TAGS=()
+EXTRA_MANIFEST_TAGS=()
+
+case "${MODE}" in
+  release)
+    RELEASE_VERSION=$(buildkite-agent meta-data get release-version --default "" | sed 's/^v//')
+    if [ -z "${RELEASE_VERSION}" ]; then
+      echo "ERROR: release-version metadata not set"
+      exit 1
+    fi
+    PRIMARY_TAGS=("latest" "v${RELEASE_VERSION}")
+    ;;
+  nightly)
+    # Commit-pinned manifest reuses nightly-{x86_64,aarch64} arch images.
+    PRIMARY_TAGS=("nightly")
+    EXTRA_MANIFEST_TAGS=("nightly-${COMMIT}")
+    ;;
+  *)
+    echo "Usage: $0 {release|nightly}"
+    exit 2
+    ;;
+esac
 
 echo "========================================"
-echo "Publishing release images v${RELEASE_VERSION}"
+echo "Publishing ${MODE} images to ${DOCKERHUB_REPO}"
 echo "  Commit: ${COMMIT}"
+echo "  Primary tags: ${PRIMARY_TAGS[*]}"
+if [ ${#EXTRA_MANIFEST_TAGS[@]} -gt 0 ]; then
+  echo "  Extra manifests: ${EXTRA_MANIFEST_TAGS[*]}"
+fi
 echo "========================================"
 
-# Login to ECR to pull staging images
 aws ecr-public get-login-password --region us-east-1 | \
   docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
 
-docker pull "${ECR_REPO}:${COMMIT}-x86_64"
+SRC_X86="${ECR_REPO}:${COMMIT}-x86_64"
-docker pull "${ECR_REPO}:${COMMIT}-aarch64"
+SRC_ARM="${ECR_REPO}:${COMMIT}-aarch64"
 
-docker tag "${ECR_REPO}:${COMMIT}-x86_64" "${DOCKERHUB_REPO}:latest-x86_64"
+docker pull "${SRC_X86}"
-docker tag "${ECR_REPO}:${COMMIT}-x86_64" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-x86_64"
+docker pull "${SRC_ARM}"
-docker push "${DOCKERHUB_REPO}:latest-x86_64"
-docker push "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-x86_64"
 
-docker tag "${ECR_REPO}:${COMMIT}-aarch64" "${DOCKERHUB_REPO}:latest-aarch64"
+publish_manifest() {
-docker tag "${ECR_REPO}:${COMMIT}-aarch64" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-aarch64"
+  local tag="$1"
-docker push "${DOCKERHUB_REPO}:latest-aarch64"
+  local arch_tag_base="$2"
-docker push "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-aarch64"
+  docker manifest rm "${DOCKERHUB_REPO}:${tag}" || true
+  docker manifest create \
+    "${DOCKERHUB_REPO}:${tag}" \
+    "${DOCKERHUB_REPO}:${arch_tag_base}-x86_64" \
+    "${DOCKERHUB_REPO}:${arch_tag_base}-aarch64"
+  docker manifest push "${DOCKERHUB_REPO}:${tag}"
+}
 
-docker manifest rm "${DOCKERHUB_REPO}:latest" || true
+for tag in "${PRIMARY_TAGS[@]}"; do
-docker manifest rm "${DOCKERHUB_REPO}:v${RELEASE_VERSION}" || true
+  docker tag "${SRC_X86}" "${DOCKERHUB_REPO}:${tag}-x86_64"
-docker manifest create "${DOCKERHUB_REPO}:latest" "${DOCKERHUB_REPO}:latest-x86_64" "${DOCKERHUB_REPO}:latest-aarch64"
+  docker tag "${SRC_ARM}" "${DOCKERHUB_REPO}:${tag}-aarch64"
-docker manifest create "${DOCKERHUB_REPO}:v${RELEASE_VERSION}" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-x86_64" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-aarch64"
+  docker push "${DOCKERHUB_REPO}:${tag}-x86_64"
-docker manifest push "${DOCKERHUB_REPO}:latest"
+  docker push "${DOCKERHUB_REPO}:${tag}-aarch64"
-docker manifest push "${DOCKERHUB_REPO}:v${RELEASE_VERSION}"
+  publish_manifest "${tag}" "${tag}"
+done
+
+ARCH_BASE="${PRIMARY_TAGS[0]}"
+for tag in "${EXTRA_MANIFEST_TAGS[@]}"; do
+  publish_manifest "${tag}" "${ARCH_BASE}"
+done
 
 echo ""
-echo "Successfully published release images for v${RELEASE_VERSION}"
+echo "Successfully published ${MODE} images"

.claude/skills/precheck-pr/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: precheck-pr
-description: Self-check your branch before creating a PR — catch dead code, verify accuracy/perf claims, validate PR title format, and confirm merge readiness. Use when the user says "precheck", "self review", "pre-submit check", or "check my PR before I open it." Never posts to GitHub.
+description: Self-check your branch before creating a PR — catch dead code, prevent new model-specific Python examples, verify accuracy/perf claims, validate PR title format, and confirm merge readiness. Use when the user says "precheck", "self review", "pre-submit check", or "check my PR before I open it." Never posts to GitHub.
 ---
 
 # PR Pre-Check
@@ -65,6 +65,8 @@ Ask: "Quick mode or full mode?" Then walk the checklist for the detected PR type
 
 **Also run the Code-Quality sweep on every PR**, regardless of type or mode: the five diff-scoped checks in [references/code-quality.md](references/code-quality.md) — kwargs fragility, broad-except swallow, `Any`/wrong type hints, hot-path `.clone()`/`deepcopy`, and event-loop blocking — plus the advisory conventions (log level, structured logging, synchronization, cleanup, dependencies, naming) in [checklists.md](references/checklists.md). These count only lines the PR *adds* — the pre-existing backlog across the repo is out of scope.
 
+**Also run the Examples-Policy check on every PR** using [references/examples-policy.md](references/examples-policy.md). Inspect only Python paths introduced by the diff. Treat a new model-, checkpoint-, vendor-, or family-specific Python example as blocking; do not report pre-existing example debt.
+
 ### Step 5: Print Report
 
 ```
@@ -77,6 +79,7 @@ Pre-check report for <branch>
   ─────────────────  ──────
   PR title format    ✓
   Code quality       ⚠ 1 broad except, 2 Any hints
+  Examples policy    ✗ new model-specific Python example
   PR desc integrity  ✓
   Registry/config    ✓
   Dead code          ⚠ 2 warnings

.claude/skills/precheck-pr/references/checklists.md

@@ -47,6 +47,19 @@ Diff-scoped sweep for five fragility patterns that are pervasive in the existing
 
 Roll the ⚠/✗ counts into the report as a single **Code quality** dimension row.
 
+---
+
+## All PRs: Examples Policy (run on every PR, quick & full)
+
+Inspect Python files introduced under `examples/` using
+[examples-policy.md](examples-policy.md). This is a diff-scoped ratchet: existing
+model-specific examples may be edited or removed without triggering the check.
+
+- [ ] **No new model-specific Python examples:** the diff adds, copies, or renames no Python file whose path or behavior is specific to one model, checkpoint, vendor, or model family. ✗ for any violation. Route inference through a shared task/protocol entrypoint, model request behavior through `vllm_omni/model_extras`, and commands through task docs or `recipes/`.
+- [ ] **New generic example code is justified:** any new task-neutral Python file under `examples/` is an actual user-facing entrypoint rather than reusable production logic, a benchmark, an app, a setup tool, or a reproducer. ⚠ when the placement is questionable.
+
+Roll the result into the report as a separate **Examples policy** dimension.
+
 ### Conventions (eyeball — no grep)
 
 Judgment calls that don't grep cleanly — apply them while reading the diff, not as a mechanical check:

.claude/skills/precheck-pr/references/examples-policy.md

@@ -0,0 +1,68 @@
+# Examples Python Policy
+
+Prevent new model-specific Python examples without forcing cleanup of the
+existing backlog. Run this check on every PR in quick and full modes.
+
+## Scope
+
+Set the merge base as in the main workflow, then list Python paths introduced
+by the PR, including additions, copies, and renames:
+
+```bash
+BASE="$(git merge-base HEAD origin/main)"
+git diff --name-status --diff-filter=ACR --find-renames --find-copies \
+  "${BASE}"...HEAD -- 'examples/**/*.py'
+```
+
+Inspect destination paths for copies and renames. Existing model-specific files
+that are only modified or deleted are grandfathered and must not be reported.
+
+## Blocking classification
+
+Mark ✗ when a newly introduced Python file does any of the following:
+
+- Names or scopes a path to one model, checkpoint, vendor, or model family.
+- Hard-codes a model identifier, model-specific prompt, request shape, output
+  conversion, or launch configuration behind an otherwise generic filename.
+- Duplicates an existing task/protocol runner to demonstrate one model.
+- Adds a per-model helper below a generic task hub such as `text_to_speech/`.
+
+Do not infer safety from a generic filename alone. For example, a file named
+`text_to_audio.py` is still model-specific if it only implements one model's
+contract.
+
+## Acceptable additions
+
+A new Python example may pass when it is a genuinely model-neutral,
+user-facing task or protocol entrypoint. It must accept model selection through
+configuration and keep model-specific behavior out of the script.
+
+Use ⚠ when the file is model-neutral but reusable logic appears misplaced under
+`examples/`. Route code by purpose:
+
+| Code purpose | Preferred location |
+|---|---|
+| Model prompt, defaults, request/output adaptation | `vllm_omni/model_extras/` or production model modules |
+| Runnable model command and validation evidence | Task documentation or `recipes/` |
+| UI or interactive application | `apps/` |
+| Benchmark or evaluator | `benchmarks/` |
+| Regression reproducer | `tests/` |
+| Download, conversion, or setup utility | `tools/` |
+
+If a model cannot use an existing shared runner, report the missing generic
+capability. Do not solve the gap by adding a model-specific Python example.
+
+## Report format
+
+Add one row to the pre-check report:
+
+```text
+Examples policy    ✓ no new Python example paths
+Examples policy    ⚠ generic helper belongs in tools/
+Examples policy    ✗ examples/offline_inference/new_model/end2end.py is model-specific
+Examples policy    ✗ examples/offline_inference/x_to_y_model_name.py is model-specific
+```
+
+For a blocker, name the path, the model-specific behavior, and the appropriate
+shared runner or destination. Do not propose deleting unrelated existing
+examples as part of the PR.

.claude/skills/review-pr/SKILL.md

@@ -97,6 +97,7 @@ If docs and live code disagree, verify the code/tests and report the drift.
 | [test-quality-evaluation.md](references/checks/test-quality-evaluation.md) | Tests change, are absent for risky code, or may not exercise production behavior. |
 | [tests-docs-checklist.md](references/checks/tests-docs-checklist.md) | Coverage, CI markers, examples, user docs, or PR evidence need review. |
 | [verification.md](references/checks/verification.md) | Hardware, a server, or a runnable affected path is available for active verification. |
+| [examples-policy.md](../precheck-pr/references/examples-policy.md) | The PR adds, copies, or renames Python under `examples/`; apply the canonical policy shared with `precheck-pr`. |
 
 ### Delivery and reviewer coordination
 
@@ -152,6 +153,13 @@ contract they protect or use only the applicable evidence checks.
 Apply every category in [general-checks.md](references/process/general-checks.md) before
 lower-priority comments.
 
+If the diff census contains an added, copied, or renamed Python path under
+`examples/`, read and apply the canonical
+[examples policy](../precheck-pr/references/examples-policy.md). Treat a new
+model-specific Python example as blocking. Do not flag model-specific example
+debt that the PR only modifies or removes, and do not run the rest of the
+author-oriented `precheck-pr` workflow.
+
 For each changed value or behavior, trace:
 
 ```text

.gitignore

@@ -46,6 +46,8 @@ htmlcov/
 .cache
 nosetests.xml
 coverage.xml
+coverage-*.xml
+coverage-*.xml.gz
 *.cover
 *.py,cover
 .hypothesis/

.readthedocs.yml

@@ -11,10 +11,15 @@ build:
     post_checkout:
       - git fetch --unshallow || true
     pre_install:
+      # `pre_install` runs before RTD upgrades pip, and the seeded pip 23.1
+      # rejects the `typing_extensions` wheel torch needs (fixed in pip 23.2).
+      - python -m pip install --upgrade pip
       # Pre-install CPU torch so the subsequent `pip install .[docs]` does not
       # pull ~2 GB of CUDA 13 libs (cuBLAS/cuDNN/NCCL/...) that exceed RTD's
-      # 1 GB memory cap and time out the mkdocs build.
+      # 1 GB memory cap and time out the mkdocs build. Wheel-only: `--index-url`
-      - python -m pip install --index-url https://download.pytorch.org/whl/cpu torch
+      # replaces PyPI, so an sdist fallback cannot reach its build deps. The
+      # quotes keep YAML from reading `:all:` as a mapping key.
+      - "python -m pip install --only-binary=:all: --index-url https://download.pytorch.org/whl/cpu torch"
 
 mkdocs:
   configuration: mkdocs.yml

README.md

@@ -16,6 +16,7 @@ Easy, fast, and cheap omni-modality model serving for everyone
 ---
 
 *Latest News* 🔥
+- [2026/08] [VeRL-Omni](https://github.com/verl-project/verl-omni) `v0.2.0` is released: faster diffusion RL powered by vLLM-Omni (request-level/step-wise batching with FA3), rebuilt Qwen3-Omni multimodal training (DPO & GSPO), plus LTX-2.3, Qwen-Image-Edit support and more. See the [release notes](https://github.com/verl-project/verl-omni/releases/tag/v0.2.0).
 - [2026/08] We released [0.26.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.26.0) - aligned with the vLLM 0.26 release line, featuring [MiniMax H3](recipes/MiniMaxAI/MiniMax-H3.md) joint video/audio generation, an experimental full-duplex realtime runtime for [MiniCPM-o 4.5](recipes/OpenBMB/MiniCPM-o-4_5.md), distributed layerwise diffusion offload, and broader model, hardware, streaming, TTS, and quantization support.
 - [2026/07] We released [0.24.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.24.0) - aligned with the vLLM 0.24 release line, expanding production-ready coverage across TTS, speech, diffusion, image/video generation, and robot-policy serving, with major Omni stage runtime refactoring, diffusion request-level batching, async output materialization, quantization/cache/memory improvements, and broad CUDA/ROCm/XPU/NPU support.
 - [2026/06] Starting with [0.14.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.14.0), vLLM-Omni publishes a stable release aligned with every even-numbered upstream vLLM minor version. [0.16.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.16.0), [0.18.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.18.0), [0.20.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.20.0), and [0.22.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.22.0) continued this cadence, expanding omni and world-model support with [NVIDIA Cosmos3](recipes/cosmos3/Cosmos3-Nano.md) and DreamZero, adding models such as MiniCPM-o 4.5, MOSS-TTS, and Lance, and advancing TTS, diffusion, distributed execution, quantization, RL integration through [VeRL-Omni](https://github.com/verl-project/verl-omni), and CUDA/ROCm/MUSA/NPU/XPU coverage.

benchmarks/diffusion/quantization_quality.py

@@ -158,7 +158,6 @@ def _build_omni_kwargs(args, quantization=None):
 def _generate_image(omni, args, prompt, seed):
     """Generate a single image and return (PIL.Image, time_seconds, memory_gib)."""
     from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-    from vllm_omni.outputs import OmniRequestOutput
     from vllm_omni.platforms import current_omni_platform
 
     generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(seed)
@@ -176,7 +175,7 @@ def _generate_image(omni, args, prompt, seed):
     elapsed = time.perf_counter() - start
     peak_mem = torch.accelerator.max_memory_allocated() / (1024**3)
 
-    req_out = OmniRequestOutput.unwrap_result(outputs)
+    req_out = outputs[0] if isinstance(outputs, list) else outputs
     if not req_out.images:
         raise ValueError("Could not extract image output from result.")
     img = req_out.images[0]
@@ -206,15 +205,15 @@ def _generate_video(omni, args, prompt, seed):
     elapsed = time.perf_counter() - start
     peak_mem = torch.accelerator.max_memory_allocated() / (1024**3)
 
-    first = outputs[0]
+    if isinstance(outputs, list) and isinstance(outputs[0], OmniRequestOutput):
-    if hasattr(first, "request_output") and isinstance(first.request_output, list):
+        first = outputs[0]
-        inner = first.request_output[0]
+    elif isinstance(outputs, OmniRequestOutput):
-        if isinstance(inner, OmniRequestOutput) and hasattr(inner, "images"):
+        first = outputs
-            frames = inner.images[0] if inner.images else None
+    else:
-        else:
+        raise ValueError("Could not extract video frames from output.")
-            frames = inner
+
-    elif hasattr(first, "images") and first.images:
+    if hasattr(first, "images"):
-        frames = first.images[0]
+        frames = first.images[0] if first.images else None
     else:
         raise ValueError("Could not extract video frames from output.")
 

benchmarks/glm_image/vllm-omni/inference.py

@@ -300,7 +300,7 @@ def benchmark(args: argparse.Namespace) -> None:
         output_idx = 0
         for stage_outputs in omni.generate(all_prompts, sampling_params_list, py_generator=True):
             if stage_outputs.final_output_type == "image":
-                request_output = stage_outputs.request_output
+                request_output = stage_outputs
                 request_id = getattr(request_output, "request_id", "")
 
                 images = getattr(request_output, "images", [])

benchmarks/lingbot_video/dense_pipeline_parity.py

@@ -167,9 +167,11 @@ def main() -> None:
     ]
     native_cmd = [
         sys.executable,
-        "examples/offline_inference/text_to_video/text_to_video_lingbot.py",
+        "examples/offline_inference/text_to_video/text_to_video.py",
         "--model",
         args.model,
+        "--model-class-name",
+        "LingBotVideoPipeline",
         "--prompt",
         args.prompt,
         "--output",

benchmarks/tts/README.md

@@ -139,6 +139,34 @@ python benchmarks/tts/bench_tts.py \
     --output-dir ./results
 ```
 
+#### Fixed IndexTTS 2.5 performance sweep
+
+IndexTTS 2.5 is registered in `model_configs.yaml` like the other TTS models.
+Use `--served-model-name` when the server was launched from a local native
+bundle instead of the registry key. The command below reproduces the acceptance
+workload: Seed-TTS Eval EN, `n=500`, concurrency `4/8/16/32`, five warmups, and
+dataset seed 0.
+
+```bash
+python benchmarks/tts/bench_tts.py \
+    --model IndexTeam/IndexTTS-2.5 \
+    --served-model-name /path/to/indextts-2.5 \
+    --task voice_clone --locale en \
+    --dataset-path /path/to/seedtts_testset \
+    --host 127.0.0.1 --port 8092 \
+    --concurrency 4 8 16 32 \
+    --num-prompts 500 --num-warmups 5 \
+    --output-dir ./results/indextts2_5 \
+    -- \
+    --tokenizer /path/to/indextts-2.5/qwen0.6bemo4-merge \
+    --seed 0 --metric-percentiles 50,95,99 --disable-tqdm --save-detailed
+```
+
+The wrapper benchmarks an already-running server. For reproducible quality
+comparisons, add `--request-seed 42`; production-performance runs should omit
+it. The trailing `--seed 0` controls Seed-TTS row selection rather than model
+sampling.
+
 ### 4. Plot a sweep
 
 ```bash

benchmarks/tts/bench_tts.py

@@ -84,12 +84,17 @@ def build_bench_args(
     result_filename: str | None,
     extra_cli_args: list[str],
     output_len: int | None = None,
+    served_model_name: str | None = None,
+    num_warmups: int = 2,
+    request_seed: int | None = None,
 ) -> list[str]:
     """Build the ``vllm bench serve --omni`` command for one (task, concurrency) run."""
     dataset_name = _TASK_TO_DATASET[task]
     backend: str = model_cfg["backend"]
     endpoint: str = model_cfg["endpoint"]
-    task_extra_body: dict[str, Any] = (model_cfg.get("task_extra_body") or {}).get(task) or {}
+    task_extra_body: dict[str, Any] = dict((model_cfg.get("task_extra_body") or {}).get(task) or {})
+    if request_seed is not None:
+        task_extra_body["seed"] = request_seed
 
     # Resolve dataset path
     if dataset_path:
@@ -109,7 +114,7 @@ def build_bench_args(
         "--port",
         str(port),
         "--model",
-        model,
+        served_model_name or model,
         "--backend",
         backend,
         "--endpoint",
@@ -119,7 +124,7 @@ def build_bench_args(
         "--num-prompts",
         str(num_prompts),
         "--num-warmups",
-        "2",
+        str(num_warmups),
         "--percentile-metrics",
         "ttft,e2el,audio_rtf,audio_ttfp,audio_duration,audio_underrun",
     ]
@@ -152,7 +157,8 @@ def build_bench_args(
         if result_filename:
             cmd += ["--result-filename", result_filename]
 
-    cmd += extra_cli_args
+    passthrough = extra_cli_args[1:] if extra_cli_args[:1] == ["--"] else extra_cli_args
+    cmd += passthrough
     return cmd
 
 
@@ -225,6 +231,12 @@ def main() -> None:
     parser.add_argument(
         "--model", required=True, help="HuggingFace model ID (e.g. Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice)"
     )
+    parser.add_argument(
+        "--served-model-name",
+        default=None,
+        help="Model name/path accepted by the running server. Use this when --model selects a registry entry "
+        "but the server was launched from a local bundle path.",
+    )
     parser.add_argument("--task", default="all", help="Task type: voice_clone | default_voice | voice_design | all")
     parser.add_argument("--locale", default="en", choices=["en", "zh"])
     parser.add_argument("--concurrency", type=int, nargs="+", default=[1, 4], metavar="N")
@@ -236,6 +248,13 @@ def main() -> None:
         metavar="N",
         help="Number of prompts per run. If one value, applied to all concurrency levels.",
     )
+    parser.add_argument("--num-warmups", type=int, default=2, help="Warmup requests before each measured run")
+    parser.add_argument(
+        "--request-seed",
+        type=int,
+        default=None,
+        help="Optional model sampling seed added to every request body for reproducible quality runs",
+    )
     parser.add_argument(
         "--dataset-path", default=None, help="Root of seed-tts-eval dataset (required for voice_clone/default_voice)"
     )
@@ -256,6 +275,9 @@ def main() -> None:
     parser.add_argument("extra", nargs=argparse.REMAINDER, help="Extra args passed directly to vllm bench serve")
     args = parser.parse_args()
 
+    if args.num_warmups < 0:
+        parser.error("--num-warmups cannot be negative")
+
     model_configs = load_model_configs(Path(args.model_configs))
     if args.model not in model_configs:
         known = "\n  ".join(model_configs.keys())
@@ -308,6 +330,9 @@ def main() -> None:
                 result_filename=result_filename,
                 extra_cli_args=args.extra or [],
                 output_len=args.output_len,
+                served_model_name=args.served_model_name,
+                num_warmups=args.num_warmups,
+                request_seed=args.request_seed,
             )
             result = run_one_benchmark(cmd)
             if result is not None:

benchmarks/tts/model_configs.yaml

@@ -7,6 +7,17 @@
 # is tracked here.
 
 models:
+  IndexTeam/IndexTTS-2.5:
+    supported_tasks: [voice_clone]
+    backend: openai-audio-speech
+    endpoint: /v1/audio/speech
+    trust_remote_code: true
+    task_extra_body:
+      voice_clone:
+        extra_params:
+          lang: en
+          text_normalization: true
+
   # -CustomVoice checkpoints lack speaker_encoder weights, so voice_clone is
   # NOT supported (an attempt raises ValueError from _extract_speaker_embedding
   # at model runtime).  Use -Base for voice_clone.

docker/Dockerfile.ci

@@ -1,7 +1,24 @@
-# Target vllm version: v0.26.0 (releases/v0.26.0 branch).
+# Target vllm version: v0.27.0 (releases/v0.27.0 branch).
 # Keep VLLM_BASE_TAG aligned with the target vLLM release on every rebase.
+#
+# Release-boundary form: there is no precompiled-wheel pin here, because the
+# base image already ships exactly the target vLLM. Everything the runtime
+# needs — torch, flashinfer, nvJitLink, numpy — comes from that image as one
+# self-consistent set.
+#
+# This block reappears only when the rebase targets a vLLM commit that is NOT a
+# released tag. During the 0.27 rebase, targeting v0.27.0rc1 on top of the
+# v0.26.0 base image required reinstalling vLLM from wheels.vllm.ai and then
+# hand-repairing the fallout: pinning torch ahead of flashinfer (installing it
+# after bound flashinfer's TVM FFI runtime to the old torch and aborted engine
+# core init), registering libnvJitLink.so.13 with ldconfig, and re-pinning numpy
+# under numba's ceiling. All of it was compensation for a base image and a vLLM
+# that disagreed. At a real release boundary they agree, so none of it is
+# needed. If you pin an untagged commit again, restore the reinstall block —
+# agent/tasks/20_update_wheel_url.sh fails the run rather than letting the
+# Dockerfile silently ship the wrong vLLM.
 ARG VLLM_BASE_IMAGE=vllm/vllm-openai
-ARG VLLM_BASE_TAG=v0.26.0
+ARG VLLM_BASE_TAG=v0.27.0
 
 # We use registry-based layer caching for the dependencies,
 # where the tag is determined based on the hash of the contents

docker/Dockerfile.npu

@@ -1,5 +1,5 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.26.0
+ARG VLLM_ASCEND_TAG=v0.27.0
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
 
 # WORKDIR /vllm-workspace/vllm

docker/Dockerfile.npu.a3

@@ -1,5 +1,5 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.26.0-a3
+ARG VLLM_ASCEND_TAG=v0.27.0-a3
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
 
 # WORKDIR /vllm-workspace/vllm

docker/Dockerfile.npu.ci

@@ -1,5 +1,5 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.26.0
+ARG VLLM_ASCEND_TAG=v0.27.0
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
 
 ARG APP_DIR=/vllm-workspace/vllm-omni

docker/Dockerfile.npu.ci.a3

@@ -1,5 +1,5 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.26.0-a3
+ARG VLLM_ASCEND_TAG=v0.27.0-a3
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
 
 ARG APP_DIR=/vllm-workspace/vllm-omni

docker/Dockerfile.rocm

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.26.0
+ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.27.0
 FROM ${BASE_IMAGE} AS base
 
 # Declare a variable to know if we want to use the nightly build or the stable build.
@@ -10,7 +10,7 @@ FROM ${BASE_IMAGE} AS base
 #    we should swap over to use stable release ASAP.
 #    We should avoid relying on custom commits.
 ARG USE_NIGHTLY_BUILD=0
-ARG VLLM_VERSION_OR_COMMIT_HASH=v0.26.0
+ARG VLLM_VERSION_OR_COMMIT_HASH=v0.27.0
 ARG ARG_PYTORCH_ROCM_ARCH
 ENV PYTORCH_ROCM_ARCH=${ARG_PYTORCH_ROCM_ARCH:-${PYTORCH_ROCM_ARCH}}
 
@@ -79,9 +79,16 @@ RUN printf '%s\n' \
     '    os._exit(_exit_code)' \
     > ${COMMON_WORKDIR}/vllm-omni/conftest.py
 
+# Some optional dependencies pull in the CPU ONNX Runtime distribution. The
+# CPU and ROCm wheels own the same import path and cannot coexist, so remove
+# the CPU wheel and reinstall the ROCm wheel after dependency resolution.
 RUN cd ${COMMON_WORKDIR}/vllm-omni && \
     uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" \
         --no-cache-dir ".[dev]" --no-build-isolation && \
+    python3 -m pip uninstall -y onnxruntime && \
+    python3 -m pip install --no-cache-dir --force-reinstall --no-deps \
+        "onnxruntime-rocm>=1.22.2" && \
+    python3 -c "import importlib.metadata as m; import onnxruntime as ort; assert 'ROCMExecutionProvider' in ort.get_available_providers(), ort.get_available_providers(); assert m.version('onnxruntime-rocm'); assert not any(d.metadata['Name'] == 'onnxruntime' for d in m.distributions())" && \
     python3 -c "from torchcodec.decoders import VideoDecoder"
 
 RUN ln -sf /usr/bin/python3 /usr/bin/python

docker/Dockerfile.xpu

@@ -88,7 +88,7 @@ ENV UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
 ENV UV_INDEX_STRATEGY="unsafe-best-match"
 ENV UV_LINK_MODE="copy"
 
-ARG VLLM_VERSION=v0.26.0
+ARG VLLM_VERSION=v0.27.0
 RUN git clone -b ${VLLM_VERSION} https://github.com/vllm-project/vllm
 WORKDIR /workspace/vllm
 
@@ -108,7 +108,7 @@ ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
 
 RUN --mount=type=cache,target=/root/.cache/uv \
     uv pip uninstall triton triton-xpu && \
-    uv pip install triton-xpu==3.7.1 && \
+    uv pip install triton-xpu==3.7.2 && \
     uv pip install --no-build-isolation --no-deps .
 
 CMD ["/bin/bash"]
@@ -144,12 +144,13 @@ WORKDIR /workspace/vllm-omni
 COPY . .
 
 ENV VLLM_OMNI_TARGET_DEVICE=xpu
+# Keep the upstream oneCCL packages paired with the XPU torch runtime.
 RUN --mount=type=cache,target=/root/.cache/uv \
     uv pip install --no-build-isolation \
     --extra-index-url https://mirrors.ustc.edu.cn/pypi/simple/ \
     ".[dev]" && \
-    uv pip uninstall triton oneccl oneccl-devel && \
+    uv pip uninstall triton && \
-    uv pip install triton-xpu==3.7.1 --reinstall
+    uv pip install triton-xpu==3.7.2 --reinstall
 
 FROM vllm-omni AS vllm-omni-openai
 

docs/.nav.yml

@@ -40,30 +40,12 @@ nav:
   - Models:
     - models/supported_models.md
   - Features:
-    - Sleep Mode: features/sleep_mode.md
+    - Overview: features/README.md
-    - Diffusion Features:
+    - Runtime and Stage Execution:
-      - Overview: user_guide/diffusion_features.md
+      - Execution Modes and Streaming: user_guide/diffusion/execution_modes.md
-      - Feature Compatibility: user_guide/feature_compatibility.md
+      - Sleep Mode: features/sleep_mode.md
-      - Regional Compilation: user_guide/diffusion/regional_compilation.md
-      - Cache Acceleration:
-        - TeaCache: user_guide/diffusion/cache_acceleration/teacache.md
-        - Cache-DiT: user_guide/diffusion/cache_acceleration/cache_dit.md
-      - Attention Backends: user_guide/diffusion/attention_backends.md
-      - Frame Interpolation: user_guide/diffusion/frame_interpolation.md
-      - Parallelism:
-        - Overview: user_guide/diffusion/parallelism/overview.md
-        - CFG Parallel: user_guide/diffusion/parallelism/cfg_parallel.md
-        - Expert Parallel: user_guide/diffusion/parallelism/expert_parallel.md
-        - Hybrid Sharded Data Parallel: user_guide/diffusion/parallelism/hsdp.md
-        - Sequence Parallel: user_guide/diffusion/parallelism/sequence_parallel.md
-        - Tensor Parallel: user_guide/diffusion/parallelism/tensor_parallel.md
-        - VAE Parallelism: user_guide/diffusion/parallelism/vae_parallelism.md
-      - CPU Offloading: user_guide/diffusion/cpu_offload.md
-      - LoRA: user_guide/diffusion/lora.md
-      - Custom Pipeline: features/custom_pipeline.md
-      - Execution Modes: user_guide/diffusion/execution_modes.md
     - Quantization:
-      - Quantization: user_guide/quantization/overview.md
+      - Overview: user_guide/quantization/overview.md
       - Online Quantization: user_guide/quantization/online.md
       - Quantized KV Cache: user_guide/quantization/quantized_kvcache.md
       - FP8 W8A8: user_guide/quantization/fp8.md
@@ -72,9 +54,43 @@ nav:
       - MXFP8 W8A8: user_guide/quantization/mxfp8.md
       - MXFP4 W4A4: user_guide/quantization/mxfp4.md
       - ModelOpt: user_guide/quantization/modelopt.md
-      - GGUF: user_guide/quantization/gguf.md
       - AutoRound: user_guide/quantization/autoround.md
       - msModelSlim: user_guide/quantization/msmodelslim.md
+    - Diffusion Acceleration:
+      - Overview and Compatibility:
+        - Overview: user_guide/diffusion_features.md
+        - Feature Compatibility: user_guide/feature_compatibility.md
+      - CPU Offloading:
+        - Overview: user_guide/diffusion/cpu_offload.md
+        - Model-Level: user_guide/diffusion/offloader/cpu_offload.md
+        - Layerwise: user_guide/diffusion/offloader/layerwise_offload.md
+        - Distributed Layerwise: user_guide/diffusion/offloader/distributed_layerwise_offload.md
+      - Cache Acceleration:
+        - TeaCache: user_guide/diffusion/cache_acceleration/teacache.md
+        - Cache-DiT: user_guide/diffusion/cache_acceleration/cache_dit.md
+      - Parallelism:
+        - Overview: user_guide/diffusion/parallelism/overview.md
+        - CFG Parallel: user_guide/diffusion/parallelism/cfg_parallel.md
+        - Expert Parallel: user_guide/diffusion/parallelism/expert_parallel.md
+        - Hybrid Sharded Data Parallel: user_guide/diffusion/parallelism/hsdp.md
+        - Sequence Parallel: user_guide/diffusion/parallelism/sequence_parallel.md
+        - Tensor Parallel: user_guide/diffusion/parallelism/tensor_parallel.md
+        - VAE Parallelism: user_guide/diffusion/parallelism/vae_parallelism.md
+      - Attention Backends:
+        - Overview: user_guide/diffusion/attention_backends.md
+        - Dense Backends: user_guide/diffusion/attention_backends/dense_backends.md
+        - TRTLLM: user_guide/diffusion/attention_backends/trtllm.md
+        - SageAttention: user_guide/diffusion/attention_backends/sage.md
+        - Hugging Face Hub: user_guide/diffusion/attention_backends/huggingface_hub.md
+        - RainFusion: user_guide/diffusion/attention_backends/rainfusion.md
+      - Regional Compilation: user_guide/diffusion/regional_compilation.md
+      - Frame Interpolation: user_guide/diffusion/frame_interpolation.md
+      - Startup and Loading: user_guide/diffusion/startup_and_loading.md
+      - LoRA: user_guide/diffusion/lora.md
+    - Experimental:
+      - Session State Manager: features/session_state_manager.md
+  - Integrations:
+    - Custom Diffusion Pipeline: features/custom_pipeline.md
     - ComfyUI: features/comfyui.md
   - Fault Tolerance:
     - Failure Modes: user_guide/fault_injection_reliability_matrix.md
@@ -108,6 +124,7 @@ nav:
           - Shared Memory Connector: design/feature/omni_connectors/shared_memory_connector.md
           - Yuanrong Store Connector: design/feature/omni_connectors/yuanrong_connector.md
           - Yuanrong Transfer Engine Connector: design/feature/omni_connectors/yuanrong_transfer_engine_connector.md
+      - Quantization: design/feature/quantization.md
       - Diffusion Acceleration:
         - Parallelism:
           - CFG-Parallel: design/feature/cfg_parallel.md
@@ -117,13 +134,17 @@ nav:
           - Sequence Parallel: design/feature/sequence_parallel.md
           - Tensor Parallel: design/feature/tensor_parallel.md
           - VAE Patch Parallelism: design/feature/vae_parallel.md
-        - Attention Backends:
+        - Attention Optimization:
+          - Backend Selection: design/feature/attention_backend_selection.md
           - Skip-Softmax: design/feature/skip_softmax.md
-        - Quantization: design/feature/quantization.md
+        - CPU Offloading:
+          - Overview: design/feature/offloader/README.md
+          - Model-Level: design/feature/offloader/cpu_offload.md
+          - Layerwise: design/feature/offloader/layerwise_offload.md
+          - Distributed Layerwise: design/feature/offloader/distributed_layerwise_offload.md
         - Cache-DiT: design/feature/cache_dit.md
         - TeaCache: design/feature/teacache.md
         - Diffusion Continuous Batching: design/feature/diffusion_continuous_batching.md
-        - Distributed Layerwise Offload: design/feature/distributed_layerwise_offload.md
     - Infrastructure and Performance:
       - Prometheus Metrics: design/metrics.md
       - Speech Generation Performance Optimizations: design/qwen3_omni_tts_performance_optimization.md
@@ -157,7 +178,8 @@ nav:
 - CLI Reference: cli
 - Community:
   - Governance: community/governance.md
-  - community/*
+  - Contact Us: community/contact_us.md
+  - Community Meetings: community/meetings.md
   - Slack: https://slack.vllm.ai
   - Blog: https://blog.vllm.ai
   - Forum: https://discuss.vllm.ai

docs/community/contact_us.md

@@ -1,5 +1,9 @@
 # Contact Us
 
-- For technical questions and feature requests, please use GitHub [Issues](https://github.com/vllm-project/vllm-omni/issues)
+- For bugs and feature requests, search or open a GitHub [issue](https://github.com/vllm-project/vllm-omni/issues).
-- For coordinating contributions and development and discussing with other users and developers, please join `sig-omni` channel in our [Slack](https://slack.vllm.ai/) or use the [vLLM Forum](https://discuss.vllm.ai/)
+- For technical and usage questions, join the `sig-omni` channel in [vLLM Slack](https://slack.vllm.ai/) or use the [vLLM Forum](https://discuss.vllm.ai/).
-- For security disclosures, please use GitHub's [Security Advisories](https://github.com/vllm-project/vllm-omni/security/advisories) feature
+- For coordinating contributions and development, use `sig-omni`, the forum, or join a [community meeting](meetings.md).
+
+## Security Disclosures
+
+Do not share vulnerability details in public GitHub issues, pull requests, discussions, or Slack channels. To arrange a private report, send a direct message to one of the [lead maintainers](governance.md#lead-maintainers) in [vLLM Slack](https://slack.vllm.ai/). If you cannot send a direct message, you may post a detail-free request in `sig-omni` asking a lead maintainer to contact you privately; do not include any vulnerability details in that request.

docs/community/governance.md

@@ -52,31 +52,11 @@ Committers have write access and merge rights. They typically have deep expertis
 - [@ZJY0516](https://github.com/ZJY0516): Diffusion attention backend, kernel fusion, and CustomOp
 - [@yuanheng-zhao](https://github.com/yuanheng-zhao): Diffusion cache, offload, and Omni Support
 
-## Diffusion Workload Division
+## Reviewer Routing
 
-This section breaks down diffusion-related responsibilities. If you have PRs touching these areas, please ping the listed owners for review.
+For path-based reviewer routing, consult [`.github/CODEOWNERS`](https://github.com/vllm-project/vllm-omni/blob/main/.github/CODEOWNERS). Detailed subsystem boundaries and safe-change guidance live in the [module design documents](../design/index.md#module-design-documents).
 
-### Runtime-related
+`CODEOWNERS` is operational routing metadata, not a governance or approval policy. Being listed does not grant committer status or merge rights, and it does not mean every listed reviewer must approve a change. The maintainer roster above is authoritative for project roles and merge rights.
-
-- Diffusion models: @RuixiangMa, @wtomin, @xuechendi
-
-### Optimization
-
-- Parallel: @RuixiangMa, @wtomin
-- Attention backend: @ZJY0516
-- Cache: @yuanheng-zhao, @RuixiangMa, @SamitHuang
-- Offload: @yuanheng-zhao
-- Intel GPU support: @xuechendi
-- Quantization: @lishunyang12, @david6666666
-- Kernel fusion / communication-computation: @ZJY0516
-
-### Docs & Test
-
-- Diffusion docs: @RuixiangMa, @wtomin
-
-## Meetings
-
-Committers hold **bi-weekly meetings** to discuss future directions and collaborations of the project.
 
 ## Committer Nomination Process
 

docs/community/meetings.md

@@ -0,0 +1,8 @@
+# Community Meetings
+
+vLLM-Omni hosts developer-facing meetings for Chinese- and English-language audiences. Schedules and access details can change, so use these maintained documents as the source of truth:
+
+- [Chinese-language meeting document](https://docs.google.com/document/d/1pdUBiS_7mdOUNDtdwy-9OUf7jsMWN324BIbps5olDME/edit?tab=t.0)
+- [English-language meeting document](https://docs.google.com/document/d/19AlD08T8uJ7wqfs8VtYCi0iGMTwlTYVVlAxFZ3S7OIg/edit?tab=t.0)
+
+Each document contains the current schedule, access details, agendas, and past meeting notes. For broader vLLM community events, see the [vLLM events calendar](https://vllm.ai/events/).

docs/community/meetups.md

@@ -1 +0,0 @@
-# Meetups

docs/community/volunteers.md

@@ -1,12 +0,0 @@
-# Volunteers for Bugfix and CI
-
-We encourage you to check current docs and [issues](https://github.com/vllm-project/vllm-omni/issues) to find possible solutions for your questions. If non of these can solve it, please propose an issue to describe your questions about bug or CI problems for developing.
-
-If you have urgent need for locating and solving bugfix or CI problems, please find community volunteers below.
-
-| Dec 4-Dec 12 | Dec 15-Dec 19 | Dec 22-Dec 26 | Dec 29- Jan 2, 2026| Jan 5-Jan 9 | Jan 12-Jan 16 |
-|----------|----------|----------|----------|----------|----------|
-| <a href="https://github.com/congw729">Conw729</a> | <a href="https://github.com/yinpeiqi">yinpeiqi</a> | <a href="https://github.com/tzhouam">tzhouam</a> | <a href="https://github.com/SamitHuang">SamitHuang</a> | <a href="https://github.com/gcanlin">gcanlin</a> | <a href="https://github.com/natureofnature">natureofnature</a> |
-| <a href="https://github.com/david6666666">david6666666</a> | <a href="https://github.com/R2-Y">R2-Y</a> | <a href="https://github.com/hsliuustc0106">hsliuustc0106</a> | <a href="https://github.com/Gaohan123">Gaohan123</a> | <a href="https://github.com/ZJY0516">ZJY0516</a> | <a href="https://github.com/qibaoyuan">qibaoyuan</a> |
-
-We kindly welcome more contributors to fix bugs and contribute new features!

docs/contributing/README.md

@@ -3,7 +3,7 @@
 Thank you for your interest in contributing to vLLM-Omni! This document provides guidelines and instructions for contributing.
 
 !!! note
-    We host weekly developer-facing online meetings to discuss milestones and updates **every Tuesday at 19:30 PDT**. Meeting link as well as the past meeting notes can be found [here](https://tinyurl.com/vllm-omni-meeting).
+    vLLM-Omni hosts developer-facing meetings for Chinese- and English-language audiences. See [Community Meetings](../community/meetings.md) for current schedules, access details, agendas, and past notes.
 
 ## Getting Started
 
@@ -122,7 +122,7 @@ Repository skills guide the agent, but they do not replace contributor judgment,
 If you encounter a bug or have a feature request, please search existing issues first to see if it has already been reported. If not, please file a new issue, providing as much relevant information as possible.
 
 !!! important
-    If you discover a security vulnerability, please report it by creating a GitHub issue with the `security` label.
+    Do not report suspected security vulnerabilities through a public issue, pull request, discussion, or Slack channel. Follow the [security disclosure instructions](../community/contact_us.md#security-disclosures) to arrange a private report.
 
 ## Pull Requests & Code Reviews
 

docs/contributing/ci/ci_settings.md

@@ -327,6 +327,96 @@ On PR builds, `upload_pipeline.py` logs `skip '…' (no changes under …)` for
 - Use **groups** for dashboard readability; keep **E2E Test** as the group name CUDA diff filtering expects for `--e2e` nightly runs on main.
 - Prefix labels by model domain: **Omni ·**, **TTS ·**, **Diffusion ·**, **Simple ·**, etc., matching existing steps.
 
+### Per-model coverage
+
+Pilot (v1): two Merge-tier (L3) jobs in `.buildkite/cuda/test-merge.yml` upload
+per-model, per-entry-mode coverage as Buildkite artifacts — "Diffusion · Bagel
+Test" and "TTS · Qwen3-TTS Base Test". Both run on a single GPU so the pilot is
+cheap to reproduce. This is a pilot, not a rollout: no dashboard/visualization
+lives in this repo, and nothing is gated on coverage.
+
+**Naming convention**: `coverage-<model_id>-<mode>-<step_id>.xml.gz`, where
+`<model_id>` is the model's directory name under
+`vllm_omni/diffusion/models/<model_id>/` or
+`vllm_omni/model_executor/models/<model_id>/` (e.g. `z_image`, `qwen3_omni`),
+`<mode>` is `online` or `offline`, and `<step_id>` is `BUILDKITE_STEP_ID`
+(`local` outside Buildkite). `gunzip` before feeding a report to a Cobertura
+consumer.
+
+Reports are gzipped because `--cov=vllm_omni` sets coverage's `source`, which
+makes it walk the package and emit one `<line>` element per statement for every
+file — including the ones a single model's tests never import. That inventory,
+not the coverage, is the size: a Bagel report is 7.1 MB, of which 96.5% is
+`<line>` elements and 77% belongs to the 840 of 1169 files that run never
+imported. A run covering zero lines produces the same 7 MB. gzip takes it to
+roughly 410 KB and changes nothing about the data.
+
+`<step_id>` is what keeps two steps that cover the same model apart. Artifacts are
+scoped to the build, and a nightly build carries the ready, merge and nightly tiers
+at once, so several same-model steps land in one namespace — `qwen3_tts` already has
+both a Base and a CustomVoice job, and `minicpmo_4_5` a base and a duplex one. Two
+uploads on one path do not fail loudly: an exact-name download can report the path
+as ambiguous, and a glob download fetches every match concurrently and renames each
+onto the same destination, so whichever finishes last wins. The step id remains the
+logical step identifier across retries, while Buildkite's default artifact lookup
+selects only the latest attempt — `BUILDKITE_JOB_ID` would instead give every
+attempt its own filename, leaving a failed attempt's partial report to be picked up
+alongside the real one. A step using `parallelism`/matrix would need
+`BUILDKITE_PARALLEL_JOB` added, since its jobs share a step id; neither pilot is
+one.
+
+**Opting in a new model**: replace the job's combined `pytest` command with
+[`run_cov_split.sh`](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/common/scripts/run_cov_split.sh),
+which runs each mode, names the reports, uploads them, and fails the step if
+either half or the upload failed:
+
+```yaml
+commands:
+  - |
+    .buildkite/common/scripts/run_cov_split.sh \
+      --model-id <model_id> \
+      --offline <offline test path> \
+      --online <online test path> \
+      -- <the job's existing pytest flags, e.g. -m '...' --run-level '...'>
+```
+
+Everything after `--` is forwarded verbatim to both runs with quoting preserved,
+so an expression like `-m 'advanced_model and cuda'` survives intact and a job can
+pass anything else it needs. `--offline`/`--online` are repeatable and only one is
+required, so a job with tests in a single mode runs just that half. Runtime is
+bounded solely by the step's `timeout_in_minutes` — raise it, since the split
+loads the model once per mode.
+
+Before splitting, confirm each half actually collects at least one test under the job's forwarded filters — pytest exits with code 5 ("no tests collected") if a half is empty, which the script reports as a failure, red-ing a job that used to pass as one combined invocation.
+
+Also check the tests' `num_cards` against the job's `mirror_hardwares` preset.
+`cuda_marks` turns `num_cards > 1` into `skipif(device_count() < num_cards)`, so a
+test asking for more GPUs than the preset provides is silently skipped and its
+coverage artifact comes back empty. Splitting such a job produces a file that
+looks fine and measures nothing — "Diffusion · Z Image Test" is the current
+example (online tests want 4 cards, the job runs on `l4_1`), which is why it is
+not a coverage pilot.
+
+Online-mode coverage depends on the `[tool.coverage.run]` `patch`/`sigterm`
+settings in `pyproject.toml`: online tests launch the server with
+`subprocess.Popen` and stop it with SIGTERM, and without those settings the XML
+reflects only the pytest parent process, not the server's code paths.
+
+The upload depends on `buildkite-agent` being callable inside the container. The
+`kubernetes` presets (`h100_*`, `*_npu_*`) provide it; the `docker` ones (`l4_*`)
+only do because they set `mount-buildkite-agent: true`. A new docker preset that
+runs a coverage job needs the same.
+
+List both `run_cov_split.sh` and `pyproject.toml` in every opted-in job's
+`source_file_dependencies` — both change what the job measures, so without them a
+change there is filtered out of normal PR builds and only surfaces in a later
+nightly. `tests/buildkite/test_upload_pipeline.py` asserts this for the pilots.
+Editing only the surrounding CI YAML still does not schedule them, so a PR that
+touches just the wiring needs a full E2E run (or the commands run on a GPU host)
+to produce artifacts. When checking a new model's
+artifacts, compare `lines-covered` between the online and offline XML rather than
+just confirming both files exist.
+
 ### Validation checklist
 
 | Check | Command / location |

docs/design/architecture_overview.md

@@ -406,7 +406,7 @@ the source for configuration, compatibility, and implementation details.
   stage contract.
 * **Quantization and memory efficiency:** [Quantization](feature/quantization.md)
   resolves per-pipeline or per-component quantization configurations, while
-  [Distributed Layerwise Offload](feature/distributed_layerwise_offload.md)
+  [Distributed Layerwise Offload](feature/offloader/distributed_layerwise_offload.md)
   streams diffusion blocks from host memory within the existing parallel
   topology.
 

docs/design/feature/attention_backend_selection.md

@@ -0,0 +1,89 @@
+# Diffusion Attention Backend Selection
+
+This document defines the selection and extension contract for diffusion
+attention backends. User-facing backend choices, installation, and tuning are
+in the [attention backend guides](../../user_guide/diffusion/attention_backends.md).
+
+## Scope
+
+The contract applies to DiT and other diffusion attention layers. It is
+separate from vLLM's autoregressive attention selector.
+
+The selection path has four responsibilities:
+
+1. normalize user configuration into `AttentionConfig` and `AttentionSpec`;
+2. resolve a spec for an attention role;
+3. ask the active platform to validate an explicit backend or choose a
+   hardware default; and
+4. load the selected `AttentionBackend` class from the registry path.
+
+## Resolution contract
+
+`get_attn_backend_for_role()` resolves configuration in this order:
+
+1. exact `per_role[role]`;
+2. category `per_role[role_category]`;
+3. `default`;
+4. platform default.
+
+An explicit resolution returns both the backend class and its
+`AttentionSpec`. A platform-default resolution returns the class and `None`.
+Layers must therefore treat the spec as optional and must not infer that a
+missing spec means SDPA.
+
+Class resolution is cached by backend name, head size, and whether TRTLLM may
+be selected automatically. Logging is cached separately by role and source so
+the startup record identifies why each role received its backend.
+
+## Registry and platform boundary
+
+`DiffusionAttentionBackendEnum` maps stable configuration names to default
+qualified class paths. `register_diffusion_backend()` may replace a path at
+runtime without changing the public enum value.
+
+The active `OmniPlatform` owns compatibility policy through
+`get_diffusion_attn_backend_cls()`. It must:
+
+- validate explicit selections and fail with an actionable error when the
+  requested kernel cannot run;
+- choose only an available, compatible default when no backend is explicit;
+- account for head size and other platform-visible constraints; and
+- return a qualified class path implementing `AttentionBackend`.
+
+The selector must not duplicate device capability or package-availability
+policy that belongs to the platform.
+
+## Typed backend options
+
+Backend-specific settings remain typed fields on `AttentionSpec` rather than
+unstructured keyword dictionaries:
+
+- `quant` is consumed by FlashInfer and TRTLLM with backend-specific value
+  validation;
+- `skip_softmax` is consumed by TRTLLM; and
+- `block_sparse` is consumed by block-sparse backends such as RainFusion.
+
+A backend reads only the fields it owns and rejects incompatible values. New
+options should be added to a shared typed spec only when more than one backend
+shares their semantics; otherwise add a dedicated typed block.
+
+## Model integration contract
+
+Each diffusion attention call declares a stable role and, when appropriate, a
+broader category. Model-specific roles permit precise overrides; categories
+preserve common `self` or `cross` policies. A model also decides whether its
+path is compatible with automatic TRTLLM selection, because only the model
+knows whether masking and packing satisfy the kernel contract.
+
+## Adding or changing a backend
+
+An implementation change is complete only when it:
+
+1. implements the `AttentionBackend` interface and exposes a stable enum name;
+2. adds platform validation and default routing where appropriate;
+3. defines and validates any typed configuration;
+4. tests explicit selection, default selection, and incompatible paths;
+5. documents installation, fallback behavior, and quality implications in the
+   corresponding user guide; and
+6. updates the overview matrix without moving algorithm details back into the
+   landing page.

docs/design/feature/offloader/README.md

@@ -0,0 +1,80 @@
+# CPU Offloading
+
+This document defines the shared architecture for diffusion CPU offloading.
+The three strategies have separate user and design pages:
+
+| Strategy | User guide | Design contract |
+| --- | --- | --- |
+| Model-level | [Guide](../../../user_guide/diffusion/offloader/cpu_offload.md) | [Design](cpu_offload.md) |
+| Layerwise | [Guide](../../../user_guide/diffusion/offloader/layerwise_offload.md) | [Design](layerwise_offload.md) |
+| Distributed layerwise | [Guide](../../../user_guide/diffusion/offloader/distributed_layerwise_offload.md) | [Design](distributed_layerwise_offload.md) |
+
+## Strategy selection
+
+`OffloadConfig.from_od_config()` converts diffusion configuration into one
+`OffloadStrategy`. If callers set multiple flags, selection is deterministic:
+
+1. `DISTRIBUTED_LAYERWISE`;
+2. `LAYERWISE`;
+3. `MODEL_LEVEL`; and
+4. `NONE`.
+
+The factory logs when a higher-priority strategy disables another. It derives
+parallel and HSDP state from `DiffusionParallelConfig`; callers do not provide
+a separate offload group size.
+
+`get_offload_backend()` then validates platform offload support, resolves the
+device, and creates exactly one backend. Returning `None` means offloading is
+disabled or unsupported and must not leave partially installed hooks.
+
+## Shared lifecycle
+
+Every backend implements `OffloadBackend`:
+
+- `enable(pipeline)` discovers modules, establishes initial residency, and
+  installs hooks;
+- `disable()` removes owned hooks and resources; and
+- `is_enabled()` reports lifecycle state.
+
+`disable()` does not promise to restore the pipeline's original device
+placement. The caller owns any subsequent rematerialization.
+
+Hooks are registered through `HookRegistry` and `ModelHook`; offload backends
+must use distinct hook names and remove only hooks they own.
+
+## Discovery and topology
+
+Pipeline component discovery prefers `SupportsComponentDiscovery` declarations:
+
+- `_dit_modules`;
+- `_encoder_modules`;
+- `_vae_modules`; and
+- `_resident_modules`.
+
+Dotted paths are supported. Legacy pipelines may use the fallback scan of
+well-known attribute names, but new integrations should declare components
+explicitly.
+
+Ordinary layerwise offload obtains topology from declared
+`_layerwise_offload_blocks_attrs` (with a deprecated singular-name
+compatibility path). Distributed layerwise offload may instead use a pipeline
+`OffloadPlan`, then falls back to declared block attributes and its documented
+compatibility discovery. Discovery metadata describes structure only; the
+backend remains responsible for transfer, synchronization, and storage
+ownership.
+
+## Cross-strategy invariants
+
+- At most one strategy owns offload hooks for a pipeline.
+- A parameter has one authoritative host representation while offloaded.
+- Device storage is not freed until dependent compute or transfer work is
+  complete.
+- Non-persistent and model-specific buffers remain correct after movement or
+  checkpoint rematerialization.
+- Unsupported parallel or loading combinations fail before hooks mutate the
+  model.
+- Platform streams, events, synchronization, and cache management go through
+  the vLLM-Omni platform abstraction.
+
+The diffusion [Offloader module design](../../module/diffusion/offloader.md)
+describes how these feature contracts fit into the larger diffusion runtime.

docs/design/feature/offloader/cpu_offload.md

@@ -0,0 +1,56 @@
+# Model-Level Offload
+
+Model-level offload enforces mutual exclusion between pipeline component
+groups. For commands and model support, see the
+[model-level user guide](../../../user_guide/diffusion/offloader/cpu_offload.md).
+
+## Generic hook path
+
+`ModelLevelOffloadBackend.enable()` discovers DiTs, encoders, and VAEs. It
+makes encoders and VAEs device resident for initialization, then installs
+`SequentialOffloadHook` instances:
+
+- each DiT hook offloads all encoders and other DiTs before loading itself;
+- each encoder hook offloads every DiT before loading itself; and
+- VAEs are not part of the mutual-exclusion hook set.
+
+The hook moves parameters and buffers without recursively calling
+`module.to()`. This avoids recursion through modules that retain references to
+other transformer blocks. Host tensors are pinned when configured; DTensors
+are not pinned through the ordinary tensor path.
+
+Before forward returns control to the model, the hook synchronizes the active
+platform so compute cannot race the transfer. XPU uses blocking device-to-host
+copies because its allocator cannot safely evict storage while an asynchronous
+copy is pending.
+
+## Split-model delegation
+
+If a pipeline provides callable `enable_omni_model_cpu_offload`, the backend
+delegates instead of installing generic encoder/DiT hooks. This supports models
+whose mutually exclusive components are nested inside one transformer, such
+as a reasoner and generator.
+
+The delegated pipeline owns its internal contexts, but must preserve the same
+contract:
+
+- only the executing component is device resident;
+- enable and disable are idempotent;
+- transfers use the supplied device, pinning, and HSDP settings; and
+- disabling removes all model-local offload hooks or contexts.
+
+The delegation is currently duck typed. If another independent implementation
+is added, formalize the enable/disable pair as a protocol rather than expanding
+attribute-name heuristics.
+
+## Extension invariants
+
+- New ordinary pipelines declare both DiT and encoder groups through
+  `SupportsComponentDiscovery`.
+- Models with multiple DiTs offload the inactive DiTs as well as encoders.
+- Empty or parameterless modules do not trigger transfers.
+- Hook removal does not move modules; callers must not assume original
+  residency is restored.
+
+Shared selection and lifecycle behavior is defined in the
+[CPU Offloading design](README.md).

docs/design/feature/offloader/distributed_layerwise_offload.md

@@ -7,7 +7,7 @@ either shard those host-side weights across an existing parallel group or keep
 the standard loader's rank-local weights and avoid an additional collective.
 
 For user-facing commands, see the
-[distributed layerwise offloading guide](../../user_guide/diffusion/cpu_offload.md)
+[distributed layerwise offloading guide](../../../user_guide/diffusion/offloader/distributed_layerwise_offload.md)
 and the [Cosmos3 recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/cosmos3/Cosmos3-DistOffload.md).
 
 ## Status
@@ -56,7 +56,7 @@ AllGather:                [full N+1]             [full N+2]
 Buffers:    [current slot]       [prefetch slot]       [current slot]
 ```
 
-![DLO double-buffer prefetch pipeline](../figures/dlo/dlo_pipeline.gif)
+![DLO double-buffer prefetch pipeline](../../figures/dlo/dlo_pipeline.gif)
 
 The backend uses two shared device buffers, so accelerator weight residency is
 bounded by the largest streamed blocks rather than the complete model.

docs/design/feature/offloader/layerwise_offload.md

@@ -0,0 +1,57 @@
+# Layerwise Offload
+
+Layerwise offload streams transformer blocks through one accelerator. For
+commands and model support, see the
+[layerwise user guide](../../../user_guide/diffusion/offloader/layerwise_offload.md).
+
+## Hook ring
+
+`LayerWiseOffloadBackend` discovers every streamable block and registers one
+`LayerwiseOffloadHook` per block. Hooks form a ring: each hook owns the pinned
+host representation of its registered block and a reference to the next block
+it must prefetch; the final hook prefetches the first block for the next
+denoising iteration.
+
+During initialization, parameters and buffers are grouped by dtype and copied
+into contiguous pinned CPU tensors. Original tensor storage is replaced with
+an empty or meta placeholder while identity and module structure are retained.
+Metadata records names, offsets, shapes, and dtypes for rematerialization.
+
+## Forward lifecycle
+
+For each block:
+
+1. the previous hook's completion event ensures this block is materialized;
+2. the copy stream asynchronously allocates and fills the next block's device
+   tensors;
+3. parameter and buffer storage is rebound to views of those tensors;
+4. the current block computes on the normal stream; and
+5. its post-forward hook replaces device storage with placeholders after all
+   dependent work is safe.
+
+The copy stream waits on the compute stream before reusing storage, and the
+compute stream waits on the per-block prefetch event before forward. These
+dependencies, not incidental global synchronization, define correctness.
+
+## Topology contract
+
+A DiT declares ordered block-container attributes using
+`_layerwise_offload_blocks_attrs` or an `OffloadPlan`. Each resolved item must
+be an executable `nn.Module`; ordering must match forward execution. Multiple
+containers are concatenated in execution order.
+
+Non-block DiT modules, encoders, VAEs, and declared resident modules are not
+streamed by this backend.
+
+## Invariants and limitations
+
+- Each offloaded tensor has one pinned host backing store.
+- Parameter identity is preserved when storage is rebound.
+- Blocks with heterogeneous shapes and dtypes use their recorded metadata;
+  implementations must not assume identical blocks.
+- The ring is initialized before the first forward so block zero is available.
+- This backend is single-device; distributed weight reconstruction belongs to
+  [Distributed Layerwise Offload](distributed_layerwise_offload.md).
+
+Shared selection and lifecycle behavior is defined in the
+[CPU Offloading design](README.md).

docs/design/feature/quantization.md

@@ -186,7 +186,7 @@ Distributed layerwise offload has an additional boundary: its sharded mmap
 AllGather path is not compatible with online quantization because the path
 expects already materialized checkpoint weights. Use the rank-local
 `--dlo-no-use-allgather` path or a compatible pre-quantized checkpoint when
-combining these features. See the [DLO design document](distributed_layerwise_offload.md)
+combining these features. See the [DLO design document](offloader/distributed_layerwise_offload.md)
 for the full compatibility matrix.
 
 ## Adding a quantization backend

docs/design/feature/skip_softmax.md

@@ -2,8 +2,10 @@
 
 Skip-Softmax is the sparse-attention mode of the `TRTLLM_ATTN` backend. Usage — the config keys
 and how to pick an operating point — is in
-[Diffusion Attention Backends](../../user_guide/diffusion/attention_backends.md). This page explains
+[TRTLLM Attention](../../user_guide/diffusion/attention_backends/trtllm.md#skip-softmax).
-the algorithm.
+The shared selector contract is documented in
+[Diffusion Attention Backend Selection](attention_backend_selection.md). This
+page explains the algorithm.
 
 ## Motivation
 

docs/design/index.md

@@ -11,6 +11,10 @@ in the active navigation.
 
 ## Feature Design Documents
 
+For user-facing configuration and current compatibility, see the
+[Features overview](../features/README.md). A design document defines an
+implementation contract; it is not, by itself, a general support claim.
+
 ### Runtime and stage execution
 
 - [Disaggregated Inference](feature/disaggregated_inference.md)
@@ -30,6 +34,10 @@ in the active navigation.
 - [Yuanrong Store Connector](feature/omni_connectors/yuanrong_connector.md)
 - [Yuanrong Transfer Engine Connector](feature/omni_connectors/yuanrong_transfer_engine_connector.md)
 
+### Quantization
+
+- [Quantization](feature/quantization.md)
+
 ### Diffusion acceleration
 
 #### Parallelism
@@ -42,18 +50,25 @@ in the active navigation.
 - [Tensor Parallel](feature/tensor_parallel.md)
 - [VAE Patch Parallelism](feature/vae_parallel.md)
 
-#### Attention Backends
+#### Attention optimization
 
+The [Diffusion Attention Backends](../user_guide/diffusion/attention_backends.md)
+guides list selectable backends, platform defaults, installation, and tuning.
+The design contracts separate selection mechanics from backend algorithms:
+
+- [Attention Backend Selection](feature/attention_backend_selection.md)
 - [Skip-Softmax](feature/skip_softmax.md)
 
-#### Quantization
+#### CPU offloading
 
-- [Quantization](feature/quantization.md)
+- [Overview and Shared Contracts](feature/offloader/README.md)
+- [Model-Level Offload](feature/offloader/cpu_offload.md)
+- [Layerwise Offload](feature/offloader/layerwise_offload.md)
+- [Distributed Layerwise Offload](feature/offloader/distributed_layerwise_offload.md)
 
 - [Cache-DiT](feature/cache_dit.md)
 - [TeaCache](feature/teacache.md)
 - [Diffusion Continuous Batching](feature/diffusion_continuous_batching.md)
-- [Distributed Layerwise Offload](feature/distributed_layerwise_offload.md)
 
 ## Infrastructure and Performance
 

docs/design/metrics.md

@@ -1,6 +1,6 @@
 # Prometheus Metrics Design
 
-This document describes how vLLM-Omni exposes Prometheus metrics for multi-stage pipelines, the constraints that shaped the design, and how the pipeline-level metrics coexist with upstream vLLM per-engine metrics. This iteration covers the text and audio surface; diffusion / image / video families land in a follow-up.
+This document describes how vLLM-Omni exposes Prometheus metrics for multi-stage pipelines, the constraints that shaped the design, and how the pipeline-level metrics coexist with upstream vLLM per-engine metrics.
 
 ## Objectives
 

docs/features/README.md

@@ -0,0 +1,67 @@
+# Features
+
+Use this section to choose and configure vLLM-Omni runtime and optimization
+features. The navigation follows the active
+[Feature Design](../design/index.md#feature-design-documents) taxonomy whenever
+an implementation contract has a user-facing workflow.
+
+!!! note "User guides, design documents, and recipes"
+
+    User guides explain how to enable a feature and where it is supported.
+    Design documents define internal contracts and do not, by themselves,
+    imply general or production support. For model-specific launch flags and
+    hardware requirements, use the linked examples and validated recipes from
+    [Supported Models](../models/supported_models.md).
+
+## Runtime and Stage Execution
+
+| Goal | User guide | Related design contract |
+| --- | --- | --- |
+| Choose serial, batched, step-wise, or streaming diffusion execution | [Execution Modes and Streaming](../user_guide/diffusion/execution_modes.md) | [Diffusion Continuous Batching](../design/feature/diffusion_continuous_batching.md), [Async Diffusion Output](../design/feature/async_diffusion_output.md) |
+| Reclaim stage memory without restarting the server | [Sleep Mode](sleep_mode.md) | Runtime lifecycle behavior is documented in the user guide |
+
+Some runtime designs are deliberately not promoted as standalone User Guide
+features yet:
+
+- [Disaggregated Inference](../design/feature/disaggregated_inference.md) and
+  [OmniConnector implementations](../design/index.md#communication) describe
+  topology and transport contracts. Practical deployment configuration remains
+  under [Pipeline and deploy configurations](../configuration/stage_configs.md).
+- [Async Chunk](../design/feature/async_chunk.md) and
+  [Async Omni Output Materialization](../design/feature/omni_async_output_materialization.md)
+  are model- and pipeline-dependent. Use the selected model's deploy
+  configuration and recipe for the supported settings.
+- [Automatic Prefix Caching](../design/feature/prefix_caching.md) remains a
+  design-level contract until its user-facing configuration and compatibility
+  surface is consolidated.
+
+## Quantization
+
+Quantization is a cross-model feature rather than a diffusion-only
+optimization. The unified [`quantization_config` guide](../user_guide/quantization/overview.md)
+covers diffusion-only models, multi-stage omni/TTS models, and multi-stage
+diffusion models. Its [design contract](../design/feature/quantization.md)
+defines the shared configuration and backend extension points.
+
+## Diffusion Acceleration
+
+| Goal | User guide | Related design contract |
+| --- | --- | --- |
+| Compare acceleration methods and supported combinations | [Overview](../user_guide/diffusion_features.md), [Feature Compatibility](../user_guide/feature_compatibility.md) | [Diffusion acceleration designs](../design/index.md#diffusion-acceleration) |
+| Move weights between host and device memory | [CPU Offloading](../user_guide/diffusion/cpu_offload.md) | [CPU Offloading](../design/feature/offloader/README.md) |
+| Reuse denoising computation | [Cache Acceleration](../user_guide/diffusion/cache_acceleration/cache_dit.md) | [Cache-DiT](../design/feature/cache_dit.md), [TeaCache](../design/feature/teacache.md) |
+| Distribute diffusion work across devices | [Parallelism](../user_guide/diffusion/parallelism/overview.md) | [Parallelism designs](../design/index.md#parallelism) |
+| Select dense, sparse, or quantized attention paths | [Attention Backends](../user_guide/diffusion/attention_backends.md) | [Attention Backend Selection](../design/feature/attention_backend_selection.md) |
+| Compile repeated diffusion regions | [Regional Compilation](../user_guide/diffusion/regional_compilation.md) | User-facing optimization guide |
+| Add generated video frames | [Frame Interpolation](../user_guide/diffusion/frame_interpolation.md) | User-facing extension guide |
+| Reduce diffusion model startup time | [Startup and Loading](../user_guide/diffusion/startup_and_loading.md) | User-facing loading guide |
+| Apply diffusion adapters | [LoRA](../user_guide/diffusion/lora.md) | User-facing extension guide |
+
+The [Pipeline Parallelism guide](../user_guide/diffusion/parallelism/pipeline_parallel.md)
+remains available by direct link, but it is not promoted in the primary User
+Guide navigation until its user-facing support and placement are settled.
+
+## Experimental
+
+[Session State Manager](session_state_manager.md) is opt-in and experimental.
+Its APIs and compatibility may change without notice.

docs/models/supported_models.md

@@ -42,7 +42,8 @@ th {
 | `Wan22VACEPipeline` | Wan2.1-VACE | `Wan-AI/Wan2.1-VACE-1.3B-diffusers`, `Wan-AI/Wan2.1-VACE-14B-diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
 | `Wan22VACEPipeline` | Wan2.2-VACE | `Pyros13/Wan2.2-VACE-Fun-A14B-Diffusers` | ✅︎ |   |   |   | — |
 | `LTX2Pipeline` | LTX-2 / LTX-2.3 one-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | | — |
-| `LTX2DistilledPipeline` | LTX-2 distilled two-stage T2V and I2V | `rootonchair/LTX-2-19b-distilled` | ✅︎ | ✅︎ | | | — |
+| `LTX2TwoStagePipeline` | LTX-2 / LTX-2.3 ordinary two-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` + matching Lightricks LoRA and upsampler | ✅︎ | ✅︎ | | | — |
+| `LTX2DistilledPipeline` | LTX-2 / LTX-2.3 full-distilled two-stage T2V and I2V | `rootonchair/LTX-2-19b-distilled`, `diffusers/LTX-2.3-Distilled-Diffusers` + `Lightricks/LTX-2.3` upsampler | ✅︎ | ✅︎ | | | — |
 | `LingBotVideoPipeline` | LingBot-Video dense and MoE T2I, T2V, TI2V | `robbyant/lingbot-video-dense-1.3b`, `robbyant/lingbot-video-moe-30b-a3b` | ✅︎ | | | | — |
 | `MiniMaxH3Pipeline` | MiniMax H3 T2VA, first/last-keyframe FL2VA, and mixed-reference Ref2VA | `MiniMaxAI/MiniMax-H3` | ✅︎ | ✅︎ | | | [Published](https://recipes.vllm.ai/MiniMaxAI/MiniMax-H3) |
 | `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
@@ -58,6 +59,7 @@ th {
 | `CosyVoice3Model` | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `OmniVoiceModel` | OmniVoice | `k2-fsa/OmniVoice` | ✅︎ | | | | — |
 | `VoxCPM2TalkerForConditionalGeneration` | VoxCPM2 | `openbmb/VoxCPM2` | ✅︎ | | | | — |
+| `DotsTTSForConditionalGeneration` | dots.tts | `rednote-hilab/dots.tts-soar` | ✅︎ | | | | — |
 | `MammothModa2ForConditionalGeneration` | MammothModa2-Preview | `bytedance-research/MammothModa2-Preview` | ✅︎ | ✅︎ | | | — |
 | `MammothModa2ForConditionalGeneration` | MammothModa2-Dev (AR-only image understanding) | `bytedance-research/MammothModa2-Dev` | ✅︎ | | | | — |
 | `Flux2KleinPipeline` | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B`, `black-forest-labs/FLUX.2-klein-9B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
@@ -80,6 +82,7 @@ th {
 | `HiggsAudioV2ForConditionalGeneration` | Higgs-Audio v2 | `bosonai/higgs-audio-v2-generation-3B-base` | ✅︎ | | | | — |
 | `HiggsMultimodalQwen3ForConditionalGeneration` | Higgs-Audio v3 (TTS) | `bosonai/higgs-audio-v3-tts-4b` | ✅︎ | | | | [Published](https://recipes.vllm.ai/bosonai/higgs-audio-v3-tts-4b) |
 | `IndexTTS2TalkerForConditionalGeneration` | IndexTTS-2 | `IndexTeam/IndexTTS-2` | ✅︎ | | | | — |
+| `IndexTTS25TalkerForConditionalGeneration` | IndexTTS-2.5 | native `checkpoints/` bundle | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/IndexTeam/IndexTTS-2_5.md) |
 | `NextStep11Pipeline` | NextStep-1.1 | `stepfun-ai/NextStep-1.1` | ✅︎ | ✅︎ | | ✅︎ | — |
 | `MiMoAudioModel` | MiMo-Audio-7B-Instruct | `XiaomiMiMo/MiMo-Audio-7B-Instruct` | ✅︎ | ✅︎ | | | — |
 | `MiMoV2ASRForCausalLM` | MiMo-V2.5-ASR | `XiaomiMiMo/MiMo-V2.5-ASR` | ✅︎ | ✅︎ | | | — |

docs/usage/faq.md

@@ -10,12 +10,12 @@ A: Refer to [GPU memory calculation and configuration](../configuration/gpu_memo
 
 > Q: I encounter some bugs or CI problems, which is urgent. How can I solve it?
 
-A: At first, you can check current [issues](https://github.com/vllm-project/vllm-omni/issues) to find possible solutions. If none of these satisfy your demand and it is urgent, please find these [volunteers](https://docs.vllm.ai/projects/vllm-omni/en/latest/community/volunteers/) for help.
+A: Search the current [issues](https://github.com/vllm-project/vllm-omni/issues) first. If no existing issue matches a bug or CI failure, open a [bug report](https://github.com/vllm-project/vllm-omni/issues/new?template=400-bug-report.yml) with a minimal reproducer, environment details, and relevant logs. For technical or usage questions, use the `sig-omni` channel in [vLLM Slack](https://slack.vllm.ai/) or the [vLLM Forum](https://discuss.vllm.ai/).
 
 > Q: Does vLLM-Omni support AWQ or any other quantization?
 
-A: We plan to introduce GGUF FP8 prequantized models and online FP8 quantization in version 0.16.0. Support for other quantization types will follow in future releases. For details, please see our [Q1 quantization roadmap](https://github.com/vllm-project/vllm-omni/issues/1057).
+A: AWQ is available through the inherited vLLM quantization registry, and vLLM-Omni supports several additional methods. Actual support and validation vary by model, component, hardware, and checkpoint. See the [quantization overview](../user_guide/quantization/overview.md) for validated combinations and method-specific guides.
 
 > Q: Does vLLM-Omni support multimodal streaming input and output?
 
-A: Not yet. We already put it on the [Roadmap](https://github.com/vllm-project/vllm-omni/issues/165). Please stay tuned!
+A: Yes, for specific models and serving paths. Current capabilities include [streaming video input](../serving/video_stream_api.md), [streaming diffusion output](../user_guide/diffusion/execution_modes.md#streaming-output), and experimental full-duplex real-time audio described in the [project overview](../README.md). Check the model and API documentation because input and output support is model-specific.

docs/usage/metrics.md

@@ -1,23 +1,23 @@
 # Production Metrics
 
-vLLM-Omni exposes Prometheus metrics via the `/metrics` endpoint on the OpenAI-compatible API server. This page covers the text and audio surface; diffusion / image / video metrics are tracked in a follow-up PR.
+vLLM-Omni exposes Prometheus metrics via the `/metrics` endpoint on the OpenAI-compatible API server.
 
 ```bash
 vllm-omni serve Qwen/Qwen3-Omni-30B-A3B-Instruct --port 8000 --log-stats
 curl http://localhost:8000/metrics
 ```
 
-**`--log-stats` is required to populate metric data.** Without the flag, the endpoint still returns `200 OK` but the upstream `vllm:*` wrap is not registered at all, and the 15 `vllm:omni_*` families are registered as placeholders with no data written to them. This keeps the runtime cost essentially zero for deployments that don't need monitoring. With the flag, all ~80 families populate.
+**`--log-stats` is required to populate metric data.** Without the flag, the endpoint still returns `200 OK` but the upstream `vllm:*` wrap is not registered at all, and the 15 `vllm_omni:*` families are registered as placeholders with no data written to them. This keeps the runtime cost essentially zero for deployments that don't need monitoring. With the flag, all ~80 families populate.
 
 ## Metric Namespaces
 
 | Prefix | Source | Present when |
 |--------|--------|--------------|
-| `vllm:omni_` | vLLM-Omni orchestrator / audio modality / cross-stage transfer | Pipeline-dependent |
+| `vllm_omni:` | vLLM-Omni orchestrator / audio modality / cross-stage transfer | Pipeline-dependent |
 | `vllm:` | Upstream vLLM engine, wrapped by `OmniPrometheusStatLogger` to expose `{stage, replica}` | Pipeline includes an LLM (AR) stage |
 | `http_` / `process_` | Uvicorn / Python runtime | Always |
 
-## Pipeline-Level Metrics (`vllm:omni_`)
+## Pipeline-Level Metrics (`vllm_omni:`)
 
 Defined in `vllm_omni/metrics/prometheus.py`. Track request lifecycle across the full multi-stage pipeline.
 
@@ -25,42 +25,66 @@ Defined in `vllm_omni/metrics/prometheus.py`. Track request lifecycle across the
 
 | Metric | Type | Labels | Description |
 |--------|------|--------|-------------|
-| `vllm:omni_num_requests_running` | Gauge | `model_name` | Pipeline-global in-flight requests (dispatched to engine, not yet finalized) |
+| `vllm_omni:num_requests_running` | Gauge | `model_name` | Pipeline-global in-flight requests (dispatched to engine, not yet finalized) |
-| `vllm:omni_num_requests_waiting` | Gauge | `model_name` | Requests waiting in the Orchestrator queue |
+| `vllm_omni:num_requests_waiting` | Gauge | `model_name` | Requests waiting in the Orchestrator queue |
-| `vllm:omni_requests_success_total` | Counter | `model_name`, `finished_reason` | Total requests by completion reason. `finished_reason` ∈ {`stop`, `length`, `abort`, ...} mirroring upstream `vllm:request_success_total`; aborts cover client disconnect / cancellation paths in addition to upstream `FinishReason.ABORT` |
+| `vllm_omni:requests_success_total` | Counter | `model_name`, `finished_reason` | Total requests by completion reason. `finished_reason` ∈ {`stop`, `length`, `abort`, ...} mirroring upstream `vllm:request_success_total`; aborts cover client disconnect / cancellation paths in addition to upstream `FinishReason.ABORT` |
 
 ### Latency
 
 | Metric | Type | Labels | Description |
 |--------|------|--------|-------------|
-| `vllm:omni_e2e_request_latency_s` | Histogram | `model_name` | Pipeline-global end-to-end request latency in seconds |
+| `vllm_omni:e2e_request_latency_s` | Histogram | `model_name` | Pipeline-global end-to-end request latency in seconds |
 
-## Audio Modality Metrics (`vllm:omni_`)
+## Audio Modality Metrics (`vllm_omni:`)
 
 Emitted at request finalize, except for `audio_ttfp_s` (streaming-hook at the first audio packet) and `audio_underrun_s` / `audio_continuity_ok_total` (streaming finalize, after the chunk stream is exhausted). All carry `{model_name, stage, replica}` plus the listed extra label.
 
 | Metric | Type | Extra label | Description |
 |--------|------|-------------|-------------|
-| `vllm:omni_audio_ttfp_s` | Histogram | — | Time from request arrival to first audio packet/frame |
+| `vllm_omni:audio_ttfp_s` | Histogram | — | Time from request arrival to first audio packet/frame |
-| `vllm:omni_audio_duration_s` | Histogram | — | Audio content duration (`audio_frames / sample_rate`) |
+| `vllm_omni:audio_duration_s` | Histogram | — | Audio content duration (`audio_frames / sample_rate`) |
-| `vllm:omni_audio_rtf` | Histogram | — | Real-time factor (`stage_gen_time_s / audio_duration_s`); streaming TTS SLO red line `< 1`; uses `RTF_BUCKETS` |
+| `vllm_omni:audio_rtf` | Histogram | — | Real-time factor (`stage_gen_time_s / audio_duration_s`); streaming TTS SLO red line `< 1`; uses `RTF_BUCKETS` |
-| `vllm:omni_audio_frames_total` | Counter | — | Cumulative audio frame count; throughput via `rate()` |
+| `vllm_omni:audio_frames_total` | Counter | — | Cumulative audio frame count; throughput via `rate()` |
-| `vllm:omni_audio_underrun_s` | Histogram | — | Per-request worst-case player deficit; `> 0` indicates listener heard silent gaps |
+| `vllm_omni:audio_underrun_s` | Histogram | — | Per-request worst-case player deficit; `> 0` indicates listener heard silent gaps |
-| `vllm:omni_audio_continuity_ok_total` | Counter | `threshold_ms` | Incremented when the request's worst underrun stayed below `threshold_ms` |
+| `vllm_omni:audio_continuity_ok_total` | Counter | `threshold_ms` | Incremented when the request's worst underrun stayed below `threshold_ms` |
-| `vllm:omni_audio_skipped_requests_total` | Counter | `reason` | Silent-loss counter — code2wav rejected malformed codec input and returned `200 OK` with empty audio |
+| `vllm_omni:audio_skipped_requests_total` | Counter | `reason` | Silent-loss counter — code2wav rejected malformed codec input and returned `200 OK` with empty audio |
 
 The continuity math comes from `vllm_omni/benchmarks/audio_continuity.py::compute_continuity_stats` so the server-side observation aligns with the bench-side definition.
 
-## Cross-Stage Transfer Metrics (`vllm:omni_`)
+## Diffusion Metrics (`vllm_omni:`)
+
+Per-request timing breakdowns for diffusion (image/video) stages. Emitted at request finalize when `stage_metrics.diffusion_metrics` is present. All carry `{model_name, stage, replica}`.
+
+| Metric | Type | Description |
+|--------|------|-------------|
+| `vllm_omni:diffusion_exec_s` | Histogram | DiT forward pass execution time per request in seconds |
+| `vllm_omni:diffusion_exec_per_step_s` | Histogram | DiT forward pass execution time per denoising step in seconds (`exec_s / num_inference_steps`) |
+| `vllm_omni:diffusion_preprocess_s` | Histogram | Diffusion input preprocessing time per request in seconds |
+| `vllm_omni:diffusion_postprocess_s` | Histogram | Diffusion output postprocessing (VAE decode) time per request in seconds |
+
+`diffusion_exec_s` uses `SECONDS_BUCKETS` (50 ms → 300 s); the other three use `SECONDS_FAST_BUCKETS` (1 ms → 60 s). `diffusion_exec_per_step_s` is skipped when `num_inference_steps` is not set.
+
+### Example PromQL
+
+```promql
+# P99 DiT exec time by stage
+histogram_quantile(0.99, rate(vllm_omni:diffusion_exec_s_bucket[5m]))
+
+# Ratio of preprocess to total exec wall time
+rate(vllm_omni:diffusion_preprocess_s_sum[5m])
+  / rate(vllm_omni:diffusion_exec_s_sum[5m])
+```
+
+## Cross-Stage Transfer Metrics (`vllm_omni:`)
 
 Per-physical-transfer histograms tracking the data hop between adjacent stages. Labels `{model_name, from_stage, from_replica, to_stage, to_replica}` let dashboards attribute latency to specific replica edges. `from_replica` / `to_replica` are resolved from the orchestrator's sticky-routing binding (`stage_pool.get_bound_replica_id(request_id)`), so no extra plumbing through `TransferEdgeStats` is needed.
 
 | Metric | Type | Description |
 |--------|------|-------------|
-| `vllm:omni_transfer_size_bytes` | Histogram | Per-transfer payload size in bytes |
+| `vllm_omni:transfer_size_bytes` | Histogram | Per-transfer payload size in bytes |
-| `vllm:omni_transfer_tx_s` | Histogram | Sender-side time (serialize + submit to connector) |
+| `vllm_omni:transfer_tx_s` | Histogram | Sender-side time (serialize + submit to connector) |
-| `vllm:omni_transfer_rx_s` | Histogram | Receiver-side time (recv + deserialize) |
+| `vllm_omni:transfer_rx_s` | Histogram | Receiver-side time (recv + deserialize) |
-| `vllm:omni_transfer_in_flight_s` | Histogram | Network in-flight time (TX done → RX recv start) |
+| `vllm_omni:transfer_in_flight_s` | Histogram | Network in-flight time (TX done → RX recv start) |
 
 ## vLLM Engine Metrics (`vllm:`)
 
@@ -83,9 +107,9 @@ For the full list of upstream metrics, see [the vLLM docs](https://github.com/vl
 
 | Metric group | Multi-stage LLM (Qwen3-Omni) |
 |---|---|
-| `vllm:omni_` request tracking + latency | With `--log-stats` |
+| `vllm_omni:` request tracking + latency | With `--log-stats` |
-| `vllm:omni_` audio modality | With `--log-stats`, if pipeline has a talker stage |
+| `vllm_omni:` audio modality | With `--log-stats`, if pipeline has a talker stage |
-| `vllm:omni_` transfer | With `--log-stats`, if pipeline has ≥ 2 stages |
+| `vllm_omni:` transfer | With `--log-stats`, if pipeline has ≥ 2 stages |
 | `vllm:` engine metrics (per `(stage, replica)`) | With `--log-stats` |
 | `vllm:` MFU metrics | With `--log-stats --enable-mfu-metrics` |
 
@@ -94,5 +118,5 @@ For the full list of upstream metrics, see [the vLLM docs](https://github.com/vl
 - All time-bearing metrics use the `_s` suffix (values in seconds). Buckets are `SECONDS_BUCKETS` for e2e / generation-style values and `SECONDS_FAST_BUCKETS` (1 ms → 60 s) for the fine-grained transfer and audio-underrun values.
 - Counters use the `_total` suffix (auto-appended by `prometheus_client`).
 - Sizes use the `_bytes` suffix.
-- All omni-specific families are prefixed `vllm:omni_`. The upstream `unregister_vllm_metrics()` function is monkey-patched (see `vllm_omni/patch.py`) to a scoped version that still strips upstream `vllm:*` collectors so multi-engine init within one process does not crash on duplicate registration, but preserves anything prefixed `vllm:omni_` / `vllm_omni`.
+- All omni-specific families are prefixed `vllm_omni:`. The upstream `unregister_vllm_metrics()` function is monkey-patched (see `vllm_omni/patch.py`) to a scoped version that still strips upstream `vllm:*` collectors so multi-engine init within one process does not crash on duplicate registration, but preserves anything prefixed `vllm_omni:`.
-- Text and audio first-output use distinct families (`vllm:time_to_first_token_seconds` reused from upstream for text; `vllm:omni_audio_ttfp_s` for audio) rather than a single metric with a `modality` label.
+- Text and audio first-output use distinct families (`vllm:time_to_first_token_seconds` reused from upstream for text; `vllm_omni:audio_ttfp_s` for audio) rather than a single metric with a `modality` label.

docs/user_guide/diffusion/attention_backends.md

@@ -1,494 +1,192 @@
 # Diffusion Attention Backends
 
-This document describes the diffusion attention backends available in vLLM-Omni, how to select them globally and per-role, the per-platform defaults, and how to use SageAttention.
+Use this page to select and configure a diffusion attention backend. Backend-
+specific installation, tuning, and compatibility details live in separate
+guides so that the selection contract stays easy to scan.
 
-## Overview
+Diffusion attention backend selection applies to DiT attention in image and
+video generation models. It does **not** change autoregressive LLM attention,
+which uses vLLM's attention selector.
 
-Diffusion attention backend selection is resolved in `vllm_omni.diffusion.attention.selector`. It looks up the backend from a structured `AttentionConfig` carried on `OmniDiffusionConfig` and falls back to the platform default when nothing is configured.
+For the internal selector, registry, and platform contract, see
+[Attention Backend Selection](../../design/feature/attention_backend_selection.md).
 
-This backend is used by diffusion attention layers such as the DiT attention in video and image generation models. It does **not** affect autoregressive (LLM) attention paths — those go through vLLM's own attention backend selector.
+## Choose a guide
 
-The full set of backends and their platform defaults is in the **Backend Options** and **Platform Defaults** sections below. If no attention backend is configured, vLLM-Omni asks the current platform to choose the default.
+| Need | Guide |
+| --- | --- |
+| Select a conservative or platform-native dense kernel | [Dense Backends](attention_backends/dense_backends.md) |
+| Use FlashInfer trtllm-gen FMHA, Skip-Softmax, or TRTLLM SAGE quantization | [TRTLLM Attention](attention_backends/trtllm.md) |
+| Install and use SageAttention 2.2 or SageAttention3 | [SageAttention](attention_backends/sage.md) |
+| Match training or rollout kernels loaded from Hugging Face | [Hugging Face Hub Backends](attention_backends/huggingface_hub.md) |
+| Use block-sparse video attention on Ascend NPU | [RainFusion](attention_backends/rainfusion.md) |
 
-## Backend Options
+## Backend options
-
-| Value | Notes |
-|---|---|
-| `TRTLLM_ATTN` | FlashInfer's trtllm-gen FMHA (TensorRT-LLM's generated kernels, vendored by FlashInfer). Dense BF16, GQA native, `head_dim=128`. Datacenter Blackwell only (sm_100 / sm_103). Packed paths can provide `cu_seqlens` directly. Supports optional **Skip-Softmax** sparse attention — see below. Requires `flashinfer`. |
-| `FLASH_ATTN` | Wraps FlashAttention 4 on Blackwell when `flash-attn-4` is installed, then falls back to FlashAttention 3/2. Default on Hopper / Ada / Ampere when a compatible FlashAttention package is installed. |
-| `CUDNN_ATTN` | Pins `sdpa_kernel([CUDNN_ATTENTION])`. Default on Blackwell (sm_10x / sm_12x) with cuDNN ≥ 9.5. Wins on mask-heavy DiTs (HunyuanVideo-1.5: 2× e2e vs SDPA). |
-| `FLASHINFER_ATTN` | Uses FlashInfer's batch-prefill wrapper and supports mixed Q/K and V input dtypes through backend-specific configuration. Used as Blackwell fallback when cuDNN is unavailable. Requires `flashinfer` >= 0.6.16rc1 for mixed-dtype configurations. |
-| `TORCH_SDPA` | PyTorch `scaled_dot_product_attention` with the default backend dispatcher. Most conservative; always available. |
-| `SAGE_ATTN` | SageAttention 2.2 — INT8-quantized attention with FP16 accumulation. Lossy but typically visually indistinguishable on diffusion outputs. Requires `sageattention`. |
-| `SAGE_ATTN_3` | Requires `sageattn3` from `SageAttention/sageattention3_blackwell`. CUDA only, intended for Blackwell GPUs, with GQA/MQA requests falling back to PyTorch SDPA. |
-| `FLASH_ATTN_HUB` | FlashAttention 2 from HuggingFace `kernels` library. Useful for train/rollout alignment. |
-| `FLASH_ATTN_3_HUB` | FlashAttention 3 from HuggingFace `kernels` library. CUDA Hopper (sm_90+) only; falls back to `FLASH_ATTN_HUB` on older GPUs. |
-| `RAINFUSION_ATTN` | MindIE-SD **RainFusion** block-sparse video attention — see [below](#rainfusion_attn-backend-and-block-sparse-video-attention). Ascend NPU only; requires `mindiesd`. Delegates to `FLASH_ATTN` for anything that is not a packed video sequence. |
 
+| Value | Family | Primary use | Detail |
+| --- | --- | --- | --- |
+| `TORCH_SDPA` | Dense | Conservative reference; always available | [Dense Backends](attention_backends/dense_backends.md#torch_sdpa) |
+| `FLASH_ATTN` | Dense | FlashAttention 4/3/2 depending on the installed package and GPU | [Dense Backends](attention_backends/dense_backends.md#flash_attn) |
+| `CUDNN_ATTN` | Dense | Mask-heavy DiTs on Blackwell with cuDNN 9.5 or newer | [Dense Backends](attention_backends/dense_backends.md#cudnn_attn) |
+| `FLASHINFER_ATTN` | Dense or quantized | FlashInfer batch prefill; optional mixed Q/K and V dtypes | [Dense Backends](attention_backends/dense_backends.md#flashinfer_attn) |
+| `TRTLLM_ATTN` | Dense, sparse, or quantized | Datacenter Blackwell with `head_dim=128` and compatible packed paths | [TRTLLM Attention](attention_backends/trtllm.md) |
+| `SAGE_ATTN` | Quantized | SageAttention 2.2 INT8 attention | [SageAttention](attention_backends/sage.md#sage_attn) |
+| `SAGE_ATTN_3` | Quantized | SageAttention3 on Blackwell | [SageAttention](attention_backends/sage.md#sage_attn_3) |
+| `FLASH_ATTN_HUB` | Hub kernel | FlashAttention 2 from Hugging Face `kernels` | [Hugging Face Hub Backends](attention_backends/huggingface_hub.md) |
+| `FLASH_ATTN_3_HUB` | Hub kernel | FlashAttention 3 from Hugging Face `kernels` on Hopper or newer | [Hugging Face Hub Backends](attention_backends/huggingface_hub.md) |
+| `RAINFUSION_ATTN` | Block sparse | MindIE-SD RainFusion video attention on Ascend NPU | [RainFusion](attention_backends/rainfusion.md) |
 
 ## Configuration
 
-Diffusion attention backends can be configured three ways, in priority order:
+Backend selection is resolved in this order:
 
-1. **`--diffusion-attention-config`** — structured per-role config (highest priority).
+1. `--diffusion-attention-config` per-role configuration.
-2. **`--diffusion-attention-backend` / `DIFFUSION_ATTENTION_BACKEND` env var** — global shorthand that sets the default backend.
+2. `--diffusion-attention-backend` or `DIFFUSION_ATTENTION_BACKEND` as a
-3. **Platform default** — used when nothing is configured.
+   global default.
+3. The current platform's default.
 
-`--diffusion-attention-backend` is shorthand for `--diffusion-attention-config.default.backend`. It may be combined with `--diffusion-attention-config.per_role.*` overrides, but is mutually exclusive with `--diffusion-attention-config.default.backend`.
+`--diffusion-attention-backend` is shorthand for
+`--diffusion-attention-config.default.backend`. Do not pass it together with an
+explicit `default.backend` in the structured configuration.
 
 ### Global default
 
-Set the default backend for every diffusion attention layer:
-
 ```bash
-# CLI flag
+vllm-omni serve <model> --diffusion-attention-backend FLASH_ATTN
-vllm-omni serve <model> --diffusion-attention-backend SAGE_ATTN
 
-# Environment variable (also recognized for backwards compatibility)
+# Backwards-compatible environment variable
-export DIFFUSION_ATTENTION_BACKEND=SAGE_ATTN
+export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
 ```
 
 ### Per-role configuration
 
-Roles are free-form strings declared by each diffusion model. The two common categories are `"self"` and `"cross"`; model-specific roles (e.g. `"ltx2.audio_to_video"`) may also be declared. A role string is matched in this order:
+Roles are declared by each diffusion model. Common categories are `self` and
+`cross`; a model may also use a more specific role such as
+`ltx2.audio_to_video`. Resolution order is:
 
-1. Exact `per_role[role]` match
+1. Exact `per_role[role]` match.
-2. `per_role[role_category]` fallback (e.g. `"ltx2.audio_to_video"` → `"cross"`)
+2. Category `per_role[role_category]` match.
-3. `default`
+3. `default`.
-4. Platform default
+4. Platform default.
-
-Use vLLM-style dotted flags or one JSON blob:
 
 ```bash
 # Dotted flags
 vllm-omni serve <model> \
-    --diffusion-attention-config.default.backend FLASH_ATTN \
+  --diffusion-attention-config.default.backend FLASH_ATTN \
-    --diffusion-attention-config.per_role.cross.backend TORCH_SDPA
+  --diffusion-attention-config.per_role.cross.backend TORCH_SDPA
 
-# JSON
+# Equivalent JSON
 vllm-omni serve <model> \
-    --diffusion-attention-config '{"default":{"backend":"FLASH_ATTN"},"per_role":{"cross":{"backend":"TORCH_SDPA"}}}'
+  --diffusion-attention-config \
+  '{"default":{"backend":"FLASH_ATTN"},"per_role":{"cross":{"backend":"TORCH_SDPA"}}}'
 ```
 
-A backend that needs configuration exposes it as a typed field on the spec:
+### Python API
-`TRTLLM_ATTN`'s Skip-Softmax (see [below](#trtllm_attn-backend-and-skip-softmax)) and
-`RAINFUSION_ATTN`'s RainFusion block (see [below](#rainfusion_attn-backend-and-block-sparse-video-attention)):
-
-```bash
---diffusion-attention-config.default.backend TRTLLM_ATTN \
---diffusion-attention-config.default.skip_softmax.target_sparsity 0.5
-```
-
-### Programmatic API
-
-When constructing `OmniDiffusionConfig` directly:
 
 ```python
-from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, AttnQuantSpec, OmniDiffusionConfig
+from vllm_omni.diffusion.data import (
+    AttentionConfig,
+    AttentionSpec,
+    OmniDiffusionConfig,
+)
 
 config = OmniDiffusionConfig(
     diffusion_attention_config=AttentionConfig(
         default=AttentionSpec(backend="FLASH_ATTN"),
-        per_role={
+        per_role={"cross": AttentionSpec(backend="TORCH_SDPA")},
-            "cross": AttentionSpec(backend="TORCH_SDPA"),
-        },
     ),
     ...,
 )
 ```
 
-A plain dict is also accepted and normalized to `AttentionConfig`.
+Backend-specific typed blocks are documented with their consumers:
 
-#### FlashInfer QK16/V8 quantized attention
+- `quant`: [FlashInfer](attention_backends/dense_backends.md#flashinfer-quantized-attention)
+  and [TRTLLM SAGE](attention_backends/trtllm.md#sage-quantization).
+- `skip_softmax`: [TRTLLM Skip-Softmax](attention_backends/trtllm.md#skip-softmax).
+- `block_sparse`: [RainFusion](attention_backends/rainfusion.md#configuration).
 
-`FLASHINFER_ATTN` accepts quantization options through `AttentionSpec.quant`. For QK16/V8:
+## Platform defaults
-
-```python
-config = OmniDiffusionConfig(
-    diffusion_attention_config=AttentionConfig(
-        default=AttentionSpec(
-            backend="FLASHINFER_ATTN",
-            quant=AttnQuantSpec(
-                dtype_qk="bfloat16",
-                dtype_vo="fp8_e4m3",
-            ),
-        ),
-    ),
-    ...,
-)
-```
-
-`dtype_qk` configures the Q and K input dtypes. `dtype_vo` configures the V input dtype.
-
-## Platform Defaults
 
 ### Blackwell (sm_100 / sm_103 / sm_120 / sm_121)
 
-Auto-route preference, in order:
+The CUDA auto-route preference is:
 
-1. `TRTLLM_ATTN` — on **datacenter** Blackwell (sm_100 / sm_103) when `flashinfer` is installed, the model's `head_dim` is 128, **and the model declares a compatible packed/mask-free path**
+1. `TRTLLM_ATTN` on datacenter Blackwell (sm_100/sm_103) when FlashInfer is
-2. `CUDNN_ATTN` — when cuDNN ≥ 9.5 is available (ships in PyTorch 2.5+ wheels)
+   available, `head_dim=128`, and the model declares a compatible packed or
-3. `FLASHINFER_ATTN` — when `flashinfer` is installed but cuDNN < 9.5
+   mask-free path.
-4. `FLASH_ATTN` — when `flash-attn` is installed with the Blackwell CUTE kernel
+2. `CUDNN_ATTN` when cuDNN 9.5 or newer is available.
-5. `TORCH_SDPA` — last resort
+3. `FLASHINFER_ATTN` when FlashInfer is available but cuDNN is too old.
+4. `FLASH_ATTN` when a compatible package is installed.
+5. `TORCH_SDPA`.
 
-`TRTLLM_ATTN` is skipped on workstation Blackwell (sm_120 / sm_121) and for any `head_dim != 128`, so those GPUs keep the `CUDNN_ATTN` route described below.
+`TRTLLM_ATTN` is not auto-selected on workstation Blackwell
+(sm_120/sm_121), for other head dimensions, or for paths that require an
+unsupported mask.
 
-`TRTLLM_ATTN` outranks `CUDNN_ATTN` on datacenter Blackwell for compatible packed/mask-free pipelines. Workstation Blackwell (sm_120 / sm_121) and pipelines that require attention masks retain their normal fallback.
+### Hopper, Ada, and Ampere
 
-The startup log line `Defaulting to diffusion attention backend CUDNN_ATTN (Blackwell sm_120, cuDNN 91002)` confirms the route.
+The CUDA auto-route uses `FLASH_ATTN` when available and otherwise falls back
+to `TORCH_SDPA`. `CUDNN_ATTN` and `FLASHINFER_ATTN` remain explicit options.
 
-**Why CUDNN_ATTN by default**: on mask-heavy diffusion models (HunyuanVideo-1.5, Qwen-Image), cuDNN's pinned FMHA kernel sidesteps a PyTorch SDPA dispatch quirk where the unpinned dispatcher picks `EFFICIENT_ATTENTION` (~25 ms) for masked calls instead of cuDNN (~11 ms). The pin gives 2× e2e on HV-1.5 with no regression on lighter models.
+Other platforms validate an explicit backend and choose their own default
+through the platform implementation. Check the startup log to confirm the
+resolved backend.
 
-### Hopper (sm_90) / Ada (sm_89) / Ampere (sm_80–sm_86)
+## Choosing a backend manually
 
-Auto-route preference:
+Override the platform default when you need:
 
-1. `FLASH_ATTN` — when `flash-attn` is installed
+- a correctness reference (`TORCH_SDPA`);
-2. `TORCH_SDPA` — fallback
+- a backend-specific workaround;
+- training/rollout kernel alignment (Hub backends); or
+- an explicitly validated sparse or quantized speedup.
 
-`CUDNN_ATTN` and `FLASHINFER_ATTN` are still selectable via env var on these GPUs but are not in the auto-route — FlashAttention 2 is the well-tuned path on pre-Blackwell hardware.
+The startup log prints the resolved backend and whether it came from explicit
+configuration or platform defaulting. If no resolution message appears, check
+earlier logs for diffusion-stage initialization failures.
 
-## TRTLLM_ATTN Backend and Skip-Softmax
+## Reference benchmark
 
-`TRTLLM_ATTN` runs FlashInfer's trtllm-gen FMHA. Dense it is a BF16 backend like the others; on top of
+The following BF16 results were measured on an sm_120 RTX Pro 6000 Blackwell
-that it can enable **Skip-Softmax**, a sparse-attention mode that trades a little fidelity for speed
+with the same prompt and seed across runs. Treat them as reference results, not
-(algorithm: [design doc](../../design/feature/skip_softmax.md)).
+portable guarantees.
 
-Enable it through the typed `skip_softmax` block on the attention spec:
+| Model | Shape | `TORCH_SDPA` | `CUDNN_ATTN` | `FLASHINFER_ATTN` |
-
+| --- | --- | ---: | ---: | ---: |
-| Key | Valid values | Meaning |
-|---|---|---|
-| `target_sparsity` | finite, `[0, 1]` | Operating point on a calibrated curve. Needs a calibration for the model. |
-| `threshold` | finite, `≥ 0` | Direct threshold, no calibration needed. Mutually exclusive with `target_sparsity`. |
-| `disabled_until_timestep` | finite, `[0, 1]` | Holds early, high-noise steps dense; skip turns on once the normalized timestep `t ≤ D`. |
-
-```bash
-vllm-omni serve Wan-AI/Wan2.2-T2V-A14B-Diffusers \
-  --diffusion-attention-config '{"default": {"backend": "TRTLLM_ATTN",
-      "skip_softmax": {"target_sparsity": 0.65, "disabled_until_timestep": 0.86}}}'
-```
-
-Programmatically the same block is a typed `SkipSoftmaxSpec` (values validated at construction):
-
-```python
-from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, SkipSoftmaxSpec
-
-AttentionConfig(
-    default=AttentionSpec(
-        backend="TRTLLM_ATTN",
-        skip_softmax=SkipSoftmaxSpec(target_sparsity=0.65, disabled_until_timestep=0.86),
-    ),
-)
-```
-
-**Start at `target_sparsity=0.65, disabled_until_timestep=0.86`** and raise `target_sparsity` only
-as far as your quality bar allows — the output diverges from dense as it climbs. The gain grows with
-sequence length, since Skip-Softmax only accelerates attention. Dense BF16 is `TRTLLM_ATTN` with no
-`skip_softmax` block.
-
-Requires datacenter Blackwell with `head_dim == 128`; elsewhere, or without FlashInfer, selecting
-`TRTLLM_ATTN` raises rather than silently degrading.
-
-## TRTLLM_ATTN SAGE Quantization
-
-On top of dense/Skip-Softmax, `TRTLLM_ATTN` can run **SAGE** (SageAttention) — the trtllm-gen
-FMHA kernel with per-block quantized Q/K and per-channel fp8 V, so the two attention matmuls run
-in low precision. Enable it through the typed `quant` block on the attention spec:
-
-| Key | Valid values | Meaning |
-|---|---|---|
-| `dtype_qk` | `int8`, `fp8_e4m3` | Quantization dtype for Q and K. Absent ⇒ dense (SAGE off). |
-| `q_block_size` | `1`, `4`, `16` | Per-token block size for Q scales (default `1`). |
-| `k_block_size` | `1`, `4`, `16` | Per-token block size for K scales (default `16`). |
-
-V is always quantized per-channel to fp8_e4m3 and K-smoothing is applied inside the routine, so
-neither is a user knob. Only block sizes `1`, `4`, `16` have compiled kernels.
-
-The `quant` block is a shared `AttnQuantSpec` consumed by more than one backend; each reads the
-fields that apply and rejects the rest. `TRTLLM_ATTN` (SAGE) reads `dtype_qk` (`int8` / `fp8_e4m3`),
-`q_block_size`, and `k_block_size`; `FLASHINFER_ATTN` reads `dtype_qk` (`float16` / `bfloat16`),
-`dtype_vo`, and `flashinfer_backend` (see its section). A `dtype_qk` from the wrong set raises.
-
-```bash
-vllm-omni serve Wan-AI/Wan2.2-T2V-A14B-Diffusers \
-  --diffusion-attention-config '{"default": {"backend": "TRTLLM_ATTN",
-      "quant": {"dtype_qk": "fp8_e4m3", "q_block_size": 1, "k_block_size": 16}}}'
-```
-
-```python
-from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, AttnQuantSpec
-
-AttentionConfig(
-    default=AttentionSpec(
-        backend="TRTLLM_ATTN",
-        quant=AttnQuantSpec(dtype_qk="fp8_e4m3", q_block_size=1, k_block_size=16),
-    ),
-)
-```
-
-**Requirements.** Needs FlashInfer ≥ 0.6.16rc1. Kernel
-availability is arch-dependent: `fp8_e4m3` QK has kernels on both **SM100** (B200) and **SM103**
-(B300); `int8` QK kernels are compiled for **SM100 only**.
-
-## RAINFUSION_ATTN Backend and Block-Sparse Video Attention
-
-`RAINFUSION_ATTN` runs MindIE-SD's RainFusion (`rf_v2`) kernel on Ascend NPU. It pools each
-128-token block of keys, ranks them per query block, and computes attention against only the
-top-scoring ones. Video tokens are rearranged into `(t, h, w)` order first, so the kept blocks are
-spatiotemporal neighbours rather than arbitrary rows of the packed sequence.
-
-Only the video segment is sparsified. The prefix rows (text, visual conditions, audio) and the
-first-frame blocks are always kept dense, which is why the realized sparsity is lower than the
-nominal `sparsity` you configure. Anything the kernel cannot handle — a warmup denoise step, an
-exempt layer, a sequence with no published video geometry, a video segment under 32 blocks —
-delegates to `FLASH_ATTN`, so a model can select this backend unconditionally.
-
-Enable it through the typed `block_sparse` block on the attention spec, shared by every
-block-sparse backend:
-
-| Key | Valid values | Meaning |
-|---|---|---|
-| `sparsity` | finite, `[0, 1]` | Nominal fraction of key blocks dropped per query block. `0` disables sparsity. Defaults to `0.8`. |
-| `start_step` | `≥ 0` | Keep the first N denoise steps dense. Layout is decided early, so these steps dominate structural fidelity. |
-| `skip_layers` | index selector, e.g. `"0-3,38"` | DiT blocks that always stay dense. |
-
-```bash
-vllm-omni serve MiniMaxAI/MiniMax-H3 \
-  --diffusion-attention-config '{"default": {"backend": "RAINFUSION_ATTN",
-      "block_sparse": {"sparsity": 0.8, "start_step": 0}}}'
-```
-
-Programmatically the same block is a typed `BlockSparseSpec` (values validated at construction):
-
-```python
-from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, BlockSparseSpec
-
-AttentionConfig(
-    default=AttentionSpec(
-        backend="RAINFUSION_ATTN",
-        block_sparse=BlockSparseSpec(sparsity=0.8, start_step=0, skip_layers="0-1"),
-    ),
-)
-```
-
-**Tune in the order `start_step` → `sparsity` → `skip_layers`.** Raise `start_step` first: it is the
-cheapest way to recover structure, because the early high-noise steps decide global layout while the
-later steps only refine texture. Then walk `sparsity` up to your quality bar. Reach for
-`skip_layers` last, once an A/B against dense at the same seed points at specific blocks.
-
-Requires Ascend NPU with `mindiesd`; selecting it on any other platform raises. It is also
-incompatible with ring sequence parallelism, since `rf_v2` needs the whole key sequence to rank
-blocks — use Ulysses SP (`ring_degree=1`).
-
-### Which geometries run sparse
-
-MindIE-SD rf_v2 handles arbitrary video grids by spatially rearranging video first and, when
-necessary, promoting a real-video suffix to its always-kept prefix segment. It never pads: rf_v2
-does not consume an attention mask for padded keys. The remaining video rows stay sparse, while the
-promoted rows remain visible to every query and key.
-
-For example, MiniMax-H3 at 1344x768 has grid `(62, 24, 42)`: its 62496 video rows leave a 32-row
-128-block residue, and its two non-8-wide latent columns form a 2928-row rearranged suffix.
-MindIE-SD promotes 2976 rows, leaving 59520 sparse rows (`465 × 128`). This keeps the block mask
-and kernel tiling aligned, at the cost of retaining more blocks.
-
-8x8-aligned spatial grids remain preferable for performance. A latent `h` or `w` not divisible by 8
-creates an always-kept residual suffix and lowers realized sparsity. "Multiple of 8" on the latent
-grid means **width and height that are multiples of 256**. This path requires a MindIE-SD release
-that implements protected video tails for rf_v2.
-
-## End-to-End Benchmark (BF16, sm_120 RTX Pro 6000 Blackwell)
-
-Same prompt and seed across runs. `Total generation time` from `text_to_video.py` / `text_to_image.py`.
-
-| Model | Shape | TORCH_SDPA | CUDNN_ATTN | FLASHINFER_ATTN |
-|---|---|---|---|---|
 | HunyuanVideo-1.5 (T2V) | 480p / 33f / 50 steps | 147.05 s | **73.02 s** | 127.84 s |
 | Wan 2.2 14B (T2V) | 480p / 33f / 40 steps | 117.75 s | 117.17 s | **115.07 s** |
 | Qwen-Image (T2I) | 1024² / 50 steps | 17.41 s | **15.14 s** | 16.02 s |
 | FLUX.2-dev (T2I) | 1024² / 50 steps, TP=2 | 53.62 s | **53.30 s** | 54.94 s |
 
-Pattern: mask-heavy DiTs (HV-1.5, Qwen-Image) favor `CUDNN_ATTN`; lighter-mask DiTs and TP-saturated configs (Wan 2.2, FLUX.2 TP=2) tie within noise.
+Mask-heavy DiTs favored `CUDNN_ATTN`; lighter-mask or TP-saturated workloads
+were close enough that users should benchmark their exact model and shape.
 
-## Known Limitations
+## Compatibility anchors
 
-### LTX-2.0: `CUDNN_ATTN` crashes under torch.compile
+The following headings preserve links to sections that moved into dedicated
+guides.
 
-LTX-2's audio attention has a symbolic head_dim under torch.compile tracing. cuDNN's SDPA backend selector rejects symbolic dims and Dynamo aborts compilation. Tracked in [#3121](https://github.com/vllm-project/vllm-omni/issues/3121).
+## TRTLLM_ATTN Backend and Skip-Softmax
 
-**Workaround**: explicitly select `FLASHINFER_ATTN` or `TORCH_SDPA` for LTX-2.0:
+See [TRTLLM Attention](attention_backends/trtllm.md#skip-softmax).
 
-```bash
+## TRTLLM_ATTN SAGE Quantization
-DIFFUSION_ATTENTION_BACKEND=FLASHINFER_ATTN python examples/offline_inference/text_to_video/text_to_video.py \
-    --model Lightricks/LTX-2 ...
-```
 
-### FlashAttention-4 on Blackwell
+See [TRTLLM Attention](attention_backends/trtllm.md#sage-quantization).
 
-Install the optional CUDA 13 extra to use the CuTe-based FA4 path:
+## RAINFUSION_ATTN Backend and Block-Sparse Video Attention
 
-```bash
+See [RainFusion](attention_backends/rainfusion.md).
-pip install 'vllm-omni[fa4]'
-```
-
-`FLASH_ATTN` prefers `flash_attn.cute` on Blackwell and falls back to FA3/FA2
-when it is unavailable. Version `4.0.0b18` is required; earlier beta
-wheels had known JIT failures on Blackwell.
-
-## Choosing a Backend Manually
-
-### When to override the default
-
-- **Quality validation**: compare a new backend against `TORCH_SDPA` as the reference, since SDPA's default dispatcher is the most extensively tested.
-- **Lossy speedup hunting**: try `SAGE_ATTN` (INT8 quantized) on diffusion outputs — typically indistinguishable visually but always validate.
-- **Workaround for known issues**: see Known Limitations above.
-
-### Verifying which backend is in use
-
-The startup log prints one of:
-
-```
-Using diffusion attention backend 'CUDNN_ATTN'           # explicit override
-Defaulting to diffusion attention backend CUDNN_ATTN ... # auto-route
-Defaulting to diffusion attention backend SDPA           # nothing else available
-```
-
-If you don't see one of these, the model didn't reach diffusion stage init — check earlier logs for failures.
 
 ## SageAttention Installation
 
-vLLM-Omni expects SageAttention to be installed into the same Python environment as vLLM-Omni.
+See [SageAttention](attention_backends/sage.md#installation).
-
-Build from source:
-
-```bash
-git clone https://github.com/thu-ml/SageAttention.git
-cd SageAttention
-
-export EXT_PARALLEL=4 NVCC_APPEND_FLAGS="--threads 8" MAX_JOBS=32
-pip install . --no-build-isolation
-```
-
-Quick check:
-
-```bash
-python -c "import sageattention; print(sageattention.__file__)"
-```
 
 ## SageAttention3 Installation
 
-vLLM-Omni expects SageAttention3 to be installed into the same Python environment as vLLM-Omni.
+See [SageAttention](attention_backends/sage.md#sageattention3-installation).
-
-Build from source:
-
-```bash
-git clone https://github.com/thu-ml/SageAttention.git
-cd SageAttention/sageattention3_blackwell
-python setup.py install
-```
-
-Quick check:
-
-```bash
-python -c "import sageattn3; print(sageattn3.__file__)"
-```
-
-Notes:
-
-- `SAGE_ATTN_3` is only selected on CUDA when `sageattn3` is importable and the GPU is Blackwell-class.
-- SageAttention3's Blackwell kernel assumes `Hq == Hkv`. In vLLM-Omni, GQA/MQA diffusion requests fall back to PyTorch SDPA for correctness.
 
 ## HuggingFace Kernels Hub Backends
 
-To achieve perfect numerical consistency between **training** (typically using HuggingFace Diffusers with Hub kernels) and **serving/rollout** (using vLLM-Omni), you can use the Hub-based attention backends. This eliminates numerical drift / sampling divergence caused by executing different local kernel versions during rollout.
+See [Hugging Face Hub Backends](attention_backends/huggingface_hub.md).
-
-The following backend options are supported:
-- `FLASH_ATTN_HUB` (HuggingFace `kernels-community/flash-attn2`)
-- `FLASH_ATTN_3_HUB` (HuggingFace `kernels-community/flash-attn3`, Hopper sm_90+ only)
-
-### Installation
-
-To use these backends, you must install the `kernels` library:
-
-```bash
-pip install kernels==0.14.1
-```
-
-If the `kernels` library is not available in the environment, vLLM-Omni will log a warning and fall back gracefully to the corresponding local backend implementations (`FLASH_ATTN`). On CUDA GPUs below Hopper (compute capability < 9.0), `FLASH_ATTN_3_HUB` falls back to `FLASH_ATTN_HUB`.
-
-### Usage
-
-Select a Hub backend using the global CLI flag or environment variables:
-
-```bash
-# Environment variable
-export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN_HUB
-
-# CLI flag
-vllm-omni serve <model> --diffusion-attention-backend FLASH_ATTN_HUB
-```
-
-## Usage Examples
-
-### Default (auto-route)
-
-```bash
-python examples/offline_inference/text_to_video/text_to_video.py \
-    --model hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v \
-    --prompt "A dog running across a field of golden wheat." \
-    --height 480 --width 832 --num-frames 33 \
-    --num-inference-steps 50 --seed 42 --guidance-scale 6.0 \
-    --output hv15.mp4
-```
-
-On Blackwell this picks `CUDNN_ATTN` automatically. Check the log for the `Defaulting to ...` line.
-
-### Explicit backend selection
-
-```bash
-DIFFUSION_ATTENTION_BACKEND=FLASHINFER_ATTN python examples/offline_inference/text_to_video/text_to_video.py \
-    --model Lightricks/LTX-2 \
-    --prompt "A dog running across a field of golden wheat." \
-    --height 480 --width 832 --num-frames 33 \
-    --num-inference-steps 40 --seed 42 --guidance-scale 4.0 \
-    --output ltx2.mp4
-```
-
-### SageAttention (lossy)
-
-```bash
-DIFFUSION_ATTENTION_BACKEND=SAGE_ATTN python examples/offline_inference/text_to_video/text_to_video.py \
-    --model hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v \
-    --prompt "A dog running across a field of golden wheat." \
-    --height 480 --width 832 --num-frames 33 \
-    --num-inference-steps 30 --seed 42 --guidance-scale 6.0 \
-    --tensor-parallel-size 2 \
-    --output hv15_sage.mp4
-```
-
-Example: Wan2.2 TI2V 5B
-
-```bash
-DIFFUSION_ATTENTION_BACKEND=SAGE_ATTN python examples/offline_inference/text_to_video/text_to_video.py \
-    --model Wan-AI/Wan2.2-TI2V-5B-Diffusers \
-    --prompt "A dog running across a field of golden wheat." \
-    --height 704 --width 1280 --num-frames 49 \
-    --num-inference-steps 30 --seed 42 --guidance-scale 5.0 \
-    --tensor-parallel-size 2 \
-    --output outputs/wan22_sage.mp4
-```
-
-### Enable SageAttention3
-
-Example:
-
-```bash
-DIFFUSION_ATTENTION_BACKEND=SAGE_ATTN_3 python examples/offline_inference/text_to_video/text_to_video.py \
-    --model hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v \
-    --prompt "A dog running across a field of golden wheat." \
-    --height 480 --width 832 --num-frames 33 \
-    --num-inference-steps 30 --seed 42 --guidance-scale 6.0 \
-    --tensor-parallel-size 2 \
-    --output outputs/hv15_sage3.mp4
-```

docs/user_guide/diffusion/attention_backends/dense_backends.md

@@ -0,0 +1,110 @@
+# Dense Attention Backends
+
+Use these backends when you want dense attention without backend-specific
+sparsity. Start with `TORCH_SDPA` for correctness comparisons, then benchmark
+the fastest compatible kernel for your model, shape, and hardware.
+
+For selection precedence and per-role configuration, see the
+[attention backend overview](../attention_backends.md).
+
+## `TORCH_SDPA`
+
+`TORCH_SDPA` calls PyTorch `scaled_dot_product_attention` and lets PyTorch's
+dispatcher choose the implementation. It is always available and is the most
+conservative reference when validating another backend.
+
+```bash
+vllm-omni serve <model> --diffusion-attention-backend TORCH_SDPA
+```
+
+## `FLASH_ATTN`
+
+`FLASH_ATTN` uses the installed FlashAttention implementation. On Blackwell it
+prefers the CuTe-based FlashAttention 4 path and falls back to
+FlashAttention 3 or 2 when that path is unavailable. On Hopper, Ada, and
+Ampere it is the preferred automatic route when a compatible package is
+installed.
+
+```bash
+vllm-omni serve <model> --diffusion-attention-backend FLASH_ATTN
+```
+
+### FlashAttention 4 on Blackwell
+
+Install the optional CUDA 13 extra:
+
+```bash
+pip install 'vllm-omni[fa4]'
+```
+
+Version `4.0.0b18` is required; earlier beta wheels had known JIT failures on
+Blackwell. If the CuTe path is unavailable, the backend falls back to the
+compatible FlashAttention 3 or 2 path.
+
+## `CUDNN_ATTN`
+
+`CUDNN_ATTN` pins PyTorch SDPA to `CUDNN_ATTENTION`. It is particularly useful
+for mask-heavy DiTs and is automatically preferred on Blackwell when cuDNN
+9.5 or newer is available and the higher-priority TRTLLM route is not
+compatible.
+
+```bash
+vllm-omni serve <model> --diffusion-attention-backend CUDNN_ATTN
+```
+
+### LTX-2.0 limitation
+
+LTX-2 audio attention has a symbolic head dimension during `torch.compile`
+tracing. The cuDNN SDPA selector rejects that symbolic dimension and Dynamo
+aborts compilation. This is tracked in
+[issue #3121](https://github.com/vllm-project/vllm-omni/issues/3121).
+
+Use `FLASHINFER_ATTN` or `TORCH_SDPA` as a workaround:
+
+```bash
+DIFFUSION_ATTENTION_BACKEND=FLASHINFER_ATTN \
+  python examples/offline_inference/text_to_video/text_to_video.py \
+  --model Lightricks/LTX-2 ...
+```
+
+## `FLASHINFER_ATTN`
+
+`FLASHINFER_ATTN` uses FlashInfer's batch-prefill wrapper. It is an explicit
+option on CUDA platforms and an automatic Blackwell fallback when FlashInfer
+is installed but cuDNN is too old for `CUDNN_ATTN`.
+
+```bash
+vllm-omni serve <model> --diffusion-attention-backend FLASHINFER_ATTN
+```
+
+### FlashInfer quantized attention
+
+The backend accepts an `AttentionSpec.quant` block. For QK16/V8, keep Q and K
+in FP16 or BF16 and use FP8 E4M3 for V:
+
+```python
+from vllm_omni.diffusion.data import (
+    AttentionConfig,
+    AttentionSpec,
+    AttnQuantSpec,
+    OmniDiffusionConfig,
+)
+
+config = OmniDiffusionConfig(
+    diffusion_attention_config=AttentionConfig(
+        default=AttentionSpec(
+            backend="FLASHINFER_ATTN",
+            quant=AttnQuantSpec(
+                dtype_qk="bfloat16",
+                dtype_vo="fp8_e4m3",
+            ),
+        ),
+    ),
+    ...,
+)
+```
+
+`dtype_qk` controls Q and K; `dtype_vo` controls V. Mixed-dtype
+configurations require FlashInfer 0.6.16rc1 or newer. The shared quantization
+schema is also consumed by TRTLLM, but each backend validates its own allowed
+fields and values; see [TRTLLM SAGE quantization](trtllm.md#sage-quantization).

docs/user_guide/diffusion/attention_backends/huggingface_hub.md

@@ -0,0 +1,32 @@
+# Hugging Face Hub Attention Backends
+
+Hub backends load the same published kernels commonly used with Hugging Face
+Diffusers. They are useful when training and serving must use matching kernel
+implementations to reduce numerical drift and sampling divergence.
+
+| Backend | Kernel | Platform |
+| --- | --- | --- |
+| `FLASH_ATTN_HUB` | `kernels-community/flash-attn2` | Compatible CUDA GPUs |
+| `FLASH_ATTN_3_HUB` | `kernels-community/flash-attn3` | Hopper sm_90 or newer |
+
+## Installation
+
+```bash
+pip install kernels==0.14.1
+```
+
+If `kernels` is unavailable, vLLM-Omni warns and falls back to the
+corresponding local FlashAttention implementation. On CUDA devices older than
+Hopper, `FLASH_ATTN_3_HUB` falls back to `FLASH_ATTN_HUB`.
+
+## Usage
+
+```bash
+export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN_HUB
+
+# Equivalent CLI selection
+vllm-omni serve <model> --diffusion-attention-backend FLASH_ATTN_HUB
+```
+
+For per-role kernel alignment, configure the Hub backend on only the relevant
+role; see the [attention backend overview](../attention_backends.md#per-role-configuration).

docs/user_guide/diffusion/attention_backends/rainfusion.md

@@ -0,0 +1,72 @@
+# RainFusion Attention
+
+`RAINFUSION_ATTN` runs MindIE-SD RainFusion (`rf_v2`) block-sparse video
+attention on Ascend NPU. It pools 128-token key blocks, ranks them per query
+block, and attends to the highest-scoring blocks after arranging video tokens
+in `(t, h, w)` order.
+
+Only the video segment is sparse. Prefix rows such as text, visual conditions,
+and audio, plus first-frame blocks, remain dense. Unsupported calls—including
+warmup steps, skipped layers, missing video geometry, and video segments below
+32 blocks—delegate to `FLASH_ATTN`, so compatible models can select RainFusion
+globally.
+
+## Configuration
+
+| Key | Valid values | Meaning |
+| --- | --- | --- |
+| `sparsity` | finite, `[0, 1]` | Nominal dropped-key-block fraction; default `0.8`; `0` disables sparsity |
+| `start_step` | integer, `>= 0` | Number of early denoise steps kept dense |
+| `skip_layers` | selector such as `"0-3,38"` | DiT blocks kept dense |
+
+```bash
+vllm-omni serve MiniMaxAI/MiniMax-H3 \
+  --diffusion-attention-config '{"default":{"backend":"RAINFUSION_ATTN",\
+    "block_sparse":{"sparsity":0.8,"start_step":0}}}'
+```
+
+```python
+from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, BlockSparseSpec
+
+config = AttentionConfig(
+    default=AttentionSpec(
+        backend="RAINFUSION_ATTN",
+        block_sparse=BlockSparseSpec(
+            sparsity=0.8,
+            start_step=0,
+            skip_layers="0-1",
+        ),
+    ),
+)
+```
+
+Tune `start_step`, then `sparsity`, then `skip_layers`. Increasing the dense
+early-step window is usually the cheapest way to recover global structure;
+use layer exclusions only after a same-seed dense comparison identifies
+sensitive blocks.
+
+## Requirements and compatibility
+
+RainFusion requires Ascend NPU and `mindiesd`. Selecting it on another
+platform raises. It is incompatible with ring sequence parallelism because
+the kernel needs the complete key sequence for ranking; use Ulysses sequence
+parallelism with `ring_degree=1`.
+
+## Geometry handling
+
+RainFusion handles arbitrary video grids by rearranging video spatially and,
+when necessary, promoting a real-video suffix to the always-kept prefix. It
+does not pad because `rf_v2` does not consume a padding attention mask.
+
+For MiniMax-H3 at 1344x768, grid `(62, 24, 42)` produces 62,496 video rows.
+The implementation promotes 2,976 rows and leaves 59,520 sparse rows
+(`465 x 128`) so the mask and kernel tiling remain aligned.
+
+Spatial grids aligned to 8x8 are still preferable. A latent height or width
+not divisible by 8 creates an always-kept suffix and reduces realized
+sparsity. At the image level, that means width and height divisible by 256.
+Protected video tails require a compatible MindIE-SD release.
+
+For common configuration and selector behavior, see the
+[attention backend overview](../attention_backends.md) and the
+[backend selection design](../../../design/feature/attention_backend_selection.md).

docs/user_guide/diffusion/attention_backends/sage.md

@@ -0,0 +1,63 @@
+# SageAttention
+
+SageAttention backends provide lossy low-precision attention for diffusion
+models. Validate output quality against `TORCH_SDPA` at the same seed before
+using either backend in production.
+
+## `SAGE_ATTN`
+
+`SAGE_ATTN` uses SageAttention 2.2 with INT8-quantized attention and FP16
+accumulation.
+
+### Installation
+
+Install SageAttention into the same environment as vLLM-Omni:
+
+```bash
+git clone https://github.com/thu-ml/SageAttention.git
+cd SageAttention
+export EXT_PARALLEL=4 NVCC_APPEND_FLAGS="--threads 8" MAX_JOBS=32
+pip install . --no-build-isolation
+```
+
+Verify the installation:
+
+```bash
+python -c "import sageattention; print(sageattention.__file__)"
+```
+
+Select it globally:
+
+```bash
+vllm-omni serve <model> --diffusion-attention-backend SAGE_ATTN
+```
+
+## `SAGE_ATTN_3`
+
+`SAGE_ATTN_3` uses the SageAttention3 Blackwell implementation.
+
+### SageAttention3 installation
+
+```bash
+git clone https://github.com/thu-ml/SageAttention.git
+cd SageAttention/sageattention3_blackwell
+python setup.py install
+```
+
+Verify the installation:
+
+```bash
+python -c "import sageattn3; print(sageattn3.__file__)"
+```
+
+```bash
+vllm-omni serve <model> --diffusion-attention-backend SAGE_ATTN_3
+```
+
+`SAGE_ATTN_3` requires CUDA, an importable `sageattn3`, and a Blackwell-class
+GPU. Its kernel assumes the query-head count equals the key/value-head count.
+GQA and MQA diffusion calls therefore fall back to PyTorch SDPA for
+correctness.
+
+For common configuration and platform routing, see the
+[attention backend overview](../attention_backends.md).

docs/user_guide/diffusion/attention_backends/trtllm.md

@@ -0,0 +1,103 @@
+# TRTLLM Attention
+
+`TRTLLM_ATTN` runs FlashInfer's trtllm-gen FMHA kernels. It supports dense
+BF16 attention plus optional Skip-Softmax sparsity or SAGE quantization.
+
+For common selection and per-role configuration, see the
+[attention backend overview](../attention_backends.md).
+
+## Requirements
+
+Selecting `TRTLLM_ATTN` requires:
+
+- datacenter Blackwell, sm_100 or sm_103;
+- `head_dim=128`;
+- FlashInfer; and
+- a model-declared packed or mask-free compatible path.
+
+It is not supported on workstation Blackwell (sm_120/sm_121). An incompatible
+explicit selection raises instead of silently changing the backend.
+
+## Dense mode
+
+Select `TRTLLM_ATTN` without `skip_softmax` or `quant` to run dense BF16:
+
+```bash
+vllm-omni serve <model> --diffusion-attention-backend TRTLLM_ATTN
+```
+
+## Skip-Softmax
+
+Skip-Softmax trades some fidelity for speed by skipping low-contribution
+attention work. See the [feature design](../../../design/feature/skip_softmax.md)
+for the algorithm and implementation contract.
+
+| Key | Valid values | Meaning |
+| --- | --- | --- |
+| `target_sparsity` | finite, `[0, 1]` | Point on a model-calibrated curve |
+| `threshold` | finite, `>= 0` | Direct threshold; mutually exclusive with `target_sparsity` |
+| `disabled_until_timestep` | finite, `[0, 1]` | Keep early high-noise steps dense until normalized `t <= D` |
+
+```bash
+vllm-omni serve Wan-AI/Wan2.2-T2V-A14B-Diffusers \
+  --diffusion-attention-config '{"default":{"backend":"TRTLLM_ATTN",\
+    "skip_softmax":{"target_sparsity":0.65,"disabled_until_timestep":0.86}}}'
+```
+
+```python
+from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, SkipSoftmaxSpec
+
+config = AttentionConfig(
+    default=AttentionSpec(
+        backend="TRTLLM_ATTN",
+        skip_softmax=SkipSoftmaxSpec(
+            target_sparsity=0.65,
+            disabled_until_timestep=0.86,
+        ),
+    ),
+)
+```
+
+Start with `target_sparsity=0.65` and `disabled_until_timestep=0.86`. Increase
+sparsity only after comparing quality against dense output at the same seed.
+The benefit grows with sequence length because only attention is accelerated.
+
+## SAGE quantization
+
+TRTLLM SAGE quantizes Q/K per block and V per channel so both attention
+matrix multiplications use low precision.
+
+| Key | Valid values | Meaning |
+| --- | --- | --- |
+| `dtype_qk` | `int8`, `fp8_e4m3` | Q/K quantization dtype; absent means dense |
+| `q_block_size` | `1`, `4`, `16` | Q scale block size; default `1` |
+| `k_block_size` | `1`, `4`, `16` | K scale block size; default `16` |
+
+V is always quantized per channel to FP8 E4M3 and K-smoothing is internal.
+Only block sizes with compiled kernels are accepted.
+
+```bash
+vllm-omni serve Wan-AI/Wan2.2-T2V-A14B-Diffusers \
+  --diffusion-attention-config '{"default":{"backend":"TRTLLM_ATTN",\
+    "quant":{"dtype_qk":"fp8_e4m3","q_block_size":1,"k_block_size":16}}}'
+```
+
+```python
+from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, AttnQuantSpec
+
+config = AttentionConfig(
+    default=AttentionSpec(
+        backend="TRTLLM_ATTN",
+        quant=AttnQuantSpec(
+            dtype_qk="fp8_e4m3",
+            q_block_size=1,
+            k_block_size=16,
+        ),
+    ),
+)
+```
+
+This path requires FlashInfer 0.6.16rc1 or newer. FP8 Q/K kernels are
+available on sm_100 and sm_103; INT8 Q/K kernels are compiled for sm_100 only.
+The shared `AttnQuantSpec` is backend typed: values intended for
+`FLASHINFER_ATTN` are rejected here and vice versa.

docs/user_guide/diffusion/cpu_offload.md

@@ -1,389 +1,75 @@
 # CPU Offloading for Diffusion Models
 
-## Overview
+vLLM-Omni provides three CPU-offload strategies for diffusion models. Choose
+the coarsest strategy that meets your memory target: finer-grained streaming
+saves more device memory but adds more transfer and setup overhead.
 
-vLLM-Omni provides two offloading strategies to reduce GPU memory usage for diffusion models:
+For the shared factory, discovery, and lifecycle contract, see the
+[CPU Offloading design](../../design/feature/offloader/README.md).
 
-1. **Model-level (Sequential) Offloading**: Mutual exclusion between DiT model and encoder - only one is on GPU at a time.
+## Choose a strategy
-2. **Layerwise (Blockwise) Offloading**: Keeps only one transformer block on GPU at a time with compute-memory overlap.
 
-Both strategies use pinned memory for faster CPU-GPU transfers. The strategies are **mutually exclusive** for now - if both are enabled, layerwise takes priority.
+| Strategy | Device residency | Parallel scope | Guide |
+| --- | --- | --- | --- |
+| Model-level (sequential) | One pipeline component group at a time | Single device | [Model-Level Offloading](offloader/cpu_offload.md) |
+| Layerwise (blockwise) | One transformer block, with next-block prefetch | Single device | [Layerwise Offloading](offloader/layerwise_offload.md) |
+| Distributed layerwise | Fixed two-block device buffer; optional host-weight sharding and AllGather | Multiple GPU/NPU ranks | [Distributed Layerwise Offloading](offloader/distributed_layerwise_offload.md) |
 
+All strategies use pinned host memory for faster transfers where applicable.
+Configuration priority is:
+
+1. Distributed layerwise offloading.
+2. Layerwise offloading.
+3. Model-level offloading.
+
+Treat the flags as mutually exclusive. If more than one is enabled, the
+higher-priority strategy is selected.
+
+## Quick selection
+
+- Use [model-level offloading](offloader/cpu_offload.md) when swapping whole
+  encoders and DiTs is enough to fit and phase-boundary transfers are
+  acceptable.
+- Use [layerwise offloading](offloader/layerwise_offload.md) for compute-heavy video
+  DiTs where block transfers can overlap computation.
+- Use [distributed layerwise offloading](offloader/distributed_layerwise_offload.md)
+  when a multi-rank deployment also needs bounded device residency and,
+  optionally, sharded host weights.
+
+## Supported models
+
+| Architecture | Example models | DiT class | Model-level | Layerwise | Distributed layerwise | Layerwise block attributes |
+| --- | --- | --- | :---: | :---: | :---: | --- |
+| Flux2Pipeline | `black-forest-labs/FLUX.2-dev` | `Flux2Transformer2DModel` | yes | yes | — | `transformer_blocks`, `single_transformer_blocks` |
+| LongCatImagePipeline | `meituan-longcat/LongCat-Image` | `LongCatImageTransformer2DModel` | — | yes | — | `transformer_blocks`, `single_transformer_blocks` |
+| NextStep11Pipeline | `stepfun-ai/NextStep-1.1` | `NextStepModel` | — | yes | — | `layers` |
+| OvisImagePipeline | `AIDC-AI/Ovis-Image-7B` | `OvisImageTransformer2DModel` | — | yes | — | `transformer` |
+| QwenImagePipeline | `Qwen/Qwen-Image` | `QwenImageTransformer2DModel` | yes | yes | — | `transformer_blocks` |
+| StableDiffusionXLPipeline | `stabilityai/stable-diffusion-xl-base-1.0` | `SDXLUNet2DConditionModel` | yes | yes | — | `down_blocks`, `up_blocks` |
+| StableDiffusion3Pipeline | `stabilityai/stable-diffusion-3.5-medium` | `SD3Transformer2DModel` | — | yes | — | `transformer_blocks` |
+| Wan22I2VPipeline | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | `WanTransformer3DModel` | yes | yes | — | `blocks` |
+| Wan22Pipeline | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | `WanTransformer3DModel` | yes | yes | — | `blocks` |
+| SoulXSingerPipeline / SoulXSingerSVCPipeline | `Soul-AILab/SoulX-Singer` | `DiffLlama` | yes | yes | — | `layers` |
+| BagelPipeline | `ByteDance-Seed/BAGEL-7B-MoT` | `Qwen2MoTModel` | — | yes | — | `layers`, customized modules |
+| Cosmos3OmniDiffusersPipeline | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | `Cosmos3VFMTransformer`, `Cosmos3LanguageModel` | yes | yes | yes | `layers`, `gen_layers` |
+
+Model-level support requires discoverable DiT and encoder components.
+Layerwise support requires transformer block topology. Distributed support
+reuses that topology but still requires validation for the model, checkpoint,
+and parallel configuration.
+
+## Compatibility anchors
+
+These headings preserve links to sections that moved into dedicated guides.
 
 ## Model-level (Sequential) Offloading
 
-### How It Works
+See [Model-Level Offloading](offloader/cpu_offload.md).
-
-Model-level offloading implements mutual exclusion between DiT transformer and encoder modules using pre forward hooks:
-
-- **When encoders run**: DiT transformer is offloaded to CPU
-- **When DiT runs**: Encoders are offloaded to CPU, if more than one dit models, only one loaded on GPU, others get offloaded to CPU.
-- **VAE**: Stays resident on GPU
-
-Before each module's forward pass, the hook automatically moves it to GPU while offloading the other module group to CPU. Transfers use pinned memory for speed.
-
-### Usage
-
-**Python API:**
-```python
-from vllm_omni import Omni
-
-m = Omni(model="Wan-AI/Wan2.2-T2V-A14B-Diffusers", enable_cpu_offload=True)
-```
-
-**CLI:**
-```bash
-vllm serve Wan-AI/Wan2.2-T2V-A14B-Diffusers --omni --enable-cpu-offload
-```
-
-### To Support a Model
-
-Implement the `SupportsComponentDiscovery` protocol to declare which
-submodules serve as pipeline components (used by offloading, HSDP
-sharding, and other framework features):
-
-```python
-from typing import ClassVar
-from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery
-
-class MyPipeline(nn.Module, SupportsComponentDiscovery):
-    _dit_modules: ClassVar[list[str]] = ["transformer"]
-    _encoder_modules: ClassVar[list[str]] = ["text_encoder", "vision_model"]
-    _vae_modules: ClassVar[list[str]] = ["vae"]
-    _resident_modules: ClassVar[list[str]] = []  # optional
-
-    def __init__(self):
-        super().__init__()
-        self.transformer = ...     # DiT — stays on GPU during denoising
-        self.text_encoder = ...    # Encoder — offloaded to CPU during denoising
-        self.vision_model = ...    # Encoder — offloaded to CPU during denoising
-        self.vae = ...             # VAE — always on GPU
-```
-
-- `_dit_modules`: attribute names of denoising submodules (kept on GPU
-  during the diffusion loop).
-- `_encoder_modules`: attribute names of encoder/vision submodules
-  (offloaded to CPU during the diffusion loop).
-- `_vae_modules`: attribute names of VAE(s) (always kept on GPU, not
-  part of the mutual exclusion hooks).
-- `_resident_modules`: attribute names of small submodules that must
-  stay on GPU during layerwise offloading (e.g. embedders, connectors).
-  Optional — defaults to `[]`.
-
-All attribute names support dotted paths for nested submodules
-(e.g. `"pipe.transformer"`, `"bagel.time_embedder"`).
-
-Both DiT and encoder lists are needed because the offload hooks use
-mutual exclusion: when one group runs, the other moves to CPU.
-
-### Limitations
-- Cold start latency increases
-- Adds overhead from CPU-GPU transfers between encoder and denoising phases
-- Support single GPU only for now
-
-
-### Component offloading for split models (e.g. Cosmos3)
-
-Some models split their transformer into mutually-exclusive *components* that run
-in different phases of a single forward pass rather than as separate pipeline
-components -- e.g. Cosmos3's understanding (reasoner) component runs once per
-generation while the generation (generator) component runs every denoising step.
-Such models have no separate text encoder to swap against, so the transformer
-owns a small model-local offload path and wraps each phase with
-`with self._offload_context(name):`
-
-```python
-class Cosmos3VFMTransformer(nn.Module):
-    def forward(self, ...):
-        with self._offload_context("reasoner"):
-            ...  # understanding pass, runs once
-        with self._offload_context("generator"):
-            ...  # denoising pass, runs every step
-```
-
-Model-level offloading then keeps exactly one component GPU-resident at a time
-(the other on CPU), reusing the same `SequentialOffloadHook` `.to()` movers. The
-pipeline opts in by exposing `enable_omni_model_cpu_offload` (which drives the
-transformer's `enable_model_cpu_offload` and pins the VAE). Layerwise offloading
-works for these models too -- each component declares its own block container via
-`_layerwise_offload_blocks_attrs`.
-
 
 ## Layerwise (Blockwise) Offloading
 
-### How It Works
+See [Layerwise Offloading](offloader/layerwise_offload.md).
-
-Layerwise offloading keeps only one transformer block on GPU at a time.
-
-As each block completes, the next block is prefetched to GPU while the current block is freed. The pre and forward hooks utilized by layerwise offloading apply a separate CUDA stream (`copy_stream`) to overlap weight transfer with computation, and retain flattened tensors in pinned CPU memory for block parameters re-materialization. Encoders, VAE, and non-block DiT modules (embeddings, norms) always stay on GPU.
-
-**Execution Flow:**
-
-| Block | Pre-forward Hook | Forward | Post-forward Hook |
-|-------|------------------|---------|-------------------|
-| block-0 | Prefetch block-1 (async) | Compute block-0 | Free block-0 |
-| block-1 | Prefetch block-2 (async) | Compute block-1 | Free block-1 |
-| ... | ... | ... | ... |
-| block-(n-1) | **Prefetch block-0** (async) | Compute block-(n-1) | Free block-(n-1) |
-
-Each transformer block has a `LayerwiseOffloadHook` that prefetches the next block before forward and frees the current block after forward.
-
-Layerwise offloading is primarily recommended for large **video generation models** where the compute cost per block is high enough to effectively overlap with memory prefetch operations. For example, Wan2.2 T2V and I2V pipelines.
-
-### Usage
-
-**Python API:**
-```python
-from vllm_omni import Omni
-
-# Text-to-video
-m = Omni(model="Wan-AI/Wan2.2-T2V-A14B-Diffusers", enable_layerwise_offload=True)
-
-# Or image-to-video
-m = Omni(model="Wan-AI/Wan2.2-I2V-A14B-Diffusers", enable_layerwise_offload=True)
-```
-
-**CLI:**
-```bash
-# Text-to-video
-vllm serve Wan-AI/Wan2.2-T2V-A14B-Diffusers --omni --enable-layerwise-offload
-
-# Or image-to-video
-vllm serve Wan-AI/Wan2.2-I2V-A14B-Diffusers --omni --enable-layerwise-offload
-```
-
-### To Support a Model
-
-Models must define the blocks attribute name for layerwise offloading:
-
-```python
-class WanTransformer3DModel(nn.Module):
-    _layerwise_offload_blocks_attrs = ["blocks"]  # Attribute names containing transformer blocks
-
-    def __init__(self):
-        self.blocks = nn.ModuleList([...])  # Transformer blocks
-```
-
-For models with multiple block types:
-
-```python
-class Flux2Transformer2DModel(nn.Module):
-    _layerwise_offload_blocks_attrs = ["transformer_blocks", "single_transformer_blocks"]
-```
-
-### Limitations
-- Cold start latency increases because offloaded components must be moved to CPU
-  during setup; layerwise offload may add extra weight consolidation and pinning
-  work.
-- Performance depends on compute cost and H2D bandwidth as well
-- Support single GPU only for now
-
 
 ## Distributed Layerwise Offloading
 
-### How It Works
+See [Distributed Layerwise Offloading](offloader/distributed_layerwise_offload.md).
-
-Distributed layerwise offloading extends single-GPU layerwise offloading to
-multi-device deployments. In the default mode, each rank stores only
-**1/dp_size** of the model weights in host memory; full layer weights are
-reconstructed at runtime via **AllGather** on a dedicated communication stream,
-overlapped with computation via a fixed double-buffer scheme. The
-`--dlo-no-use-allgather` mode instead streams standard-loader rank-local
-weights independently and does not shard weights across ranks.
-
-For the implementation design and compatibility matrix, see
-[Distributed Layerwise Offload](../../design/feature/distributed_layerwise_offload.md).
-
-**Key features:**
-- **Weight sharding + AllGather**: each rank stores 1/dp_size of weights,
-  reconstructed per-layer via `all_gather_into_tensor`
-- **Fixed double-buffer**: exactly 2 transformer blocks on each device at any
-  time, regardless of model size
-- **DP multi-concurrency**: N concurrent requests processed in parallel
-  (AllGather only gathers weight shards, which are request-independent)
-- **mmap weight loading**: weights loaded as mmap views pointing to shared OS
-  page cache, eliminating O(dp_size × model_size) RSS during model creation
-- **Platform-agnostic**: works on NVIDIA GPU (CUDA/NCCL) and Ascend NPU
-  (CANN/HCCL) via vLLM-Omni's platform abstraction
-
-**Execution Flow (per device):**
-
-```
-Compute Stream:  [Layer N]          [Layer N+1]          [Layer N+2]
-H2D Stream:      [H2D Shard N+1]   [H2D Shard N+2]
-AllGather:       [AG N+1]          [AG N+2]
-Slot Usage:      Slot0: Layer N    Slot1: Layer N+1
-```
-
-### Usage
-
-**CLI:**
-```bash
-# 4× GPU/NPU with AllGather (recommended)
-vllm serve /path/to/model --omni \
-  --enable-distributed-layerwise-offload \
-  --data-parallel-size 4
-
-# Without DLO AllGather (each rank streams standard-loader rank-local weights)
-vllm serve /path/to/model --omni \
-  --enable-distributed-layerwise-offload \
-  --data-parallel-size 4 \
-  --dlo-no-use-allgather
-
-# With SP instead of DP (long sequences)
-vllm serve /path/to/model --omni \
-  --enable-distributed-layerwise-offload \
-  --usp 4
-```
-
-**Python API:**
-```python
-from vllm_omni import Omni
-
-m = Omni(
-    model="/path/to/model",
-    enable_distributed_layerwise_offload=True,
-    dlo_use_allgather=True,  # default
-)
-```
-
-### CLI Flags
-
-| Flag | Description | Default |
-|------|-------------|---------|
-| `--enable-distributed-layerwise-offload` | Enable DLO | `false` |
-| `--data-parallel-size N` | Number of DP ranks; DLO weight-sharding group when AllGather is enabled | `1` |
-| `--dlo-use-allgather` | Shard + AllGather (saves CPU, requires concurrent requests) | `true` |
-| `--dlo-no-use-allgather` | Use standard-loader rank-local weights; no DLO sharding or AllGather | `false` |
-
-### How model weights are loaded (mmap path)
-
-When DLO + AllGather is active, the offloader:
-
-1. **Saves non-persistent buffers** (e.g. RoPE `inv_freq`, timestep `freqs`)
-   from the normally-created transformer
-2. **Converts to meta device** via `to_empty(device="meta")`, releasing random
-   initialization weights
-3. **Loads checkpoint weights as mmap views** via `safe_open().get_tensor()`,
-   which return views into the OS page cache (shared across all ranks, 0 RSS)
-4. **Calls `post_load_weights()`** to apply model-specific dtype conversions
-   (e.g. Cosmos3's `time_embedder` → FP32)
-5. **Restores non-persistent buffers** from saved copies
-
-This approach requires **zero model-specific code changes** — no pipeline or
-transformer modifications are needed.
-
-### OffloadPlan (declarative topology metadata)
-
-Models can optionally declare an `OffloadPlan` class variable to provide
-topology metadata (block attribute names, submodules to offload) without
-any offload-specific logic:
-
-```python
-from vllm_omni.diffusion.offloader import OffloadPlan
-
-class MyPipeline(nn.Module):
-    _dit_modules = ["transformer"]
-    _offload_plan = OffloadPlan(
-        block_attrs={"transformer": ("blocks",)},
-        offload_submodules={"context_encoder": "layers"},
-    )
-```
-
-When not declared, the offloader falls back to `_layerwise_offload_blocks_attrs`
-and heuristic attribute search.  This is backward-compatible — existing models
-work without any changes.
-
-### DP Multi-concurrency
-
-When `--data-parallel-size > 1` and AllGather is enabled, the scheduler batches
-up to `dp_size` requests per denoise step.  Each DP rank processes a different
-request while AllGather synchronizes only weight shards (request-independent).
-
-**Requirements for concurrent requests:**
-- `num_inference_steps` must be specified explicitly (None is not allowed)
-- All concurrent requests must have the same `num_inference_steps` value
-- These constraints exist because AllGather is a collective that requires
-  every rank to participate at each denoise step
-
-### Limitations
-- Online quantization (FP8) is incompatible with mmap loading — falls back to
-  regular loading only when the model is outside the DLO+AllGather mmap path;
-  online quantization with DLO+AllGather is rejected. Use
-  `--dlo-no-use-allgather` or disable online quantization.
-- Tensor Parallel > 1 is rejected in the DLO+AllGather mmap path because TP-aware
-  weight-loader callbacks are bypassed. The no-AllGather path retains the
-  standard TP loader and is experimental.
-- HSDP + AllGather is rejected because it would double-shard parameters. HSDP
-  with no-AllGather is accepted at configuration level, but has limited
-  end-to-end validation.
-- `num_inference_steps=None` is not allowed in DP multi-concurrency mode
-
-**Module Discovery**
-
-The offloader discovers pipeline components in two ways:
-
-1. **Protocol-based** (preferred): If the pipeline implements
-    `SupportsComponentDiscovery`, its `_dit_modules`, `_encoder_modules`,
-    `_vae_modules`, and `_resident_modules` class variables are used
-    directly.  All attribute names support dotted paths (e.g.
-    `"pipe.transformer"`, `"bagel.time_embedder"`) for nested submodules.
-
-2. **Fallback attribute scan**: Otherwise, the offloader scans for
-    well-known attribute names:
-    - **DiT modules**: `transformer`, `transformer_2`, `dit`, `sr_dit`, `language_model`, `transformer_blocks`, `model`
-    - **Encoders**: `text_encoder`, `text_encoder_2`, `text_encoder_3`, `image_encoder`
-    - **VAE**: `vae`, `audio_vae`
-
-**Hook System**
-
-Both strategies use vLLM-Omni's hook registry system (`HookRegistry` and `ModelHook`) to register pre/post forward callbacks on modules, enabling automatic swapping without modifying model code.
-
-**Backend Architecture**
-
-```
-OffloadBackend (base class)
-├── ModelLevelOffloadBackend → uses SequentialOffloadHook (.to() swap)
-│                              (delegates to a pipeline's enable_omni_model_cpu_offload
-│                               for split models like Cosmos3)
-├── LayerWiseOffloadBackend → uses LayerwiseOffloadHook
-│                          (single-GPU, full weights on host)
-└── DistributedLayerwiseOffloadBackend → uses DistributedLayerwiseOffloadHook
-                                         (multi-GPU, sharded AllGather or rank-local mode)
-```
-
-Factory function `get_offload_backend()` selects the appropriate backend based on
-configuration.
-
-For split models, `ModelLevelOffloadBackend.enable()` detects a pipeline's
-`enable_omni_model_cpu_offload` hook and delegates to it; Cosmos3 then swaps its
-reasoner/generator components inside the model forward pass.
-
-
-## Supported Models
-
-| Architecture | Example Models | DiT Class | Model-Level Offload | Layerwise Offload | Distributed Layerwise Offload | Blocks Attrs (Layerwise specific) |
-|--------------|----------------|-----------|---------------------|-------------------|-------------------------------|-----------------------------------|
-| Flux2Pipeline | `black-forest-labs/FLUX.2-dev` | `Flux2Transformer2DModel` | ✓ | ✓ | - | `"transformer_blocks"`, `"single_transformer_blocks"` |
-| LongCatImagePipeline | `meituan-longcat/LongCat-Image` | `LongCatImageTransformer2DModel` | - | ✓ | - | `"transformer_blocks"`, `"single_transformer_blocks"` |
-| NextStep11Pipeline | `stepfun-ai/NextStep-1.1` | `NextStepModel` | - | ✓ | - | `"layers"` |
-| OvisImagePipeline | `AIDC-AI/Ovis-Image-7B` | `OvisImageTransformer2DModel` | - | ✓ | - | `"transformer"` |
-| QwenImagePipeline | `Qwen/Qwen-Image` | `QwenImageTransformer2DModel` | ✓ | ✓ | - | `"transformer_blocks"` |
-| StableDiffusionXLPipeline | `stabilityai/stable-diffusion-xl-base-1.0` | `SDXLUNet2DConditionModel` | ✓ | ✓ | - | `"down_blocks"`, `"up_blocks"` |
-| StableDiffusion3Pipeline | `stabilityai/stable-diffusion-3.5-medium` | `SD3Transformer2DModel` | - | ✓ | - | `"transformer_blocks"` |
-| Wan22I2VPipeline | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | `WanTransformer3DModel` | ✓ | ✓ | - | `"blocks"` |
-| Wan22Pipeline | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | `WanTransformer3DModel` | ✓ | ✓ | - | `"blocks"` |
-| SoulXSingerPipeline / SoulXSingerSVCPipeline | `Soul-AILab/SoulX-Singer` | `DiffLlama` (`cfm_decoder.model.diff_estimator`) | ✓ | ✓ | - | `"layers"` |
-| BagelPipeline | `ByteDance-Seed/BAGEL-7B-MoT` | `Qwen2MoTModel` | - | ✓ | - | `"layers"`, `"customized modules"` |
-| Cosmos3OmniDiffusersPipeline | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | `Cosmos3VFMTransformer`, `Cosmos3LanguageModel` | ✓ | ✓ | ✓ | `"layers"`, `"gen_layers"` |
-
-**Notes:**
-- Model-Level Offloading is expected to be supported by all common diffusion models (DiT and encoders) naturally
-- Layerwise Offloading requires DiT class to define `_layerwise_offload_blocks_attrs` pointing to transformer blocks
-- Distributed Layerwise Offloading reuses the layerwise block topology, but the
-  model and parallelism combination still needs validation. AllGather mode
-  additionally requires a compatible checkpoint/mmap layout; no-AllGather mode
-  retains the standard loader's rank-local tensors. See the
-  [DLO design](../../design/feature/distributed_layerwise_offload.md) and
-  [Cosmos3 DistOffload recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/cosmos3/Cosmos3-DistOffload.md)
-  for usage examples.

docs/user_guide/diffusion/lora.md

@@ -1,14 +1,18 @@
 # LoRA (Low-Rank Adaptation) Guide
 
-LoRA (Low-Rank Adaptation) enables fine-tuning diffusion models by adding trainable low-rank matrices to existing model weights. vLLM-Omni currently supports PEFT-style LoRA adapters, allowing you to customize model behavior without modifying the base model weights.
+LoRA (Low-Rank Adaptation) enables fine-tuning diffusion models by adding trainable low-rank matrices to existing model weights. vLLM-Omni supports two LoRA backends: **PEFT** for PEFT-style adapters, and **Distill** usually for few-steps inference. **PEFT** backend allows you to customize model behavior without modifying the base model weights. **Distill** backend fuses LoRA weights into the base model at initialization.
 
 ## Overview
 
-LoRA adapters are lightweight, model-specific fine-tuning weights that can be dynamically loaded and applied to diffusion models. vLLM-Omni uses a unified LoRA handling mechanism similar to vLLM with LRU cache management.
+LoRA adapters are lightweight, model-specific fine-tuning weights that can be applied to diffusion models in two ways:
+
+- **PEFT backend** (`--lora-backend peft`, default): Loads a PEFT-format adapter folder via `DiffusionLoRAManager`. Adapters are cached (LRU) and activated per request via `LoRARequest`. It uses a unified LoRA handling mechanisms similar to vLLM with LRU cache management.
+- **Distill backend** (`--lora-backend distill`): Calls `pipeline.load_lora_weights` once at initialization to fuse one or more concrete checkpoint files directly into the base weights. Typically used for distilled few-step LoRAs (e.g. Lightning, LightX2V).
 
 ## LoRA Adapter Format
 
-LoRA adapters must be in **PEFT (Parameter-Efficient Fine-Tuning)** format. A typical LoRA adapter directory structure:
+### PEFT (Parameter-Efficient Fine-Tuning) format (default)
+A typical PEFT-format LoRA adapter directory structure:
 
 ```
 lora_adapter/
@@ -25,19 +29,20 @@ The `adapter_config.json` file contains metadata about the LoRA adapter, includi
 
 ### Offline Inference
 
-#### Pre-loaded LoRA
+#### PEFT backend: pre-loaded LoRA
 
-Load a LoRA adapter at initialization. This adapter is pre-loaded into the cache and can be activated by requests:
+Load a PEFT-format LoRA adapter at initialization. The adapter is pre-loaded into the cache and can be activated per request:
 
 ```python
 from vllm_omni import Omni
 from vllm_omni.lora.request import LoRARequest
 
-lora_path="/path/to/lora_adapter"
+lora_path = "/path/to/lora_adapter"
 
 omni = Omni(
     model="stabilityai/stable-diffusion-3.5-medium",
-    lora_path=lora_path
+    lora_path=lora_path,
+    lora_backend="peft",  # default, can be omitted
 )
 
 lora_request = LoRARequest(
@@ -53,6 +58,98 @@ outputs = omni.generate(
 )
 ```
 
+#### Distill backend: fuse distilled LoRA at init
+
+For distilled few-step LoRAs, pass `lora_backend="distill"` together with one or more concrete `.safetensors` files. The weights are fused into the base model once at init; subsequent `generate()` calls do not need a `LoRARequest`.
+
+##### Supported pipelines
+
+For Qwen-Image and Wan pipelines, the distill backend calls `pipeline.load_lora_weights(...)` during worker initialization.
+
+| Pipeline | Supported distilled LoRA repo | Notes |
+|----------|--------------------------------|-------|
+| `QwenImagePipeline` | `lightx2v/Qwen-Image-2512-Lightning` | Used with Qwen-Image-2512 Lightning-style few-step inference. |
+| `Wan22Pipeline` | `lightx2v/Wan2.1-Distill-Loras`, `lightx2v/Wan2.2-Distill-Loras` | Wan2.1 uses one LoRA file. For dual-transformer Wan2.2 MoE, pass high-noise then low-noise LoRA files. |
+| `Wan22I2VPipeline` | `lightx2v/Wan2.2-Distill-Loras` | For dual-transformer Wan2.2 MoE, pass high-noise then low-noise LoRA files. |
+
+Other diffusion pipelines are not currently listed as supporting distilled LoRA. Use the PEFT backend for request-time adapters, or bake converted weights into a local Diffusers directory before serving.
+
+Single-file example (Qwen-Image-Lightning):
+
+```python
+from vllm_omni import Omni
+
+omni = Omni(
+    model="Qwen/Qwen-Image-2512",
+    lora_path="/path/to/Qwen-Image-2512-Lightning.safetensors",
+    lora_backend="distill",
+)
+
+outputs = omni.generate(prompt="A piece of cheesecake")
+```
+
+Multi-file example (Wan2.2 MoE, high + low noise):
+
+```python
+from vllm_omni import Omni
+
+omni = Omni(
+    model="Wan-AI/Wan2.2-T2V-A14B-Diffusers",
+    lora_path=[
+        "/path/to/wan2.2_high_noise_lora.safetensors",   # -> transformer
+        "/path/to/wan2.2_low_noise_lora.safetensors",    # -> transformer_2
+    ],
+    lora_backend="distill",
+)
+```
+
+The CLI examples under `examples/offline_inference/` accept the same flags, e.g.:
+
+```bash
+python examples/offline_inference/text_to_video/text_to_video.py \
+  --model Wan-AI/Wan2.2-T2V-A14B-Diffusers \
+  --lora-backend distill \
+  --lora-path /path/to/high.safetensors /path/to/low.safetensors \
+  --prompt "A cat playing with yarn"
+```
+
+##### Online serving
+
+Distilled LoRAs can also be fused when an online diffusion server starts. The
+adapter remains active for the lifetime of that server, so requests do not pass
+a per-request `lora` field.
+
+```bash
+vllm serve Qwen/Qwen-Image-2512 \
+  --omni \
+  --port 8091 \
+  --lora-backend distill \
+  --lora-path /path/to/Qwen-Image-2512-Lightning-4steps.safetensors
+```
+
+Then send a normal generation request with the sampling settings expected by
+the distilled checkpoint:
+
+```bash
+curl -s http://localhost:8091/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -d '{
+    "messages": [
+      {"role": "user", "content": "A Chinese female college student, around 20 years old, with a very short haircut that conveys a gentle, artistic vibe. Her hair naturally falls to partially cover her cheeks, projecting a tomboyish yet charming demeanor. She has cool-toned fair skin and delicate features, with a slightly shy yet subtly confident expression—her mouth crooked in a playful, youthful smirk. She wears an off-shoulder top, revealing one shoulder, with a well-proportioned figure. The image is framed as a close-up selfie: she dominates the foreground, while the background clearly shows her dormitory—a neatly made bed with white linens on the top bunk, a tidy study desk with organized stationery, and wooden cabinets and drawers. The photo is captured on a smartphone under soft, even ambient lighting, with natural tones, high clarity, and a bright, lively atmosphere full of youthful, everyday energy."}
+    ],
+    "extra_body": {
+      "height": 1024,
+      "width": 1024,
+      "num_inference_steps": 4,
+      "true_cfg_scale": 1.0,
+      "seed": 42
+    }
+  }' | jq -r '.choices[0].message.content[0].image_url.url' | cut -d',' -f2 | base64 -d > qwen_image_distill_lora_output.png
+```
+
+For Wan2.2 MoE serving, pass the high-noise checkpoint first and the low-noise
+checkpoint second to `--lora-path`.
+
 !!! note "Server-side Path Requirement"
     The LoRA adapter path (`local_path`) must be readable on the **server** machine. If your client and server are on different machines, ensure the LoRA adapter is accessible via a shared mount or copied to the server.
 
@@ -141,6 +238,7 @@ Notes:
 
 - This route avoids runtime LoRA loading changes in vLLM-Omni when you choose to bake converted weights into a local Diffusers directory.
 - Output quality and speed depend on the replacement checkpoints and sampling params you choose.
+- If you only need to fuse distilled LoRAs into a Wan2.2 checkpoint at load time (without the full LightX2V convert + assemble pipeline), you can instead pass them directly via `--lora-backend distill --lora-path <high>.safetensors <low>.safetensors`. See the [Distill backend](#distill-backend-fuse-distilled-lora-at-init) section above.
 
 
 ## See Also

docs/user_guide/diffusion/offloader/cpu_offload.md

@@ -0,0 +1,81 @@
+# Model-Level Offloading
+
+Model-level, or sequential, offloading keeps only the pipeline component group
+currently executing on the accelerator. It is the simplest offload strategy
+and is selected with `--enable-cpu-offload`.
+
+## How it works
+
+Pre-forward hooks enforce mutual exclusion between DiT and encoder modules:
+
+- before an encoder runs, the DiT moves to CPU;
+- before a DiT runs, encoders and other DiTs move to CPU; and
+- VAE modules remain on the accelerator.
+
+Pinned host memory reduces transfer overhead. Transfers occur at phase
+boundaries, so cold-start and encoder-to-denoiser transitions become slower.
+
+## Usage
+
+```python
+from vllm_omni import Omni
+
+omni = Omni(
+    model="Wan-AI/Wan2.2-T2V-A14B-Diffusers",
+    enable_cpu_offload=True,
+)
+```
+
+```bash
+vllm serve Wan-AI/Wan2.2-T2V-A14B-Diffusers \
+  --omni --enable-cpu-offload
+```
+
+## Model integration
+
+Pipelines should implement `SupportsComponentDiscovery`:
+
+```python
+from typing import ClassVar
+
+from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery
+
+
+class MyPipeline(nn.Module, SupportsComponentDiscovery):
+    _dit_modules: ClassVar[list[str]] = ["transformer"]
+    _encoder_modules: ClassVar[list[str]] = ["text_encoder", "vision_model"]
+    _vae_modules: ClassVar[list[str]] = ["vae"]
+    _resident_modules: ClassVar[list[str]] = []
+```
+
+All entries may be dotted paths. DiT and encoder lists are both required for
+mutual exclusion. VAE modules are pinned but not swapped; resident modules are
+small modules that must stay on the accelerator for layerwise paths.
+
+## Split-model components
+
+Some models, such as Cosmos3, split one transformer into mutually exclusive
+components that run in different phases. The pipeline exposes
+`enable_omni_model_cpu_offload`, and the backend delegates to the model-local
+contexts:
+
+```python
+class Cosmos3VFMTransformer(nn.Module):
+    def forward(self, ...):
+        with self._offload_context("reasoner"):
+            ...
+        with self._offload_context("generator"):
+            ...
+```
+
+This preserves the same invariant—exactly one component is device resident—
+while reusing sequential `.to()` movers.
+
+## Limitations
+
+- Single device only.
+- Higher cold-start latency.
+- Transfers between encoder and denoising phases add latency.
+
+See the [model-level design](../../../design/feature/offloader/cpu_offload.md)
+for lifecycle and extension invariants.

docs/user_guide/diffusion/offloader/distributed_layerwise_offload.md

@@ -0,0 +1,118 @@
+# Distributed Layerwise Offloading
+
+Distributed layerwise offloading (DLO) extends block streaming to multi-device
+deployments. With AllGather enabled, each rank stores roughly `1 / dp_size` of
+the host weights and reconstructs each layer at runtime. Without AllGather,
+each rank streams its standard-loader rank-local weights independently.
+
+See the [DLO feature design](../../../design/feature/offloader/distributed_layerwise_offload.md)
+for the implementation contract and compatibility matrix.
+
+## Execution model
+
+DLO overlaps three operations with a fixed two-block device buffer:
+
+```text
+Compute stream:  [Layer N]          [Layer N+1]        [Layer N+2]
+H2D stream:      [H2D shard N+1]    [H2D shard N+2]
+AllGather:       [AG N+1]           [AG N+2]
+Slots:           slot 0: Layer N    slot 1: Layer N+1
+```
+
+AllGather communicates only request-independent weight shards, so data-
+parallel ranks may process different requests concurrently.
+
+## Usage
+
+```bash
+# Four ranks with sharded host weights and AllGather
+vllm serve /path/to/model --omni \
+  --enable-distributed-layerwise-offload \
+  --data-parallel-size 4
+
+# Standard-loader rank-local weights, without DLO AllGather
+vllm serve /path/to/model --omni \
+  --enable-distributed-layerwise-offload \
+  --data-parallel-size 4 \
+  --dlo-no-use-allgather
+
+# Sequence parallel deployment
+vllm serve /path/to/model --omni \
+  --enable-distributed-layerwise-offload \
+  --usp 4
+```
+
+```python
+from vllm_omni import Omni
+
+omni = Omni(
+    model="/path/to/model",
+    enable_distributed_layerwise_offload=True,
+    dlo_use_allgather=True,
+)
+```
+
+## Flags
+
+| Flag | Meaning | Default |
+| --- | --- | --- |
+| `--enable-distributed-layerwise-offload` | Enable DLO | `false` |
+| `--data-parallel-size N` | DP ranks and AllGather weight-sharding group | `1` |
+| `--dlo-use-allgather` | Shard host weights and reconstruct with AllGather | `true` |
+| `--dlo-no-use-allgather` | Keep standard-loader rank-local weights | `false` |
+| `--dlo-resident-layers N` | Keep N leading main-DiT blocks on device; requires no-AllGather and model-declared resident paths | `0` |
+
+## mmap weight loading
+
+The DLO plus AllGather path:
+
+1. saves non-persistent buffers such as RoPE frequencies;
+2. moves the normally created transformer to the meta device;
+3. loads checkpoint tensors as mmap views backed by the shared OS page cache;
+4. calls model-specific `post_load_weights()` conversions; and
+5. restores the saved non-persistent buffers.
+
+This avoids one full checkpoint RSS copy per rank and does not require
+model-specific loading code.
+
+## Declarative topology
+
+Models may declare an `OffloadPlan` instead of embedding offload logic:
+
+```python
+from vllm_omni.diffusion.offloader import OffloadPlan
+
+
+class MyPipeline(nn.Module):
+    _dit_modules = ["transformer"]
+    _offload_plan = OffloadPlan(
+        block_attrs={"transformer": ("blocks",)},
+        offload_submodules={"context_encoder": "layers"},
+    )
+```
+
+When no plan exists, discovery falls back to
+`_layerwise_offload_blocks_attrs` and then heuristic attribute lookup.
+
+## Data-parallel concurrency
+
+With `data_parallel_size > 1` and AllGather enabled, the scheduler can process
+up to `dp_size` requests per denoising step. Every concurrent request must set
+the same explicit `num_inference_steps`; `None` is rejected because every rank
+must enter each collective.
+
+## Limitations
+
+- Online FP8 quantization is rejected with the DLO plus AllGather mmap path.
+  Use `--dlo-no-use-allgather` or disable online quantization.
+- Tensor parallel size greater than one is rejected in the mmap path because
+  it bypasses TP-aware loader callbacks. The no-AllGather path retains the
+  standard loader and remains experimental with TP.
+- HSDP plus AllGather is rejected to avoid double sharding. HSDP without
+  AllGather has limited end-to-end validation.
+- Resident leading layers require `--dlo-no-use-allgather` and a model
+  `OffloadPlan` that declares eligible `resident_dit_paths`.
+- DP concurrency requires an explicit, identical inference-step count.
+
+See the [Cosmos3 DistOffload recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/cosmos3/Cosmos3-DistOffload.md)
+for an end-to-end example.

docs/user_guide/diffusion/offloader/layerwise_offload.md

@@ -0,0 +1,65 @@
+# Layerwise Offloading
+
+Layerwise, or blockwise, offloading keeps one transformer block on the
+accelerator and prefetches the next block while the current block computes.
+It is best suited to compute-heavy video DiTs whose block execution can hide
+host-to-device transfers.
+
+## Execution flow
+
+Each block has a pre-forward and post-forward hook. Parameters are consolidated
+in pinned host tensors and rematerialized for execution on a dedicated copy
+stream.
+
+| Block | Pre-forward hook | Forward | Post-forward hook |
+| --- | --- | --- | --- |
+| block 0 | Prefetch block 1 | Compute block 0 | Free block 0 |
+| block 1 | Prefetch block 2 | Compute block 1 | Free block 1 |
+| ... | ... | ... | ... |
+| last block | Prefetch block 0 | Compute last block | Free last block |
+
+Encoders, VAE modules, and non-block DiT modules such as embeddings and norms
+remain device resident.
+
+## Usage
+
+```python
+from vllm_omni import Omni
+
+omni = Omni(
+    model="Wan-AI/Wan2.2-T2V-A14B-Diffusers",
+    enable_layerwise_offload=True,
+)
+```
+
+```bash
+vllm serve Wan-AI/Wan2.2-T2V-A14B-Diffusers \
+  --omni --enable-layerwise-offload
+```
+
+## Model integration
+
+Transformer classes declare containers of executable blocks:
+
+```python
+class WanTransformer3DModel(nn.Module):
+    _layerwise_offload_blocks_attrs = ["blocks"]
+
+
+class Flux2Transformer2DModel(nn.Module):
+    _layerwise_offload_blocks_attrs = [
+        "transformer_blocks",
+        "single_transformer_blocks",
+    ]
+```
+
+See the [layerwise design](../../../design/feature/offloader/layerwise_offload.md) for
+the discovery and hook invariants. `OffloadPlan` is a separate declarative
+topology path for distributed layerwise offload.
+
+## Limitations
+
+- Single device only.
+- Setup consolidates and pins block parameters, increasing cold-start time.
+- Performance depends on block compute time and host-to-device bandwidth;
+  lightweight blocks may not hide transfers.

docs/user_guide/diffusion/parallelism/hsdp.md

@@ -64,6 +64,41 @@ omni = Omni(
 )
 ```
 
+### Combined with Online FP8 Quantization
+
+HSDP can be combined with online FP8 quantization for diffusion models that
+support both features. HSDP shards the quantized weights across GPUs to reduce
+the per-GPU weight memory footprint.
+
+```python
+from vllm_omni import Omni
+from vllm_omni.diffusion.data import DiffusionParallelConfig
+
+omni = Omni(
+    model="<your-model>",
+    quantization="fp8",
+    parallel_config=DiffusionParallelConfig(
+        use_hsdp=True,
+        hsdp_shard_size=2,
+    ),
+)
+```
+
+For online serving:
+
+```bash
+vllm serve <your-model> \
+    --omni \
+    --quantization fp8 \
+    --use-hsdp \
+    --hsdp-shard-size 2
+```
+
+The model must support both HSDP and online FP8 quantization. HSDP remains
+incompatible with Tensor Parallelism. See the
+[FP8 Quantization Guide](../../quantization/fp8.md) for supported models and
+hardware.
+
 ---
 
 ## Example Script
@@ -147,3 +182,4 @@ For detailed instructions on adding HSDP support to new models, see the [HSDP Co
 1. ✅ **Enable HSDP** - Set `use_hsdp=True` and `hsdp_shard_size` to reduce per-GPU memory for large models
 2. ✅ **Combine with SP** - Use together with `ulysses_degree` for video models requiring both memory reduction and sequence parallelism
 3. ⚠️ **Incompatible with TP** - `tensor_parallel_size` must be 1 when HSDP is enabled
+4. ✅ **Combine with online FP8** - Use `quantization="fp8"` or `--quantization fp8` to reduce the sharded weight memory footprint

docs/user_guide/diffusion/startup_and_loading.md

@@ -0,0 +1,60 @@
+# Diffusion Startup and Loading
+
+Large diffusion models can take several minutes to load at startup. vLLM-Omni
+loads safetensors shards in parallel to reduce this initialization time.
+
+Multi-thread weight loading is enabled by default with four threads. No
+configuration is needed for the default behavior.
+
+## Configuration
+
+| Parameter | CLI flag | Default | Description |
+| --- | --- | --- | --- |
+| `enable_multithread_weight_load` | `--disable-multithread-weight-load` | `True` | Pass the flag to disable multi-thread loading |
+| `num_weight_load_threads` | `--num-weight-load-threads` | `4` | Number of parallel weight-loading threads |
+
+!!! tip
+
+    The default balances startup speed and disk I/O contention. Fast NVMe
+    storage may benefit from more threads, while network storage or hard disks
+    may not.
+
+## Online Serving
+
+```bash
+# Default: multi-thread loading with four threads
+vllm serve Qwen/Qwen-Image --omni --port 8091
+
+# Increase the thread count
+vllm serve Wan-AI/Wan2.2-I2V-A14B-Diffusers --omni \
+  --num-weight-load-threads 8
+
+# Disable multi-thread loading
+vllm serve Qwen/Qwen-Image --omni --disable-multithread-weight-load
+```
+
+## Offline Inference
+
+```python
+from vllm_omni import Omni
+
+# Default: multi-thread loading with four threads
+omni = Omni(model="Qwen/Qwen-Image")
+
+# Increase the thread count
+omni = Omni(
+    model="Wan-AI/Wan2.2-I2V-A14B-Diffusers",
+    num_weight_load_threads=8,
+)
+```
+
+## Reference Benchmarks
+
+The following measurements were collected on NVIDIA H800 hardware. Treat them
+as reference results rather than a guarantee for other storage or hardware
+configurations.
+
+| Model | Sequential loading | Multi-thread loading | Speedup |
+| --- | ---: | ---: | ---: |
+| **Qwen/Qwen-Image** (53.7 GiB) | 168 s | 27 s | **6.2x** |
+| **Wan-AI/Wan2.2-I2V-A14B-Diffusers** (64.5 GiB) | 283 s | 56 s | **5.1x** |

docs/user_guide/diffusion_features.md

@@ -49,7 +49,7 @@ Parallelism methods distribute computation across GPUs without quality loss (mat
 
 | Method | Description | Best For |
 |--------|-------------|----------|
-| **[Multi-Thread Weight Loading](#multi-thread-weight-loading)** | Loads safetensors shards in parallel using a thread pool | All diffusion models; reduces startup from minutes to seconds |
+| **[Multi-Thread Weight Loading](diffusion/startup_and_loading.md)** | Loads safetensors shards in parallel using a thread pool | All diffusion models; reduces startup from minutes to seconds |
 
 **Note:** Some acceleration methods can be combined together for optimized performance. See [Feature Compatibility Table](#feature-compatibility) and [Feature Compatibility Tutorial](feature_compatibility.md) for detailed configuration examples.
 
@@ -100,11 +100,16 @@ The following tables show which models support each feature:
 
 - **🔀SP (Ulysses & Ring)**: Includes both Ulysses-SP and Ring-Attention methods
 - ✅ = Fully supported
+- ✅* = Supported with the constraint listed below the table
 - ❌ = Not supported
+- ❓ = Not verified; not recommended
 
 > Notes:
 
-> 1. CPU Offload has two methods: Module-wise (default for models with DiT + text encoder) and Layerwise. The tables below show **Layerwise support** only. Split models like Cosmos3 (no separate text encoder) swap their reasoner/generator components for module-wise offload; see the [CPU Offload Guide](diffusion/cpu_offload.md).
+> 1. CPU Offload has three strategies: model-level, layerwise, and distributed
+>    layerwise. The tables below show **layerwise support** only. Split models
+>    like Cosmos3 swap their reasoner/generator components for model-level
+>    offload; see the [CPU Offload Guide](diffusion/cpu_offload.md).
 > 2. The **💾Quantization** column is collapsed for readability. See [Quantization](quantization/overview.md) for per-method and per-model support details.
 
 ### ImageGen
@@ -119,7 +124,7 @@ The following tables show which models support each feature:
 | **FLUX.2-dev**           |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |          ✅           |       ❌        |        ❌         |
 | **GLM-Image**            |     ❌     |     ❌      |           ❌           |       ✅        |         ✅         |          ❌          |   ✅    |             ❌             |          ❌           |       ❌        |        ❌         |
 | **Hidream-I1-Full**        |     ❌     |     ❌      |           ❌           |       ❌        |         ✅         |          ❌          |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
-| **HunyuanImage3**        |     ❌     |     ✅      |           ❌           |       ❌        |         ✅         |          ❌          |   ❌    |             ❌             |          ❌           |       ✅        |        ❌         |
+| **HunyuanImage3**        |     ❌     |     ✅      |           ❌           |       ❌        |         ✅         |          ❌          |   ❌    |             ❌             |          ❌           |       ✅        |        ✅*        |
 | **Krea 2**               |     ❌     |     ✅      |           ❌           |       ❌        |         ❌         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **LongCat-Image**        |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
 | **LongCat-Image-Edit**   |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
@@ -145,6 +150,7 @@ The following tables show which models support each feature:
 > 2. `Tongyi-MAI/Z-Image-Turbo` and `SII-GAIR/daVinci-MagiHuman-Base-1080p` are distilled models with minimal NFEs; CFG-Parallel is not necessary.
 > 3. Cosmos3 T2I uses `Cosmos3OmniDiffusersPipeline` with `modalities=["image"]`. Model-level CPU offload swaps the nested UND reasoner and GEN generator pathways; layerwise offload remains available for blockwise GEN/UND offload.
 > 4. Krea 2 currently supports single-GPU inference plus LoRA, Cache-DiT, HSDP, CPU/layerwise offload, and VAE-patch-parallel (decode). TP/SP/CFG-Parallel are not yet wired. The few-step distilled (Turbo) checkpoint uses `is_distilled=true` (fixed timestep shift `mu=1.15`); generate at 2048x2048 by default with `num_inference_steps≈8` and `guidance_scale=0`. The Raw checkpoint uses 1024x1024, `num_inference_steps=28`, and `guidance_scale=4.5`.
+> 5. HunyuanImage3 supports step execution. Multi-request step execution requires `TORCH_SDPA`; see [Diffusion Execution Modes](diffusion/execution_modes.md#step-execution).
 
 ### VideoGen
 
@@ -155,13 +161,16 @@ The following tables show which models support each feature:
 | **Wan2.1-VACE**              |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **LTX-2**                    |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
 | **LTX-2.3**                  |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
-| **Helios**                   |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |          ❌           |       ❌        |        ❌         |
+| **Helios**                   |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |          ❌           |       ❌        |        ✅*        |
 | **HunyuanVideo-1.5 T2V I2V** |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |  ✅ (encode/decode)   |       ✅        |        ❌         |
 | **DreamID-Omni**             |     ❌     |     ❌      |           ❌           |       ✅        |         ❌         |         ❌         |   ✅    |             ✅             |          ❌           |       ❌        |        ❌         |
 | **Cosmos3**                  |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |  ✅ (encode/decode)   |       ✅        |        ❌         |
 | **LongCat-Video-Avatar-1.5** |     ❌     |     ❌      |           ❌           |       ❌        |         ❌         |         ❌         |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
 | **MiniMax-H3**               | ✅ (FL2VA) |     ✅      |           ✅           |       ❌        |       ✅ (DiT/TE)  |         ❌         |   ✅    |             ✅             |       ✅ (tile)       |      ✅ (DiT)      |        ❌         |
 
+> **Step execution note:** Helios supports single-request step execution only;
+> use `max_num_seqs=1`.
+
 **Frame Interpolation Support**
 
 - **Supported**: Wan2.2 text-to-video, image-to-video, and TI2V pipelines
@@ -190,7 +199,7 @@ The following tables show which models support each feature:
 | **🔀Ring-Attn** | ✅ | ✅ | ✅ | | | | | | | | | | | |
 | **🔀CFG-Parallel** | ✅ | ✅ | ✅ | ✅ | | | | | | | | | | |
 | **🔀Tensor Parallel** | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | | | | |
-| **🔀HSDP** | ❓ | ❓ | ❓ | ❓ | ❓ | ❌ | | | | | | | | |
+| **🔀HSDP** | ❓ | ❓ | ✅ | ❓ | ✅ | ❌ | | | | | | | | |
 | **🔀Expert Parallel** | ❓ | ❓ | ❓ | ❓ | ❓ | ❓ | ❓ | | | | | | | |
 | **💾CPU Offloading (Layerwise)** | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | | | | | | |
 | **💾CPU Offloading (Module-wise)** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❓ | ❓ | ❌ | | | | | |
@@ -201,97 +210,43 @@ The following tables show which models support each feature:
 
 !!! info
 
-    1. Tensor Parallel and HSDP are not compatible.
+    1. HSDP can be combined with Ulysses-SP or CFG-Parallel. Tensor Parallel
+       and HSDP are not compatible; other HSDP combinations in the table
+       remain unverified.
     2. TeaCache and Cache-DiT are not compatible.
     3. CPU Offloading (Layerwise) and CPU Offloading (Module-wise) are not compatible.
-    4. CPU Offloading (Layerwise) supports single-card for now.
+    4. The CPU Offloading (Layerwise) row describes local layerwise offload.
-    5. Using FP8-Quant as an example of qunatization methods.
+       Multi-device Distributed Layerwise Offload has a separate topology and
+       compatibility matrix in the [Distributed Layerwise Offloading guide](diffusion/offloader/distributed_layerwise_offload.md).
+    5. The compatibility matrix uses FP8 as the representative quantization method.
     6. Step Execution is not compatible with any diffusion cache backend. LoRA is supported, but each scheduled batch must use a single adapter (requests with different `lora_request` or `lora_scale` are kept in separate batches).
 
 
 ## Multi-Thread Weight Loading
 
-Large diffusion models can take several minutes to load weights at startup (e.g., ~3 min for Qwen-Image, ~5 min for Wan2.2 I2V 14B). Multi-thread weight loading speeds up this process by loading safetensors shards in parallel using a thread pool instead of sequentially.
+The loading guide now lives at [Diffusion Startup and
-
+Loading](diffusion/startup_and_loading.md). This heading remains so existing
-This optimization is **enabled by default** with 4 threads. No configuration is needed for the default behavior.
+links to this section continue to work.
-
-### Configuration
-
-| Parameter | CLI Flag | Default | Description |
-|-----------|----------|---------|-------------|
-| `enable_multithread_weight_load` | `--disable-multithread-weight-load` | `True` (enabled) | Pass the flag to disable multi-thread loading |
-| `num_weight_load_threads` | `--num-weight-load-threads` | `4` | Number of threads for parallel weight loading |
-
-!!! tip
-    The default of 4 threads balances speed and disk I/O contention. On fast NVMe storage you may benefit from more threads (e.g., 8). On HDD or network storage, the default of 4 avoids saturating I/O bandwidth.
-
-### Online Serving
-
-```bash
-# Default (multi-thread enabled, 4 threads)
-vllm serve Qwen/Qwen-Image --omni --port 8091
-
-# Custom thread count
-vllm serve Wan-AI/Wan2.2-I2V-A14B-Diffusers --omni --num-weight-load-threads 8
-
-# Disable multi-thread loading
-vllm serve Qwen/Qwen-Image --omni --disable-multithread-weight-load
-```
-
-### Offline Inference
-
-```python
-from vllm_omni import Omni
-
-# Default (multi-thread enabled, 4 threads)
-omni = Omni(model="Qwen/Qwen-Image")
-
-# Custom thread count
-omni = Omni(
-    model="Wan-AI/Wan2.2-I2V-A14B-Diffusers",
-    num_weight_load_threads=8,
-)
-```
-
-### Benchmarks
-
-Measured on NVIDIA H800:
-
-| Model | Before | After | Speedup |
-|-------|--------|-------|---------|
-| **Qwen/Qwen-Image** (53.7 GiB) | 168s | 27s | **6.2x** |
-| **Wan-AI/Wan2.2-I2V-A14B-Diffusers** (64.5 GiB) | 283s | 56s | **5.1x** |
 
 ## Learn More
 
-**Cache Acceleration:**
+The Diffusion Acceleration navigation groups the remaining guides as follows:
 
-- **[TeaCache Configuration Guide](diffusion/cache_acceleration/teacache.md)** - Parameter tuning, performance tips, troubleshooting
+| Area | Guide |
-- **[Cache-DiT Advanced Guide](diffusion/cache_acceleration/cache_dit.md)** - DBCache, TaylorSeer, SCM techniques and optimization
+| --- | --- |
+| Compatibility | [Feature Compatibility](feature_compatibility.md) |
+| CPU offloading | [CPU Offloading](diffusion/cpu_offload.md) |
+| Cache acceleration | [TeaCache](diffusion/cache_acceleration/teacache.md), [Cache-DiT](diffusion/cache_acceleration/cache_dit.md) |
+| Parallelism | [Parallelism Overview](diffusion/parallelism/overview.md) |
+| Attention | [Attention Backends](diffusion/attention_backends.md) |
+| Compilation | [Regional Compilation](diffusion/regional_compilation.md) |
+| Video extension | [Frame Interpolation](diffusion/frame_interpolation.md) |
+| Startup | [Startup and Loading](diffusion/startup_and_loading.md) |
+| Adapters | [LoRA](diffusion/lora.md) |
 
-**Parallelism Methods:**
+Related cross-model and runtime features are documented separately:
 
-- **[Parallelism Overview](diffusion/parallelism/overview.md)** - Tensor Parallelism, Sequence Parallelism, CFG Parallelism, Pipeline Parallelism, HSDP, and Expert Parallelism
+- [Quantization](quantization/overview.md) covers diffusion-only models,
-
+  multi-stage omni/TTS models, and multi-stage diffusion models.
-**Memory Optimization:**
+- [Execution Modes and Streaming](diffusion/execution_modes.md) covers the
-
+  diffusion runtime, including batching, step execution, and streaming output.
-- **[CPU Offload Guide](diffusion/cpu_offload.md)** - Offload model components to CPU, reduce GPU memory usage
-- **[VAE Parallelism Guide](diffusion/parallelism/vae_parallelism.md)** - Distribute VAE decode work across GPUs for high-resolution images and videos
-- **[Quantization](quantization/overview.md)** - Quantization methods for diffusion, multi-stage omni/TTS, and multi-stage diffusion models
-
-**Extensions:**
-
-- **[LoRA Inference Guide](diffusion/lora.md)** - Low-Rank Adaptation for style customization and fine-tuning
-- **[Frame Interpolation Guide](diffusion/frame_interpolation.md)** - Worker-side post-generation video frame interpolation for smoother motion
-
-**Execution Modes:**
-
-- **[Diffusion Execution Modes](diffusion/execution_modes.md)** - Configure request batching, step execution, continuous batching, and streaming output
-
-**Startup Optimization:**
-
-- **[Multi-Thread Weight Loading](#multi-thread-weight-loading)** - Speed up model startup by loading safetensors shards in parallel
-
-**Advanced Topics:**
-
-- **[Feature Compatibility](feature_compatibility.md)** - How to combine multiple features for maximum performance

docs/user_guide/examples/offline_inference/text_to_image.md

@@ -88,8 +88,9 @@ python text_to_image.py \
 | `--vae-use-tiling` | flag | off | Enable VAE tiling for memory optimization |
 | `--cfg-parallel-size` | int | `1` | Set to `2` to enable CFG Parallel |
 | `--enable-cpu-offload` | flag | off | Enable CPU offloading for diffusion models |
-| `--lora-path` | str | — | Path to PEFT LoRA adapter folder |
+| `--lora-path` | str | — | Path to PEFT LoRA adapter folder or checkpoint file |
 | `--lora-scale` | float | `1.0` | Scale factor for LoRA weights |
+| `--lora-backend` | str |`"peft"`| LoRA backend for loading LoRA adapters. Default: peft. Available options: peft, distill |
 
 **NextStep-1.1 specific arguments:**
 

docs/user_guide/examples/offline_inference/text_to_video.md

@@ -12,6 +12,7 @@ For backend selection and SageAttention usage, see the [Diffusion Attention Back
 | Model | Default Resolution | Default Frames | Default Steps | Guidance | VRAM (BF16) |
 |---|---|---|---|---|---|
 | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | 720x1280 | 81 | 40 | 4.0 | ~60 GiB |
+| `robbyant/lingbot-video-dense-1.3b` / `robbyant/lingbot-video-moe-30b-a3b` | 192x320 | 9 | 2 | 3.0 | ~68 GiB (MoE smoke) |
 | `Lightricks/LTX-2` | 512x768 | 121 | 40 | video 3.0 / audio 7.0 | Model-dependent |
 | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v` | 480x832 | 121 | 50 | 6.0 | 1×A100 80GB |
 | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | 720x1280 | 121 | 50 | 6.0 | FP8 + VAE tiling required |
@@ -35,6 +36,25 @@ python text_to_video.py \
   --output t2v_out.mp4
 ```
 
+### LingBot-Video
+
+The shared runner recognizes both official LingBot checkpoint IDs and selects
+the native `LingBotVideoPipeline` automatically:
+
+```bash
+python text_to_video.py \
+  --model robbyant/lingbot-video-dense-1.3b \
+  --prompt "a robotic arm picks up a red block" \
+  --height 192 --width 320 --num-frames 9 --num-inference-steps 2 \
+  --guidance-scale 3.0 --flow-shift 3.0 --fps 24 \
+  --output lingbot_t2v.mp4
+```
+
+Use `robbyant/lingbot-video-moe-30b-a3b` for the MoE checkpoint. For local
+paths without `lingbot` in the directory name, add
+`--model-class-name LingBotVideoPipeline`. Model-only options use
+`--extra-body`, for example `'{"batch_cfg": true, "output_type": "np"}'`.
+
 ### LTX-2
 
 ```bash
@@ -125,6 +145,9 @@ python text_to_video.py \
 - `--audio-sample-rate`: fallback audio sample rate when the pipeline returns audio.
 - `--quantization`: quantization method (`fp8` for FP8, `gguf` for GGUF).
 - `--flow-shift`: scheduler flow_shift parameter.
+- `--lora-path`: path to PEFT LoRA adapter folder or checkpoint file.
+- `--lora-scale`: scale factor for LoRA weights.
+- `--lora-backend`: backend for loading LoRA adapters. Default: peft. Available options: peft, distill.
 
 ### Wan2.2-specific
 

docs/user_guide/feature_compatibility.md

@@ -165,7 +165,7 @@ vllm serve Qwen/Qwen-Image --omni --port 8091 \
 
 - **Tensor Parallelism — Text Encoder Not Sharded**: TP currently only shards the DiT blocks. Each TP rank retains a **full copy of the text encoder weights**, leading to significant GPU memory overhead proportional to TP degree. Tracked in [Issue #771](https://github.com/vllm-project/vllm-omni/issues/771).
 
-- **CPU Offloading — Two Modes Are Mutually Exclusive**: Model-level offload (`enable_cpu_offload`) and layerwise offload (`enable_layerwise_offload`) cannot be used simultaneously. If both are set, layerwise takes priority and model-level is silently ignored.
+- **CPU Offloading — Strategies Are Mutually Exclusive**: Distributed layerwise, layerwise, and model-level offload should not be enabled together. Selection priority is distributed layerwise, then layerwise, then model-level.
 
 - **CPU Offloading — VAE stays on GPU**: Both offloading strategies keep the VAE on GPU at all times. For high-resolution generation, VAE decode can still cause OOM. Mitigate by combining with `vae_use_tiling=True` or VAE Patch Parallelism.
 

docs/user_guide/quantization/fp8.md

@@ -80,12 +80,14 @@ warmup_quack_fp8([(14040, 2048, 6144), (14040, 2048, 2048)])
 
 ## Model Type Support
 
-### Diffusion Model (Qwen-Image, Wan2.2)
+### Diffusion Models
 
 | Model | HF models | Online | Pre-calibrated | Recommendation | `ignored_layers` | Text-Encoder quantization |
 |-------|-----------|:-------:|:------:|----------------|------------------|------------------|
 | Qwen-Image | `Qwen/Qwen-Image`, `Qwen/Qwen-Image-2512` | Yes | Yes | Skip sensitive image-stream MLPs when quality regresses | `img_mlp` | |
 | Wan2.2 | Wan2.2 diffusion pipelines | Not validated | Not validated | Validate against BF16 before documenting as supported | TBD | |
+| LTX-2 | `Lightricks/LTX-2`, `rootonchair/LTX-2-19b-distilled` | Yes | Not validated | Transformer only; use dynamic phase LoRA for ordinary two-stage | None | |
+| LTX-2.3 | `diffusers/LTX-2.3-Diffusers`, `diffusers/LTX-2.3-Distilled-Diffusers` | Yes | Not validated | Transformer only; use dynamic phase LoRA for ordinary two-stage | None | |
 | Z-Image | `Tongyi-MAI/Z-Image-Turbo` | Yes | Yes | All layers | None | ✅︎ |
 | FLUX.1 | `black-forest-labs/FLUX.1-dev`, `black-forest-labs/FLUX.1-schnell` | Yes | Yes | All layers | None | |
 | FLUX.2-dev | `black-forest-labs/FLUX.2-dev` | Yes | Not validated | All layers | None | ✅︎ |

docs/user_guide/quantization/overview.md

@@ -12,20 +12,20 @@ For the internal architecture and backend extension points, see the
 
 | Mode | Guide | Description | Methods |
 |------|-------|-------------|---------|
-| Online quantization | [Online Quantization](online.md) | vLLM-Omni computes quantized weights and scales while loading the model. | FP8 W8A8, Int8 W8A8, MXFP8 W8A8, MXFP4 W4A4 |
+| Online quantization | [Online Quantization](online.md) | vLLM-Omni computes quantized weights and scales while loading the model. | FP8 W8A8, Int8 W8A8, BitsAndBytes W4, MXFP8 W8A8, MXFP4 W4A4 |
 | Runtime attention quantization | [Quantized KV Cache](quantized_kvcache.md) | vLLM-Omni dynamically quantizes eligible diffusion Flash Attention tensors during inference. | FP8 FA |
-| Pre-quantized checkpoints | Method-specific guides | The checkpoint or an offline quantizer provides quantized weights and scales before serving. | ModelOpt, GGUF, AutoRound, msModelSlim, serialized Int8, offline MXFP8, offline MXFP4 DualScale |
+| Pre-quantized checkpoints | Method-specific guides | The checkpoint or an offline quantizer provides quantized weights and scales before serving. | ModelOpt, AutoRound, msModelSlim, serialized Int8, offline MXFP8, offline MXFP4 DualScale |
 
 ## Hardware Support
 
-| Device | FP8 W8A8 | Int8 W8A8 | ModelOpt | MXFP8 W8A8 | MXFP4 W4A4 | GGUF | AutoRound | msModelSlim |
+| Device | FP8 W8A8 | Int8 W8A8 | BitsAndBytes W4 | ModelOpt | MXFP8 W8A8 | MXFP4 W4A4 | AutoRound | msModelSlim |
-|--------|----------|-----------|----------|------------|------------|------|-----------|-------------|
+|--------|----------|-----------|------------------|----------|------------|------------|-----------|-------------|
-| NVIDIA Blackwell GPU (SM 100+) | ✅ | ✅ | ✅ | ⭕ | ⭕ | ✅ | ✅ | ❌ |
+| NVIDIA Blackwell GPU (SM 100+) | ✅ | ✅ | ✅ | ✅ | ⭕ | ⭕ | ✅ | ❌ |
-| NVIDIA Ada/Hopper GPU (SM 89+) | ✅ | ✅ | ✅ | ⭕ | ⭕ | ✅ | ✅ | ❌ |
+| NVIDIA Ada/Hopper GPU (SM 89+) | ✅ | ✅ | ✅ | ✅ | ⭕ | ⭕ | ✅ | ❌ |
-| NVIDIA Ampere GPU (SM 80+) | ✅ | ✅ | ⭕ | ⭕ | ⭕ | ✅ | ✅ | ❌ |
+| NVIDIA Ampere GPU (SM 80+) | ✅ | ✅ | ✅ | ⭕ | ⭕ | ⭕ | ✅ | ❌ |
-| AMD ROCm | ⭕ | ⭕ | ⭕ | ⭕ | ✅ | ⭕ | ⭕ | ❌ |
+| AMD ROCm | ⭕ | ⭕ | ❌ | ⭕ | ⭕ | ✅ | ⭕ | ❌ |
-| Intel XPU | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ | ✅ | ❌ |
+| Intel XPU | ⭕ | ⭕ | ❌ | ⭕ | ⭕ | ⭕ | ✅ | ❌ |
-| Ascend NPU | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ |
+| Ascend NPU | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ |
 
 Legend: `✅` supported, `❌` unsupported, `⭕` not verified in this
 guide. FP8 on Ampere may use a weight-only path where available.
@@ -43,10 +43,10 @@ otherwise.
 |--------|-------|------|----------------|--------|
 | FP8 W8A8 | [FP8](fp8.md) | Online W8A8 or checkpoint FP8 | Qwen-Image; Wan2.2 is not validated | Validated for Qwen-Image family and other DiT models |
 | Int8 W8A8 | [Int8](int8.md) | Online or serialized W8A8 | Qwen-Image; Wan2.2 is not validated | Validated for Qwen-Image and Z-Image |
+| BitsAndBytes W4 | [BitsAndBytes](bitsandbytes.md) | Online W4 weight-only | Z-Image-Turbo; Qwen-Image/Wan2.2 not validated | Validated for Z-Image-Turbo |
 | ModelOpt | [ModelOpt](modelopt.md) | Pre-quantized FP8 checkpoints | Qwen-Image, Z-Image, FLUX.2, HunyuanImage-3.0 | Validated for ModelOpt FP8 diffusion checkpoints |
 | MXFP8 W8A8 | [MXFP8](mxfp8.md) | Online W8A8 or offline pre-quantized | Wan2.2-T2V-A14B, I2V-A14B, TI2V-5B | Ascend NPU only; validated for Wan2.2 |
 | MXFP4 W4A4 | [MXFP4](mxfp4.md) | `mxfp4`: online single-scale only; `mxfp4_dualscale`: online or offline dual-scale (offline recommended) | Wan2.2-T2V-A14B, I2V-A14B | Ascend NPU only; validated for Wan2.2 A14B cascade models; TI2V-5B not supported; offline `mxfp4_dualscale` uses calibrated `mul_scale` for best accuracy |
-| GGUF | [GGUF](gguf.md) | Pre-quantized transformer weights | Qwen-Image | Validated where a model-specific GGUF adapter exists |
 | AutoRound | [AutoRound](autoround.md) | Pre-quantized W4A16 checkpoints | FLUX.1-dev; Qwen-Image/Wan2.2 not validated | Checkpoint-driven |
 | msModelSlim | [msModelSlim](msmodelslim.md) | Pre-quantized Ascend checkpoints | Wan2.2 recipe; HunyuanImage-3.0 inference target | Ascend/NPU path |
 
@@ -61,9 +61,9 @@ in BF16 unless the model guide explicitly adds support.
 |--------|-------|-------|----------------|--------|
 | ModelOpt | [ModelOpt](modelopt.md) | Thinker or language-model checkpoint config | Qwen3-Omni thinker | ModelOpt checkpoint path |
 | Int8 | [Int8](int8.md) | Not currently validated for omni/TTS stages | Qwen3-Omni, Qwen3-TTS | Not validated |
+| BitsAndBytes | [BitsAndBytes](bitsandbytes.md) | Not currently validated for omni/TTS stages | Qwen3-Omni, Qwen3-TTS | Not validated |
 | MXFP8 | [MXFP8](mxfp8.md) | Not currently validated for omni/TTS stages | Qwen3-Omni, Qwen3-TTS | Not validated |
 | MXFP4 | [MXFP4](mxfp4.md) | Not currently validated for omni/TTS stages | Qwen3-Omni, Qwen3-TTS | Not validated |
-| GGUF | [GGUF](gguf.md) | Not currently validated for omni/TTS stages | Qwen3-Omni, Qwen3-TTS | Not validated |
 | AutoRound | [AutoRound](autoround.md) | Thinker or language-model checkpoint config | Qwen2.5-Omni, Qwen3-Omni | Supported through AutoRound checkpoints |
 | msModelSlim | [msModelSlim](msmodelslim.md) | Not currently validated for omni/TTS stages | Qwen3-Omni, Qwen3-TTS | Not validated |
 
@@ -76,10 +76,10 @@ attached to the intended stage rather than applied globally.
 |--------|-------|-------|----------------|--------|
 | FP8 | [FP8](fp8.md) | Stage-specific DiT or transformer module | BAGEL, GLM-Image | Requires model-specific validation |
 | Int8 | [Int8](int8.md) | Stage-specific DiT or transformer module | BAGEL, GLM-Image | Requires model-specific validation |
+| BitsAndBytes | [BitsAndBytes](bitsandbytes.md) | Stage-specific transformer or DiT module | BAGEL, GLM-Image | Not validated |
 | ModelOpt | [ModelOpt](modelopt.md) | Checkpoint-defined diffusion stage | BAGEL, GLM-Image | Requires model-specific validation |
 | MXFP8 | [MXFP8](mxfp8.md) | Stage-specific DiT or transformer module | BAGEL, GLM-Image | Not validated |
 | MXFP4 | [MXFP4](mxfp4.md) | Stage-specific DiT or transformer module | BAGEL, GLM-Image | Not validated |
-| GGUF | [GGUF](gguf.md) | Stage-specific transformer weights | BAGEL, GLM-Image | No validated adapter listed |
 | AutoRound | [AutoRound](autoround.md) | Checkpoint-defined stage | BAGEL, GLM-Image | No validated checkpoint listed |
 | msModelSlim | [msModelSlim](msmodelslim.md) | Ascend-generated stage weights | GLM-Image | Requires model-specific adaptation |
 
@@ -106,7 +106,7 @@ config = build_quant_config({
 
 | Component | Default quantized? | Notes |
 |-----------|--------------------|-------|
-| Diffusion transformer | Yes | Primary target for FP8, Int8, ModelOpt, MXFP8, MXFP4, GGUF, AutoRound, and msModelSlim |
+| Diffusion transformer | Yes | Primary target for FP8, Int8, BitsAndBytes, ModelOpt, MXFP8, MXFP4, AutoRound, and msModelSlim |
 | Text encoder | No | Keep BF16 unless a method-specific guide documents support |
 | VAE | No | Keep BF16; storage-only paths are method-specific |
 | Scheduler/tokenizer | No | Loaded from the base model repository |
@@ -139,8 +139,8 @@ from vllm_omni.quantization import build_quant_config
 
 build_quant_config("fp8")
 build_quant_config({"method": "fp8", "activation_scheme": "static"})
+build_quant_config("bitsandbytes")
 build_quant_config("auto-round", bits=4, group_size=128)
-build_quant_config({"method": "gguf", "gguf_model": "/path/to/model.gguf"})
 build_quant_config({"transformer": {"method": "fp8"}, "vae": None})
 build_quant_config(None)
 ```

examples/offline_inference/aura_omni/end2end.py

@@ -131,7 +131,7 @@ def build_sampling_params() -> list[SamplingParams]:
 
 
 def save_stage_output(stage_outputs, output_dir: str) -> None:
-    output = stage_outputs.request_output
+    output = stage_outputs
     request_id = output.request_id
     if stage_outputs.final_output_type == "text":
         text_output = output.outputs[0].text

examples/offline_inference/covo_audio/end2end.py

@@ -59,7 +59,7 @@ def main(args):
 
     omni = Omni(
         model=args.model_name,
-        stage_configs_path=args.stage_configs_path,
+        deploy_config=args.deploy_config,
         log_stats=args.log_stats,
         stage_init_timeout=args.stage_init_timeout,
         batch_timeout=args.batch_timeout,
@@ -104,7 +104,7 @@ def main(args):
     os.makedirs(output_dir, exist_ok=True)
 
     for stage_outputs in omni_outputs:
-        output = stage_outputs.request_output
+        output = stage_outputs
         if stage_outputs.final_output_type == "text":
             request_id = output.request_id
             text_output = output.outputs[0].text
@@ -167,10 +167,10 @@ def parse_args():
         help="Sampling rate for audio loading (default: 16000).",
     )
     parser.add_argument(
-        "--stage-configs-path",
+        "--deploy-config",
         type=str,
         default=None,
-        help="Path to stage configs YAML file.",
+        help="Path to a deploy config YAML file.",
     )
     parser.add_argument(
         "--log-stats",

examples/offline_inference/custom_pipeline/image_to_image/image_edit.py

@@ -228,7 +228,7 @@ async def main():
     if not outputs:
         raise ValueError("No output produced from omni.generate()")
 
-    first_out = outputs[0].request_output
+    first_out = outputs[0]
     req_out: OmniRequestOutput = first_out
 
     # Verify trajectory data (from custom pipeline)

examples/offline_inference/dynin_omni/end2end.py

@@ -741,7 +741,7 @@ def make_mmu_prompt(
 
 def iter_mm_outputs(outputs: list[Any]):
     for omni_out in outputs:
-        req_out = getattr(omni_out, "request_output", None)
+        req_out = omni_out
         req_list = req_out if isinstance(req_out, list) else [req_out]
         for item in req_list:
             if item is None:
@@ -876,7 +876,7 @@ def parse_args(repo_root: Path) -> argparse.Namespace:
     parser.add_argument("--task", type=str, required=True, choices=TASK_CHOICES)
     parser.add_argument("--model", type=str, required=True, help="HF repo id or local model directory.")
     parser.add_argument(
-        "--stage-config-path",
+        "--deploy-config",
         type=str,
         default=str(repo_root / "vllm_omni/deploy/dynin_omni.yaml"),
         help="Path to deploy config yaml.",
@@ -1394,8 +1394,8 @@ def main() -> None:
 
     from vllm_omni.entrypoints.omni import Omni
 
-    stage_config_path = str(Path(args.stage_config_path).expanduser())
+    deploy_config_path = str(Path(args.deploy_config).expanduser())
-    omni = Omni(model=model_source, stage_configs_path=stage_config_path, dtype=args.dtype)
+    omni = Omni(model=model_source, deploy_config=deploy_config_path, dtype=args.dtype)
     sampling_params_list = [
         SamplingParams(max_tokens=int(args.max_tokens_per_stage), temperature=0.0, top_p=1.0, detokenize=False)
         for _ in range(omni.num_stages)

examples/offline_inference/hunyuan_image3/end2end.py

@@ -73,7 +73,6 @@ def parse_args():
     )
     parser.add_argument("--sys-type", type=str, default=None, help="Override system prompt type.")
     parser.add_argument("--deploy-config", type=str, default=None, help="Custom deploy YAML path.")
-    parser.add_argument("--stage-configs-path", type=str, default=None, help="Custom legacy stage config YAML path.")
     parser.add_argument(
         "--stream",
         action="store_true",
@@ -146,13 +145,7 @@ def main():
     else:
         bot_task = args.bot_task
 
-    if args.deploy_config is not None and args.stage_configs_path is not None:
+    deploy_config = args.deploy_config or _MODALITY_DEFAULT_DEPLOY_CONFIG[args.modality]
-        raise ValueError("--deploy-config and --stage-configs-path are mutually exclusive.")
-
-    deploy_config = args.deploy_config
-    stage_configs_path = args.stage_configs_path
-    if deploy_config is None and stage_configs_path is None:
-        deploy_config = _MODALITY_DEFAULT_DEPLOY_CONFIG[args.modality]
 
     omni_kwargs = {
         "model": args.model,
@@ -168,10 +161,7 @@ def main():
 
     if additional_config is not None:
         omni_kwargs["additional_config"] = additional_config
-    if deploy_config is not None:
+    omni_kwargs["deploy_config"] = deploy_config
-        omni_kwargs["deploy_config"] = deploy_config
-    else:
-        omni_kwargs["stage_configs_path"] = stage_configs_path
 
     omni = Omni(**omni_kwargs)
 
@@ -272,10 +262,7 @@ def main():
     print(f"  Modality: {args.modality}")
     print(f"  Prompt task: {task}")
     print(f"  Bot task: {bot_task}")
-    if deploy_config is not None:
+    print(f"  Deploy config: {deploy_config}")
-        print(f"  Deploy config: {deploy_config}")
-    else:
-        print(f"  Stage config: {stage_configs_path}")
     print(f"  Num stages: {omni.num_stages}")
     if args.modality in ("text2img", "img2img"):
         print(f"  Inference steps: {args.steps}")
@@ -303,7 +290,7 @@ def main():
     )
     img_idx = 0
     for req_output in omni_outputs:
-        ro = getattr(req_output, "request_output", None)
+        ro = req_output
         stage_id = getattr(req_output, "stage_id", None)
 
         # AR stage text — each CompletionOutput.text is already a delta when

examples/offline_inference/image_to_image/image_edit.py

@@ -621,7 +621,7 @@ def main():
         images = getattr(output, "images", None)
         if images:
             break
-        req_out = getattr(output, "request_output", None)
+        req_out = output
         images = getattr(req_out, "images", None) if req_out is not None else None
         if images:
             break

examples/offline_inference/image_to_video/image_to_video.py

@@ -60,8 +60,12 @@ from vllm_omni.diffusion.data import DiffusionParallelConfig
 from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args
 from vllm_omni.entrypoints.omni import Omni
 from vllm_omni.inputs.data import OmniDiffusionSamplingParams
+from vllm_omni.lora.request import LoRARequest
+from vllm_omni.lora.utils import stable_lora_int_id
+from vllm_omni.model_extras import (
+    build_image_to_video_prompt as build_model_image_to_video_prompt,
+)
 from vllm_omni.model_extras import (
-    build_image_to_video_prompt,
     get_extra_body_params,
     get_model_class_name,
 )
@@ -83,6 +87,22 @@ def parse_json_object(value: str, flag_name: str = "argument") -> dict[str, Any]
 parse_profiler_config = functools.partial(parse_json_object, flag_name="--profiler-config")
 
 
+def build_image_to_video_prompt(
+    prompt: str,
+    negative_prompt: str | None,
+    media_inputs: dict[str, Any],
+) -> dict[str, Any]:
+    """Build the canonical request envelope for the shared I2V example."""
+    result: dict[str, Any] = {
+        "prompt": prompt,
+        "modalities": ["video"],
+        "multi_modal_data": media_inputs,
+    }
+    if negative_prompt is not None:
+        result["negative_prompt"] = negative_prompt
+    return result
+
+
 def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(
         description=(
@@ -294,6 +314,30 @@ def parse_args() -> argparse.Namespace:
         default=None,
         help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
     )
+    parser.add_argument(
+        "--lora-path",
+        type=str,
+        nargs="+",
+        default=None,
+        help="Path to a LoRA adapter folder (PEFT format) or concrete LoRA checkpoint files. "
+        "For Wan2.2 MoE models, pass the high-noise checkpoint first and the low-noise checkpoint second.",
+    )
+    parser.add_argument(
+        "--lora-scale",
+        type=float,
+        default=1.0,
+        help="Scale factor for PEFT LoRA weights (default: 1.0).",
+    )
+    parser.add_argument(
+        "--lora-backend",
+        type=str,
+        default="peft",
+        choices=["peft", "distill"],
+        help=(
+            "LoRA backend. 'peft' loads a PEFT-format adapter for request-time activation; "
+            "'distill' fuses one or more concrete LoRA checkpoint files at initialization."
+        ),
+    )
     parser.add_argument(
         "--extra-body",
         type=functools.partial(parse_json_object, flag_name="--extra-body"),
@@ -487,6 +531,12 @@ def main():
         omni_kwargs["flow_shift"] = flow_shift
     if args.quantization is not None:
         omni_kwargs["quantization"] = args.quantization
+    if args.lora_path is not None:
+        lora_path = args.lora_path
+        if len(lora_path) == 1:
+            lora_path = lora_path[0]
+        omni_kwargs["lora_path"] = lora_path
+        omni_kwargs["lora_backend"] = args.lora_backend
     # Cosmos3 loads its (gated) guardrail models at build time, so the guardrails
     # gate is an engine-level config (offline analog of the server's --no-guardrails).
     if args.extra_body and "guardrails" in args.extra_body:
@@ -517,19 +567,39 @@ def main():
     print(f"  Video size: {width}x{height}")
     print(f"{'=' * 60}\n")
 
+    lora_request = None
+    if args.lora_path and args.lora_backend == "peft":
+        if len(args.lora_path) != 1:
+            raise ValueError("Only one LoRA path is expected for PEFT backend.")
+
+        lora_path = args.lora_path[0]
+        lora_request = LoRARequest(
+            lora_name=Path(lora_path).stem,
+            lora_int_id=stable_lora_int_id(lora_path),
+            lora_path=lora_path,
+        )
+
     negative_prompt = args.negative_prompt
     if negative_prompt is None and not is_ltx2:
         # Preserve the historical empty-prompt behavior for non-LTX examples.
         negative_prompt = ""
     prompt_dict = build_image_to_video_prompt(
-        model_class_name=model_class_name,
         prompt=args.prompt,
         negative_prompt=negative_prompt,
         media_inputs=media_inputs,
+    )
+    prompt_dict = build_model_image_to_video_prompt(
+        model_class_name=model_class_name,
+        prompt=prompt_dict,
         height=height,
         width=width,
         num_frames=num_frames,
     )
+    sampling_extra_args = {"sample_solver": args.sample_solver}
+    if lora_request is not None:
+        sampling_extra_args["lora_request"] = lora_request
+        sampling_extra_args["lora_scale"] = args.lora_scale
+
     sampling_params = OmniDiffusionSamplingParams(
         height=height,
         width=width,
@@ -540,7 +610,7 @@ def main():
         num_inference_steps=num_inference_steps,
         num_frames=num_frames,
         frame_rate=frame_rate,
-        extra_args={"sample_solver": args.sample_solver},
+        extra_args=sampling_extra_args,
     )
     if flow_shift is not None:
         sampling_params.extra_args["flow_shift"] = flow_shift
@@ -576,8 +646,8 @@ def main():
         if frames.multimodal_output and "audio" in frames.multimodal_output:
             audio = frames.multimodal_output["audio"]
             audio_sample_rate = frames.multimodal_output.get("audio_sample_rate", audio_sample_rate)
-        if frames.is_pipeline_output and frames.request_output is not None:
+        if frames.is_pipeline_output and frames is not None:
-            inner_output = frames.request_output
+            inner_output = frames
             if isinstance(inner_output, OmniRequestOutput):
                 if inner_output.multimodal_output and "audio" in inner_output.multimodal_output:
                     audio = inner_output.multimodal_output["audio"]

examples/offline_inference/mammothmodal2_preview/run_mammothmoda2_image_summarize.py

@@ -0,0 +1,117 @@
+"""
+Offline inference example: MammothModa2 image summarization (single AR stage).
+
+Example:
+  uv run python examples/offline_inference/mammothmodal2_preview/run_mammothmoda2_image_summarize.py \
+    --model path/to/MammothModa2-Preview \
+    --deploy-config vllm_omni/deploy/mammoth_moda2_ar.yaml \
+    --image /path/to/input.jpg \
+    --question "Please summarize the content of this image."
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+
+from PIL import Image
+from vllm import SamplingParams
+from vllm.multimodal.image import convert_image_mode
+
+from vllm_omni import Omni
+
+DEFAULT_SYSTEM = "You are a helpful assistant."
+DEFAULT_QUESTION = "Please summarize the content of this image."
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="MammothModa2 image summarization (offline, AR only).")
+    parser.add_argument("--model", type=str, required=True, help="Path to model directory or model id.")
+    parser.add_argument("--deploy-config", type=str, required=True, help="Path to deploy config yaml.")
+    parser.add_argument("--image", type=str, required=True, help="Path to input image.")
+    parser.add_argument("--question", type=str, default=DEFAULT_QUESTION, help="Question/instruction for the model.")
+    parser.add_argument("--system", type=str, default=DEFAULT_SYSTEM, help="System prompt.")
+    parser.add_argument(
+        "--max-tokens",
+        type=int,
+        default=512,
+        help="Max new tokens to generate.",
+    )
+    parser.add_argument("--temperature", type=float, default=0.2)
+    parser.add_argument("--top-p", type=float, default=0.9)
+    parser.add_argument("--seed", type=int, default=42)
+    parser.add_argument("--trust-remote-code", action="store_true")
+    parser.add_argument(
+        "--enable-diffusion-pipeline-profiler",
+        action="store_true",
+        help="Enable diffusion pipeline profiler to display stage durations.",
+    )
+    return parser.parse_args()
+
+
+def build_prompt(system: str, question: str) -> str:
+    return (
+        f"<|im_start|>system\n{system}<|im_end|>\n"
+        "<|im_start|>user\n"
+        "<|vision_start|><|image_pad|><|vision_end|>"
+        f"{question}<|im_end|>\n"
+        "<|im_start|>assistant\n"
+    )
+
+
+def main() -> None:
+    args = parse_args()
+
+    if not os.path.exists(args.image):
+        raise FileNotFoundError(f"Image file not found: {args.image}")
+
+    pil_image = Image.open(args.image)
+    image_data = convert_image_mode(pil_image, "RGB")
+    prompt = build_prompt(args.system, args.question)
+
+    omni = Omni(
+        model=args.model,
+        deploy_config=args.deploy_config,
+        trust_remote_code=args.trust_remote_code,
+        enable_diffusion_pipeline_profiler=args.enable_diffusion_pipeline_profiler,
+    )
+    try:
+        sp = SamplingParams(
+            temperature=float(args.temperature),
+            top_p=float(args.top_p),
+            top_k=-1,
+            max_tokens=int(args.max_tokens),
+            seed=int(args.seed),
+            detokenize=True,
+        )
+        # NOTE: omni.generate() returns a Generator[OmniRequestOutput, None, None].
+        # Consume it inside the try block so the worker isn't closed early.
+        outputs = list(
+            omni.generate(
+                [
+                    {
+                        "prompt": prompt,
+                        "multi_modal_data": {"image": image_data},
+                        "additional_information": {"omni_task": ["chat"]},
+                    }
+                ],
+                [sp],
+            )
+        )
+    finally:
+        omni.close()
+
+    lines: list[str] = []
+    for stage_outputs in outputs:
+        ro = stage_outputs
+        text = ro.outputs[0].text if getattr(ro, "outputs", None) else str(ro)
+        lines.append(f"request_id: {getattr(ro, 'request_id', 'unknown')}\n")
+        lines.append("answer:\n")
+        lines.append(text.strip() + "\n")
+        lines.append("\n")
+
+    print("\n".join(lines))
+
+
+if __name__ == "__main__":
+    main()

examples/offline_inference/mimo_audio/end2end.py

@@ -297,7 +297,7 @@ def main(args):
     os.makedirs(output_dir, exist_ok=True)
 
     for stage_outputs in omni_outputs:
-        output = stage_outputs.request_output
+        output = stage_outputs
         if stage_outputs.final_output_type == "text":
             request_id = output.request_id
             text_output = output.outputs[0].text

examples/offline_inference/ming_flash_omni/end2end.py

@@ -347,7 +347,7 @@ def main(args):
         omni.start_profile(stages=args.profiler_stages)
 
     for stage_outputs in omni.generate(prompts, sampling_params_list):
-        output = stage_outputs.request_output
+        output = stage_outputs
         if stage_outputs.final_output_type == "text":
             request_id = output.request_id
             text_output = output.outputs[0].text

examples/offline_inference/minicpmo/end2end.py

@@ -452,12 +452,6 @@ def parse_args():
         default=None,
         help="Path to a .txt file with one prompt per line (preferred).",
     )
-    parser.add_argument(
-        "--stage-configs-path",
-        type=str,
-        default=None,
-        help="Path to a stage configs file (deprecated; prefer --deploy-config).",
-    )
     parser.add_argument(
         "--deploy-config",
         type=str,

examples/offline_inference/nemotron_voicechat/end2end.py

@@ -0,0 +1,229 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Offline speech-to-speech with NVIDIA-NemotronLabs-VoiceChat-11B.
+
+Feeds a 16 kHz user utterance (plus a spoken-style system prompt) through the
+3-stage nemotron_voicechat pipeline and writes the agent's reply as text and a
+22.05 kHz WAV.
+
+Usage:
+    python end2end.py \
+        --checkpoint /path/to/NVIDIA-NemotronLabs-VoiceChat-11B \
+        --wav /path/to/user_question_16k.wav \
+        --output-dir results/nemotron_voicechat
+
+The system prompt defaults to the NeMo reference default. The text tokenizer
+resolves from the ``nvidia/NVIDIA-Nemotron-Nano-9B-v2`` HF id (or the
+``NEMOTRON_VOICECHAT_LLM_PATH`` env var for air-gapped runs).
+
+Sizing the input WAV: the timeline is frame-locked, so the reply budget IS the
+input duration — the acoustic channel trails the text channel, and a reply
+that has not finished when the frames run out is truncated SILENTLY (no error;
+ASR of the WAV will just disagree with the emitted text). Leave generous
+trailing silence after the question (a question ending at ~4.5 s truncated in
+an 8 s WAV but completed cleanly in a 16 s one). The script warns when the
+text channel is still emitting non-PAD tokens near the last frame.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from pathlib import Path
+
+import numpy as np
+import soundfile as sf
+
+# Warn if the text channel is still speaking within this many final frames
+# (12 frames = ~1 s) — the frame-locked reply likely got cut off.
+TRUNCATION_GUARD_FRAMES = 12
+
+# The NeMo reference offline script's default system prompt.
+DEFAULT_SYSTEM_PROMPT = (
+    "You are an AI voice assistant developed by NVIDIA. "
+    "Your name is NVIDIA Voice Chat. "
+    "Answer in a spoken, conversational style rather than a written one. "
+    "Do not repeat the same sentence over and over again. "
+    "Start the conversation by greeting the user."
+)
+
+
+def load_wav_16k_mono(path: str) -> np.ndarray:
+    wav, sr = sf.read(path, dtype="float32")
+    if wav.ndim > 1:
+        wav = wav.mean(axis=1)
+    if sr != 16000:
+        from vllm.multimodal.audio import resample_audio_scipy
+
+        wav = resample_audio_scipy(wav, orig_sr=sr, target_sr=16000)
+    return wav.astype(np.float32)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--checkpoint", required=True, help="NVIDIA-NemotronLabs-VoiceChat-11B directory")
+    parser.add_argument("--wav", required=True, help="user speech (any sr; resampled to 16 kHz mono)")
+    parser.add_argument("--output-dir", default="results/nemotron_voicechat")
+    parser.add_argument("--system-prompt", default=DEFAULT_SYSTEM_PROMPT)
+    parser.add_argument(
+        "--stage-configs-path",
+        default=None,
+        help=(
+            "Deploy yaml override. Pass vllm_omni/deploy/nemotron_labs_voicechat_streaming.yaml "
+            "for the async-chunk streaming pipeline (stages overlap; audio starts decoding "
+            "before the thinker finishes). Default: the bundled non-streaming yaml (bit-exact "
+            "parity path)."
+        ),
+    )
+    args = parser.parse_args()
+
+    if not Path(args.wav).is_file():
+        raise FileNotFoundError(f"Input WAV not found: {args.wav}")
+    ckpt = Path(args.checkpoint)
+    raw_cfg = json.loads((ckpt / "config.json").read_text())
+    stt_cfg = raw_cfg["model"]["stt"]["model"]
+
+    from transformers import AutoTokenizer
+    from vllm.sampling_params import SamplingParams
+
+    from vllm_omni.entrypoints.omni import Omni
+    from vllm_omni.model_executor.models.nemotron_voicechat.nemotron_voicechat_thinker import (
+        compute_acoustic_frame_count,
+    )
+
+    tok_ref = os.environ.get("NEMOTRON_VOICECHAT_LLM_PATH") or stt_cfg.get(
+        "pretrained_llm", "nvidia/NVIDIA-Nemotron-Nano-9B-v2"
+    )
+    tokenizer = AutoTokenizer.from_pretrained(tok_ref, trust_remote_code=False)
+    bos_id = tokenizer.convert_tokens_to_ids(stt_cfg.get("bos_token", "<s>"))
+    eos_id = tokenizer.convert_tokens_to_ids(stt_cfg.get("eos_token", "</s>"))
+    pad_id = tokenizer.convert_tokens_to_ids(stt_cfg.get("pad_token", "<SPECIAL_12>"))
+    # NeMo encode_system_prompt: [bos] + tokens + [eos].
+    prompt_ids = [bos_id] + tokenizer.encode(args.system_prompt, add_special_tokens=False) + [eos_id]
+
+    wav = load_wav_16k_mono(args.wav)
+    n_frames = compute_acoustic_frame_count(stt_cfg, int(wav.shape[0]))
+    print(f"input: {wav.shape[0]} samples @16kHz -> {n_frames} acoustic frames; prompt {len(prompt_ids)} tokens")
+
+    inputs = {
+        # vLLM prompt = system prompt + one placeholder position (acoustic
+        # frame 0); the thinker replaces every position with fused embeddings.
+        "prompt_token_ids": prompt_ids + [pad_id],
+        "additional_information": {
+            # Plain float list: raw ndarrays do not survive the engine-core
+            # message serialization.
+            "nvc_audio": wav.astype(np.float32).tolist(),
+            "nvc_sr": 16000,
+            "nvc_prompt_token_ids": prompt_ids,
+            "nvc_expected_frames": n_frames,
+        },
+    }
+    sampling_params_list = [
+        # thinker: greedy, frame-locked generation length. NO min_tokens: the
+        # tokenizer's eos_token is <SPECIAL_12> — the frame-locked PAD/silence
+        # token — and vLLM's min_tokens masks "EOS" to -inf, which would forbid
+        # silence and force the agent to babble through the whole utterance.
+        SamplingParams(temperature=0.0, max_tokens=n_frames, ignore_eos=True, seed=0),
+        # talker: placeholder loop, stops on the stage's stop token. 16383:
+        # the 1-token placeholder prompt + max_tokens must fit the stage's
+        # max_model_len (16384).
+        SamplingParams(temperature=0.0, max_tokens=16383, stop_token_ids=[1], detokenize=False, seed=0),
+        # code2wav: single decode step.
+        SamplingParams(temperature=0.0, max_tokens=1, detokenize=False, seed=0),
+    ]
+
+    omni_kwargs: dict = {}
+    if args.stage_configs_path:
+        omni_kwargs["stage_configs_path"] = args.stage_configs_path
+    omni = Omni(model=str(ckpt), trust_remote_code=True, **omni_kwargs)
+    outputs = omni.generate(inputs, sampling_params_list)
+
+    out_dir = Path(args.output_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+    stem = Path(args.wav).stem
+
+    agent_text = None
+    audio = None
+    sr = 22050
+    for stage_outputs in outputs:
+        # One entry per final stage (text from the thinker, audio from code2wav);
+        # mirrors the qwen2_5_omni offline example's collection loop.
+        output = getattr(stage_outputs, "request_output", stage_outputs)
+        final_type = getattr(stage_outputs, "final_output_type", None)
+        completions = getattr(output, "outputs", None) or []
+        if not completions:
+            continue
+        completion = completions[0]
+        if final_type in (None, "text"):
+            text = getattr(completion, "text", None)
+            if text:
+                agent_text = text
+            token_ids = getattr(completion, "token_ids", None)
+            if token_ids:
+                (out_dir / f"{stem}_text_tokens.json").write_text(json.dumps(list(map(int, token_ids))))
+                # Frame lock: the reply budget IS the input length. If the text
+                # channel is still mid-sentence when the frames run out, the
+                # spoken answer is silently truncated — warn instead.
+                tail = list(map(int, token_ids))[-TRUNCATION_GUARD_FRAMES:]
+                if any(t != pad_id for t in tail):
+                    print(
+                        f"WARNING: the text channel emits non-PAD tokens within the last "
+                        f"{TRUNCATION_GUARD_FRAMES} frames ({TRUNCATION_GUARD_FRAMES * 80} ms) — the reply "
+                        "likely ran out of acoustic frames and is cut off mid-sentence. "
+                        "Pad the input WAV with more trailing silence and rerun."
+                    )
+            # Best-effort: dump the function-channel debug timeline if the
+            # engine surfaced the request info on this output.
+            info = getattr(output, "additional_information", None)
+            func_timeline = info.get("nvc_function_tokens") if isinstance(info, dict) else None
+            if func_timeline is not None:
+                values = func_timeline.tolist() if hasattr(func_timeline, "tolist") else list(func_timeline)
+                (out_dir / f"{stem}_function_tokens.json").write_text(json.dumps([int(v) for v in values]))
+                print(f"function-channel timeline: {len(values)} entries")
+        if final_type in (None, "audio"):
+            # multimodal_output is a MultimodalPayload (Mapping) — duck-type it.
+            mm = getattr(completion, "multimodal_output", None)
+            get = getattr(mm, "get", None)
+            if not callable(get):
+                continue
+            candidate = None
+            for key in ("audio", "model_outputs", "wav", "waveform"):
+                value = get(key)
+                if value is not None:
+                    candidate = value
+                    break
+            if isinstance(candidate, list):
+                candidate = candidate[0] if candidate else None
+            if candidate is not None:
+                arr = np.asarray(
+                    candidate.float().cpu() if hasattr(candidate, "cpu") else candidate, dtype=np.float32
+                ).reshape(-1)
+                if arr.size > 0:
+                    audio = arr
+                sr_val = get("sr")
+                if sr_val is not None:
+                    sr_item = sr_val[0] if isinstance(sr_val, list) else sr_val
+                    sr = int(sr_item.item() if hasattr(sr_item, "item") else sr_item)
+
+    if agent_text is not None:
+        # The frame-locked text channel emits PAD tokens on silent frames;
+        # strip them (NeMo's post-inference detokenizer does the same).
+        pad_token = stt_cfg.get("pad_token", "<SPECIAL_12>")
+        agent_text = " ".join(agent_text.replace(pad_token, " ").split())
+        text_path = out_dir / f"{stem}_output.txt"
+        text_path.write_text(agent_text + "\n")
+        print(f"agent text -> {text_path}\n{agent_text}")
+    else:
+        print("WARNING: no agent text in outputs")
+
+    if audio is not None and audio.size > 0:
+        wav_path = out_dir / f"{stem}_output.wav"
+        sf.write(wav_path, audio, sr)
+        print(f"agent audio -> {wav_path} ({audio.size / sr:.2f}s @ {sr} Hz)")
+    else:
+        raise RuntimeError("No audio produced by the code2wav stage.")
+
+
+if __name__ == "__main__":
+    main()

examples/offline_inference/qwen2_5_omni/end2end.py

@@ -392,7 +392,7 @@ def main(args):
     total_requests = len(prompts)
     processed_count = 0
     for stage_outputs in omni_generator:
-        output = stage_outputs.request_output
+        output = stage_outputs
         if stage_outputs.final_output_type == "text":
             request_id = output.request_id
             text_output = output.outputs[0].text

examples/offline_inference/qwen3_omni/end2end.py

@@ -367,7 +367,7 @@ def main(args):
     print(f"query type: {args.query_type}")
 
     for stage_outputs in omni_generator:
-        output = stage_outputs.request_output
+        output = stage_outputs
         if stage_outputs.final_output_type == "text":
             request_id = output.request_id
             text_output = output.outputs[0].text
@@ -484,10 +484,10 @@ def parse_args():
         help="Path to a .txt file with one prompt per line (preferred).",
     )
     parser.add_argument(
-        "--stage-configs-path",
+        "--deploy-config",
         type=str,
         default=None,
-        help="Path to a stage configs file.",
+        help="Path to a deploy config YAML.",
     )
     parser.add_argument(
         "--video-path",

examples/offline_inference/qwen3_omni/end2end_async_chunk.py

@@ -237,7 +237,7 @@ async def run_single_request(
             sampling_params_list=sampling_params_list,
             output_modalities=output_modalities,
         ):
-            output = omni_output.request_output
+            output = omni_output
             if omni_output.final_output_type == "text":
                 if stage_0_first_output_ts is None:
                     stage_0_first_output_ts = time.perf_counter()

examples/offline_inference/speech_to_video/speech_to_video.py

@@ -26,7 +26,6 @@ import torch
 from vllm_omni.diffusion.data import DiffusionParallelConfig
 from vllm_omni.entrypoints.omni import Omni
 from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-from vllm_omni.outputs import OmniRequestOutput
 from vllm_omni.platforms import current_omni_platform
 
 
@@ -352,8 +351,8 @@ def main():
         else:
             print("[Profiler] No valid profiling data returned.")
 
-    # Extract output from result
+    # omni.generate() returns a list for sync calls; unwrap single-result list.
-    output = OmniRequestOutput.unwrap_result(result)
+    output = result[0] if isinstance(result, list) else result
 
     if not output.images:
         raise ValueError("No video frames found in OmniRequestOutput.")

examples/offline_inference/step_audio2/end2end.py

@@ -232,15 +232,8 @@ def main(args):
         raise ValueError(f"Unknown query type: {args.query_type}")
 
     # Initialize vLLM-Omni with Step-Audio2
-    # Resolve deploy config path. Keep explicit stage_configs_path for legacy
+    # Resolve an explicit deploy config or use the bundled task default.
-    # custom configs, but use bundled deploy configs by default.
+    if args.deploy_config:
-    deploy_config_path = None
-    stage_config_path = None
-    if args.stage_configs_path and args.deploy_config:
-        raise ValueError("--stage-configs-path and --deploy-config are mutually exclusive")
-    if args.stage_configs_path:
-        stage_config_path = args.stage_configs_path
-    elif args.deploy_config:
         deploy_config_path = args.deploy_config
     else:
         configs_dir = Path(__file__).parent.parent.parent.parent / "vllm_omni/deploy"
@@ -262,10 +255,7 @@ def main(args):
         "ray_address": args.ray_address,
         "trust_remote_code": True,
     }
-    if stage_config_path is not None:
+    omni_kwargs["deploy_config"] = deploy_config_path
-        omni_kwargs["stage_configs_path"] = stage_config_path
-    else:
-        omni_kwargs["deploy_config"] = deploy_config_path
 
     omni_llm = Omni(**omni_kwargs)
 
@@ -345,7 +335,7 @@ def main(args):
     # Process outputs from each stage
     for stage_idx, stage_outputs in enumerate(omni_outputs):
         # request_output may be a single object or a list
-        raw_output = stage_outputs.request_output
+        raw_output = stage_outputs
         outputs_list = raw_output if isinstance(raw_output, list) else [raw_output]
 
         if stage_outputs.final_output_type == "text":
@@ -418,12 +408,6 @@ def parse_args():
             "  - HuggingFace ID: stepfun-ai/Step-Audio-2-mini"
         ),
     )
-    parser.add_argument(
-        "--stage-configs-path",
-        type=str,
-        default=None,
-        help="Path to a legacy stage config YAML file (default: use bundled deploy config)",
-    )
     parser.add_argument(
         "--deploy-config",
         type=str,

examples/offline_inference/text_to_audio/text_to_audio.py

@@ -43,6 +43,7 @@ from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args
 from vllm_omni.entrypoints.omni import Omni
 from vllm_omni.inputs.data import OmniDiffusionSamplingParams
 from vllm_omni.model_extras import get_extra_body_params, get_model_class_name
+from vllm_omni.outputs import OmniRequestOutput
 from vllm_omni.platforms import current_omni_platform
 
 AUDIOX_TASKS = ("t2a", "t2m", "v2a", "v2m", "tv2a", "tv2m")
@@ -388,9 +389,9 @@ def main():
         raise ValueError("No output generated from omni.generate()")
 
     output = outputs[0]
-    if not hasattr(output, "request_output") or not output.request_output:
+    if not isinstance(output, OmniRequestOutput) or not output:
         raise ValueError("No request_output found in OmniRequestOutput")
-    request_output = output.request_output
+    request_output = output
     if not hasattr(request_output, "multimodal_output"):
         raise ValueError("No multimodal_output found in request_output")
 

examples/offline_inference/text_to_image/README.md

@@ -57,8 +57,9 @@ python examples/offline_inference/text_to_image/text_to_image.py \
   --output lingbot_t2i.png
 ```
 
-The LingBot prompt builder selects the `image` output modality and requests one
+The shared T2I entry point selects the `image` output modality. The LingBot
-frame. Passing an input image is not supported in LingBot T2I mode.
+pipeline validates that T2I requests produce one frame and do not include an
+input image.
 
 ## Quick Start
 
@@ -109,8 +110,9 @@ python text_to_image.py \
 | `--ring-degree` | int | `1` | Ring sequence parallel degree for hybrid Ulysses + Ring inference |
 | `--ulysses-mode` | str | `"strict"` | Ulysses SP mode: `"strict"` or `"advanced_uaa"` |
 | `--enable-cpu-offload` | flag | off | Enable CPU offloading for diffusion models |
-| `--lora-path` | str | — | Path to PEFT LoRA adapter folder |
+| `--lora-path` | str | — | Path to PEFT LoRA adapter folder or checkpoint file |
 | `--lora-scale` | float | `1.0` | Scale factor for LoRA weights |
+| `--lora-backend` | str |`"peft"`| LoRA backend for loading LoRA adapters. Default: peft. Available options: peft, distill |
 | `--use-system-prompt` | str | `None` | System prompt preset: `en_unified`, `en_vanilla`, `en_recaption`, `en_think_recaption`, `dynamic`, `None`, or custom text. Recommended: `en_unified`. Only for HunyuanImage-3.0.|
 | `--system-prompt` | str | `None` | Custom system prompt text. Only used when `--use-system-prompt` is set to `custom`. Only for HunyuanImage-3.0.|
 | `--auxiliary-text-encoder` | str | `None` | Supplementary auxiliary text encoder parameters model name or path (especially for Hidream-l1-full). |

examples/offline_inference/text_to_image/gradio_demo.py

@@ -115,7 +115,7 @@ def build_demo(args: argparse.Namespace) -> gr.Blocks:
         )
         images_outputs = []
         for output in outputs:
-            req_out = output.request_output
+            req_out = output
             if not isinstance(req_out, OmniRequestOutput) or not hasattr(req_out, "images"):
                 raise ValueError("Invalid request_output structure or missing 'images' key")
             images = req_out.images

examples/offline_inference/text_to_image/text_to_image.py

@@ -21,7 +21,9 @@ from vllm_omni.inputs.data import OmniDiffusionSamplingParams
 from vllm_omni.lora.request import LoRARequest
 from vllm_omni.lora.utils import stable_lora_int_id
 from vllm_omni.model_extras import (
-    build_text_to_image_prompt,
+    build_text_to_image_prompt as build_model_text_to_image_prompt,
+)
+from vllm_omni.model_extras import (
     get_extra_body_params,
     get_model_class_name,
     should_init_extra_args_for_non_diffusion_stages,
@@ -56,6 +58,17 @@ def parse_json_object(value: str, flag_name: str = "argument") -> dict[str, Any]
 parse_profiler_config = functools.partial(parse_json_object, flag_name="--profiler-config")
 
 
+def build_text_to_image_prompt(prompt: str, negative_prompt: str | None) -> dict[str, Any]:
+    """Build the canonical request envelope for the shared T2I example."""
+    result: dict[str, Any] = {
+        "prompt": prompt,
+        "modalities": ["image"],
+    }
+    if negative_prompt is not None:
+        result["negative_prompt"] = negative_prompt
+    return result
+
+
 def _normalize_images_for_save(images: list[Any]) -> list[Any]:
     """Convert NumPy diffusion outputs to PIL images before saving."""
     normalized = []
@@ -79,12 +92,6 @@ def parse_args() -> argparse.Namespace:
         "meituan-longcat/LongCat-Image, OvisAI/Ovis-Image, "
         "stabilityai/stable-diffusion-3.5-medium, Tongyi-MAI/Z-Image-Turbo and etc.",
     )
-    parser.add_argument(
-        "--stage-configs-path",
-        type=str,
-        default=None,
-        help="[Deprecated] Path to a legacy stage_args-format YAML. Prefer --deploy-config.",
-    )
     parser.add_argument(
         "--deploy-config",
         type=str,
@@ -260,6 +267,7 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument(
         "--lora-path",
         type=str,
+        nargs="+",
         default=None,
         help="Path to LoRA adapter folder (PEFT format). Loaded at initialization and used for generation.",
     )
@@ -269,6 +277,15 @@ def parse_args() -> argparse.Namespace:
         default=1.0,
         help="Scale factor for LoRA weights (default: 1.0).",
     )
+    parser.add_argument(
+        "--lora-backend",
+        type=str,
+        default="peft",
+        choices=["peft", "distill"],
+        help="LoRA backend for loading LoRA adapters. Default: peft"
+        "'peft' loads a PEFT-format adapter folder, used e.g. for RL"
+        "'distill' fuses one or more concrete LoRA checkpoint files, used e.g. for distilled few-step LoRAs",
+    )
     parser.add_argument(
         "--vae-patch-parallel-size",
         type=int,
@@ -404,8 +421,12 @@ def main():
     # Prepare LoRA kwargs for Omni initialization
     lora_args: dict[str, Any] = {}
     if args.lora_path:
-        lora_args["lora_path"] = args.lora_path
+        lora_path = args.lora_path
-        print(f"Using LoRA from: {args.lora_path}")
+        if len(lora_path) == 1:
+            lora_path = lora_path[0]
+        lora_args["lora_path"] = lora_path
+        lora_args["lora_backend"] = args.lora_backend
+        print(f"Using LoRA from: {lora_path} with backend: {args.lora_backend}")
 
     # Build quantization kwargs: use quantization_config dict when
     # ignored_layers is specified so the list flows through OmniDiffusionConfig
@@ -448,8 +469,6 @@ def main():
         omni_kwargs["tensor_parallel_size"] = args.tensor_parallel_size
     if args.enforce_eager is not None:
         omni_kwargs["enforce_eager"] = args.enforce_eager
-    if args.stage_configs_path:
-        omni_kwargs["stage_configs_path"] = args.stage_configs_path
     if args.deploy_config:
         omni_kwargs["deploy_config"] = args.deploy_config
     if use_nextstep:
@@ -488,26 +507,33 @@ def main():
     print(f"  Image size: {args.width}x{args.height}")
     if args.lora_path:
         print(f"  LoRA: scale={args.lora_scale}")
-    if args.stage_configs_path:
+    if args.deploy_config:
-        print(f"  stage-configs-path: {args.stage_configs_path}")
+        print(f"  deploy-config: {args.deploy_config}")
     print(f"{'=' * 60}\n")
 
     # Build LoRA request when --lora-path is set
     lora_request = None
-    if args.lora_path:
+    if args.lora_path and args.lora_backend == "peft":
-        lora_request_id = stable_lora_int_id(args.lora_path)
+        if len(args.lora_path) != 1:
+            raise ValueError("Only one LoRA path is expected for PEFT backend.")
+
+        lora_path = args.lora_path[0]
+        lora_request_id = stable_lora_int_id(lora_path)
         lora_request = LoRARequest(
-            lora_name=Path(args.lora_path).stem,
+            lora_name=Path(lora_path).stem,
             lora_int_id=lora_request_id,
-            lora_path=args.lora_path,
+            lora_path=lora_path,
         )
 
     generation_start = time.perf_counter()
 
     prompt_dict = build_text_to_image_prompt(
-        model_class_name=model_class_name,
         prompt=args.prompt,
         negative_prompt=args.negative_prompt,
+    )
+    prompt_dict = build_model_text_to_image_prompt(
+        model_class_name=model_class_name,
+        prompt=prompt_dict,
         height=args.height,
         width=args.width,
     )
@@ -622,7 +648,7 @@ def main():
         images = getattr(output, "images", None)
         if images:
             break
-        req_out = getattr(output, "request_output", None)
+        req_out = output
         images = getattr(req_out, "images", None) if req_out is not None else None
         if images:
             break

examples/offline_inference/text_to_speech/README.md

@@ -23,7 +23,9 @@ list of supported architectures across all modalities, see
 | OmniVoice | `k2-fsa/OmniVoice` | 2 (gen + dec) | ✓ | — | voice design, language hint | 24 kHz |
 | Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | 2 (talker + code2wav) | ✓ (Base) | ✓ | 3 task variants | 24 kHz |
 | VoxCPM2 | `openbmb/VoxCPM2` | single (native AR) | ✓ | ✓ (online) | continuation | 48 kHz |
+| dots.tts | `rednote-hilab/dots.tts-soar` | single (native AR) | — (not wired yet) | — | — | 48 kHz |
 | IndexTTS-2 | `IndexTeam/IndexTTS-2` | 2 (AR talker + S2Mel DiT + BigVGAN) | ✓ (required) | — | emotion control (`--emo-audio`, `--emo-text`, `--emo-vector`) | 22.05 kHz |
+| IndexTTS-2.5 | native `checkpoints/` bundle | 2 (AR talker + EnhancedCodec + S2Mel DiT + BigVGAN) | ✓ (required) | — | multilingual (`--lang`) + emotion control | 22.05 kHz |
 | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | varies | ✓ | ✓ | voice presets | 24 kHz |
 
 ## Common Quick Start
@@ -371,18 +373,10 @@ python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
 Streaming requires `async_chunk: true` in the stage config.
 
 ### Word Timestamps
-Generate a WAV offline and a JSON sidecar with word-level timestamps from
+Word-level timestamps are currently a serving-path feature: launch
-`Qwen/Qwen3-ForcedAligner-0.6B`:
+`vllm-omni serve` with `--forced-aligner` and request `word_timestamps`
-```bash
+(see `examples/online_serving/text_to_speech/README.md`). An offline
-python examples/offline_inference/text_to_speech/qwen3_tts/word_timestamps.py \
+example is not provided in this release.
-    --model Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
-    --forced-aligner Qwen/Qwen3-ForcedAligner-0.6B \
-    --text "Hello world." \
-    --output-dir /tmp/qwen3_tts_timestamps
-```
-The script writes `qwen3_tts_word_timestamps.wav` and
-`qwen3_tts_word_timestamps.json`. On machines without a local CUDA toolkit,
-set `VLLM_USE_FLASHINFER_SAMPLER=0` to avoid FlashInfer sampler JIT.
 
 ### Batched decoding
 The Code2Wav stage supports batched decoding through the SpeechTokenizer. Pass multiple prompts via `--txt-prompts` and set `--batch-size` accordingly. To raise `max_num_seqs` on either stage, point `--stage-configs-path` at a stage configs YAML with the desired values (see `vllm_omni/model_executor/stage_configs/` for templates):
@@ -437,11 +431,39 @@ Streaming is exposed through the online OpenAI Speech API (`stream=true`). See [
 
 ---
 
-## IndexTTS-2
+## dots.tts
 
-2-stage TTS pipeline (GPT AR talker + S2Mel CFM DiT + BigVGAN vocoder) at 22.05 kHz. Every request requires reference audio for zero-shot voice cloning. Supports emotion conditioning via audio, text, or 8-dim vector.
+Single-stage native AR TTS at 48 kHz (rednote-hilab). Pipeline: `Qwen2.5-1.5B base LM → DiT (10-step Euler flow matching) → patch_encoder AR loopback → AudioVAE (streaming decode)`. Same "vLLM-native base LM + side-path computation" pattern as VoxCPM2, with a plain Qwen2 backbone instead of MiniCPM4 and no FSQ / residual LM stage.
 
 ### Quick start
+```bash
+python examples/offline_inference/text_to_speech/dots_tts/end2end.py \
+    --model rednote-hilab/dots.tts-soar \
+    --text "Hello, this is a test of dots TTS running on vLLM Omni."
+```
+
+### Voice cloning
+Not wired in this release — generation is zero-shot only. The CAM++ x-vector speaker encoder weights load, but `end2end.py` has no `--ref-audio`/`--ref-text` flags yet.
+
+### Streaming
+The AudioVAE decoder has an internal streaming path (`init_stream_state` / `stream_step` / `stream_flush`) used to avoid boundary artifacts between 160 ms patches, but it is not yet exposed through an online serving endpoint or example.
+
+### Notes
+- Output: 48 kHz mono WAV.
+- Deploy config: `vllm_omni/deploy/dots_tts.yaml` (auto-loaded by HF `model_type`).
+- Checkpoints: `rednote-hilab/dots.tts-soar` is the validated default. `dots.tts-base` shares the same architecture but is unvalidated in this repo. `dots.tts-mf` (MeanFlow, 2-4 step) is not supported yet.
+- Known limitation: no CUDA graph capture and no batched side-path computation yet, so concurrent requests do not currently scale (each request's DiT Euler steps run serially). See `recipes/rednote-hilab/dots.tts.md` for details and the roadmap.
+
+---
+
+## IndexTTS-2 and IndexTTS-2.5
+
+Both versions use a 2-stage TTS pipeline and produce 22.05 kHz mono audio. Stage 0 is a GPT AR talker. IndexTTS-2 Stage 1 uses RepCodec semantic embeddings, GPT latent, S2Mel CFM/DiT, and BigVGAN. IndexTTS-2.5 replaces RepCodec with EnhancedCodec and uses the official code-only path (`use_gpt_latent=false`). Every request requires reference audio for zero-shot voice cloning.
+
+### Quick start
+
+IndexTTS-2:
+
 ```bash
 python examples/offline_inference/text_to_speech/indextts2/end2end.py \
     --model IndexTeam/IndexTTS-2 \
@@ -449,6 +471,24 @@ python examples/offline_inference/text_to_speech/indextts2/end2end.py \
     --ref-audio /path/to/reference.wav
 ```
 
+IndexTTS-2 uses its existing frontend and does not require the IndexTTS-2.5 text dependencies. Before running IndexTTS-2.5, install its optional frontend dependencies:
+
+```bash
+pip install 'vllm-omni[indextts2]'
+```
+
+IndexTTS-2.5 with a native repository/lab bundle whose model root contains `checkpoints/`:
+
+```bash
+python examples/offline_inference/text_to_speech/indextts2/end2end.py \
+    --model /path/to/indextts-2.5 \
+    --model-version 2.5 \
+    --lang ja \
+    --text "こんにちは、音声合成のテストです。" \
+    --ref-audio /path/to/reference.wav
+
+```
+
 ### Emotion control
 ```bash
 # Emotion from reference audio
@@ -474,10 +514,13 @@ python examples/offline_inference/text_to_speech/indextts2/end2end.py \
 ```
 
 ### Notes
-- `--ref-audio` is **required** — IndexTTS-2 does not support text-only synthesis.
+- `--ref-audio` is **required** — neither version provides text-only synthesis.
-- Stage 0 (AR Talker): GPT-2 generates mel codes from text + reference audio.
+- IndexTTS-2.5 Stage 0 (AR Talker) generates mel codes from text + reference audio using plain vLLM sampling. It does not reproduce the official default `num_beams=3` beam search. For parity comparisons, run upstream with `num_beams=1`; output quality can differ from the official beam-search result.
-- Stage 1 (S2Mel + BigVGAN): CFM DiT converts mel codes to waveform at 22.05 kHz.
+- IndexTTS-2.5 accepts language codes such as `zh`, `en`, `zhen` (mixed Chinese/English), `ja`, and `yue`. `Mandarin` is a vLLM-Omni convenience alias for `zh`. Japanese (`ja`) uses `fugashi` tokenization and produces audio, but it does not automatically expand numbers, dates, or percentages; write those inputs as readable Japanese text before calling the model. Use `--no-text-normalization` only when the input is already normalized.
-- Deploy config: `vllm_omni/deploy/indextts2.yaml`. Stage 1 runs with `enforce_eager: true` (DiT has dynamic shapes).
+- A seed controls Stage 0 AR sampling and per-request CFM noise. Different concurrent batch compositions do not guarantee a bit-identical waveform.
+- Stage 1 (semantic codec + S2Mel + BigVGAN): CFM/DiT converts the complete semantic-code sequence to waveform.
+- Deploy configs: `vllm_omni/deploy/indextts2.yaml` and `indextts2_5.yaml`.
+- `async_chunk=false` is intentional for IndexTTS-2.5 correctness: S2Mel consumes the completed semantic sequence.
 
 ---
 

examples/offline_inference/text_to_speech/cosyvoice3/end2end.py

@@ -168,7 +168,7 @@ def run_e2e():
     print(f"Received {len(outputs)} outputs.")
     for i, output in enumerate(outputs):
         try:
-            ro = output.request_output
+            ro = output
             if ro is None:
                 print("No request_output found.")
                 continue