upd

Signed-off-by: hsliu_ustc <hsliu_ustc@noreply.gitcode.com>

.buildkite/bootstrap-amd-omni.sh

@@ -56,6 +56,88 @@ check_run_all_label() {
     fi
 }
 
+is_docs_only_change() {
+    local file_path
+    local has_any=0
+
+    while IFS= read -r file_path; do
+        [[ -z "${file_path}" ]] && continue
+        has_any=1
+
+        if [[ "${file_path}" == docs/* ]]; then
+            continue
+        fi
+        if [[ "${file_path}" == *.md ]]; then
+            continue
+        fi
+        if [[ "${file_path}" == "mkdocs.yml" ]]; then
+            continue
+        fi
+        return 1
+    done
+
+    [[ "${has_any}" -eq 1 ]]
+}
+
+resolve_skip_ci() {
+    local is_pr_build=0
+    local files
+    local base_branch base_ref
+
+    if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" && -n "${BUILDKITE_PULL_REQUEST:-}" ]]; then
+        is_pr_build=1
+    fi
+
+    if [[ "${is_pr_build}" -eq 1 ]]; then
+        base_branch="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}"
+        if ! git rev-parse --verify "origin/${base_branch}" >/dev/null 2>&1; then
+            echo "resolve_skip_ci: origin/${base_branch} not found locally; trying fetch" >&2
+            git fetch --depth=200 origin "${base_branch}" >/dev/null 2>&1 || true
+        fi
+
+        base_ref=""
+        if git rev-parse --verify "origin/${base_branch}" >/dev/null 2>&1; then
+            base_ref="origin/${base_branch}"
+        elif git rev-parse --verify "${base_branch}" >/dev/null 2>&1; then
+            base_ref="${base_branch}"
+        else
+            echo "resolve_skip_ci: cannot resolve PR base ${base_branch}; skip-ci=0" >&2
+            echo -n 0
+            return 0
+        fi
+
+        if ! files="$(git diff --name-only "${base_ref}...${BUILDKITE_COMMIT}" 2>/dev/null)"; then
+            echo "resolve_skip_ci: failed to compute PR changed files; skip-ci=0" >&2
+            echo -n 0
+            return 0
+        fi
+    elif [[ "${BUILDKITE_BRANCH:-}" == "main" ]]; then
+        if ! git rev-parse --verify "${BUILDKITE_COMMIT}^" >/dev/null 2>&1; then
+            echo "resolve_skip_ci: commit has no parent on main; skip-ci=0" >&2
+            echo -n 0
+            return 0
+        fi
+        if ! files="$(git diff --name-only "${BUILDKITE_COMMIT}^..${BUILDKITE_COMMIT}" 2>/dev/null)"; then
+            echo "resolve_skip_ci: failed to compute main changed files; skip-ci=0" >&2
+            echo -n 0
+            return 0
+        fi
+    else
+        echo "resolve_skip_ci: not PR/main build; skip-ci=0" >&2
+        echo -n 0
+        return 0
+    fi
+
+    if is_docs_only_change <<< "${files}"; then
+        echo "resolve_skip_ci: docs-only change detected; skip-ci=1" >&2
+        echo -n 1
+        return 0
+    fi
+
+    echo "resolve_skip_ci: non-doc changes detected; skip-ci=0" >&2
+    echo -n 0
+}
+
 if [[ -z "${COV_ENABLED:-}" ]]; then
     COV_ENABLED=0
 fi
@@ -91,8 +173,24 @@ upload_pipeline() {
 
     cd .buildkite
 
-    # Select test definition file: merge suite for main, ready suite for PRs
-    if [[ $BUILDKITE_BRANCH == "main" ]]; then
+    # Select test definition file: merge suite for main, ready suite for PRs.
+    # For debugging, DEBUG_TEST_YAML can override the selection — accepts
+    # "merge" or "ready" (case-insensitive).
+    if [[ -n "${DEBUG_TEST_YAML:-}" ]]; then
+        case "${DEBUG_TEST_YAML,,}" in
+            merge)
+                TEST_YAML="test-amd-merge.yml"
+                ;;
+            ready)
+                TEST_YAML="test-amd-ready.yaml"
+                ;;
+            *)
+                echo "ERROR: DEBUG_TEST_YAML must be 'merge' or 'ready', got '$DEBUG_TEST_YAML'" >&2
+                exit 1
+                ;;
+        esac
+        echo "DEBUG_TEST_YAML override: using $TEST_YAML"
+    elif [[ $BUILDKITE_BRANCH == "main" ]]; then
         TEST_YAML="test-amd-merge.yml"
     else
         TEST_YAML="test-amd-ready.yaml"
@@ -140,30 +238,13 @@ fi
 # Early exit start: skip pipeline if conditions are met
 # ----------------------------------------------------------------------
 
-# skip pipeline if all changed files are under docs/
+# Match CUDA Buildkite skip-ci behavior for docs/markdown-only changes.
 if [[ "${DOCS_ONLY_DISABLE}" != "1" ]]; then
-  if [[ -n "${file_diff:-}" ]]; then
-    docs_only=1
-    # Robust iteration over newline-separated file_diff
-    while IFS= read -r f; do
-      [[ -z "$f" ]] && continue
-      # **Policy:** only skip if *every* path starts with docs/
-      if [[ "$f" != docs/* ]]; then
-        docs_only=0
-        break
-      fi
-    done < <(printf '%s\n' "$file_diff" | tr ' ' '\n' | tr -d '\r')
-
-    if [[ "$docs_only" -eq 1 ]]; then
-      buildkite-agent annotate ":memo: CI skipped — docs/** only changes detected
-
-\`\`\`
-${file_diff}
-\`\`\`" --style "info" || true
-      echo "[docs-only] All changes are under docs/. Exiting before pipeline upload."
-      exit 0
+    SKIP_CI="$(resolve_skip_ci)"
+    if [[ "${SKIP_CI}" == "1" ]]; then
+        echo "[docs-only] Docs/markdown-only changes detected. Exiting before pipeline upload."
+        exit 0
     fi
-  fi
 fi
 
 # ----------------------------------------------------------------------

.buildkite/pipeline-intel.yaml

@@ -10,7 +10,7 @@ steps:
           DOCKER_BUILDKIT: "1"
           # Buildkite will automatically replace this with the actual commit hash
           VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
-          VLLM_VERSION: "v0.19.0"
+          VLLM_VERSION: "v0.20.0"
         priority: 100
         timeout_in_minutes: 60
-        soft_fail: true
+        soft_fail: false

.buildkite/pipeline-npu-a3.yaml

@@ -0,0 +1,110 @@
+# steps:
+#   - label: ":buildkit: Build and Push NPU Test Image - A3"
+#     key: image-build-a3
+#     agents:
+#       queue: "ascend-a3"
+#       resource_class: "npu-2"
+#     plugins:
+#       - kubernetes:
+#           metadata:
+#             annotations:
+#               vault.hashicorp.com/agent-init-first: "true"
+#               vault.hashicorp.com/agent-inject: "true"
+#               vault.hashicorp.com/agent-inject-perms-ca.pem: "0400"
+#               vault.hashicorp.com/agent-inject-perms-cert.pem: "0400"
+#               vault.hashicorp.com/agent-inject-perms-config.json: "0400"
+#               vault.hashicorp.com/agent-inject-perms-key.pem: "0400"
+#               vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd
+#               vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd
+#               vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd
+#               vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd
+#               vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}"
+#               vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}"
+#               vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}"
+#               vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}"
+#               vault.hashicorp.com/agent-pre-populate-only: "true"
+#               vault.hashicorp.com/agent-run-as-group: "1000"
+#               vault.hashicorp.com/agent-run-as-user: "1000"
+#               vault.hashicorp.com/agent-service-account-token-volume-name: token-vol
+#               vault.hashicorp.com/role: ascend-gha-runners
+#               vault.hashicorp.com/secret-volume-path: /home/user/.docker/
+#               vault.hashicorp.com/tls-skip-verify: "true"
+#           podSpecPatch:
+#             volumes:
+#             - name: token-vol
+#               projected:
+#                 defaultMode: 420
+#                 sources:
+#                 - serviceAccountToken:
+#                     audience: api
+#                     expirationSeconds: 600
+#                     path: token
+#     env:
+#       VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
+#       IMAGE_NAME: "vllm-omni-ci-npu"
+#       IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
+#       BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
+#     command: |
+#       set -ex
+
+#       echo "--- Building and pushing NPU Test Image"
+#       echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
+#       echo "buildkitd address: $${BUILDKITD_ADDR}"
+
+#       git config --global url."https://gh-proxy.test.osinfra.cn/https://github.com/".insteadOf "https://github.com/"
+
+#       if ! command -v buildctl &> /dev/null; then
+#         echo "Installing buildctl..."
+#         mkdir -p /tmp/buildkit
+#         BUILDKIT_VERSION="v0.29.0"
+#         wget -q "https://gh-proxy.test.osinfra.cn/https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz
+#         tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
+#         cp /tmp/buildkit/bin/buildctl /usr/local/bin/
+#       fi
+
+#       export DOCKER_CONFIG=/home/user/.docker
+#       buildctl \
+#         --addr="$${BUILDKITD_ADDR}" \
+#         --tlscacert=/home/user/.docker/ca.pem \
+#         --tlscert=/home/user/.docker/cert.pem \
+#         --tlskey=/home/user/.docker/key.pem \
+#         build \
+#         --frontend dockerfile.v0 \
+#         --local context=. \
+#         --local dockerfile=./docker \
+#         --opt filename=Dockerfile.npu.ci.a3 \
+#         --secret id=dockerconfig,src=/home/user/.docker/config.json \
+#         --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \
+#         --progress=plain
+
+#       echo "--- Image pushed successfully"
+#       echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
+
+#   # - label: "Upload Ready Pipeline - A3"
+#   #   depends_on: image-build-a3
+#   #   key: upload-ready-pipeline-a3
+#   #   if: build.branch != "main" && build.pull_request.labels includes "npu-test"
+#   #   agents:
+#   #     queue: "ascend-a3"
+#   #     resource_class: "npu-2"
+#   #   command: |
+#   #     buildkite-agent pipeline upload .buildkite/test-npu-ready.yaml
+
+  # # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
+  # - label: "Upload Nightly Pipeline"
+  #   depends_on: image-build
+  #   key: upload-nightly-pipeline
+  #   if: >-
+  #     (build.branch == "main" && build.env("NIGHTLY") == "1") ||
+  #     (build.branch != "main" && (
+  #       build.pull_request.labels includes "nightly-test" ||
+  #       build.pull_request.labels includes "omni-test" ||
+  #       build.pull_request.labels includes "tts-test" ||
+  #       build.pull_request.labels includes "diffusion-x2iat-test" ||
+  #       build.pull_request.labels includes "diffusion-x2v-test"
+  #     ))
+  #   commands:
+  #     - buildkite-agent pipeline upload .buildkite/test-npu-nightly.yaml
+  #   agents:
+  #     queue: "ascend-a3"
+  #     resource_class: "npu-2"

.buildkite/pipeline-npu.yaml

@@ -0,0 +1,108 @@
+steps:
+  - label: ":buildkit: Build and Push NPU Test Image - A2"
+    key: image-build
+    agents:
+      queue: "ascend-a2b3"
+      resource_class: "npu-1"
+    plugins:
+      - kubernetes:
+          metadata:
+            annotations:
+              vault.hashicorp.com/agent-init-first: "true"
+              vault.hashicorp.com/agent-inject: "true"
+              vault.hashicorp.com/agent-inject-perms-ca.pem: "0400"
+              vault.hashicorp.com/agent-inject-perms-cert.pem: "0400"
+              vault.hashicorp.com/agent-inject-perms-config.json: "0400"
+              vault.hashicorp.com/agent-inject-perms-key.pem: "0400"
+              vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd
+              vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd
+              vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd
+              vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd
+              vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}"
+              vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}"
+              vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}"
+              vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}"
+              vault.hashicorp.com/agent-pre-populate-only: "true"
+              vault.hashicorp.com/agent-run-as-group: "1000"
+              vault.hashicorp.com/agent-run-as-user: "1000"
+              vault.hashicorp.com/agent-service-account-token-volume-name: token-vol
+              vault.hashicorp.com/role: ascend-gha-runners
+              vault.hashicorp.com/secret-volume-path: /home/user/.docker/
+              vault.hashicorp.com/tls-skip-verify: "true"
+          podSpecPatch:
+            volumes:
+            - name: token-vol
+              projected:
+                defaultMode: 420
+                sources:
+                - serviceAccountToken:
+                    audience: api
+                    expirationSeconds: 600
+                    path: token
+    env:
+      VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
+      IMAGE_NAME: "vllm-omni-ci-npu"
+      IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
+      BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
+    command: |
+      set -ex
+
+      echo "--- Building and pushing NPU Test Image"
+      echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
+      echo "buildkitd address: $${BUILDKITD_ADDR}"
+
+      if ! command -v buildctl &> /dev/null; then
+        echo "Installing buildctl..."
+        mkdir -p /tmp/buildkit
+        BUILDKIT_VERSION="v0.29.0"
+        wget -q "https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz
+        tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
+        cp /tmp/buildkit/bin/buildctl /usr/local/bin/
+      fi
+
+      export DOCKER_CONFIG=/home/user/.docker
+      buildctl \
+        --addr="$${BUILDKITD_ADDR}" \
+        --tlscacert=/home/user/.docker/ca.pem \
+        --tlscert=/home/user/.docker/cert.pem \
+        --tlskey=/home/user/.docker/key.pem \
+        build \
+        --frontend dockerfile.v0 \
+        --local context=. \
+        --local dockerfile=./docker \
+        --opt filename=Dockerfile.npu.ci \
+        --secret id=dockerconfig,src=/home/user/.docker/config.json \
+        --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \
+        --progress=plain
+
+      echo "--- Image pushed successfully"
+      echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
+
+  # - label: "Upload Ready Pipeline"
+  #   depends_on: image-build
+  #   key: upload-ready-pipeline
+  #   if: build.branch != "main" && build.pull_request.labels includes "npu-test"
+  #   agents:
+  #     queue: "ascend-a2b3"
+  #     resource_class: "npu-1"
+  #   command: |
+  #     buildkite-agent pipeline upload .buildkite/test-npu-ready.yaml
+
+  # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
+  - label: "Upload Nightly Pipeline"
+    depends_on: image-build
+    key: upload-nightly-pipeline
+    if: >-
+      (build.branch == "main" && build.env("NIGHTLY") == "1") ||
+      (build.branch != "main" && (
+        build.pull_request.labels includes "nightly-test" ||
+        build.pull_request.labels includes "omni-test" ||
+        build.pull_request.labels includes "tts-test" ||
+        build.pull_request.labels includes "diffusion-x2iat-test" ||
+        build.pull_request.labels includes "diffusion-x2v-test"
+      ))
+    commands:
+      - buildkite-agent pipeline upload .buildkite/test-npu-nightly.yaml
+    agents:
+      queue: "ascend-a2b3"
+      resource_class: "npu-1"

.buildkite/scripts/upload_pipeline_with_skip_ci.sh

@@ -1,10 +1,18 @@
 #!/usr/bin/env bash
-# Evaluate docs-only skip-ci and upload continuation steps from the same `.buildkite/pipeline.yml`
+# Evaluate docs-only skip-ci and upload continuation steps from the given pipeline YAML
 # (YAML document after the first `---`). Buildkite `if` is evaluated at upload time.
+#
+# Usage: upload_pipeline_with_skip_ci.sh [pipeline_yaml]
+#   pipeline_yaml: path relative to repo root or absolute (default: .buildkite/pipeline.yml)
 set -euo pipefail
 
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
-PIPELINE_YML="${ROOT}/.buildkite/pipeline.yml"
+PIPELINE_ARG="${1:-.buildkite/pipeline.yml}"
+if [[ "${PIPELINE_ARG}" = /* ]]; then
+  PIPELINE_YML="${PIPELINE_ARG}"
+else
+  PIPELINE_YML="${ROOT}/${PIPELINE_ARG}"
+fi
 
 # Prints a single digit to stdout: 1 = skip image CI, 0 = run. Logs go to stderr.
 is_docs_only_change() {
@@ -21,7 +29,7 @@ is_docs_only_change() {
     if [[ "${file_path}" == *.md ]]; then
       continue
     fi
-    if [[ "${file_path}" == "mkdocs.yaml" ]]; then
+    if [[ "${file_path}" == "mkdocs.yml" ]]; then
       continue
     fi
     return 1
@@ -104,12 +112,13 @@ import pathlib
 path = pathlib.Path(os.environ["PIPELINE_YML"])
 text = path.read_text(encoding="utf-8")
 sep = "\n---\n"
-if sep not in text:
-    raise SystemExit(
-        "upload_pipeline_with_skip_ci: .buildkite/pipeline.yml must contain a '\\n---\\n' separator "
-        "(document 1 = bootstrap, document 2 = uploaded steps)"
-    )
-_, continuation = text.split(sep, 1)
+# Two supported layouts:
+#   - multi-doc: doc 1 = bootstrap (loaded by Buildkite), doc 2 (after `---`) = uploaded steps.
+#   - single-doc: whole file is the uploaded steps; caller (UI init step) is the bootstrap.
+if sep in text:
+    _, continuation = text.split(sep, 1)
+else:
+    continuation = text
 
 skip = os.environ.get("SKIP_CI") == "1"
 # When docs-only skip-ci: skip default CI image, but still build for L4 nightly (PR label nightly-test or
@@ -126,7 +135,7 @@ if skip:
 else:
     rep = "'true'"
     ready_rep = "'build.branch != \"main\" && build.pull_request.labels includes \"ready\"'"
-    merge_rep = "'(build.branch == \"main\" && build.env(\"NIGHTLY\") != \"1\") || (build.branch != \"main\" && build.pull_request.labels includes \"merge-test\")'"
+    merge_rep = "'(build.branch == \"main\" && build.env(\"NIGHTLY\") != \"1\" && build.env(\"WEEKLY\") != \"1\") || (build.branch != \"main\" && build.pull_request.labels includes \"merge-test\")'"
 rendered = (
     continuation
     .replace("__IMAGE_BUILD_IF__", rep)

.buildkite/test-amd-merge.yml

@@ -7,16 +7,9 @@ steps:
   grade: Blocking
   commands:
     - export VLLM_ROCM_USE_AITER=0
-    - "timeout 20m pytest -v -s -m 'core_model and cpu' --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml"
-
-- label: "Diffusion Model Test"
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - timeout 20m pytest -s -v tests/e2e/offline_inference/test_t2i_model.py -m "advanced_model and diffusion" --run-level "advanced_model"
+    # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
+    # that is not supported on CPU
+    - "timeout 60m pytest -v -s -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
 
 - label: "Diffusion Images API LoRA E2E"
   agent_pool: mi325_1
@@ -24,7 +17,7 @@ steps:
   mirror_hardwares: [amdproduction]
   grade: Blocking
   commands:
-    - timeout 20m pytest -s -v tests/e2e/online_serving/test_images_generations_lora.py
+    - timeout 20m pytest -s -v tests/diffusion/lora/test_images_generations_lora.py
 
 - label: "Diffusion Model CPU offloading Test"
   agent_pool: mi325_1
@@ -37,9 +30,9 @@ steps:
     - |
       timeout 20m bash -c '
         set +e
-        pytest -s -v tests/e2e/offline_inference/test_diffusion_cpu_offload.py
+        pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
         EXIT1=\$?
-        pytest -s -v tests/e2e/offline_inference/test_diffusion_layerwise_offload.py
+        pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
         EXIT2=\$?
         exit \$((EXIT1 | EXIT2))
       '
@@ -74,7 +67,7 @@ steps:
   commands:
     - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - timeout 20m pytest -s -v tests/e2e/offline_inference/test_sequence_parallel.py
+    - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py
     - timeout 20m pytest -s -v tests/diffusion/distributed/test_ulysses_uaa_perf.py
 
 # merge-only tests
@@ -84,7 +77,15 @@ steps:
   mirror_hardwares: [amdproduction]
   grade: Blocking
   commands:
-    - pytest -s -v tests/e2e/offline_inference/test_zimage_parallelism.py
+    - pytest -s -v tests/diffusion/distributed/test_zimage_parallelism.py
+
+- label: "Diffusion Model Test"
+  agent_pool: mi325_1
+  depends_on: amd-build
+  mirror_hardwares: [amdproduction]
+  grade: Blocking
+  commands:
+    - timeout 30m pytest -s -v tests/e2e/offline_inference/test_z_image.py
 
 - label: "Diffusion GPU Worker Test"
   agent_pool: mi325_2
@@ -103,16 +104,17 @@ steps:
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
     - timeout 20m pytest -s -v tests/engine/test_async_omni_engine_abort.py
 
+# This is from test-nightly.yml.
+# This model seems to be important, so I am trying to run it here.
+# Since this is from test-nightly.yml,
+# I keep this non-blocking.
 - label: "Omni Model Test Qwen2-5-Omni"
-  agent_pool: mi325_2
+  agent_pool: mi325_4
   depends_on: amd-build
   mirror_hardwares: [amdproduction]
-  grade: Blocking
   commands:
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - timeout 20m pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py
-    - timeout 20m pytest -s -v tests/e2e/online_serving/test_qwen2_5_omni_expansion.py -m "advanced_model" --run-level "core_model"
+    - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+    - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model"
 
 - label: "Omni Model Test Qwen3-Omni"
   agent_pool: mi325_2
@@ -120,10 +122,8 @@ steps:
   mirror_hardwares: [amdproduction]
   grade: Blocking
   commands:
-    - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - export VLLM_TEST_CLEAN_GPU_MEMORY=1
-    - timeout 30m pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/online_serving/test_mimo_audio.py -m "advanced_model" --run-level "advanced_model"
+    - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/online_serving/test_mimo_audio.py -m "advanced_model" --run-level "advanced_model"
 
 - label: "Qwen3-TTS CustomVoice E2E Test"
   agent_pool: mi325_1
@@ -153,17 +153,6 @@ steps:
         pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m "advanced_model" --run-level "advanced_model"
       '
 
-- label: "Diffusion Image Edit Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - timeout 20m pytest -s -v tests/e2e/online_serving/test_image_gen_edit.py
-
 # TODO: Bagel test on ROCm is very unstable. @tjtanaa
 # Need to debug before reneable numerical changes across large PRs
 # # split Bagel Model Test with H100 (Real Weights) into three tests
@@ -174,11 +163,10 @@ steps:
 #   grade: Blocking
 #   commands:
 #     - export GPU_ARCHS=gfx942
-#     - export VLLM_TEST_CLEAN_GPU_MEMORY=1
 #     - export VLLM_LOGGING_LEVEL=DEBUG
 #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
 #     - export VLLM_ROCM_USE_AITER_RMSNORM=0
-#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel_text2img.py -m "advanced_model" --run-level "advanced_model" -k "shared_memory" -k "rocm"
+#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "advanced_model" --run-level "advanced_model" -k "shared_memory" -k "rocm"
 
 # - label: "Bagel Img2Img Model Test (2/3)"
 #   agent_pool: mi325_1
@@ -187,11 +175,10 @@ steps:
 #   grade: Blocking
 #   commands:
 #     - export GPU_ARCHS=gfx942
-#     - export VLLM_TEST_CLEAN_GPU_MEMORY=1
 #     - export VLLM_LOGGING_LEVEL=DEBUG
 #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
 #     - export VLLM_ROCM_USE_AITER_RMSNORM=0
-#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel_img2img.py -m "advanced_model" --run-level "advanced_model" -k "rocm"
+#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "advanced_model" --run-level "advanced_model" -k "rocm"
 
 # - label: "Bagel Online Serving Test (3/3)"
 #   agent_pool: mi325_1
@@ -200,12 +187,23 @@ steps:
 #   grade: Blocking
 #   commands:
 #     - export GPU_ARCHS=gfx942
-#     - export VLLM_TEST_CLEAN_GPU_MEMORY=1
 #     - export VLLM_IMAGE_FETCH_TIMEOUT=60
 #     - export VLLM_LOGGING_LEVEL=DEBUG
 #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
 #     - export VLLM_ROCM_USE_AITER_RMSNORM=0
-#     - timeout 40m pytest -s -v tests/e2e/online_serving/test_bagel_online.py -m "advanced_model" --run-level "advanced_model" -k "rocm"
+#     - timeout 40m pytest -s -v tests/e2e/online_serving/test_bagel.py -m "advanced_model" --run-level "advanced_model" -k "rocm"
+
+- label: "Diffusion · Wan22 Test"
+  agent_pool: mi325_1
+  depends_on: amd-build
+  mirror_hardwares: [amdproduction]
+  grade: Blocking
+  commands:
+    - |
+      timeout 40m bash -c "
+        export VLLM_IMAGE_FETCH_TIMEOUT=60
+        pytest -s -v tests/e2e/offline_inference/test_wan22_t2v.py tests/e2e/online_serving/test_wan22_t2v.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
+      "
 
 - label: "Voxtral-TTS E2E Test"
   agent_pool: mi325_1

.buildkite/test-amd-ready.yaml

@@ -7,7 +7,9 @@ steps:
   grade: Blocking
   commands:
     - export VLLM_ROCM_USE_AITER=0
-    - "timeout 20m pytest -v -s -m 'core_model and cpu' --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml"
+    # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
+    # that is not supported on CPU
+    - "timeout 60m pytest -v -s -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
 
 - label: "Voxtral TTS CUDA Unit Test"
   agent_pool: mi325_1
@@ -17,21 +19,13 @@ steps:
   commands:
     - timeout 10m pytest -s -v tests/model_executor/models/voxtral_tts/test_cuda_graph_acoustic_transformer.py
 
-- label: "Diffusion Model Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - timeout 30m pytest -s -v tests/e2e/offline_inference/test_t2i_model.py -m "core_model and diffusion" --run-level "core_model"
-
 - label: "Diffusion Batching Test"
   agent_pool: mi325_1
   depends_on: amd-build
   mirror_hardwares: [amdproduction]
   grade: Blocking
   commands:
-    - timeout 20m pytest -s -v tests/e2e/offline_inference/test_qwen_image_diffusion_batching.py -m "core_model and diffusion" --run-level "core_model"
+    - timeout 20m pytest -s -v tests/diffusion/batching/test_diffusion_batching.py -m "core_model and diffusion" --run-level "core_model"
 
 - label: "Custom Pipeline Test"
   agent_pool: mi325_1
@@ -52,9 +46,9 @@ steps:
     - |
       timeout 20m bash -c '
         set +e
-        pytest -s -v tests/e2e/offline_inference/test_diffusion_cpu_offload.py
+        pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
         EXIT1=\$?
-        pytest -s -v tests/e2e/offline_inference/test_diffusion_layerwise_offload.py
+        pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
         EXIT2=\$?
         exit \$((EXIT1 | EXIT2))
       '
@@ -90,7 +84,7 @@ steps:
   commands:
     - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - timeout 20m pytest -s -v tests/e2e/offline_inference/test_sequence_parallel.py -m core_model
+    - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py -m core_model
 
 - label: "Diffusion GPU Worker Test"
   agent_pool: mi325_2
@@ -100,6 +94,27 @@ steps:
   commands:
     - timeout 20m pytest -s -v tests/diffusion/test_diffusion_worker.py
 
+- label: "Diffusion · Model Test"
+  agent_pool: mi325_1
+  depends_on: amd-build
+  mirror_hardwares: [amdproduction]
+  grade: Blocking
+  commands:
+    - timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model"
+
+- label: "Diffusion · Wan22 Test"
+  agent_pool: mi325_1
+  depends_on: amd-build
+  mirror_hardwares: [amdproduction]
+  grade: Blocking
+  commands:
+    - |
+      timeout 40m bash -c "
+        export VLLM_TEST_CLEAN_GPU_MEMORY=1
+        export VLLM_IMAGE_FETCH_TIMEOUT=60
+        pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model' --run-level 'core_model'
+      "
+
 - label: "Engine Test"
   agent_pool: mi325_1
   depends_on: amd-build
@@ -120,7 +135,6 @@ steps:
 #   commands:
 #     - export VLLM_LOGGING_LEVEL=DEBUG
 #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-#     - export VLLM_TEST_CLEAN_GPU_MEMORY=1
 #     - timeout 10m pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py
 #     - timeout 20m pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m "core_model" --run-level "core_model"
 
@@ -162,17 +176,6 @@ steps:
         pytest -s -v tests/e2e/offline_inference/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model"
       '
 
-- label: "Diffusion Image Edit Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - timeout 20m pytest -s -v tests/e2e/online_serving/test_image_gen_edit.py
-
 # TODO: Bagel test on ROCm is very unstable. @tjtanaa
 # Need to debug before reneable numerical changes across large PRs
 # - label: "Bagel Text2Img Model Test"
@@ -182,11 +185,10 @@ steps:
 #   grade: Blocking
 #   commands:
 #     - export GPU_ARCHS=gfx942
-#     - export VLLM_TEST_CLEAN_GPU_MEMORY=1
 #     - export VLLM_LOGGING_LEVEL=DEBUG
 #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
 #     - export VLLM_ROCM_USE_AITER_RMSNORM=0
-#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel_text2img.py -m "core_model" --run-level "core_model" -k "rocm"
+#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"
 
 # - label: "Bagel Img2Img Model Test"
 #   agent_pool: mi325_1
@@ -195,11 +197,10 @@ steps:
 #   grade: Blocking
 #   commands:
 #     - export GPU_ARCHS=gfx942
-#     - export VLLM_TEST_CLEAN_GPU_MEMORY=1
 #     - export VLLM_LOGGING_LEVEL=DEBUG
 #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
 #     - export VLLM_ROCM_USE_AITER_RMSNORM=0
-#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel_img2img.py -m "core_model" --run-level "core_model" -k "rocm"
+#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"
 
 # - label: "Bagel Online Serving Test"
 #   agent_pool: mi325_1
@@ -208,12 +209,11 @@ steps:
 #   grade: Blocking
 #   commands:
 #     - export GPU_ARCHS=gfx942
-#     - export VLLM_TEST_CLEAN_GPU_MEMORY=1
 #     - export VLLM_IMAGE_FETCH_TIMEOUT=60
 #     - export VLLM_LOGGING_LEVEL=DEBUG
 #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
 #     - export VLLM_ROCM_USE_AITER_RMSNORM=0
-#     - timeout 40m pytest -s -v tests/e2e/online_serving/test_bagel_online.py -m "core_model" --run-level "core_model" -k "rocm"
+#     - timeout 40m pytest -s -v tests/e2e/online_serving/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"
 
 - label: "CosyVoice3-TTS E2E Test"
   agent_pool: mi325_1
@@ -223,5 +223,5 @@ steps:
   commands:
     - |
       timeout 20m bash -c '
-        pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts.py -m "core_model" --run-level "core_model"
+        pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "full_model" --run-level "core_model"
       '

.buildkite/test-amd.yaml

@@ -8,7 +8,7 @@ steps:
   grade: Blocking
   commands:
     - export GPU_ARCHS=gfx942
-    - pytest -s -v tests/e2e/offline_inference/test_t2i_model.py
+    - pytest -s -v tests/e2e/offline_inference/test_z_image.py
 
 - label: "Diffusion Images API LoRA E2E"
   timeout_in_minutes: 30
@@ -20,7 +20,7 @@ steps:
     - export GPU_ARCHS=gfx942
     - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/online_serving/test_images_generations_lora.py
+    - pytest -s -v tests/diffusion/lora/test_images_generations_lora.py
 
 - label: "Diffusion Model CPU offloading Test"
   timeout_in_minutes: 20
@@ -32,8 +32,8 @@ steps:
     - export GPU_ARCHS=gfx942
     - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/offline_inference/test_diffusion_cpu_offload.py
-    - pytest -s -v tests/e2e/offline_inference/test_diffusion_layerwise_offload.py
+    - pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
+    - pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
 
 - label: "Diffusion Cache Backend Test"
   timeout_in_minutes: 15
@@ -57,7 +57,7 @@ steps:
     - export GPU_ARCHS=gfx942
     - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/offline_inference/test_sequence_parallel.py -m core_model
+    - pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py -m core_model
 
 - label: "Diffusion Tensor Parallelism Test"
   timeout_in_minutes: 20
@@ -67,7 +67,7 @@ steps:
     - export GPU_ARCHS=gfx942
     - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/offline_inference/test_zimage_parallelism.py
+    - pytest -s -v tests/diffusion/distributed/test_zimage_parallelism.py
 
 - label: "Diffusion GPU Worker Test"
   timeout_in_minutes: 20
@@ -100,25 +100,11 @@ steps:
   commands:
     - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - export VLLM_TEST_CLEAN_GPU_MEMORY="1"
     - export VLLM_ROCM_USE_AITER=0
     - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py
     - pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py
 
 
-- label: "Diffusion Image Edit Test"
-  timeout_in_minutes: 15
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/online_serving/test_image_gen_edit.py
-
-
 - label: "Omni Sleep Mode Test"
   timeout_in_minutes: 40
   agent_pool: mi325_2
@@ -129,5 +115,4 @@ steps:
     - export GPU_ARCHS=gfx942
     - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - export VLLM_TEST_CLEAN_GPU_MEMORY="1"
-    - pytest -s -v tests/e2e/offline_inference/test_omni_sleep_mode.py -m "advanced_model and omni and MI325" --run-level "advanced_model"
+    - pytest -s -v tests/entrypoints/test_omni_sleep_mode.py -m "advanced_model and omni and MI325" --run-level "advanced_model"

.buildkite/test-nightly.yml

@@ -13,7 +13,7 @@ steps:
       - label: ":full_moon: Omni · Function Test with H100"
         timeout_in_minutes: 90
         commands:
-          - pytest -s -v tests/e2e/ -m "full_model and H100 and omni" --run-level "full_model" --ignore=tests/e2e/accuracy
+          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py
         agents:
           queue: "mithril-h100-pool"
         plugins:
@@ -169,14 +169,104 @@ steps:
                       path: /mnt/hf-cache
                       type: DirectoryOrCreate
 
-      - label: ":full_moon: Omni · Perf Test"
-        key: nightly-omni-performance
+      - label: ":full_moon: Omni · Perf Test · No Async Chunk"
+        key: nightly-omni-performance-no-async-chunk
         timeout_in_minutes: 180
         commands:
           - export BENCHMARK_DIR=tests/dfx/perf/results
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_omni.json
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
+            exit $$EXIT
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          - kubernetes:
+              podSpec:
+                containers:
+                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                    resources:
+                      limits:
+                        nvidia.com/gpu: 2
+                    volumeMounts:
+                      - name: devshm
+                        mountPath: /dev/shm
+                      - name: hf-cache
+                        mountPath: /root/.cache/huggingface
+                    env:
+                      - name: HF_HOME
+                        value: /root/.cache/huggingface
+                      - name: HF_TOKEN
+                        valueFrom:
+                          secretKeyRef:
+                            name: hf-token-secret
+                            key: token
+                nodeSelector:
+                  node.kubernetes.io/instance-type: gpu-h100-sxm
+                volumes:
+                  - name: devshm
+                    emptyDir:
+                      medium: Memory
+                  - name: hf-cache
+                    hostPath:
+                      path: /mnt/hf-cache
+                      type: DirectoryOrCreate
+
+      - label: ":full_moon: Omni · Perf Test · Async Chunk"
+        key: nightly-omni-performance-async-chunk
+        timeout_in_minutes: 180
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
+            exit $$EXIT
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          - kubernetes:
+              podSpec:
+                containers:
+                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                    resources:
+                      limits:
+                        nvidia.com/gpu: 2
+                    volumeMounts:
+                      - name: devshm
+                        mountPath: /dev/shm
+                      - name: hf-cache
+                        mountPath: /root/.cache/huggingface
+                    env:
+                      - name: HF_HOME
+                        value: /root/.cache/huggingface
+                      - name: HF_TOKEN
+                        valueFrom:
+                          secretKeyRef:
+                            name: hf-token-secret
+                            key: token
+                nodeSelector:
+                  node.kubernetes.io/instance-type: gpu-h100-sxm
+                volumes:
+                  - name: devshm
+                    emptyDir:
+                      medium: Memory
+                  - name: hf-cache
+                    hostPath:
+                      path: /mnt/hf-cache
+                      type: DirectoryOrCreate
+
+      - label: ":full_moon: Omni · Perf Test · vLLM Text"
+        key: nightly-omni-performance-vllm-text
+        timeout_in_minutes: 180
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
@@ -215,6 +305,44 @@ steps:
                       type: DirectoryOrCreate
 
 
+      - label: ":full_moon: Omni · Multi-Replica Startup Test with 4x H100"
+        timeout_in_minutes: 45
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py -m "full_model" --run-level "core_model"
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          - kubernetes:
+              podSpec:
+                containers:
+                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                    resources:
+                      limits:
+                        nvidia.com/gpu: 4
+                    volumeMounts:
+                      - name: devshm
+                        mountPath: /dev/shm
+                      - name: hf-cache
+                        mountPath: /root/.cache/huggingface
+                    env:
+                      - name: HF_HOME
+                        value: /root/.cache/huggingface
+                      - name: HF_TOKEN
+                        valueFrom:
+                          secretKeyRef:
+                            name: hf-token-secret
+                            key: token
+                nodeSelector:
+                  node.kubernetes.io/instance-type: gpu-h100-sxm
+                volumes:
+                  - name: devshm
+                    emptyDir:
+                      medium: Memory
+                  - name: hf-cache
+                    hostPath:
+                      path: /mnt/hf-cache
+                      type: DirectoryOrCreate
+
   - group: ":card_index_dividers: TTS Model Test"
     key: nightly-tts-test-group
     depends_on: upload-nightly-pipeline
@@ -224,7 +352,7 @@ steps:
         timeout_in_minutes: 90
         commands:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-          - pytest -s -v tests/e2e/ -m "full_model and L4 and omni" -k "not test_qwen2_5" --run-level "full_model" --ignore=tests/e2e/accuracy
+          - pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy
         agents:
           queue: "gpu_1_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
         plugins:
@@ -294,10 +422,65 @@ steps:
       build.pull_request.labels includes "nightly-test" ||
       build.pull_request.labels includes "diffusion-x2iat-test"
     steps:
-      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100"
+      # Use explicit file shards so single-card cases do not reserve 2+ GPUs,
+      # and Bagel feature coverage is not duplicated via a broad tests/e2e sweep.
+      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Single-GPU"
         timeout_in_minutes: 120
         commands:
-          - pytest -sv tests/e2e/ -k "not test_wan and not test_bagel_expansion and not hunyuan" -m "full_model and diffusion and H100" --run-level "full_model" --ignore=tests/e2e/accuracy
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_qwen_image_expansion.py
+            tests/e2e/online_serving/test_qwen_image_edit_expansion.py
+            tests/e2e/online_serving/test_qwen_image_layered_expansion.py
+            tests/e2e/online_serving/test_longcat_image_expansion.py
+            tests/e2e/online_serving/test_longcat_image_edit_expansion.py
+            tests/e2e/online_serving/test_flux_2_dev_expansion.py
+            tests/e2e/online_serving/test_bagel_expansion.py
+            -m "full_model and diffusion and H100 and not distributed_cuda"
+            --run-level "full_model"
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          - kubernetes:
+              podSpec:
+                containers:
+                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                    resources:
+                      limits:
+                        nvidia.com/gpu: 1
+                    volumeMounts:
+                      - name: devshm
+                        mountPath: /dev/shm
+                      - name: hf-cache
+                        mountPath: /root/.cache/huggingface
+                    env:
+                      - name: HF_HOME
+                        value: /root/.cache/huggingface
+                      - name: HF_TOKEN
+                        valueFrom:
+                          secretKeyRef:
+                            name: hf-token-secret
+                            key: token
+                nodeSelector:
+                  node.kubernetes.io/instance-type: gpu-h100-sxm
+                volumes:
+                  - name: devshm
+                    emptyDir:
+                      medium: Memory
+                  - name: hf-cache
+                    hostPath:
+                      path: /mnt/hf-cache
+                      type: DirectoryOrCreate
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Qwen-Image"
+        timeout_in_minutes: 120
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_qwen_image_expansion.py
+            tests/e2e/online_serving/test_qwen_image_edit_expansion.py
+            -m "full_model and diffusion and H100 and distributed_cuda"
+            --run-level "full_model"
         agents:
           queue: "mithril-h100-pool"
         plugins:
@@ -332,10 +515,59 @@ steps:
                       path: /mnt/hf-cache
                       type: DirectoryOrCreate
 
-      - label: ":full_moon: Diffusion X2I(&A&T) · Bagel Function Test with H100"
+      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Layered/LongCat/FLUX"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_qwen_image_layered_expansion.py
+            tests/e2e/online_serving/test_longcat_image_expansion.py
+            tests/e2e/online_serving/test_longcat_image_edit_expansion.py
+            tests/e2e/online_serving/test_flux_2_dev_expansion.py
+            -m "full_model and diffusion and H100 and distributed_cuda"
+            --run-level "full_model"
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          - kubernetes:
+              podSpec:
+                containers:
+                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                    resources:
+                      limits:
+                        nvidia.com/gpu: 2
+                    volumeMounts:
+                      - name: devshm
+                        mountPath: /dev/shm
+                      - name: hf-cache
+                        mountPath: /root/.cache/huggingface
+                    env:
+                      - name: HF_HOME
+                        value: /root/.cache/huggingface
+                      - name: HF_TOKEN
+                        valueFrom:
+                          secretKeyRef:
+                            name: hf-token-secret
+                            key: token
+                nodeSelector:
+                  node.kubernetes.io/instance-type: gpu-h100-sxm
+                volumes:
+                  - name: devshm
+                    emptyDir:
+                      medium: Memory
+                  - name: hf-cache
+                    hostPath:
+                      path: /mnt/hf-cache
+                      type: DirectoryOrCreate
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Bagel"
         timeout_in_minutes: 120
         commands:
-          - pytest -sv tests/e2e/online_serving/test_bagel_expansion.py -m "full_model and diffusion and H100" --run-level "full_model"
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_bagel_expansion.py
+            -m "full_model and diffusion and H100 and distributed_cuda"
+            --run-level "full_model"
         agents:
           queue: "mithril-h100-pool"
         plugins:
@@ -370,6 +602,7 @@ steps:
                       path: /mnt/hf-cache
                       type: DirectoryOrCreate
 
+
       - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4"
         timeout_in_minutes: 60
         commands:
@@ -391,7 +624,6 @@ steps:
       - label: ":full_moon: Diffusion X2I(&A&T) · Doc Test"
         timeout_in_minutes: 60
         commands:
-          - export VLLM_TEST_CLEAN_GPU_MEMORY="1"
           - pytest -s -v tests/examples/*/test_text_to_image.py -m "full_model and example and H100" --run-level "full_model"
         agents:
           queue: "mithril-h100-pool"
@@ -512,6 +744,7 @@ steps:
         timeout_in_minutes: 180
         commands:
           - pytest -s -v tests/e2e/accuracy/test_qwen_image*.py --run-level full_model
+          - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2i_matches_diffusers' -m full_model --run-level full_model
         agents:
           queue: "mithril-h100-pool"
         plugins:
@@ -548,28 +781,218 @@ steps:
                       path: /mnt/hf-cache
                       type: DirectoryOrCreate
 
-      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test"
-        key: nightly-diffusion-x2iat-performance
+      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image"
+        key: nightly-diffusion-x2iat-performance-qwen-image
         timeout_in_minutes: 180
         commands:
           - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
           - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
           - export CACHE_DIT_VERSION=1.3.0
-          # [HACK]: run upload in the same command block as pytest.
-          # Because `exit` aborts the entire commands list.
           - |
             set +e
             pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
-            EXIT1=$$?
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json
-            EXIT2=$$?
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2509_vllm_omni.json
-            EXIT3=$$?
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json
-            EXIT4=$$?
+            EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
             buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
-            exit $$((EXIT1 | EXIT2 | EXIT3 | EXIT4))
+            exit $$EXIT
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          - kubernetes:
+              podSpec:
+                containers:
+                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                    resources:
+                      limits:
+                        nvidia.com/gpu: 4
+                    volumeMounts:
+                      - name: devshm
+                        mountPath: /dev/shm
+                      - name: hf-cache
+                        mountPath: /root/.cache/huggingface
+                    env:
+                      - name: HF_HOME
+                        value: /root/.cache/huggingface
+                      - name: HF_TOKEN
+                        valueFrom:
+                          secretKeyRef:
+                            name: hf-token-secret
+                            key: token
+                nodeSelector:
+                  node.kubernetes.io/instance-type: gpu-h100-sxm
+                volumes:
+                  - name: devshm
+                    emptyDir:
+                      medium: Memory
+                  - name: hf-cache
+                    hostPath:
+                      path: /mnt/hf-cache
+                      type: DirectoryOrCreate
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit"
+        key: nightly-diffusion-x2iat-performance-qwen-image-edit
+        timeout_in_minutes: 180
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
+          - export CACHE_DIT_VERSION=1.3.0
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          - kubernetes:
+              podSpec:
+                containers:
+                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                    resources:
+                      limits:
+                        nvidia.com/gpu: 4
+                    volumeMounts:
+                      - name: devshm
+                        mountPath: /dev/shm
+                      - name: hf-cache
+                        mountPath: /root/.cache/huggingface
+                    env:
+                      - name: HF_HOME
+                        value: /root/.cache/huggingface
+                      - name: HF_TOKEN
+                        valueFrom:
+                          secretKeyRef:
+                            name: hf-token-secret
+                            key: token
+                nodeSelector:
+                  node.kubernetes.io/instance-type: gpu-h100-sxm
+                volumes:
+                  - name: devshm
+                    emptyDir:
+                      medium: Memory
+                  - name: hf-cache
+                    hostPath:
+                      path: /mnt/hf-cache
+                      type: DirectoryOrCreate
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit-2509"
+        key: nightly-diffusion-x2iat-performance-qwen-image-edit-2509
+        timeout_in_minutes: 180
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
+          - export CACHE_DIT_VERSION=1.3.0
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2509_vllm_omni.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          - kubernetes:
+              podSpec:
+                containers:
+                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                    resources:
+                      limits:
+                        nvidia.com/gpu: 4
+                    volumeMounts:
+                      - name: devshm
+                        mountPath: /dev/shm
+                      - name: hf-cache
+                        mountPath: /root/.cache/huggingface
+                    env:
+                      - name: HF_HOME
+                        value: /root/.cache/huggingface
+                      - name: HF_TOKEN
+                        valueFrom:
+                          secretKeyRef:
+                            name: hf-token-secret
+                            key: token
+                nodeSelector:
+                  node.kubernetes.io/instance-type: gpu-h100-sxm
+                volumes:
+                  - name: devshm
+                    emptyDir:
+                      medium: Memory
+                  - name: hf-cache
+                    hostPath:
+                      path: /mnt/hf-cache
+                      type: DirectoryOrCreate
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Layered"
+        key: nightly-diffusion-x2iat-performance-qwen-image-layered
+        timeout_in_minutes: 120
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
+          - export CACHE_DIT_VERSION=1.3.0
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          - kubernetes:
+              podSpec:
+                containers:
+                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                    resources:
+                      limits:
+                        nvidia.com/gpu: 1
+                    volumeMounts:
+                      - name: devshm
+                        mountPath: /dev/shm
+                      - name: hf-cache
+                        mountPath: /root/.cache/huggingface
+                    env:
+                      - name: HF_HOME
+                        value: /root/.cache/huggingface
+                      - name: HF_TOKEN
+                        valueFrom:
+                          secretKeyRef:
+                            name: hf-token-secret
+                            key: token
+                nodeSelector:
+                  node.kubernetes.io/instance-type: gpu-h100-sxm
+                volumes:
+                  - name: devshm
+                    emptyDir:
+                      medium: Memory
+                  - name: hf-cache
+                    hostPath:
+                      path: /mnt/hf-cache
+                      type: DirectoryOrCreate
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · HunyuanImage3 · DiT Perf Test"
+        key: nightly-hunyuan-image3-performance
+        soft_fail: true
+        timeout_in_minutes: 120
+        if: build.env("RUN_HUNYUAN_IMAGE3_PERF") == "1"
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
+          - export CACHE_DIT_VERSION=1.3.0
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp4_fp8.json
+            EXIT1=$$?
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp2_fp8_sp2.json
+            EXIT2=$$?
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp2_fp8_cfgp2.json
+            EXIT3=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$((EXIT1 | EXIT2 | EXIT3))
         agents:
           queue: "mithril-h100-pool"
         plugins:
@@ -655,7 +1078,8 @@ steps:
         timeout_in_minutes: 180
         commands:
           - pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py -m full_model --run-level full_model
-          - pytest -s -v tests/e2e/accuracy/test_ltx2_3_video_similarity.py -m advanced_model --run-level advanced_model
+          - pytest -s -v tests/e2e/accuracy/test_ltx2_3_video_similarity.py -m full_model --run-level full_model
+          - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m full_model --run-level full_model
         agents:
           queue: "mithril-h100-pool"
         plugins:
@@ -737,6 +1161,65 @@ steps:
                       path: /mnt/hf-cache
                       type: DirectoryOrCreate
 
+  - group: ":card_index_dividers: Diffusion · Quantization Test"
+    depends_on: upload-nightly-pipeline
+    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "diffusion-x2iat-test"
+    steps:
+      - label: ":full_moon: Quantization Test with H100"
+        timeout_in_minutes: 90
+        commands:
+          - timeout 30m pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100' --run-level "full_model"
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          - kubernetes:
+              podSpec:
+                containers:
+                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                    resources:
+                      limits:
+                        nvidia.com/gpu: 1
+                    volumeMounts:
+                      - name: devshm
+                        mountPath: /dev/shm
+                      - name: hf-cache
+                        mountPath: /root/.cache/huggingface
+                    env:
+                      - name: HF_HOME
+                        value: /root/.cache/huggingface
+                      - name: HF_TOKEN
+                        valueFrom:
+                          secretKeyRef:
+                            name: hf-token-secret
+                            key: token
+                nodeSelector:
+                  node.kubernetes.io/instance-type: gpu-h100-sxm
+                volumes:
+                  - name: devshm
+                    emptyDir:
+                      medium: Memory
+                  - name: hf-cache
+                    hostPath:
+                      path: /mnt/hf-cache
+                      type: DirectoryOrCreate
+
+      - label: ":full_moon: Quantization Test with L4"
+        timeout_in_minutes: 60
+        commands:
+          - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4' --run-level "full_model"
+        agents:
+          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
+        plugins:
+          - docker#v5.2.0:
+              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+              always-pull: true
+              propagate-environment: true
+              environment:
+                - "HF_HOME=/fsx/hf_cache"
+                - "HF_TOKEN"
+              volumes:
+                - "/fsx/hf_cache:/fsx/hf_cache"
+
   - label: ":bar_chart: Testcase Statistics"
     key: nightly-testcase-statistics
     timeout_in_minutes: 120
@@ -754,7 +1237,7 @@ steps:
               - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                 resources:
                   limits:
-                    nvidia.com/gpu: 2
+                    nvidia.com/gpu: 1
                 volumeMounts:
                   - name: devshm
                     mountPath: /dev/shm
@@ -783,9 +1266,14 @@ steps:
   - label: ":email: Nightly Collection & Email"
     key: nightly-perf-distribution
     depends_on:
-      - nightly-omni-performance
+      - nightly-omni-performance-no-async-chunk
+      - nightly-omni-performance-async-chunk
+      - nightly-omni-performance-vllm-text
       - nightly-tts-performance
-      - nightly-diffusion-x2iat-performance
+      - nightly-diffusion-x2iat-performance-qwen-image
+      - nightly-diffusion-x2iat-performance-qwen-image-edit
+      - nightly-diffusion-x2iat-performance-qwen-image-edit-2509
+      - nightly-diffusion-x2iat-performance-qwen-image-layered
       - nightly-diffusion-x2v-performance
       - nightly-testcase-statistics
     if: build.env("NIGHTLY") == "1"
@@ -794,8 +1282,13 @@ steps:
       - export DEFAULT_INPUT_DIR=tests/dfx/perf/results
       - export DEFAULT_OUTPUT_DIR=tests/dfx/perf/results
       - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance
-      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance
-      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-no-async-chunk
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-async-chunk
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-vllm-text
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit-2509
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-layered
       - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance
       - buildkite-agent artifact download "tests/dfx/perf/results/*.html" . --step nightly-testcase-statistics
       - python tools/nightly/generate_nightly_perf_excel.py

.buildkite/test-npu-nightly.yaml

@@ -0,0 +1,24 @@
+steps:
+  - group: ":card_index_dividers: Diffusion X2V Model Test"
+    key: nightly-diffusion-x2v-group
+    depends_on: upload-nightly-pipeline
+    if: >-
+      build.env("NIGHTLY") == "1" ||
+      build.pull_request.labels includes "nightly-test" ||
+      build.pull_request.labels includes "diffusion-x2v-test"
+    steps:
+      - label: ":full_moon: Diffusion X2V · Function Test"
+        agents:
+          queue: "ascend-a2b3"
+          resource_class: "npu-4"
+        image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}"
+        plugins:
+          - kubernetes:
+              podSpecPatch:
+                imagePullSecrets:
+                  - name: swr-secret
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          HF_TOKEN: "${HF_TOKEN}"
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k npu_i2v --run-level "full_model"

.claude/skills/add-diffusion-model/SKILL.md

@@ -515,15 +515,15 @@ After adding parallelism support, update:
 
 ### Step 11: Add CPU Offload Support
 
-Implement `SupportsModuleOffload` on your pipeline class to enable
+Implement `SupportsComponentDiscovery` on your pipeline class to enable
 `--enable-cpu-offload` and `--enable-layerwise-offload`. The protocol
 declares which submodules the offloader should manage:
 
 ```python
 from typing import ClassVar
-from vllm_omni.diffusion.models.interface import SupportsModuleOffload
+from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery
 
-class YourPipeline(nn.Module, SupportsModuleOffload):
+class YourPipeline(nn.Module, SupportsComponentDiscovery):
     _dit_modules: ClassVar[list[str]] = ["transformer"]
     _encoder_modules: ClassVar[list[str]] = ["text_encoder"]
     _vae_modules: ClassVar[list[str]] = ["vae"]
@@ -540,7 +540,7 @@ class YourPipeline(nn.Module, SupportsModuleOffload):
 All attribute names support dotted paths for nested submodules
 (e.g. `"pipe.transformer"`, `"bagel.time_embedder"`).
 
-Pipelines without `SupportsModuleOffload` fall back to scanning
+Pipelines without `SupportsComponentDiscovery` fall back to scanning
 well-known attribute names (`transformer`, `text_encoder`, `vae`,
 etc.), which fails for non-standard names.
 

.claude/skills/add-diffusion-model/references/parallelism-patterns.md

@@ -92,6 +92,7 @@ class TPSelfAttention(nn.Module):
             softmax_scale=1.0 / (self.head_dim ** 0.5),
             causal=False,
             num_kv_heads=self.to_qkv.num_kv_heads,  # Local KV heads per GPU
+            role="self",
         )
 
     def forward(self, x):

.claude/skills/add-diffusion-model/references/transformer-adaptation.md

@@ -61,6 +61,7 @@ class SelfAttentionBlock(nn.Module):
             softmax_scale=1.0 / (self.head_dim ** 0.5),
             causal=False,
             num_kv_heads=num_heads,
+            role="self",
         )
 
     def forward(self, x, attn_mask=None):
@@ -75,6 +76,24 @@ class SelfAttentionBlock(nn.Module):
         return self.to_out(out)
 ```
 
+**Declaring the attention role.** Always pass a `role` string to `Attention(...)`. It is the lookup key users target with `--diffusion-attention-config.per_role.<role>.backend` to swap kernels per site without touching model code.
+
+| Convention | When to use |
+|---|---|
+| `role="self"` | Q/K/V come from the same hidden state |
+| `role="cross"` | K/V come from a separate `encoder_hidden_states`; pair with `skip_sequence_parallel=True` if K/V is replicated across SP ranks |
+
+For multi-modal sites that don't fit `self` / `cross`, use a dot-namespaced role and provide `role_category` so user config can fall back to the generic category:
+
+```python
+self.audio_to_video_attn = Attention(
+    num_heads=num_heads, head_size=self.head_dim,
+    softmax_scale=1.0 / (self.head_dim ** 0.5), causal=False,
+    role="mymodel.audio_to_video",
+    role_category="cross",
+)
+```
+
 #### Fused QKV with TP (Advanced)
 
 For tensor parallelism, use vLLM's parallel linear layers:
@@ -104,6 +123,7 @@ class TPSelfAttention(nn.Module):
             softmax_scale=1.0 / (self.head_dim ** 0.5),
             causal=False,
             num_kv_heads=num_heads,
+            role="self",
         )
 ```
 

.claude/skills/add-diffusion-model/references/troubleshooting.md

@@ -68,7 +68,7 @@ out = out.reshape(batch, seq_len, -1)
 ### Different Output vs Diffusers Reference
 
 **Common causes**:
-1. **Attention backend difference**: FlashAttention vs SDPA may produce slightly different results. Set `DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA` to match diffusers
+1. **Attention backend difference**: FlashAttention vs SDPA may produce slightly different results. Force SDPA via `--diffusion-attention-backend TORCH_SDPA` (or `DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA`) to match diffusers; for finer-grained per-role overrides see `--diffusion-attention-config.per_role.<role>.backend`
 2. **Float precision**: vLLM-Omni may use bfloat16 where diffusers uses float32 for some operations
 3. **Missing normalization**: Check all LayerNorm/RMSNorm are preserved
 4. **Scheduler rounding**: Some schedulers have numerical sensitivity

.claude/skills/add-tts-model/SKILL.md

@@ -211,8 +211,8 @@ See `plan/voxcpm2_native_ar_design.md`.
 
 - Model files in `vllm_omni/model_executor/models/<model_name>/`
 - Stage config YAML
-- Working `end2end.py` with correct audio output
-- README.md in the example directory
+- Working `end2end.py` at `examples/offline_inference/text_to_speech/<model>/end2end.py`
+- New section in `examples/offline_inference/text_to_speech/README.md` (table row + per-model section). Do **not** create a top-level `examples/offline_inference/<model>/` dir or a per-model `README.md` inside `text_to_speech/<model>/` — the hub README is the documented surface and the mkdocs `generate_examples` hook only descends one level into `examples/<category>/`.
 
 ## Phase 3: Online Serving
 
@@ -308,11 +308,11 @@ def build_voice_clone_prompt(ref_audio_path: str, text: str, codec) -> list:
 ### Deliverables
 
 - Updated `serving_speech.py` with all 5 integration points (single commit)
-- Client scripts and server launcher
-- Gradio demo with streaming and voice cloning UI
+- Client scripts and server launcher under `examples/online_serving/text_to_speech/<model>/`
+- Gradio demo with streaming and voice cloning UI in the same dir
 - E2E online serving test (`tests/e2e/online_serving/test_<model>.py`)
 - Buildkite CI entry in `.buildkite/test-merge.yml`
-- Documentation (offline + online serving docs)
+- New section in `examples/online_serving/text_to_speech/README.md` (table row + per-model section). Do **not** create a top-level `examples/online_serving/<model>/` dir or a per-model `README.md` inside `text_to_speech/<model>/`.
 
 ### E2E test pitfalls to avoid
 

.github/CODEOWNERS

@@ -3,15 +3,64 @@
 # More info: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners
 
 # Default owners (fallback reviewers for all files)
-# later, we can add more reviewers here
-* @hsliuustc0106
+* @Gaohan123 @hsliuustc0106 @ywang96
 
-# Assign reviewers by directory
-/.github/ @owner1
-/tests/ @owner1
-/docs/ @owner1
+# Repository operations
+/.github/ @hsliuustc0106
 
-# Example: reviewers for specific directories
-# /examples/offline_inference/ @offline-inference-team
-# /vllm_omni/core/ @core-team
-# /vllm_omni/model_executor/ @model-team
+# Documentation and Recipes
+/docs/ @hsliuustc0106 @Gaohan123 @david6666666
+/recipes @hsliuustc0106 @Gaohan123 @david6666666
+
+# Skills
+/.claude/ @hsliuustc0106
+
+# Tests
+/tests/ @yenuo26
+
+# CI
+/.buildkite/ @yenuo26 @congw729
+
+# Engine and model_executor
+/vllm_omni/core/ @tzhouam @yinpeiqi @fake0fan
+/vllm_omni/engine/ @tzhouam @yinpeiqi @fake0fan
+/vllm_omni/entrypoints/ @tzhouam @yinpeiqi @fake0fan
+/vllm_omni/model_executor/ @tzhouam @gcanlin
+/vllm_omni/worker/ @tzhouam @gcanlin
+
+# Diffusion
+/vllm_omni/diffusion/ @Isotr0py @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa @david6666666
+/vllm_omni/lora/ @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa @yuanheng-zhao
+
+# Diffusion CPU offloading
+/vllm_omni/diffusion/offloader/ @Isotr0py @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa @yuanheng-zhao
+
+# Omni and TTS support
+/vllm_omni/transformers_utils/ @tzhouam @yuanheng-zhao
+/vllm_omni/model_executor/models/ @linyueqian @ZeldaHuang @princepride @yuanheng-zhao @amy-why-3459 @Bounty-hunter
+
+# Benchmarks
+/benchmarks/accuracy/ @david6666666
+/benchmarks/build_dataset/ @linyueqian @yuanheng-zhao
+/benchmarks/diffusion/ @Isotr0py @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa
+/benchmarks/distributed/ @princepride
+/benchmarks/glm_image/ @princepride @JaredforReal
+/benchmarks/tts/ @linyueqian @ZeldaHuang @princepride
+
+# Usability and Observability
+/vllm_omni/deploy/ @tzhouam @lishunyang12
+/vllm_omni/metrics/ @lishunyang12
+/vllm_omni/profiler/ @gcanlin
+
+# ComfyUI
+/apps/ @fhfuih
+
+# Large Scale Serving
+/vllm_omni/distributed/ @princepride @yuanheng-zhao @xuechendi @natureofnature @fake0fan
+
+# Quantization and configuration
+/vllm_omni/config/ @lishunyang12
+/vllm_omni/quantization/ @david6666666 @Isotr0py @lishunyang12
+
+# Hardware plugins and accelerators
+/vllm_omni/platforms/ @gcanlin @tjtanaa @xuechendi

.github/mergify.yml

@@ -0,0 +1,223 @@
+pull_request_rules:
+- name: label-documentation
+  description: Automatically apply documentation label
+  conditions:
+    - label != stale
+    - or:
+      - files~=^[^/]+\.md$
+      - files~=^docs/
+      - files~=^examples/
+  actions:
+    label:
+      add:
+        - documentation
+    comment:
+      message: "Documentation preview: https://vllm--{{number}}.org.readthedocs.build/en/{{number}}/"
+
+- name: comment-pre-commit-failure
+  description: Comment on PR when pre-commit check fails
+  conditions:
+    - status-failure=pre-commit
+    - -closed
+    - -draft
+  actions:
+    comment:
+      message: |
+        Hi @{{author}}, the pre-commit checks have failed. Please run:
+
+        ```bash
+        uv pip install pre-commit
+        pre-commit install
+        pre-commit run --all-files
+        ```
+
+        Then, commit the changes and push to your branch.
+
+        For future commits, `pre-commit` will run automatically on changed files before each commit.
+
+        > [!TIP]
+        > <details>
+        > <summary>Is <code>mypy</code> or <code>markdownlint</code> failing?</summary>
+        > <br/>
+        > <code>mypy</code> and <code>markdownlint</code> are run differently in CI. If the failure is related to either of these checks, please use the following commands to run them locally:
+        >
+        > ```bash
+        > # For mypy (substitute "3.10" with the failing version if needed)
+        > pre-commit run --hook-stage manual mypy-3.10
+        > # For markdownlint
+        > pre-commit run --hook-stage manual markdownlint
+        > ```
+        > </details>
+
+- name: comment-dco-failure
+  description: Comment on PR when DCO check fails
+  conditions:
+    - status-failure=dco
+    - -closed
+    - -draft
+  actions:
+    comment:
+      message: |
+        Hi @{{author}}, the DCO check has failed. Please click on DCO in the Checks section for instructions on how to resolve this.
+
+- name: label-ci-build
+  description: Automatically apply ci/build label
+  conditions:
+    - label != stale
+    - or:
+      - files~=^\.github/
+      - files~=\.buildkite/
+      - files~=^cmake/
+      - files=CMakeLists.txt
+      - files~=^docker/Dockerfile
+      - files~=^requirements.*\.txt
+      - files=setup.py
+  actions:
+    label:
+      add:
+        - ci/build
+
+- name: label-frontend
+  description: Automatically apply frontend label
+  conditions:
+    - label != stale
+    - files~=^vllm_omni/entrypoints/
+  actions:
+    label:
+      add:
+        - frontend
+
+- name: label-diffusion
+  description: Automatically apply diffusion label
+  conditions:
+    - label != stale
+    - files~=^vllm_omni/diffusion/
+  actions:
+    label:
+      add:
+        - diffusion
+
+- name: label-new-model
+  description: Automatically apply new-model label
+  conditions:
+    - label != stale
+    - or:
+        - files=vllm_omni/model_executor/models/registry.py
+        - files=vllm_omni/diffusion/registry.py
+  actions:
+    label:
+      add:
+        - new-model
+
+- name: label-npu
+  description: Automatically apply npu label
+  conditions:
+    - label != stale
+    - or:
+      - files~=^vllm_omni/platforms/npu/
+      - title~=(?i)NPU
+      - title~=(?i)Ascend
+  actions:
+    label:
+      add:
+        - npu
+    assign:
+      users:
+        - "gcanlin"
+
+- name: label-rocm
+  description: Automatically apply rocm label
+  conditions:
+    - label != stale
+    - or:
+      - files~=^vllm_omni/platforms/rocm/
+      - title~=(?i)AMD
+      - title~=(?i)ROCm
+  actions:
+    label:
+      add:
+        - rocm
+    assign:
+      users:
+        - "tjtanaa"
+
+- name: label-xpu
+  description: Automatically apply xpu label
+  conditions:
+    - label != stale
+    - or:
+      - files~=^vllm_omni/platforms/xpu/
+      - title~=(?i)Intel
+      - title~=(?i)XPU
+  actions:
+    label:
+      add:
+        - xpu
+    assign:
+      users:
+        - "yma11"
+
+- name: label-modelrunner
+  description: Automatically apply modelrunner label
+  conditions:
+    - label != stale
+    - or:
+      - files~=^vllm_omni/worker/
+      - title~=(?i)ModelRunner
+      - title~=(?i)worker
+  actions:
+    label:
+      add:
+        - ModelRunner
+    assign:
+      users:
+        - "gcanlin" # To check whether need to sync to NPU
+
+- name: auto-rebase if approved, ready, and 30 commits behind main
+  conditions:
+    - base = main
+    - label=ready
+    - "#approved-reviews-by >= 1"
+    - "#commits-behind >= 30"
+    - -closed
+    - -draft
+    - -conflict
+  actions:
+    rebase: {}
+
+- name: ping author on conflicts and add 'needs-rebase' label
+  conditions:
+    - label != stale
+    - conflict
+    - -closed
+  actions:
+    label:
+      add:
+        - needs-rebase
+    comment:
+      message: |
+       This pull request has merge conflicts that must be resolved before it can be
+       merged. Please rebase the PR, @{{author}}.
+
+       https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/syncing-a-fork
+
+- name: remove 'needs-rebase' label when conflict is resolved
+  conditions:
+    - -conflict
+    - -closed
+  actions:
+    label:
+      remove:
+        - needs-rebase
+
+- name: label-bug
+  description: Automatically apply bug label
+  conditions:
+    - label != stale
+    - or:
+      - title~=(?i)\bbug\b
+      - title~=(?i)\bbugfix\b
+  actions:
+    label:
+      add:
+        - bug

.gitignore

@@ -239,6 +239,7 @@ data/
 datasets/
 *.csv
 *.json
+!tests/dfx/perf/tests/*.json
 !apps/ComfyUI-vLLM-Omni/example_workflows/*.json
 *.jsonl
 *.parquet

README.md

@@ -9,19 +9,19 @@ Easy, fast, and cheap omni-modality model serving for everyone
 </h3>
 
 <p align="center">
-| <a href="https://vllm-omni.readthedocs.io/en/latest/"><b>Documentation</b></a> | <a href="https://discuss.vllm.ai"><b>User Forum</b></a> | <a href="https://slack.vllm.ai"><b>Developer Slack</b></a> | <a href="docs/assets/WeChat.jpg"><b>WeChat</b></a> | <a href="https://arxiv.org/abs/2602.02204"><b>Paper</b></a> | <a href="https://docs.google.com/presentation/d/1XJWgv79lORl8rbaVvp2d5Sqs6ZEBgAgj/edit?slide=id.p1#slide=id.p1"><b>Slides</b></a> |
+| <a href="https://vllm-omni.readthedocs.io/en/latest/"><b>Documentation</b></a> | <a href="https://discuss.vllm.ai"><b>User Forum</b></a> | <a href="https://slack.vllm.ai"><b>Developer Slack</b></a> | <a href="docs/assets/WeChat.jpg"><b>WeChat</b></a> | <a href="https://arxiv.org/abs/2602.02204"><b>Paper</b></a> | <a href="https://docs.google.com/presentation/d/111-L8zF7A1j_YI_cR8JsblofdScdRr2f/edit?usp=sharing&ouid=110473603432222024453&rtpof=true&sd=true"><b>Slides</b></a> |
 </p>
 
 
 ---
 
 *Latest News* 🔥
+- [2026/05] We released [0.20.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.20.0) - refreshes the serving/runtime stack for large-scale omni workloads, and improves diffusion model performance, quantization, and hardware readiness across CUDA, ROCm, MUSA, NPU, and XPU backends.
 - [2026/03] We released [0.18.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.18.0) - strengthens the core runtime through a large entrypoint refactor and scheduler/runtime cleanups, expands unified quantization and diffusion execution, broadens multimodal model coverage, and improves production readiness across audio, omni, image, video, RL, and multi-platform deployments.
 - [2026/03] Check out our first public [project deepdive](https://youtu.be/sgwNfsNnR9I) at the vLLM Hong Kong Meetup!
 - [2026/03] **[vllm-omni-skills](https://github.com/hsliuustc0106/vllm-omni-skills)** is a community-driven collection of AI assistant skills that help developers work with vLLM-Omni more effectively. These skills can be used with popular agentic AI coding assistants like **Cursor IDE**, **Claude**, **Codex**, and more.
 - [2026/02] We released [0.16.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.16.0) - A major alignment + capability release that rebases onto **upstream vLLM v0.16.0** and significantly expands performance, distributed execution, and production readiness across **Qwen3-Omni / Qwen3-TTS**, **Bagel**, **MiMo-Audio**, **GLM-Image** and the **Diffusion (DiT) image/video stack**—while also improving platform coverage (CUDA / ROCm / NPU / XPU), CI quality, and documentation.
 - [2026/02] We released [0.14.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.14.0) - This is the first **stable release** of vLLM-Omni that expands Omni’s diffusion / image-video generation and audio / TTS stack, improves distributed execution and memory efficiency, and broadens platform/backend coverage (GPU/ROCm/NPU/XPU). It also brings meaningful upgrades to serving APIs, profiling & benchmarking, and overall stability. Please check our latest [paper](https://arxiv.org/abs/2602.02204) for architecture design and performance results.
-- [2026/01] We released [0.12.0rc1](https://github.com/vllm-project/vllm-omni/releases/tag/v0.12.0rc1) - a major RC milestone focused on maturing the diffusion stack, strengthening OpenAI-compatible serving, expanding omni-model coverage, and improving stability across platforms (GPU/NPU/ROCm).
 - [2025/11] vLLM community officially released [vllm-project/vllm-omni](https://github.com/vllm-project/vllm-omni) in order to support omni-modality models serving.
 
 ---

benchmarks/diffusion/README.md

@@ -130,9 +130,21 @@ Warmup flags:
 
 - `--warmup-requests`: Number of warmup requests.
 - `--warmup-num-inference-steps`: Steps used during warmup.
+- `--warmup-concurrency`: Maximum concurrent warmup requests. Use this to warm
+  the same batch shape as the measured run instead of warming only batch=`1`.
 - For `--task t2v`: warmup requests are forced to use `num_frames=1` to make warmup faster and less noisy.
 
 Traffic / concurrency flags:
 
 - `--request-rate`: Target request rate (requests/second). If set to `inf`, the script sends all requests immediately.
 - `--max-concurrency`: Max number of in-flight requests (default: `1`). This can hard-cap the achieved QPS: if it is too small, requests will queue behind the semaphore, and both achieved throughput and observed SLO attainment can be skewed.
+
+### Batched warmup note
+
+For batched serving runs, warm the same in-flight shape you plan to measure.
+For example, a run with `--max-concurrency 8` should usually also use
+`--warmup-requests 8 --warmup-concurrency 8`; otherwise the first measured
+batch may still pay compile or CUDA-graph capture cost.
+
+For a Qwen-Image continuous-batching replay example, see
+[`performance_dashboard/qwen_image_serving_performance.md`](./performance_dashboard/qwen_image_serving_performance.md).

benchmarks/diffusion/diffusion_benchmark_serving.py

@@ -751,6 +751,52 @@ async def iter_requests(
         yield req
 
 
+def _make_warmup_request(
+    requests_list: list[RequestFuncInput],
+    index: int,
+    args,
+) -> RequestFuncInput:
+    warm_req = requests_list[index % len(requests_list)]
+    if args.warmup_num_inference_steps is not None:
+        warm_req = replace(
+            warm_req,
+            num_inference_steps=args.warmup_num_inference_steps,
+        )
+    if args.task == "t2v":
+        warm_req = replace(warm_req, num_frames=1)
+    return warm_req
+
+
+async def _run_warmups(
+    requests_list: list[RequestFuncInput],
+    args,
+    session: aiohttp.ClientSession,
+    request_func,
+) -> list[tuple[RequestFuncInput, RequestFuncOutput]]:
+    if not args.warmup_requests or not requests_list:
+        return []
+
+    warmup_requests = [_make_warmup_request(requests_list, i, args) for i in range(args.warmup_requests)]
+    warmup_concurrency = min(int(args.warmup_concurrency), len(warmup_requests))
+    warmup_semaphore = asyncio.Semaphore(warmup_concurrency)
+
+    print(
+        f"Running {len(warmup_requests)} warmup request(s) "
+        f"with num_inference_steps={args.warmup_num_inference_steps} "
+        f"and warmup_concurrency={warmup_concurrency}..."
+    )
+
+    async def limited_warmup_request_func(
+        req: RequestFuncInput,
+    ) -> RequestFuncOutput:
+        async with warmup_semaphore:
+            return await request_func(req, session, None)
+
+    warmup_tasks = [asyncio.create_task(limited_warmup_request_func(req)) for req in warmup_requests]
+    warmup_outputs = await asyncio.gather(*warmup_tasks)
+    return list(zip(warmup_requests, warmup_outputs))
+
+
 def calculate_metrics(
     outputs: list[RequestFuncOutput],
     total_duration: float,
@@ -900,23 +946,12 @@ async def benchmark(args):
     pbar = tqdm(total=len(requests_list), disable=args.disable_tqdm)
 
     async with aiohttp.ClientSession() as session:
-        warmup_pairs: list[tuple[RequestFuncInput, RequestFuncOutput]] = []
-        if args.warmup_requests and requests_list:
-            print(
-                f"Running {args.warmup_requests} warmup request(s) \
-                with num_inference_steps={args.warmup_num_inference_steps}..."
-            )
-            for i in range(args.warmup_requests):
-                warm_req = requests_list[i % len(requests_list)]
-                if args.warmup_num_inference_steps is not None:
-                    warm_req = replace(
-                        warm_req,
-                        num_inference_steps=args.warmup_num_inference_steps,
-                    )
-                if args.task == "t2v":
-                    warm_req = replace(warm_req, num_frames=1)
-                warm_out = await limited_request_func(warm_req, session, None)
-                warmup_pairs.append((warm_req, warm_out))
+        warmup_pairs = await _run_warmups(
+            requests_list=requests_list,
+            args=args,
+            session=session,
+            request_func=request_func,
+        )
 
         if args.slo:
             # Prefer trace-provided per-request slo_ms. Only populate when missing.
@@ -1072,6 +1107,13 @@ if __name__ == "__main__":
         default=1,
         help="num_inference_steps used for warmup requests.",
     )
+    parser.add_argument(
+        "--warmup-concurrency",
+        type=int,
+        default=1,
+        help="Maximum number of warmup requests to run concurrently. "
+        "Set this to match the real batch shape when warming up torch.compile or CUDA graphs.",
+    )
     parser.add_argument("--width", type=int, default=None, help="Image/Video width.")
     parser.add_argument("--height", type=int, default=None, help="Image/Video height.")
     parser.add_argument("--num-frames", type=int, default=None, help="Number of frames (for video).")
@@ -1135,5 +1177,4 @@ if __name__ == "__main__":
     )
 
     args = parser.parse_args()
-
     asyncio.run(benchmark(args))

benchmarks/diffusion/performance_dashboard/qwen_image_serving_performance.md

@@ -33,6 +33,12 @@ vllm serve Qwen/Qwen-Image --omni \
     --port 8091
 ```
 
+To replay step-wise continuous batching, compare
+`vllm serve Qwen/Qwen-Image --omni --port 8089 --step-execution --max-num-seqs 1`
+against the same command with `--max-num-seqs 8`. `--step-execution` is the
+feature gate, and increasing `--max-num-seqs` above `1` lets the scheduler
+keep more compatible requests active at once.
+
 ## 3.2 Key Parameters
 
 | Parameter             | Description              |
@@ -70,6 +76,7 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \
 | `--task`               | Task type (e.g., `t2i`)           |
 | `--num-prompts`        | Total number of requests          |
 | `--max-concurrency`    | Client-side concurrency           |
+| `--warmup-concurrency` | Warmup concurrency for batch-shape prewarming |
 | `--random-request-config`| JSON string defining random request |
 
 ---
@@ -133,6 +140,12 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \
     ]'
 ```
 
+For a continuous-batching A/B replay, run the same benchmark command twice
+against the two serving configs above and keep traffic plus warmup settings
+identical across both runs. If the measured run uses `--max-concurrency 8`,
+warm with `--warmup-requests 8 --warmup-concurrency 8` so the first measured
+batch does not include compile or CUDA-graph capture overhead.
+
 ---
 
 # 6. Performance Metrics
@@ -163,6 +176,7 @@ To ensure consistent and comparable benchmark results:
 * Record parallel configuration
 * Record benchmark parameters (resolution, concurrency, number of prompts)
 * Ensure no background workload on GPUs during testing
+* When comparing continuous batching, keep warmup settings the same across runs
 
 ---
 

benchmarks/diffusion/quantization_quality.py

@@ -151,7 +151,7 @@ def _generate_image(omni, args, prompt, seed):
     from vllm_omni.platforms import current_omni_platform
 
     generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(seed)
-    torch.cuda.reset_peak_memory_stats()
+    torch.accelerator.reset_peak_memory_stats()
     start = time.perf_counter()
     outputs = omni.generate(
         {"prompt": prompt},
@@ -163,7 +163,7 @@ def _generate_image(omni, args, prompt, seed):
         ),
     )
     elapsed = time.perf_counter() - start
-    peak_mem = torch.cuda.max_memory_allocated() / (1024**3)
+    peak_mem = torch.accelerator.max_memory_allocated() / (1024**3)
 
     first = outputs[0]
     req_out = first.request_output[0] if hasattr(first, "request_output") else first
@@ -178,7 +178,7 @@ def _generate_video(omni, args, prompt, seed):
     from vllm_omni.platforms import current_omni_platform
 
     generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(seed)
-    torch.cuda.reset_peak_memory_stats()
+    torch.accelerator.reset_peak_memory_stats()
     start = time.perf_counter()
     outputs = omni.generate(
         {"prompt": prompt, "negative_prompt": ""},
@@ -192,7 +192,7 @@ def _generate_video(omni, args, prompt, seed):
         ),
     )
     elapsed = time.perf_counter() - start
-    peak_mem = torch.cuda.max_memory_allocated() / (1024**3)
+    peak_mem = torch.accelerator.max_memory_allocated() / (1024**3)
 
     first = outputs[0]
     if hasattr(first, "request_output") and isinstance(first.request_output, list):
@@ -228,8 +228,8 @@ def _unload_omni(omni):
     del omni
     gc.collect()
     if torch.cuda.is_available():
-        torch.cuda.empty_cache()
-        torch.cuda.synchronize()
+        torch.accelerator.empty_cache()
+        torch.accelerator.synchronize()
 
 
 def run_benchmark(args):

benchmarks/distributed/omni_connectors/cross_node_mooncake_transfer_engine.py

@@ -614,8 +614,8 @@ Examples:
         if not torch.cuda.is_available():
             print("[ERROR] CUDA is not available but GPU mode was requested.")
             sys.exit(1)
-        if args.gpu_id >= torch.cuda.device_count():
-            print(f"[ERROR] GPU {args.gpu_id} not available. Found {torch.cuda.device_count()} GPUs.")
+        if args.gpu_id >= torch.accelerator.device_count():
+            print(f"[ERROR] GPU {args.gpu_id} not available. Found {torch.accelerator.device_count()} GPUs.")
             sys.exit(1)
         print(f"[INFO] Using GPU {args.gpu_id}: {torch.cuda.get_device_name(args.gpu_id)}")
 

benchmarks/fish-speech/bench_speaker_cache.py

@@ -3,11 +3,11 @@
 Measures TTFP improvement from DAC-code caching when using uploaded voices.
 
 Setup:
-  1. Start vllm-omni with Fish Speech S2 Pro (use our feat branch)
+  1. Start vllm-omni with Fish Speech S2 Pro
   2. Provide a reference audio file for voice cloning
 
 Usage:
-    python bench_voice_cache.py \
+    python bench_speaker_cache.py \
         --ref-audio /path/to/reference.wav \
         --ref-text "Transcript of the reference audio." \
         --num-prompts 20 \

benchmarks/tts/bench_voxcpm_offline.py

@@ -63,7 +63,7 @@ def _find_repo_root(start: Path) -> Path:
 
 REPO_ROOT = _find_repo_root(Path(__file__).resolve())
 DEFAULT_STAGE_ASYNC = REPO_ROOT / "vllm_omni" / "model_executor" / "stage_configs" / "voxcpm_async_chunk.yaml"
-DEFAULT_STAGE_SYNC = REPO_ROOT / "vllm_omni" / "model_executor" / "stage_configs" / "voxcpm.yaml"
+DEFAULT_STAGE_SYNC = REPO_ROOT / "vllm_omni" / "deploy" / "voxcpm.yaml"
 
 logger = logging.getLogger(__name__)
 

docker/Dockerfile.ci

@@ -1,5 +1,5 @@
 ARG VLLM_BASE_IMAGE=vllm/vllm-openai
-ARG VLLM_BASE_TAG=v0.20.0-cu130
+ARG VLLM_BASE_TAG=v0.20.0
 FROM ${VLLM_BASE_IMAGE}:${VLLM_BASE_TAG}
 ARG APP_DIR=/workspace/vllm-omni
 WORKDIR ${APP_DIR}

docker/Dockerfile.cuda

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=vllm/vllm-openai:v0.19.0
+ARG BASE_IMAGE=vllm/vllm-openai:v0.20.0
 FROM ${BASE_IMAGE}
 
 ARG COMMON_WORKDIR=/app

docker/Dockerfile.npu.ci

@@ -0,0 +1,42 @@
+ARG VLLM_ASCEND_IMAGE=quay.io/ascend/vllm-ascend
+ARG VLLM_ASCEND_TAG=nightly-main
+FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
+
+ARG APP_DIR=/vllm-workspace/vllm-omni
+WORKDIR ${APP_DIR}
+COPY . .
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        ffmpeg \
+        libgl1 \
+        libglib2.0-0 \
+        libgomp1 \
+        libsm6 \
+        libxext6 \
+        libxrender1 \
+        libxcb1 && \
+    rm -rf /var/cache/apt/* && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
+    source /usr/local/Ascend/nnal/atb/set_env.sh && \
+    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
+    python3 -m pip install -U pip && \
+    python3 -m pip install -v \
+      "pytest>=7.0.0" \
+      "pytest-asyncio>=0.21.0" \
+      "pytest-cov>=4.0.0" \
+      "pytest-mock>=3.10.0" \
+      "openpyxl>=3.0.0" \
+      --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
+      --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
+      --default-timeout=60 && \
+    python3 -m pip install -v -e /vllm-workspace/vllm-omni/ --no-build-isolation \
+      --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
+      --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
+      --default-timeout=60
+
+ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
+
+ENTRYPOINT []

docker/Dockerfile.npu.ci.a3

@@ -0,0 +1,42 @@
+ARG VLLM_ASCEND_IMAGE=quay.io/ascend/vllm-ascend
+ARG VLLM_ASCEND_TAG=nightly-main-a3
+FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
+
+ARG APP_DIR=/vllm-workspace/vllm-omni
+WORKDIR ${APP_DIR}
+COPY . .
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        ffmpeg \
+        libgl1 \
+        libglib2.0-0 \
+        libgomp1 \
+        libsm6 \
+        libxext6 \
+        libxrender1 \
+        libxcb1 && \
+    rm -rf /var/cache/apt/* && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
+    source /usr/local/Ascend/nnal/atb/set_env.sh && \
+    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
+    python3 -m pip install -U pip && \
+    python3 -m pip install -v \
+      "pytest>=7.0.0" \
+      "pytest-asyncio>=0.21.0" \
+      "pytest-cov>=4.0.0" \
+      "pytest-mock>=3.10.0" \
+      "openpyxl>=3.0.0" \
+      --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
+      --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
+      --default-timeout=60 && \
+    python3 -m pip install -v -e /vllm-workspace/vllm-omni/ --no-build-isolation \
+      --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
+      --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
+      --default-timeout=60
+
+ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
+
+ENTRYPOINT []

docker/Dockerfile.rocm

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.19.0
+ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.20.0
 FROM ${BASE_IMAGE} AS base
 
 # Declare a variable to know if we want to use the nightly build or the stable build.

docker/Dockerfile.xpu

@@ -14,8 +14,9 @@ RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRO
 RUN apt clean && apt-get update -y && \
     apt-get install -y --no-install-recommends --fix-missing \
     curl \
-    espeak-ng \
+    ffmpeg \
     git \
+    libsndfile1 \
     libsm6 \
     libxext6 \
     libgl1 \
@@ -28,8 +29,20 @@ RUN apt clean && apt-get update -y && \
     python3.12-dev \
     python3-pip
 
-RUN apt update && apt upgrade -y && \
-    apt install -y intel-oneapi-compiler-dpcpp-cpp-2025.3
+# Add oneAPI repo, pin oneAPI to 2025.3, then install pinned packages in one layer.
+RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \
+    echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list && \
+    printf '%s\n' \
+    'Package: intel-oneapi-* intel-deep-learning-essentials* intel-pti*' \
+    'Pin: version 2025.3*' \
+    'Pin-Priority: 1001' \
+    > /etc/apt/preferences.d/oneapi-2025.3.pref && \
+    apt-get update -y && \
+    apt-get install -y --no-install-recommends \
+    intel-oneapi-compiler-dpcpp-cpp-2025.3 \
+    intel-oneapi-mkl-devel-2025.3 \
+    intel-oneapi-dnnl-devel-2025.3 && \
+    rm -rf /var/lib/apt/lists/*
 
 # Install UMD with fixed version
 RUN mkdir neo && \
@@ -52,9 +65,9 @@ RUN curl -LsSf https://astral.sh/uv/install.sh | sh
 RUN uv venv --python ${PYTHON_VERSION} --seed ${VIRTUAL_ENV}
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
-# This oneccl contains the BMG support which is not the case for default version of oneapi 2025.2.
-ARG ONECCL_INSTALLER="intel-oneccl-2021.15.7.8_offline.sh"
-RUN wget "https://github.com/uxlfoundation/oneCCL/releases/download/2021.15.7/${ONECCL_INSTALLER}" && \
+# This oneccl contains the BMG support which is not the case for default version of oneapi 2025.3.
+ARG ONECCL_INSTALLER="intel-oneccl-2021.15.9.14_offline.sh"
+RUN wget "https://github.com/uxlfoundation/oneCCL/releases/download/2021.15.9/${ONECCL_INSTALLER}" && \
     bash "${ONECCL_INSTALLER}" -a --silent --eula accept && \
     rm "${ONECCL_INSTALLER}" && \
     echo "source /opt/intel/oneapi/setvars.sh --force" >> /root/.bashrc && \
@@ -66,6 +79,7 @@ SHELL ["bash", "-c"]
 CMD ["bash", "-c", "source /root/.bashrc && exec bash"]
 
 WORKDIR /workspace/
+
 ENV UV_HTTP_TIMEOUT=500
 
 # Configure package index for XPU
@@ -74,24 +88,18 @@ ENV UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
 ENV UV_INDEX_STRATEGY="unsafe-best-match"
 ENV UV_LINK_MODE="copy"
 
-ARG VLLM_VERSION=v0.19.0
+ARG VLLM_VERSION=v0.20.0
 RUN git clone -b ${VLLM_VERSION} https://github.com/vllm-project/vllm
 WORKDIR /workspace/vllm
 
 RUN --mount=type=cache,target=/root/.cache/uv \
     uv pip install --upgrade pip && \
-    uv pip install -r requirements/xpu.txt
-
- # used for suffix method speculative decoding
- # build deps for proto + nanobind-based extensions to set up the build environment
-RUN --mount=type=cache,target=/root/.cache/uv \
-    uv pip install grpcio-tools protobuf nanobind
- # arctic-inference is built from source which needs torch-xpu properly installed first
-RUN --mount=type=cache,target=/root/.cache/uv \
+    uv pip install -r requirements/xpu.txt && \
+    uv pip install setuptools_scm &&\
+    uv pip install grpcio-tools protobuf nanobind && \
     source /opt/intel/oneapi/setvars.sh --force && \
     source /opt/intel/oneapi/ccl/2021.15/env/vars.sh --force && \
-    export CMAKE_PREFIX_PATH="$(python -c 'import site; print(site.getsitepackages()[0])'):${CMAKE_PREFIX_PATH}" && \
-    uv pip install --no-build-isolation arctic-inference==0.1.1
+    export CMAKE_PREFIX_PATH="$(python3 -c 'import site; print(site.getsitepackages()[0])'):${CMAKE_PREFIX_PATH}"
 
 ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib/"
 
@@ -105,6 +113,9 @@ CMD ["/bin/bash"]
 
 FROM vllm-base AS vllm-openai
 
+# install nixl from source code
+RUN uv pip install nixl
+
 # install additional dependencies for openai api server
 RUN --mount=type=cache,target=/root/.cache/uv \
     uv pip install accelerate hf_transfer pytest pytest_asyncio lm_eval[api] modelscope
@@ -112,10 +123,6 @@ RUN --mount=type=cache,target=/root/.cache/uv \
 # install development dependencies (for testing)
 RUN uv pip install -e tests/vllm_test_utils
 
-# install nixl from source code
-ENV NIXL_VERSION=0.7.0
-RUN python /workspace/vllm/tools/install_nixl_from_source_ubuntu.py
-
 # ensure vllm is properly installed
 RUN python -c "import vllm, inspect; print(vllm.__file__)"
 RUN uv pip show vllm
@@ -135,7 +142,7 @@ RUN uv pip install --no-cache-dir ".[dev]" --no-build-isolation
 # FIX triton
 RUN --mount=type=cache,target=/root/.cache/uv \
     uv pip uninstall triton triton-xpu && \
-    uv pip install triton-xpu==3.6.0
+    uv pip install triton-xpu==3.7.0
 
 # remove torch bundled oneccl to avoid conflicts
 RUN --mount=type=cache,target=/root/.cache/uv \

docs/.nav.yml

@@ -16,8 +16,6 @@ nav:
     - examples/README.md
     - Offline Inference:
       - BAGEL-7B-MoT: user_guide/examples/offline_inference/bagel.md
-      - CosyVoice3: user_guide/examples/offline_inference/cosyvoice3.md
-      - Fish Speech S2 Pro: user_guide/examples/offline_inference/fish_speech.md
       - GLM-Image Multistage End-to-End Inference: user_guide/examples/offline_inference/glm_image.md
       - Helios Video Generation: user_guide/examples/offline_inference/helios.md
       - HunyuanImage-3.0 Image-to-Text Inference: user_guide/examples/offline_inference/hunyuan_image3.md
@@ -28,24 +26,22 @@ nav:
       - MiMo-Audio Offline Inference: user_guide/examples/offline_inference/mimo_audio.md
       - Qwen2.5-Omni: user_guide/examples/offline_inference/qwen2_5_omni.md
       - Qwen3-Omni: user_guide/examples/offline_inference/qwen3_omni.md
-      - Qwen3-TTS: user_guide/examples/offline_inference/qwen3_tts.md
       - Text-To-Audio: user_guide/examples/offline_inference/text_to_audio.md
       - Text-To-Image: user_guide/examples/offline_inference/text_to_image.md
+      - Text-To-Speech: user_guide/examples/offline_inference/text_to_speech.md
       - Text-To-Video: user_guide/examples/offline_inference/text_to_video.md
-      - Voxtral TTS Offline Inference: user_guide/examples/offline_inference/voxtral_tts.md
       - X-To-Video-Audio: user_guide/examples/offline_inference/x_to_video_audio.md
     - Online Serving:
       - BAGEL-7B-MoT: user_guide/examples/online_serving/bagel.md
       - vLLM-Omni Helm Chart: user_guide/examples/online_serving/chart-helm.md
-      - Diffusers Backend Adapter Example: user_guide/examples/online_serving/diffusers_pipeline_adapter.md
-      - Fish Speech S2 Pro: user_guide/examples/online_serving/fish_speech.md
+      - Diffusers Backend Adapter: user_guide/examples/online_serving/diffusers_pipeline_adapter.md
       - GLM-Image Online Serving: user_guide/examples/online_serving/glm_image.md
       - Image-To-Image: user_guide/examples/online_serving/image_to_image.md
       - Image-To-Video: user_guide/examples/online_serving/image_to_video.md
       - Online serving Example of vLLM-Omni for MiMo-Audio: user_guide/examples/online_serving/mimo_audio.md
       - Qwen2.5-Omni: user_guide/examples/online_serving/qwen2_5_omni.md
       - Qwen3-Omni: user_guide/examples/online_serving/qwen3_omni.md
-      - Qwen3-TTS: user_guide/examples/online_serving/qwen3_tts.md
+      - Text-To-Speech: user_guide/examples/online_serving/text_to_speech.md
       - Text-To-Image: user_guide/examples/online_serving/text_to_image.md
       - Text-To-Video: user_guide/examples/online_serving/text_to_video.md
   - General:
@@ -80,8 +76,10 @@ nav:
     - Quantization:
       - Overview: user_guide/quantization/overview.md
       - Online Quantization: user_guide/quantization/online.md
+      - Quantized KV Cache: user_guide/quantization/quantized_kvcache.md
       - FP8 W8A8: user_guide/quantization/fp8.md
       - Int8 W8A8: user_guide/quantization/int8.md
+      - ModelOpt: user_guide/quantization/modelopt.md
       - GGUF: user_guide/quantization/gguf.md
       - AutoRound: user_guide/quantization/autoround.md
       - msModelSlim: user_guide/quantization/msmodelslim.md
@@ -116,6 +114,7 @@ nav:
       - design/feature/async_chunk.md
       - design/feature/vae_parallel.md
       - design/feature/diffusion_step_execution.md
+      - design/feature/diffusion_continuous_batching.md
     - Module Design:
       - design/module/ar_module.md
       - design/module/dit_module.md

docs/contributing/ci/CI_5levels.md

@@ -152,6 +152,8 @@ vllm_omni/                                    tests/
 │   │                                            │   │   └── test_lora_manager.py        ✅
 │   ├── models/                                 │   ├── models/
 │   │   ├── qwen_image/                         │   │   ├── qwen_image/                 (e2e coverage)
+│   │   ├── ovis_image/                         │   │   ├── ovis_image/
+│   │   │   └── ...                             │   │   │   └── test_ovis_image.py     ✅
 │   │   ├── z_image/                            │   │   └── z_image/
 │   │   └── ...                                 │   │       └── test_zimage_tp_constraints.py  ✅
 │   └── worker/                                 │   └── worker/
@@ -232,15 +234,13 @@ vllm_omni/                                    tests/
                                                │   ├── test_qwen3_omni.py
                                                │   ├── test_qwen3_omni_expansion.py
                                                │   ├── test_mimo_audio.py
-                                               │   ├── test_image_gen_edit.py
                                                │   └── test_images_generations_lora.py
                                                └── offline_inference/                  ✅
                                                    ├── test_qwen2_5_omni.py
                                                    ├── test_qwen3_omni.py
                                                    ├── test_bagel_text2img.py
-                                                   ├── test_t2i_model.py
-                                                   ├── test_t2v_model.py
-                                                   ├── test_ovis_image.py
+                                                   ├── test_z_image.py
+                                                   ├── test_wan22.py
                                                    ├── test_zimage_tensor_parallel.py
                                                    ├── test_cache_dit.py
                                                    ├── test_teacache.py

docs/contributing/ci/test_guide.md

@@ -80,7 +80,7 @@ Our test scripts use the pytest framework. First, please use `git clone https://
 
     ```bash
     cd tests
-    pytest -s -v -m "full_model and omni and H100" --run-level=full_model
+    pytest -s -v -m "full_model and (omni or tts) and H100" --run-level=full_model
     ```
 
 === "L5 level"
@@ -91,10 +91,16 @@ Our test scripts use the pytest framework. First, please use `git clone https://
     cd tests
 
     # Stability: Qwen3-Omni
-    pytest -s -v dfx/stability/scripts/test_stability_qwen3_omni.py
+    pytest -s -v dfx/stability/scripts/test_stability_qwen3_omni.py -m slow
 
     # Stability: Wan2.2 (v1/videos diffusion benchmark loop)
-    pytest -s -v dfx/stability/scripts/test_stability_wan22.py
+    pytest -s -v dfx/stability/scripts/test_stability_wan22.py -m slow
+
+    # Reliability: Qwen3-Omni
+    pytest -s -v dfx/reliability/test_reliability_qwen3_omni.py -m slow
+
+    # Reliability: Wan2.2
+    pytest -s -v dfx/reliability/test_reliability_wan22.py -m slow
 
     ```
 

docs/contributing/ci/tests_markers.md

@@ -11,7 +11,8 @@ Defined in `pyproject.toml`:
 | `advanced_model`   | L3 tests (run on each merge to main)                 |
 | `full_model`       | L4 tests (run nightly) |
 | `diffusion`        | Diffusion model tests                                     |
-| `omni`             | Omni model tests                                          |
+| `omni`             | Omni multimodal model tests |
+| `tts`              | TTS model tests     |
 | `cache`            | Cache backend tests                                       |
 | `parallel`         | Parallelism/distributed tests                             |
 | `cpu`              | Tests that run on CPU                                     |

docs/contributing/ci/tests_style.md

@@ -54,6 +54,8 @@ vllm_omni/                                    tests/
 │   │                                            │   │   └── test_lora_manager.py        ✅
 │   ├── models/                                 │   ├── models/
 │   │   ├── qwen_image/                         │   │   ├── qwen_image/                 (e2e coverage)
+│   │   ├── ovis_image/                         │   │   ├── ovis_image/
+│   │   │   └── ...                             │   │   │   └── test_ovis_image.py     ✅
 │   │   ├── z_image/                            │   │   └── z_image/
 │   │   └── ...                                 │   │       └── test_zimage_tp_constraints.py  ✅
 │   └── worker/                                 │   └── worker/
@@ -134,15 +136,13 @@ vllm_omni/                                    tests/
                                                │   ├── test_qwen3_omni.py
                                                │   ├── test_qwen3_omni_expansion.py
                                                │   ├── test_mimo_audio.py
-                                               │   ├── test_image_gen_edit.py
                                                │   └── test_images_generations_lora.py
                                                └── offline_inference/                  ✅
                                                    ├── test_qwen2_5_omni.py
                                                    ├── test_qwen3_omni.py
                                                    ├── test_bagel_text2img.py
-                                                   ├── test_t2i_model.py
-                                                   ├── test_t2v_model.py
-                                                   ├── test_ovis_image.py
+                                                   ├── test_z_image.py
+                                                   ├── test_wan22.py
                                                    ├── test_zimage_tensor_parallel.py
                                                    ├── test_cache_dit.py
                                                    ├── test_teacache.py
@@ -213,7 +213,6 @@ E2E Online tests for Qwen3-Omni model with mix input and audio+text output.
 import os
 
 os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
-os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
 
 import threading
 from pathlib import Path

docs/contributing/model/adding_diffusion_model.md

@@ -115,17 +115,20 @@ class YourAttentionBlock(nn.Module):
 from vllm_omni.diffusion.attention.layer import Attention
 from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
 
-class YourAttentionBlock(nn.Module):
+class YourSelfAttentionBlock(nn.Module):
     def __init__(self, ...):
         super().__init__()
 
-        # Initialize vLLM-Omni's Attention layer
+        # Initialize vLLM-Omni's Attention layer.
+        # `role` lets users target this site with --diffusion-attention-config
+        # (e.g. --diffusion-attention-config.per_role.self.backend SAGE_ATTN).
         self.attn = Attention(
             num_heads=self.num_heads,
             head_size=self.head_dim,
             softmax_scale=1.0 / (self.head_dim ** 0.5),
             causal=False,  # Diffusion models typically use bidirectional attention
             num_kv_heads=self.num_kv_heads,
+            role="self",
         )
 
     def forward(self, hidden_states, encoder_hidden_states=None, attention_mask=None, ...):
@@ -133,7 +136,6 @@ class YourAttentionBlock(nn.Module):
         # Create attention metadata
         attn_metadata = AttentionMetadata(attn_mask=attention_mask)
         hidden_states = self.attn(query, key, value, attn_metadata=attn_metadata)
-
 ```
 
 **Key Points:**
@@ -141,8 +143,35 @@ class YourAttentionBlock(nn.Module):
 - **Attention layer initialization:** Done in `__init__`, not per-forward
 - **Tensor shapes:** vLLM-Omni `Attention` expects QKV to have `[B, seq, num_heads, head_dim]` shape
 - **AttentionMetadata:** Wraps attention mask and other metadata
+- **Role:** Tag every `Attention` site with a `role` string so users can configure backends per role (see below)
 
-**Attention backends:** vLLM-Omni automatically selects the attention backend given the environmental variable `DIFFUSION_ATTENTION_BACKEND`. The default attention backend is `FLASH_ATTN` for diffusion models.
+**Declaring attention roles**
+
+The `role` argument is a free-form string that identifies this attention site. Users can match it from `--diffusion-attention-config.per_role.<role>.*` to swap backends without touching model code. Two conventions cover the common cases:
+
+| Convention | When to use | Example |
+|---|---|---|
+| `"self"` | Q/K/V come from the same hidden state | DiT self-attention block |
+| `"cross"` | K/V come from a separate `encoder_hidden_states` | Text-conditioned cross-attention |
+
+For multi-modal or unusual sites, use a dot-namespaced role and pair it with `role_category` so it can fall back to the generic config when nothing model-specific is set:
+
+```python
+# A model-specific cross-attention site that user config can target
+# either as 'mymodel.audio_to_video' (exact) or as 'cross' (category fallback).
+self.audio_to_video_attn = Attention(
+    num_heads=self.num_heads,
+    head_size=self.head_dim,
+    softmax_scale=1.0 / (self.head_dim ** 0.5),
+    causal=False,
+    role="mymodel.audio_to_video",
+    role_category="cross",
+)
+```
+
+For cross-attention sites whose K/V are replicated across ranks (e.g. text encoder output), pass `skip_sequence_parallel=True` to opt this layer out of sequence-parallel sharding.
+
+**Attention backends:** When the user does not configure a backend, vLLM-Omni asks the current platform for its default (typically `FLASH_ATTN` on CUDA when available). Users override the default via `--diffusion-attention-backend`, the `DIFFUSION_ATTENTION_BACKEND` env var, or finer-grained `--diffusion-attention-config.per_role.*` flags. See [Diffusion Attention Backends](../../user_guide/diffusion/attention_backends.md) for the full configuration surface.
 
 #### 1.3: Replace Imports and Utilities
 
@@ -737,6 +766,16 @@ step-scoped phases. The reference implementation is
 Do not enable `step_execution=True` until those four methods are implemented
 and validated against the request-level path.
 
+If you want the pipeline to work with the experimental batched step-wise path
+(`max_num_seqs > 1`), also see:
+[Continuous Batching for Step-Wise Diffusion](../../design/feature/diffusion_continuous_batching.md).
+
+If you expose this in example scripts or recipes, keep it opt-in. Surface
+runtime features like `step_execution` as optional flags instead of silently
+turning them on. For Qwen-Image-style serving examples, document
+`--step-execution` as the feature gate and `--max-num-seqs N` as the
+companion batching knob.
+
 ### Cache Acceleration
 
 #### TeaCache

docs/contributing/model/adding_tts_model.md

@@ -186,6 +186,17 @@ vllm_omni/model_executor/stage_configs/
   your_model_name_async_chunk.yaml   # Streaming mode config
 ```
 
+### Example placement
+
+TTS examples live in the consolidated text-to-speech hub, **not** in their
+own top-level directory. Place per-model scripts under
+`examples/offline_inference/text_to_speech/<your_model>/` and
+`examples/online_serving/text_to_speech/<your_model>/`, and add a section
+to the hub `README.md` files (table row + per-model section) instead of a
+new per-model `README.md`. The mkdocs `generate_examples` hook treats the
+`text_to_speech/` parent as a single example, so per-model READMEs inside
+it would not be picked up — the hub README is the documented surface.
+
 **Qwen3-TTS reference files:**
 
 | File | Purpose |

docs/contributing/profiling.md

@@ -37,6 +37,8 @@ Supported fields:
 | `warmup_iterations` | Torch-profiler warmup iterations. |
 | `active_iterations` | Torch-profiler active iterations. |
 
+For detailed explanations of the fields, please refer to upstream vLLM implementation [vllm/config/profiler.py](https://github.com/vllm-project/vllm/blob/v0.20.1/vllm/config/profiler.py)
+
 ### Minimal configurations by output
 
 Only collect trace output:
@@ -124,7 +126,7 @@ profiler_config = {
     "max_iterations": 0,
     "wait_iterations": 0,
     "warmup_iterations": 0,
-    "active_iterations": 0,
+    "active_iterations": 1,
 }
 ```
 
@@ -220,7 +222,7 @@ vllm serve Wan-AI/Wan2.2-I2V-A14B-Diffusers \
     "max_iterations": 0,
     "wait_iterations": 0,
     "warmup_iterations": 0,
-    "active_iterations": 0
+    "active_iterations": 1
   }'
 ```
 

docs/design/feature/diffusion_continuous_batching.md

@@ -0,0 +1,156 @@
+# Continuous Batching for Step-Wise Diffusion
+
+!!! warning "Experimental Feature"
+    This feature is experimental. It currently applies only to native
+    diffusion pipelines running with `step_execution=True`.
+
+This document describes the batching extension built on top of
+[Diffusion Step Execution](diffusion_step_execution.md). The base
+step-execution contract is unchanged. The batching work is mainly in the
+scheduler and runner layers.
+
+## Why It Helps
+
+Step-wise execution breaks a long denoise loop into scheduler-visible units.
+That gives the runtime a place to admit other compatible requests between
+steps instead of waiting for an entire request to finish.
+
+This matters most in low-MFU or bursty serving scenarios:
+
+- one request's denoise step may not fully saturate the GPU
+- several compatible requests can share the same denoise forward pass
+- throughput and device utilization can improve without changing request-local
+  scheduler state
+
+This is **not** a guaranteed single-request latency win. The main benefit is
+usually higher utilization and better throughput when the workload contains
+multiple in-flight compatible requests.
+
+## Overview
+
+With continuous batching enabled:
+
+- the scheduler may keep multiple compatible requests active at the same time
+- the runner packs request-local step state into one `InputBatch`
+- `denoise_step()` runs on that batch
+- `step_scheduler()` and `post_decode()` still run per request
+
+The current implementation is conservative:
+
+- only compatible requests are batched together
+- request-mode diffusion still runs with `max_num_seqs=1`
+- per-request progress and completion remain independent
+
+## Enablement
+
+Use `--step-execution` as the feature gate, then increase `--max-num-seqs`
+above `1` if you want batching:
+
+```bash
+vllm serve Qwen/Qwen-Image --omni \
+  --port 8091 \
+  --step-execution \
+  --max-num-seqs 8
+```
+
+`--max-num-seqs 1` keeps the step-wise path without enabling batching.
+
+For a reproducible replay flow using the bundled serving benchmark, see the
+Qwen-Image replay commands in
+[`benchmarks/diffusion/README.md`](gh-file:benchmarks/diffusion/README.md)
+and
+[`benchmarks/diffusion/performance_dashboard/qwen_image_serving_performance.md`](gh-file:benchmarks/diffusion/performance_dashboard/qwen_image_serving_performance.md).
+
+## Scheduler
+
+The scheduler derives its batch capacity from `max_num_seqs` through
+`max_num_running_reqs`.
+
+Batch admission is gated by
+[`SamplingParamsKey`](gh-file:vllm_omni/diffusion/sched/interface.py),
+which is built from shape-sensitive and CFG-sensitive sampling fields. This is
+the core correctness rule for batching: requests are only co-batched when they
+share the same denoise tensor contract.
+
+There are two important details:
+
+- `num_inference_steps` is not part of the key, so requests with different
+  total step counts can still share a batch
+- requests also do not need to be at the same current denoise progress; active
+  requests can continue batching even when their current step indices diverge
+- admission is still FIFO, so an incompatible request at the head of the
+  waiting queue blocks later compatible requests
+
+Today that compatibility rule is still shape-sensitive. `height`, `width`,
+`num_frames`, and CFG-related fields remain part of the key, so different
+resolutions or incompatible guidance settings do **not** co-batch yet.
+
+The current batching unit is one `OmniDiffusionRequest`. Requests with
+multiple prompts do not participate in batching today.
+
+## Runner
+
+The runner keeps persistent per-request execution state in
+[`DiffusionRequestState`](gh-file:vllm_omni/diffusion/worker/utils.py),
+while the scheduler owns a separate lightweight request state for queueing and
+lifecycle tracking.
+
+For each step, the runner builds an
+[`InputBatch`](gh-file:vllm_omni/diffusion/worker/input_batch.py) from the
+active request states:
+
+- prompt embeddings and masks are normalized and padded
+- dynamic tensors such as `latents` and `timesteps` are gathered each step
+- buffers are reused when batch composition stays the same
+
+The step-wise batched path is:
+
+1. Run `prepare_encode()` for newly admitted requests.
+2. Build or refresh `InputBatch`.
+3. Run one batched `denoise_step(input_batch)`.
+4. Slice the batched `noise_pred` back per request.
+5. Run per-request `step_scheduler()`.
+6. Run `post_decode()` only for requests that finished denoising.
+7. Scatter updated latents back into persistent request state with
+   [`scatter_latents()`](gh-file:vllm_omni/diffusion/worker/input_batch.py).
+
+This keeps the shared work limited to the denoise forward pass while preserving
+request-local scheduler state and outputs.
+
+## Engine
+
+[`DiffusionEngine`](gh-file:vllm_omni/diffusion/diffusion_engine.py) provides
+the background loop and async add-request path needed for multiple requests to
+accumulate in the scheduler.
+
+This is supporting infrastructure, not the main design point. The batching
+behavior is defined by scheduler-side compatibility gating and runner-side
+batch packing.
+
+## Current Limitations
+
+- Experimental feature; use `max_num_seqs=1` for the older conservative path.
+- Only native pipelines that already support `step_execution=True`.
+- Request-mode diffusion still clamps `max_num_seqs` back to `1`.
+- Only homogeneous batches keyed by `SamplingParamsKey` are supported.
+- Multi-prompt requests are not batched.
+- `cache_backend`, KV transfer, and other request-mode extras are not wired
+  into the batched step-wise path yet.
+- Future work can relax the current same-shape restriction with richer
+  heterogeneous batching policies such as bucketing or padded execution for
+  different resolutions.
+
+## Related Files
+
+- Scheduler base:
+  [`vllm_omni/diffusion/sched/base_scheduler.py`](gh-file:vllm_omni/diffusion/sched/base_scheduler.py)
+- Scheduler interface:
+  [`vllm_omni/diffusion/sched/interface.py`](gh-file:vllm_omni/diffusion/sched/interface.py)
+- Step scheduler:
+  [`vllm_omni/diffusion/sched/step_scheduler.py`](gh-file:vllm_omni/diffusion/sched/step_scheduler.py)
+- Runner:
+  [`vllm_omni/diffusion/worker/diffusion_model_runner.py`](gh-file:vllm_omni/diffusion/worker/diffusion_model_runner.py)
+- Input batch:
+  [`vllm_omni/diffusion/worker/input_batch.py`](gh-file:vllm_omni/diffusion/worker/input_batch.py)
+- Tests:
+  [`tests/diffusion/test_diffusion_scheduler.py`](gh-file:tests/diffusion/test_diffusion_scheduler.py)

docs/design/feature/diffusion_step_execution.md

@@ -1,4 +1,4 @@
-# Diffusion Step Execution
+# Adding Step Execution Support for Diffusion Pipelines
 
 This guide documents vLLM-Omni's stepwise diffusion contract for model authors
 and contributors implementing `step_execution=True` support for a diffusion
@@ -7,12 +7,20 @@ pipeline.
 For end-user enablement, supported models, and current limitations, see
 [Step Execution](../../user_guide/diffusion/step_execution.md).
 
+This document describes the base step-execution contract only. For the
+experimental batching policy layered on top of the step-wise path, see
+[Continuous Batching for Step-Wise Diffusion](diffusion_continuous_batching.md).
+
 ## Current Support Scope
 
 `step_execution` is **not** a generic diffusion toggle. It only works for
 pipelines that implement the segmented stateful contract in
 [`vllm_omni/diffusion/models/interface.py`](gh-file:vllm_omni/diffusion/models/interface.py).
 
+This page is intentionally author-facing. Treat runtime enablement
+(`step_execution=True` in Python or `--step-execution` in serving) as an
+opt-in user knob layered on top of the implementation contract below.
+
 Current in-tree support:
 
 | Pipeline | Example models | Step execution |
@@ -22,7 +30,9 @@ Current in-tree support:
 
 Current engine/runtime limitations:
 
-- `StepScheduler` only schedules `batch_size=1`.
+- Continuous batching with `max_num_seqs > 1` is experimental and documented in
+  [Continuous Batching for Step-Wise Diffusion](diffusion_continuous_batching.md).
+  Keep `max_num_seqs=1` if you want the older conservative behavior.
 - `cache_backend` is not supported in step mode.
 - Request-mode extras such as KV transfer are not wired into step mode yet.
 - Unsupported pipelines now fail early during model loading instead of failing on the first request.

docs/design/index.md

@@ -10,6 +10,8 @@ This section contains design documents and architecture specifications for vLLM-
 
 - [Disaggregated Inference](feature/disaggregated_inference.md)
 - [Ray-based Execution](feature/ray_based_execution.md)
+- [Adding Step Execution Support for Diffusion Pipelines](feature/diffusion_step_execution.md)
+- [Continuous Batching for Step-Wise Diffusion](feature/diffusion_continuous_batching.md)
 
 ## Module Design Documents
 

docs/design/module/dit_module.md

@@ -201,7 +201,7 @@ class _BaseScheduler(SchedulerInterface):
 
 - **Shared cleanup logic**: Request-id registration, finish handling, and state removal are centralized instead of duplicated in each policy.
 
-- **Current constraint**: `max_num_running_reqs` remains `1` because the current engine path is still synchronous request-mode execution.
+- **Current constraint boundary**: `_BaseScheduler` derives `max_num_running_reqs` from `max_num_seqs`, but request-mode diffusion is still clamped back to `1` by the engine. The step-wise path can keep this above `1` for compatible-request batching.
 
 #### 2.4 Current `RequestScheduler` Policy
 
@@ -217,7 +217,7 @@ class RequestScheduler(_BaseScheduler):
 
 - **FIFO request scheduling**: Waiting requests are promoted in queue order.
 
-- **Single-request admission**: The current policy only admits one active request at a time.
+- **Single-request admission**: `RequestScheduler` still admits one active request at a time because request-mode execution completes a whole request per dispatch.
 
 - **Executor result feedback**: `update_from_output()` converts executor output into `FINISHED_COMPLETED` or `FINISHED_ERROR` and returns finished scheduler ids.
 
@@ -237,7 +237,7 @@ while True:
 
 - **No scheduler-owned IPC**: Scheduler no longer talks to workers directly.
 
-- **Conservative concurrency**: The current request-mode implementation still allows only one active request at a time.
+- **Split concurrency model**: Request-mode diffusion remains single-active-request, while the step-wise path can keep multiple compatible requests running and advance them independently between denoise steps.
 
 ---
 
@@ -455,7 +455,7 @@ To learn more about the diffusion pipeline and how to add a new diffusion pipeli
 
 #### Architecture
 
-The attention system uses a **backend selector pattern** that automatically chooses the optimal attention implementation based on hardware and model configuration.
+The attention system uses a **role-aware backend selector pattern**. Each `Attention` site declares a semantic `role` (`"self"`, `"cross"`, or a model-specific string). The selector consults the user's `AttentionConfig` to pick a backend per role and falls back to the platform default when nothing is configured.
 
 #### Backend Selection
 
@@ -463,92 +463,90 @@ The attention system uses a **backend selector pattern** that automatically choo
 
 ```python
 class Attention(nn.Module):
-    def __init__(self, num_heads, head_size, causal, softmax_scale, ...):
-        # Auto-select backend
-        self.attn_backend = get_attn_backend(-1)
-        self.attn_impl_cls = self.attn_backend.get_impl_cls()
-        self.attention = self.attn_impl_cls(...)
+    def __init__(self, num_heads, head_size, causal, softmax_scale, *,
+                 role="self", role_category=None, ...):
+        # Resolve backend for this role from the active OmniDiffusionConfig
+        config = get_current_diffusion_config_or_none()
+        attention_config = config.attention_config if config is not None else None
+        attn_backend_cls, spec = get_attn_backend_for_role(
+            role=role,
+            head_size=head_size,
+            attention_config=attention_config,
+            role_category=role_category,
+        )
+        self.attn_impl_cls = attn_backend_cls.get_impl_cls()
+        self.attention = self.attn_impl_cls(..., backend_kwargs=spec.extra if spec else None)
 ```
 
 **Available Backends**:
 
-- **FlashAttention**: Optimized CUDA kernel (FA2/FA3) - memory efficient via tiling
+- **FlashAttention** (`FLASH_ATTN`): Memory-efficient kernel that dispatches to FA2/FA3 on CUDA, AITER on ROCm, and `mindiesd` on Ascend NPU.
 
-- **SDPA**: PyTorch's scaled dot-product attention - default, cross-platform
+- **SDPA** (`TORCH_SDPA`): PyTorch's scaled dot-product attention — cross-platform fallback.
 
-- **SageAttention**: Sparse attention implementation from SageAttention library
-
-- **AscendAttention**: NPU-optimized attention for Ascend hardware
+- **SageAttention** (`SAGE_ATTN`): Quantized attention implementation from the SageAttention library.
 
 These backends provide the **kernel implementations** for attention computation. For attention-level sequence parallelism strategies (Ring Attention, Ulysses), see [Parallel Attention](#52-parallel-attention).
 
 #### Backend Selection Mechanism
 
+Selection is driven by `AttentionConfig` on `OmniDiffusionConfig`, which carries a global `default` plus a `per_role` map of `AttentionSpec` entries. A role string is resolved in this order:
+
 ```python
-def get_attn_backend(head_size: int) -> type[AttentionBackend]:
-    # Check environment variable
-    backend_name = os.environ.get("DIFFUSION_ATTENTION_BACKEND")
-
-    if backend_name:
-        return load_backend(backend_name.upper())
-
-    # Default to SDPA
-    return SDPABackend
+def get_attn_backend_for_role(role, head_size, attention_config=None, role_category=None):
+    # 1. attention_config.per_role[role]            — exact match
+    # 2. attention_config.per_role[role_category]   — category fallback
+    # 3. attention_config.default                   — global default
+    # 4. Platform default                           — hardware-specific
+    if attention_config is not None:
+        spec, source = attention_config.resolve_with_source(
+            role=role, role_category=role_category,
+        )
+        if spec is not None:
+            return load_backend(spec.backend), spec
+    return current_omni_platform.get_diffusion_attn_backend_cls(...), None
 ```
 
 **Selection Priority**:
 
-1. **Environment Variable**: `DIFFUSION_ATTENTION_BACKEND` for manual override
+1. **Per-role override** (`--diffusion-attention-config.per_role.<role>.backend`) — finest control; matched on the layer's exact `role` string.
 
-    - Valid values: `FLASH_ATTN`, `TORCH_SDPA`, `SAGE_ATTN`, `ASCEND`
+2. **Role category fallback** (`--diffusion-attention-config.per_role.<category>.backend`) — used when an exact match is missing and the layer declared `role_category` (e.g. a `"mymodel.audio_to_video"` site can fall back to `"cross"`).
 
-    - Example: `export DIFFUSION_ATTENTION_BACKEND=SAGE_ATTN`
+3. **Global default** (`--diffusion-attention-backend FLASH_ATTN`, or `--diffusion-attention-config.default.backend`, or `DIFFUSION_ATTENTION_BACKEND` env var).
 
-2. **Automatic Fallback**: Falls back to SDPA if selected backend unavailable
+4. **Platform default** — `current_omni_platform.get_diffusion_attn_backend_cls(...)` picks the best-available kernel for the hardware.
 
-3. **Hardware Detection**: Can select based on device type (NPU, CUDA, etc.)
+`AttentionSpec.extra` is forwarded to the backend constructor as `backend_kwargs`, so backend-specific parameters (e.g. SparseBlock `block_size`) can be set without changing model code.
+
+For the user-facing CLI surface, see [Diffusion Attention Backends](../../user_guide/diffusion/attention_backends.md). For the role declaration contract on the model side, see [Adding a Diffusion Model](../../contributing/model/adding_diffusion_model.md).
 
 **Backend Availability**:
 
 - **SDPA**: Always available (PyTorch built-in)
 
-- **FlashAttention**: Requires `flash-attn` package installed
+- **FlashAttention**: Requires `flash-attn` on CUDA, `aiter` on ROCm, or `mindiesd` on Ascend NPU
 
 - **SageAttention**: Requires `sage-attention` package (from THU-ML GitHub)
 
-- **AscendAttention**: Only available on Ascend NPU hardware
-
 #### Attention Backend Registry
 
-**Location**: `vllm_omni/diffusion/attention/selector.py`
+**Location**: `vllm_omni/diffusion/attention/selector.py` + `vllm_omni/platforms/<device>/platform.py`
 
-The attention system uses a **registry pattern** to manage and dynamically load attention backends. This allows for easy extension and runtime selection of backends.
-
-
-**Registry Structure**:
+Backend resolution is delegated to the active platform: `current_omni_platform.get_diffusion_attn_backend_cls(selected_backend, head_size)` returns a fully-qualified class path that `_load_backend_cls` imports lazily, with the result cached by `(backend_name, head_size)`.
 
 ```python
-# Registry mapping backend names to their module paths and class names
-_BACKEND_CONFIG = {
-    "FLASH_ATTN": {
-        "module": "vllm_omni.diffusion.attention.backends.flash_attn",
-        "class": "FlashAttentionBackend",
-    },
-    "TORCH_SDPA": {
-        "module": "vllm_omni.diffusion.attention.backends.sdpa",
-        "class": "SDPABackend",
-    },
-    "SAGE_ATTN": {
-        "module": "vllm_omni.diffusion.attention.backends.sage_attn",
-        "class": "SageAttentionBackend",
-    },
-    "ASCEND": {
-        "module": "vllm_omni.diffusion.attention.backends.ascend_attn",
-        "class": "AscendAttentionBackend",
-    },
-}
+@cache
+def _cached_get_backend_cls(backend_name: str | None, head_size: int) -> type[AttentionBackend]:
+    backend_cls_path = current_omni_platform.get_diffusion_attn_backend_cls(
+        selected_backend=backend_name,
+        head_size=head_size,
+    )
+    return _load_backend_cls(backend_cls_path)
 ```
 
+Each platform (`cuda`, `rocm`, `xpu`, `musa`, `npu`) maps backend names like `"FLASH_ATTN"`, `"TORCH_SDPA"`, `"SAGE_ATTN"` to the right backend class path for that hardware, including head-size compatibility checks. Passing `selected_backend=None` lets the platform pick its own default.
+
 #### Attention Backend Integration
 
 The `Attention` layer integrates backends through a unified interface. Here's how **FlashAttentionBackend** is integrated as an example:

docs/getting_started/installation/gpu.md

@@ -34,7 +34,7 @@ vLLM-Omni is a Python library that supports the following GPU variants. The libr
 
 ### Pre-built wheels
 
-Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, and 0.18.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).
+Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, 0.18.0, and 0.20.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).
 
 === "NVIDIA CUDA"
 

docs/getting_started/installation/gpu/cuda.inc.md

@@ -20,7 +20,7 @@ Therefore, it is recommended to install vLLM and vLLM-Omni with a **fresh new**
 
 vLLM-Omni is built based on vLLM. Please install it with command below.
 ```bash
-uv pip install vllm --torch-backend=auto
+uv pip install vllm==0.20.0 --torch-backend=auto
 ```
 
 #### Installation of vLLM-Omni
@@ -102,7 +102,7 @@ docker run --runtime nvidia --gpus 2 \
     --env "HF_TOKEN=$HF_TOKEN" \
     -p 8091:8091 \
     --ipc=host \
-    vllm/vllm-omni:v0.18.0 \
+    vllm/vllm-omni:v0.20.0 \
     --model Qwen/Qwen3-Omni-30B-A3B-Instruct --port 8091
 ```
 

docs/getting_started/installation/gpu/rocm.inc.md

@@ -13,7 +13,7 @@ vLLM-Omni current recommends the steps in under setup through Docker Images.
 
 vLLM-Omni is built based on vLLM. Please install it with command below.
 ```bash
-uv pip install vllm==0.20.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm700
+uv pip install vllm==0.20.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm721
 ```
 
 #### Installation of vLLM-Omni
@@ -37,10 +37,10 @@ uv pip install onnxruntime-rocm
 If you do not need to modify source code of vLLM, you can directly install the stable 0.20.0 release version of the library
 
 ```bash
-uv pip install vllm==0.20.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm700
+uv pip install vllm==0.20.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm721
 ```
 
-The pre-built 0.20.0 vLLM wheel targets ROCm 7.0. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
+The pre-built 0.20.0 vLLM wheel targets ROCm 7.2.1. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
 
 #### Installation of vLLM-Omni
 Since vllm-omni is rapidly evolving, it's recommended to install it from source

docs/getting_started/quickstart.md

@@ -22,7 +22,7 @@ source .venv/bin/activate
 uv pip install vllm==0.20.0 --torch-backend=auto
 
 # On ROCm
-uv pip install vllm==0.20.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm700
+uv pip install vllm==0.20.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm721
 
 git clone https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni
@@ -31,6 +31,12 @@ uv pip install -e .
 
 For additional installation methods — please see the [installation guide](installation/README.md).
 
+
+!!! note
+    It is important to install the same major & minor version of vLLM and vLLM Omni, otherwise things may not work as expected. If the versions are misaligned, you will see a warning when you import vLLM Omni.
+
+    If you are seeing strange behavior with the `vllm` command not handling the `--omni` flag correctly, you most likely have a version mismatch with vLLM < `0.20.0` and vLLM Omni `0.20.0`, as vLLM Omni no longer hijacks the vLLM entrypoint. Updating vLLM should resolve this issue.
+
 ## Offline Inference
 
 Text-to-image generation quickstart with vLLM-Omni:

docs/models/supported_models.md

@@ -32,6 +32,7 @@ th {
 | `ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `WanPipeline` | Wan2.1-T2V, Wan2.2-T2V, Wan2.2-TI2V | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`, `Wan-AI/Wan2.1-T2V-14B-Diffusers`, `Wan-AI/Wan2.2-T2V-A14B-Diffusers`, `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `WanImageToVideoPipeline` | Wan2.2-I2V | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
+| `WanSpeechToVideoPipeline` | Wan2.2-S2V | `Wan-AI/Wan2.2-S2V-14B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `Wan22VACEPipeline` | Wan2.1-VACE | `Wan-AI/Wan2.1-VACE-1.3B-diffusers`, `Wan-AI/Wan2.1-VACE-14B-diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `LTX2Pipeline` | LTX-2-T2V | `Lightricks/LTX-2` | ✅︎ | ✅︎ | | |
 | `LTX2ImageToVideoPipeline` | LTX-2-I2V | `Lightricks/LTX-2` | ✅︎ | ✅︎ | | |
@@ -53,6 +54,7 @@ th {
 | `FluxPipeline` | FLUX.1-schnell | `black-forest-labs/FLUX.1-schnell` | ✅︎ | ✅︎ | | ✅︎ |
 | `OmniGen2Pipeline` | OmniGen2 | `OmniGen2/OmniGen2` | ✅︎ | ✅︎ | | ✅︎ |
 | `StableAudioPipeline` | Stable-Audio-Open | `stabilityai/stable-audio-open-1.0` | ✅︎ | ✅︎ | | ✅︎ |
+| `AudioXPipeline` | AudioX | `zhangj1an/AudioX` | ✅︎ | ✅︎ | | |
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-CustomVoice | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-VoiceDesign | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-Base | `Qwen/Qwen3-TTS-12Hz-0.6B-Base` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
@@ -62,9 +64,11 @@ th {
 | `Flux2Pipeline` | FLUX.2-dev | `black-forest-labs/FLUX.2-dev` | ✅︎ | ✅︎ | | |
 | `FishSpeechSlowARForConditionalGeneration` | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✅︎ | ✅︎ | | |
 | `DreamIDOmniPipeline` | DreamID-Omni | `XuGuo699/DreamID-Omni` | ✅︎ | ✅︎ | | |
+| `SenseNovaU1Pipeline` | SenseNova-U1 (DiT-only) | `SenseNova/SenseNova-U1-8B-MoT` | ✅︎ | | | |
 | `HunyuanVideo15Pipeline` | HunyuanVideo-1.5-T2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | ✅︎ | ✅︎ | | |
 | `HunyuanVideo15ImageToVideoPipeline` | HunyuanVideo-1.5-I2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_i2v` | ✅︎ | ✅︎ | | |
 | `VoxtralTTSForConditionalGeneration` | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✅︎ | ✅︎ | | |
+| `CovoAudioForConditionalGeneration` | Covo-Audio-Chat | `tencent/Covo-Audio-Chat` | ✅︎ | | | |
 |`DyninOmniForConditionalGeneration` | Dynin-Omni | `snu-aidas/Dynin-Omni` | ✅︎ | | | |
-
+| `ErnieImagePipeline` | ERNIE-Image | `baidu/ERNIE-Image`, `baidu/ERNIE-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 ✅︎ indicates the model is supported on that backend. Empty cells mean not listed as supported on that backend.

docs/serving/speech_api.md

@@ -358,6 +358,31 @@ curl -X POST http://localhost:8091/v1/audio/speech \
     }' --output cloned.wav
 ```
 
+### Voice Storage & Caching
+
+Uploaded voices are persisted to disk as a single `.safetensors` file per voice
+(audio samples + metadata — name, consent, ref_text, sample_rate, created_at —
+in the file header). On server restart the directory is scanned and all
+previously uploaded voices are restored automatically, so uploads survive
+process restarts.
+
+Uploading an existing name overwrites the previous entry (a warning is logged).
+
+Feature extraction artifacts (ref_code, speaker_embedding, DAC codes, etc.)
+are cached in-process with a shared LRU so repeated requests with the same
+`voice=...` skip the extraction pipeline. The cache is a true singleton across
+all TTS model types; deleting a voice invalidates every model-type slot at
+once.
+
+**Configuration (environment variables):**
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `SPEAKER_SAMPLES_DIR` | `~/.cache/vllm-omni/speakers` | Directory for persisted uploaded speakers (`.safetensors` files). |
+| `SPEAKER_MAX_UPLOADED` | `1000` | Maximum number of uploaded speakers kept on disk. Upload requests past the cap return 400. |
+
+The in-memory LRU has a fixed 512 MiB byte budget.
+
 ## Batch Speech Generation
 
 The batch endpoint synthesizes multiple texts in a single request, returning all results as JSON with base64-encoded audio.
@@ -506,16 +531,18 @@ for result in response.json()["results"]:
 
 All items are fanned out to `generate()` concurrently. The engine's stage worker automatically batches them up to the configured `max_batch_size` and queues the rest — no client-side throttling needed.
 
-For best throughput, set both stages' `max_num_seqs` to ≥4 via `--stage-overrides`:
+For best throughput, set both stages' `max_num_seqs` above 1 via `--stage-overrides`. On the current Qwen3-TTS CustomVoice benchmark, stage 1 performed best at `max_num_seqs: 10`:
 
 ```bash
 vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
     --omni --port 8091 --trust-remote-code --enforce-eager \
-    --stage-overrides '{"0":{"max_num_seqs":4,"gpu_memory_utilization":0.2},
-                        "1":{"max_num_seqs":4,"gpu_memory_utilization":0.2}}'
+    --stage-overrides '{"0":{"max_num_seqs":10,"gpu_memory_utilization":0.2},
+                        "1":{"max_num_seqs":10,"gpu_memory_utilization":0.2}}'
 ```
 
-The bundled `qwen3_tts.yaml` uses `max_num_seqs: 1` (single request) on both stages. Bumping to 4 yields roughly 4× throughput on the talker and lets stage 1 batch chunks across in-flight requests.
+The bundled `qwen3_tts.yaml` uses a multi-request default and lets stage 1 batch chunks across in-flight requests. For latency-sensitive deployments, avoid forcing stage 1 back to `max_num_seqs: 1`; benchmark before reducing it below `10`.
+
+The bundled config also sets `initial_codec_chunk_frames: 1`. This emits only the first audio chunk early for lower TTFA, then returns to the normal `codec_chunk_frames` window so Code2Wav does not repeatedly decode tiny overlapping chunks.
 
 ## Supported Models
 
@@ -535,7 +562,7 @@ The bundled `qwen3_tts.yaml` uses `max_num_seqs: 1` (single request) on both sta
 |-------|-------------|
 | `fishaudio/s2-pro` | 4B dual-AR TTS with DAC codec (44.1 kHz). Supports text-to-speech and voice cloning. |
 
-Fish Speech uses `ref_audio` and `ref_text` for voice cloning (no `task_type` needed). The `voice` field should be set to `"default"`. See the [Fish Speech online serving example](../user_guide/examples/online_serving/fish_speech.md) for details.
+Fish Speech uses `ref_audio` and `ref_text` for voice cloning (no `task_type` needed). The `voice` field should be set to `"default"`. See the [Fish Speech section of the online TTS hub](../user_guide/examples/online_serving/text_to_speech.md#fish-speech-s2-pro) for details.
 
 ### Voxtral TTS
 
@@ -543,6 +570,30 @@ Fish Speech uses `ref_audio` and `ref_text` for voice cloning (no `task_type` ne
 |-------|-------------|
 | `mistralai/Voxtral-4B-TTS-2603` | 3B AR + FlowMatching TTS. Supports text-to-speech with preset voices. |
 
+### CosyVoice3
+
+| Model | Description |
+|-------|-------------|
+| `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | Voice cloning from `ref_audio` + `ref_text`. No built-in voice presets — upload a voice or pass `ref_audio`/`ref_text` per request. |
+
+### OmniVoice
+
+| Model | Description |
+|-------|-------------|
+| `k2-fsa/OmniVoice` | Pure-diffusion TTS. Supports voice cloning via `ref_audio` (with optional `ref_text`); no built-in voice presets. |
+
+### VoxCPM2
+
+| Model | Description |
+|-------|-------------|
+| `openbmb/VoxCPM2` | TTS + voice cloning with built-in speaker presets and uploaded-voice support. Accepts `voice` (preset or uploaded) or `ref_audio` + optional `ref_text`. |
+
+### MOSS-TTS-Nano
+
+| Model | Description |
+|-------|-------------|
+| `OpenMOSS-Team/MOSS-TTS-Nano` | Voice cloning only. Requires `ref_audio` (or an uploaded `voice`); no built-in voice presets. `ref_text` is accepted but ignored — upstream's `voice_clone` mode does not consume a transcript. |
+
 ## Error Responses
 
 ### 400 Bad Request

docs/user_guide/diffusion/attention_backends.md

@@ -1,10 +1,10 @@
 # Diffusion Attention Backends
 
-This document describes the diffusion attention backends available in vLLM-Omni, how to select them, and how to use SageAttention.
+This document describes the diffusion attention backends available in vLLM-Omni, how to select them globally and per-role, and how to use SageAttention.
 
 ## Overview
 
-Diffusion attention backend selection is controlled by the `DIFFUSION_ATTENTION_BACKEND` environment variable and resolved in `vllm_omni.diffusion.attention.selector`.
+Diffusion attention backend selection is resolved in `vllm_omni.diffusion.attention.selector`. It looks up the backend from a structured `AttentionConfig` carried on `OmniDiffusionConfig` and falls back to the platform default when nothing is configured.
 
 This backend is used by diffusion attention layers such as the DiT attention in video and image generation models.
 
@@ -14,7 +14,7 @@ On CUDA, the practical choices today are:
 - `TORCH_SDPA`: PyTorch `scaled_dot_product_attention`.
 - `SAGE_ATTN`: SageAttention backend, if `sageattention` is installed.
 
-If `DIFFUSION_ATTENTION_BACKEND` is unset, vLLM-Omni asks the current platform to choose the default backend. On CUDA, that normally means `FLASH_ATTN` when available, otherwise `TORCH_SDPA`.
+If no attention backend is configured, vLLM-Omni asks the current platform to choose the default. On CUDA, that normally means `FLASH_ATTN` when available, otherwise `TORCH_SDPA`.
 
 ## Backend Options
 
@@ -24,19 +24,77 @@ If `DIFFUSION_ATTENTION_BACKEND` is unset, vLLM-Omni asks the current platform t
 | `TORCH_SDPA` | Most conservative fallback. Useful for debugging or compatibility. |
 | `SAGE_ATTN` | Requires `sageattention`. Can improve performance on some workloads, but output quality must be validated model-by-model. |
 
-## Selection Priority
+## Configuration
 
-Diffusion attention backend selection follows this order:
+Diffusion attention backends can be configured three ways, in priority order:
 
-1. `DIFFUSION_ATTENTION_BACKEND`
-2. Platform default
+1. **`--diffusion-attention-config`** — structured per-role config (highest priority).
+2. **`--diffusion-attention-backend` / `DIFFUSION_ATTENTION_BACKEND` env var** — global shorthand that sets the default backend.
+3. **Platform default** — used when nothing is configured.
 
-Example:
+`--diffusion-attention-backend` is shorthand for `--diffusion-attention-config.default.backend`. It may be combined with `--diffusion-attention-config.per_role.*` overrides, but is mutually exclusive with `--diffusion-attention-config.default.backend`.
+
+### Global default
+
+Set the default backend for every diffusion attention layer:
 
 ```bash
+# CLI flag
+vllm-omni serve <model> --diffusion-attention-backend SAGE_ATTN
+
+# Environment variable (also recognized for backwards compatibility)
 export DIFFUSION_ATTENTION_BACKEND=SAGE_ATTN
 ```
 
+### Per-role configuration
+
+Roles are free-form strings declared by each diffusion model. The two common categories are `"self"` and `"cross"`; model-specific roles (e.g. `"ltx2.audio_to_video"`) may also be declared. A role string is matched in this order:
+
+1. Exact `per_role[role]` match
+2. `per_role[role_category]` fallback (e.g. `"ltx2.audio_to_video"` → `"cross"`)
+3. `default`
+4. Platform default
+
+Use vLLM-style dotted flags or one JSON blob:
+
+```bash
+# Dotted flags
+vllm-omni serve <model> \
+    --diffusion-attention-config.default.backend FLASH_ATTN \
+    --diffusion-attention-config.per_role.cross.backend TORCH_SDPA
+
+# JSON
+vllm-omni serve <model> \
+    --diffusion-attention-config '{"default":{"backend":"FLASH_ATTN"},"per_role":{"cross":{"backend":"TORCH_SDPA"}}}'
+```
+
+Backends may also accept backend-specific parameters via `extra`:
+
+```bash
+--diffusion-attention-config.per_role.self.backend SPARSE_BLOCK \
+--diffusion-attention-config.per_role.self.extra.block_size 128
+```
+
+### Programmatic API
+
+When constructing `OmniDiffusionConfig` directly:
+
+```python
+from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, OmniDiffusionConfig
+
+config = OmniDiffusionConfig(
+    attention_config=AttentionConfig(
+        default=AttentionSpec(backend="FLASH_ATTN"),
+        per_role={
+            "cross": AttentionSpec(backend="TORCH_SDPA"),
+        },
+    ),
+    ...,
+)
+```
+
+A plain dict is also accepted and normalized to `AttentionConfig`.
+
 ## SageAttention Installation
 
 vLLM-Omni expects SageAttention to be installed into the same Python environment as vLLM-Omni.
@@ -85,7 +143,21 @@ DIFFUSION_ATTENTION_BACKEND=SAGE_ATTN python examples/offline_inference/text_to_
     --output outputs/wan22_sage.mp4
 ```
 
-### Compare Against FlashAttention
+### Mixed backends across roles
+
+Use `FLASH_ATTN` for self-attention and `TORCH_SDPA` for cross-attention:
+
+```bash
+python examples/offline_inference/text_to_video/text_to_video.py \
+    --model Wan-AI/Wan2.2-TI2V-5B-Diffusers \
+    --prompt "A dog running across a field of golden wheat." \
+    --diffusion-attention-config.per_role.self.backend FLASH_ATTN \
+    --diffusion-attention-config.per_role.cross.backend TORCH_SDPA \
+    --tensor-parallel-size 2 \
+    --output outputs/wan22_mixed.mp4
+```
+
+### Compare against FlashAttention
 
 Unset the backend override, or explicitly use `FLASH_ATTN`:
 

docs/user_guide/diffusion/cpu_offload_diffusion.md

@@ -17,7 +17,7 @@ Both strategies use pinned memory for faster CPU-GPU transfers. The strategies a
 Model-level offloading implements mutual exclusion between DiT transformer and encoder modules using pre forward hooks:
 
 - **When encoders run**: DiT transformer is offloaded to CPU
-- **When DiT runs**: Encoders are offloaded to CPU
+- **When DiT runs**: Encoders are offloaded to CPU, if more than one dit models, only one loaded on GPU, others get offloaded to CPU.
 - **VAE**: Stays resident on GPU
 
 Before each module's forward pass, the hook automatically moves it to GPU while offloading the other module group to CPU. Transfers use pinned memory for speed.
@@ -38,14 +38,15 @@ vllm-omni serve diffusion Wan-AI/Wan2.2-T2V-A14B-Diffusers --enable-cpu-offload
 
 ### To Support a Model
 
-Implement the `SupportsModuleOffload` protocol to declare which
-submodules participate in offloading:
+Implement the `SupportsComponentDiscovery` protocol to declare which
+submodules serve as pipeline components (used by offloading, HSDP
+sharding, and other framework features):
 
 ```python
 from typing import ClassVar
-from vllm_omni.diffusion.models.interface import SupportsModuleOffload
+from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery
 
-class MyPipeline(nn.Module, SupportsModuleOffload):
+class MyPipeline(nn.Module, SupportsComponentDiscovery):
     _dit_modules: ClassVar[list[str]] = ["transformer"]
     _encoder_modules: ClassVar[list[str]] = ["text_encoder", "vision_model"]
     _vae_modules: ClassVar[list[str]] = ["vae"]
@@ -158,7 +159,7 @@ class Flux2Transformer2DModel(nn.Module):
 The offloader discovers pipeline components in two ways:
 
 1. **Protocol-based** (preferred): If the pipeline implements
-    `SupportsModuleOffload`, its `_dit_modules`, `_encoder_modules`,
+    `SupportsComponentDiscovery`, its `_dit_modules`, `_encoder_modules`,
     `_vae_modules`, and `_resident_modules` class variables are used
     directly.  All attribute names support dotted paths (e.g.
     `"pipe.transformer"`, `"bagel.time_embedder"`) for nested submodules.

docs/user_guide/diffusion/step_execution.md

@@ -8,6 +8,8 @@ implement the stepwise contract support it today.
 
 ## Quick Start
 
+### Python API
+
 ```python
 from vllm_omni import Omni
 from vllm_omni.inputs.data import OmniDiffusionSamplingParams
@@ -25,6 +27,18 @@ outputs = omni.generate(
 )
 ```
 
+### Serving
+
+```bash
+vllm serve Qwen/Qwen-Image --omni \
+  --port 8091 \
+  --step-execution \
+  --max-num-seqs 8
+```
+
+For serving, `--step-execution` enables the step-wise runtime. Continuous
+batching only becomes relevant when `--max-num-seqs > 1`.
+
 ## Supported Pipelines
 
 | Pipeline | Example models | Step execution |
@@ -32,9 +46,17 @@ outputs = omni.generate(
 | `QwenImagePipeline` | `Qwen/Qwen-Image`, `Qwen/Qwen-Image-2512` | Yes |
 | All other diffusion pipelines | `QwenImageEditPipeline`, `QwenImageEditPlusPipeline`, `QwenImageLayeredPipeline`, GLM-Image, Wan, Flux, etc. | No |
 
+!!! warning "Experimental continuous batching"
+    When `--step-execution` is enabled and `max_num_seqs > 1` is configured,
+    the step-wise path can batch
+    compatible requests together. This is experimental. Requests with
+    incompatible sampling parameters are intentionally kept in separate batches,
+    and `max_num_seqs=1` remains the conservative default.
+
 ## Current Limitations
 
-- `step_execution` currently supports `batch_size=1` only.
+- Continuous batching under `step_execution` is experimental and only batches
+  compatible requests.
 - `cache_backend` is not supported together with step execution.
 - Unsupported pipelines fail early during model loading.
 - Request-mode extras such as KV transfer are not wired into step mode yet.
@@ -45,6 +67,11 @@ Use step execution only when you specifically need the pipeline to run through
 its stepwise request state machine. For normal diffusion inference, leave it
 disabled unless your workflow depends on this mode.
 
+For Qwen-Image online serving, the usual progression is:
+
+- start with `--step-execution --max-num-seqs 1` if you only need the step-wise path
+- increase `--max-num-seqs` after that if you want the experimental compatible-request batching behavior
+
 If you are looking for general diffusion speedups, see
 [Diffusion Features Overview](../diffusion_features.md).
 
@@ -59,3 +86,7 @@ pipeline does not support step execution.
 If you want to add step execution support to a new diffusion pipeline, see the
 implementation guide:
 [Diffusion Step Execution Design](../../design/feature/diffusion_step_execution.md).
+
+If you also want that pipeline to participate in the experimental batched
+step-wise path, see:
+[Continuous Batching for Step-Wise Diffusion](../../design/feature/diffusion_continuous_batching.md).

docs/user_guide/diffusion_features.md

@@ -107,7 +107,7 @@ The following tables show which models support each feature:
 
 | Model | ⚡TeaCache | ⚡Cache-DiT | 🔀SP (Ulysses & Ring) | 🔀CFG-Parallel | 🔀Tensor-Parallel | 🔀HSDP | 💾CPU Offload (Layerwise) | 💾VAE-Patch-Parallel | 💾Quantization | 🔄Step Execution |
 |-------|:----------:|:-----------:|:---------------------:|:--------------:|:-----------------:|:------:|:------------------------:|:--------------------:|:--------------:|:----------------:|
-| **Bagel** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅  | ❌ | ❌ | ❌ |
+| **Bagel** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅  | ❌ | ❌ | ❌ |
 | **FLUX.1-dev** | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ |
 | **FLUX.1-schnell** | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ |
 | **FLUX.2-klein** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ |
@@ -127,8 +127,10 @@ The following tables show which models support each feature:
 | **Qwen-Image-Edit** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ |
 | **Qwen-Image-Edit-2509** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ✅ | ❌ | ❌ |
 | **Qwen-Image-Layered** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ |
+| **SenseNova-U1** | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ |
 | **Stable-Diffusion3.5** | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ (decode) | ❌ | ❌ |
 | **Z-Image** | ✅ | ✅ | ✅ | ❓ | ✅ (TP=2 only) | ✅ | ❌ | ✅ (decode) | ✅ | ❌ |
+| **ERNIE-Image** | ❌ | ✅ | ✅ | ❓ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
 
 > Notes:
 > 1. Nextstep_1(T2I) does not support cache acceleration methods such as TeaCache or Cache-DiT.
@@ -144,7 +146,7 @@ The following tables show which models support each feature:
 | **LTX-2.3** | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
 | **Helios** | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
 | **HunyuanVideo-1.5 T2V I2V** | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ✅ | ❌ |
-| **DreamID-Omni** | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
+| **DreamID-Omni** | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
 
 **Frame Interpolation Support**
 
@@ -155,7 +157,7 @@ The following tables show which models support each feature:
 
 | Model | ⚡TeaCache | ⚡Cache-DiT | 🔀SP (Ulysses & Ring) | 🔀CFG-Parallel | 🔀Tensor-Parallel | 🔀HSDP | 💾CPU Offload (Layerwise) | 💾VAE-Patch-Parallel | 💾Quantization | 🔄Step Execution |
 |-------|:----------:|:-----------:|:---------------------:|:--------------:|:-----------------:|:------:|:------------------------:|:--------------------:|:--------------:|:----------------:|
-| **Stable-Audio-Open** | ✅ | ❌ | ❓ | ❓ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ |
+| **Stable-Audio-Open** | ✅ | ❌ | ❓ | ❓ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ |
 
 
 ## Feature Compatibility

docs/user_guide/examples/offline_inference/cosyvoice3.md

@@ -1,81 +0,0 @@
-# CosyVoice3
-
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/cosyvoice3>.
-
-
-## Setup
-
-Install dependencies:
-```
-uv pip install -e .
-```
-
-> **Note:** This includes required libraries such as `soundfile`,
-> `onnxruntime`, `x-transformers`, and `einops` via
-> `requirements/common.txt` and platform-specific requirements files.
-
-Download the model snapshot:
-```
-from huggingface_hub import snapshot_download
-snapshot_download('FunAudioLLM/Fun-CosyVoice3-0.5B-2512', local_dir='pretrained_models/Fun-CosyVoice3-0.5B')
-```
-
-Add `config.json` in `pretrained_models/Fun-CosyVoice3-0.5B/`:
-```json
-{
-    "model_type": "cosyvoice3",
-    "architectures": [
-        "CosyVoice3Model"
-    ]
-}
-```
-
-> **Why `config.json` is required:**
-> `AutoConfig.register("cosyvoice3", CosyVoice3Config)` only registers a class mapping.
-> The loader still needs `model_type: "cosyvoice3"` from `config.json` to select that class.
-> If no `config.json` is present, model type cannot be inferred automatically.
-> If your downloaded checkpoint already includes a valid `config.json` with
-> `model_type: "cosyvoice3"`, this manual step can be skipped.
-
-Run the offline verification script:
-```
-python examples/offline_inference/cosyvoice3/verify_e2e_cosyvoice.py \
-  --model pretrained_models/Fun-CosyVoice3-0.5B \
-  --tokenizer pretrained_models/Fun-CosyVoice3-0.5B/CosyVoice-BlankEN
-```
-
-## Implementation Overview
-
-CosyVoice3 runs as a **2-stage Omni pipeline**.
-
-- **Stage 0** (`talker`) converts text + prompt audio to speech tokens.
-- **Stage 1** (`code2wav`) converts speech tokens + prompt features to waveform via flow matching and HiFiGAN.
-
-Key components live in `vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py`.
-
-- `CosyVoice3MultiModalProcessor` builds the multimodal inputs:
-    - Tokenizes `prompt` and `prompt_text`.
-    - Extracts speech tokens and mel features from the prompt audio.
-    - Extracts a speaker embedding.
-- `CosyVoice3Model` implements both stages:
-    - Stage 0 uses `CosyVoice3LM` and outputs speech tokens + conditioning features.
-    - Stage 1 runs the flow model (DiT-based CFM) and HiFiGAN to synthesize waveform.
-
-Pipeline topology lives in `vllm_omni/model_executor/models/cosyvoice3/pipeline.py`;
-runtime tunables (batch size, memory limits, sampling) live in
-`vllm_omni/deploy/cosyvoice3.yaml`. The deploy config auto-loads by
-HF `model_type` and defaults to `async_chunk: true` (shared-memory
-streaming). Pass `--no-async-chunk` on `vllm serve` to switch to the
-legacy sync path where stage 1 runs `text2flow` over the full
-speech-token sequence.
-
-- Stage 0 emits latent speech tokens.
-- Stage 1 consumes them via `sync_process_input_func` (sync mode) or the
-  shared-memory connector (async-chunk mode) and outputs audio.
-
-## Example materials
-
-??? abstract "verify_e2e_cosyvoice.py"
-    ``````py
-    --8<-- "examples/offline_inference/cosyvoice3/verify_e2e_cosyvoice.py"
-    ``````

docs/user_guide/examples/offline_inference/fish_speech.md

@@ -1,85 +0,0 @@
-# Fish Speech S2 Pro
-
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/fish_speech>.
-
-
-This directory contains an offline demo for running Fish Speech S2 Pro (`fishaudio/s2-pro`) with vLLM Omni. It supports text-only synthesis and voice cloning with reference audio.
-
-## Model Overview
-
-[Fish Speech S2 Pro](https://huggingface.co/fishaudio/s2-pro) is a 4B dual-AR text-to-speech model by FishAudio. It uses:
-
-- **Slow AR**: Generates semantic tokens autoregressively (Qwen3-based backbone)
-- **Fast AR**: Predicts residual codebook tokens from semantic tokens
-- **DAC Decoder**: Converts codec codes to 44.1 kHz audio waveform
-
-The model produces high-quality speech with voice cloning capabilities.
-
-## Prerequisites
-
-Install the `fish-speech` package for the DAC codec:
-
-```bash
-pip install fish-speech
-```
-
-## Setup
-
-Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup.
-
-## Quick Start
-
-### Text-Only Synthesis
-
-```bash
-python end2end.py --text "Hello, this is a test of the Fish Speech text to speech system."
-```
-
-Generated audio files are saved to `output_audio/` by default.
-
-### Voice Cloning
-
-Provide a reference audio file and its transcript to clone a voice:
-
-```bash
-python end2end.py \
-    --text "Hello, this is a cloned voice." \
-    --ref-audio /path/to/reference.wav \
-    --ref-text "Transcript of the reference audio."
-```
-
-## Streaming Mode
-
-Add `--streaming` to stream audio chunks progressively via `AsyncOmni` (requires `async_chunk: true` in the stage config):
-
-```bash
-python end2end.py --text "Hello, this is a test." --streaming
-```
-
-Each DAC decoder chunk is logged as it arrives. The final WAV file is written once generation completes.
-
-## Configuration
-
-| Argument | Default | Description |
-|----------|---------|-------------|
-| `--text` | `"Hello, this is a test..."` | Text to synthesize |
-| `--ref-audio` | None | Path to reference audio for voice cloning |
-| `--ref-text` | None | Transcript of the reference audio |
-| `--model` | `fishaudio/s2-pro` | HuggingFace model path |
-| `--stage-configs-path` | `fish_speech_s2_pro.yaml` | Path to stage configs YAML |
-| `--output-dir` | `output_audio` | Output directory for WAV files |
-| `--streaming` | False | Enable streaming via AsyncOmni |
-| `--stage-init-timeout` | 600 | Stage initialization timeout (seconds) |
-
-## Notes
-
-- Output audio is 44.1 kHz mono WAV.
-- The DAC codec weights (`codec.pth`) are loaded lazily from the model directory.
-- Voice cloning encodes the reference audio using the DAC codec to extract semantic codes, then prepends them as a system message.
-
-## Example materials
-
-??? abstract "end2end.py"
-    ``````py
-    --8<-- "examples/offline_inference/fish_speech/end2end.py"
-    ``````

docs/user_guide/examples/offline_inference/qwen3_tts.md

@@ -1,172 +0,0 @@
-# Qwen3-TTS
-
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/qwen3_tts>.
-
-
-This directory contains an offline demo for running Qwen3 TTS models with vLLM Omni. It builds task-specific inputs and generates WAV files locally.
-
-## Model Overview
-
-Qwen3 TTS provides multiple task variants for speech generation:
-
-- **CustomVoice**: Generate speech with a known speaker identity (speaker ID) and optional instruction.
-- **VoiceDesign**: Generate speech from text plus a descriptive instruction that designs a new voice.
-- **Base**: Voice cloning using a reference audio + reference transcript, with optional mode selection. The `ref_audio` field accepts a local file path, HTTP URL, or base64 data URL.
-
-## Setup
-Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup.
-
-### ROCm Dependencies
-
-You will need to install the dependency `onnxruntime-rocm`.
-
-```
-pip uninstall onnxruntime # should be removed before we can install onnxruntime-rocm
-pip install onnxruntime-rocm
-```
-
-## Quick Start
-
-Run a single sample for a task:
-
-```
-python end2end.py --query-type CustomVoice
-```
-
-Generated audio files are saved to `output_audio/` by default.
-
-## Task Usage
-
-### CustomVoice
-
-Single sample:
-
-```
-python end2end.py --query-type CustomVoice
-```
-
-Batch sample (multiple prompts in one run):
-
-```
-python end2end.py --query-type CustomVoice --use-batch-sample
-```
-
-### VoiceDesign
-
-Single sample:
-
-```
-python end2end.py --query-type VoiceDesign
-```
-
-Batch sample:
-
-```
-python end2end.py --query-type VoiceDesign --use-batch-sample
-```
-
-### Base (Voice Clone)
-
-Single sample:
-
-```
-python end2end.py --query-type Base
-```
-
-Batch sample:
-
-```
-python end2end.py --query-type Base --use-batch-sample
-```
-
-Mode selection for Base:
-
-- `--mode-tag icl` (default): standard mode
-- `--mode-tag xvec_only`: enable `x_vector_only_mode` in the request
-
-Examples:
-
-```
-python end2end.py --query-type Base --mode-tag icl
-```
-
-## Voice and Language Control
-
-### Supported Voices (CustomVoice)
-
-Predefined speaker voices are set via the `speaker` (or `voice_type`) field in `additional_information`. Available speakers depend on the loaded checkpoint; check `talker_config.spk_id` in the model config for the full list. Common voices include `vivian`, `ryan`, `aiden`, `ethan`, `serena` (case-insensitive).
-
-Pass the speaker name in your request:
-
-```python
-additional_information = {
-    "text": ["你好，我是通义千问"],
-    "task_type": ["CustomVoice"],
-    "speaker": ["Vivian"],  
-    "language": ["Chinese"],
-}
-```
-
-### Supported Languages
-
-The `language` field controls the codec-level language tag. Use `"Auto"` (default) for automatic detection.
-
-Supported values: `Auto`, `Chinese`, `English`, `Japanese`, `Korean`, `German`, `French`, `Russian`, `Portuguese`, `Spanish`, `Italian`.
-
-```python
-additional_information = {
-    "text": ["Hello, nice to meet you."],
-    "task_type": ["CustomVoice"],
-    "speaker": ["Aiden"],
-    "language": ["English"],
-}
-```
-
-### VoiceDesign and Base
-
-- **VoiceDesign**: Use `instruct` for natural-language voice description; no `speaker` needed.
-- **Base**: Use `ref_audio` and `ref_text` for voice cloning; `language` is optional.
-
-## Streaming Mode
-
-Add `--streaming` to stream audio chunks progressively via `AsyncOmni` (requires `async_chunk: true` in the stage config):
-
-```bash
-python end2end.py --query-type CustomVoice --streaming --output-dir /tmp/out_stream
-```
-
-Each Code2Wav chunk is logged as it arrives (default 25 frames; configurable via `codec_chunk_frames`
-in the stage config). The initial chunk size is dynamically selected based on server load for reduced
-TTFA, and can be overridden per-request via the `initial_codec_chunk_frames` API field. The final WAV file is written once generation
-completes. This demonstrates that audio data is available progressively rather than only at the end.
-
-> **Note:** Streaming uses `AsyncOmni` internally. The non-streaming path (`Omni`) is unchanged.
-
-## Batched Decoding
-
-The Code2Wav stage (stage 1) supports batched decoding, where multiple requests are decoded in a single forward pass through the SpeechTokenizer. To use it, set `max_num_seqs > 1` on both stages via `--stage-overrides` and pass multiple prompts via `--txt-prompts` with a matching `--batch-size`.
-
-```
-python end2end.py --query-type CustomVoice \
-    --txt-prompts benchmark_prompts.txt \
-    --batch-size 4 \
-    --stage-overrides '{"0":{"max_num_seqs":4,"gpu_memory_utilization":0.2},"1":{"max_num_seqs":4,"gpu_memory_utilization":0.2}}'
-```
-
-**Important:** `--batch-size` must match a CUDA graph capture size (1, 2, 4, 8, 16...) because the Talker's code predictor KV cache is sized to `max_num_seqs`, and CUDA graphs pad the batch to the next capture size. Both stages need `max_num_seqs >= batch_size` in the stage config for batching to take effect. If only stage 1 has a higher `max_num_seqs`, it won't help — stage 1 can only batch chunks from requests that are in-flight simultaneously, which requires stage 0 to also process multiple requests concurrently.
-
-## Notes
-
-- The script uses the model paths embedded in `end2end.py`. Update them if your local cache path differs.
-- Use `--output-dir` to change the output folder.
-
-## Example materials
-
-??? abstract "benchmark_prompts.txt"
-    ``````txt
-    --8<-- "examples/offline_inference/qwen3_tts/benchmark_prompts.txt"
-    ``````
-??? abstract "end2end.py"
-    ``````py
-    --8<-- "examples/offline_inference/qwen3_tts/end2end.py"
-    ``````

docs/user_guide/examples/offline_inference/text_to_audio.md

@@ -57,6 +57,8 @@ Key arguments:
 - `--hsdp-shard-size`: number of GPUs used for HSDP sharding.
 - `--hsdp-replicate-size`: number of HSDP replica groups.
 - `--output`: path to save the generated WAV file.
+- `--enable-cpu-offload`: enabling model-wise offloading to save gpu memory
+- `--enable-layerwise-offload`: enabling layerwise offloading to save gpu memory
 
 ## Example materials
 

docs/user_guide/examples/offline_inference/text_to_speech.md

@@ -0,0 +1,395 @@
+# Text-To-Speech
+
+Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_speech>.
+
+
+vLLM-Omni supports several autoregressive TTS models. They share a common
+CLI shape (`--text`, `--ref-audio`, `--ref-text`, `--output-dir`) and live
+together in this hub. Each model has its own subdirectory containing a
+single `end2end.py` script; this README is the single doc entry point.
+
+For online serving, see `examples/online_serving/<model>/`. For the full
+list of supported architectures across all modalities, see
+[Supported Models](https://github.com/vllm-project/vllm-omni/tree/main/docs/models/supported_models.md).
+
+## Supported Models
+
+| Model | HuggingFace repo | Stages | Voice cloning | Streaming | Special modes | Sample rate |
+|---|---|---|---|---|---|---|
+| VoxCPM2 | `openbmb/VoxCPM2` | single (native AR) | ✓ | — | continuation (`--ref-audio` + `--ref-text`) | 48 kHz |
+| VoxCPM | local model dir | split | ✓ | ✓ (`voxcpm_async_chunk.yaml`) | — | 24 kHz |
+| CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | 2 (talker + code2wav) | ✓ | ✓ (`async_chunk: true` default) | — | 22.05 kHz |
+| Fish Speech S2 Pro | `fishaudio/s2-pro` | dual-AR | ✓ | ✓ (`--streaming`) | — | 44.1 kHz |
+| OmniVoice | `k2-fsa/OmniVoice` | 2 (gen + dec) | ✓ | — | voice design (`--instruct`), language (`--lang`) | 24 kHz |
+| Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | 2 (talker + code2wav) | ✓ (Base) | ✓ | 3 task variants (`--query-type`) | 24 kHz |
+| Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | varies | ✓ | ✓ | voice presets (`--voice`) | 24 kHz |
+
+## Common Quick Start
+
+Most models share this invocation shape:
+
+```bash
+python examples/offline_inference/text_to_speech/<model>/end2end.py \
+    --text "Hello, this is a test." \
+    --ref-audio /path/to/reference.wav \
+    --ref-text  "Transcript of the reference audio."
+```
+
+`--ref-audio` and `--ref-text` are optional (text-only synthesis works without them) and must be provided together for voice cloning. The exotic scripts — Qwen3-TTS, Voxtral TTS, CosyVoice3 — accept additional model-specific flags documented in their per-model section below.
+
+---
+
+## VoxCPM2
+
+Single-stage native AR TTS at 48 kHz. Pipeline: `feat_encoder → MiniCPM4 → FSQ → residual_lm → LocDiT → AudioVAE`.
+
+### Prerequisites
+```bash
+pip install voxcpm
+# or, for a local source checkout:
+export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/voxcpm
+```
+
+### Quick start
+```bash
+python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \
+    --model openbmb/VoxCPM2 \
+    --text "Hello, this is a VoxCPM2 demo."
+```
+
+### Voice cloning
+Pass a reference audio for isolated cloning, or both `--ref-audio` + `--ref-text` for prompt continuation:
+```bash
+python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \
+    --text "Hello, this is a voice clone demo." \
+    --ref-audio /path/to/reference.wav \
+    --ref-text  "Transcript of the reference audio."
+```
+
+### Notes
+- Output: 48 kHz mono WAV.
+- Stage config: `vllm_omni/model_executor/stage_configs/voxcpm2.yaml` (default).
+
+---
+
+## VoxCPM
+
+Split-stage TTS. The hub example covers single TTS, single voice cloning, and streaming. Use `benchmarks/voxcpm/` for warmup, batch JSONL prompts, profiler injection, and offline TTFP / RTF measurement.
+
+### Prerequisites
+```bash
+pip install voxcpm soundfile
+# or use a local source tree:
+export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/VoxCPM/src
+```
+
+If the native VoxCPM `config.json` does not contain HF metadata such as `model_type`, prepare a persistent HF-compatible config directory and point the stage configs to it via `VLLM_OMNI_VOXCPM_HF_CONFIG_PATH`:
+
+```bash
+export VOXCPM_MODEL=/path/to/voxcpm-model
+export VLLM_OMNI_VOXCPM_HF_CONFIG_PATH=/tmp/voxcpm_hf_config
+mkdir -p "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"
+cp "$VOXCPM_MODEL/config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/config.json"
+cp "$VOXCPM_MODEL/generation_config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/generation_config.json" 2>/dev/null || true
+python3 -c 'import json, os; p=os.path.join(os.environ["VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"], "config.json"); cfg=json.load(open(p, "r", encoding="utf-8")); cfg["model_type"]="voxcpm"; cfg.setdefault("architectures", ["VoxCPMForConditionalGeneration"]); json.dump(cfg, open(p, "w", encoding="utf-8"), indent=2, ensure_ascii=False)'
+```
+
+### Quick start
+```bash
+python examples/offline_inference/text_to_speech/voxcpm/end2end.py \
+    --model "$VOXCPM_MODEL" \
+    --text "This is a split-stage VoxCPM synthesis example running on vLLM Omni."
+```
+
+### Voice cloning
+```bash
+python examples/offline_inference/text_to_speech/voxcpm/end2end.py \
+    --model "$VOXCPM_MODEL" \
+    --text "This sentence is synthesized with a cloned voice." \
+    --ref-audio /path/to/reference.wav \
+    --ref-text  "The exact transcript spoken in reference.wav."
+```
+
+### Streaming
+Pass the async-chunk stage config:
+```bash
+python examples/offline_inference/text_to_speech/voxcpm/end2end.py \
+    --model "$VOXCPM_MODEL" \
+    --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \
+    --text "This is a split-stage VoxCPM streaming example running on vLLM Omni."
+```
+
+### Notes
+- `voxcpm.yaml` is the default non-streaming stage config; `voxcpm_async_chunk.yaml` enables streaming.
+- Streaming is currently single-request oriented.
+- `--ref-text` must be the real transcript of `--ref-audio`; mismatched text degrades quality.
+- For online serving, see [`examples/online_serving/voxcpm`](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/voxcpm/README.md). For benchmark reporting, see [`benchmarks/voxcpm`](https://github.com/vllm-project/vllm-omni/tree/main/benchmarks/voxcpm/README.md).
+
+---
+
+## CosyVoice3
+
+2-stage TTS pipeline (`talker` + `code2wav`) at 22.05 kHz.
+
+### Prerequisites
+```bash
+uv pip install -e .
+# Includes soundfile, onnxruntime, x-transformers, einops via requirements.
+```
+
+Download the model snapshot:
+```python
+from huggingface_hub import snapshot_download
+snapshot_download('FunAudioLLM/Fun-CosyVoice3-0.5B-2512',
+                  local_dir='pretrained_models/Fun-CosyVoice3-0.5B')
+```
+
+If your downloaded checkpoint lacks `config.json`, add it:
+```json
+{
+    "model_type": "cosyvoice3",
+    "architectures": ["CosyVoice3Model"]
+}
+```
+This is required because `AutoConfig.register("cosyvoice3", CosyVoice3Config)` only registers the class mapping; the loader still reads `model_type` from `config.json` to select the class.
+
+### Quick start
+```bash
+python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
+    --model pretrained_models/Fun-CosyVoice3-0.5B \
+    --tokenizer pretrained_models/Fun-CosyVoice3-0.5B/CosyVoice-BlankEN
+```
+
+### Voice cloning
+Pass a reference audio. Note that CosyVoice3's `--prompt-text` is a system-style prompt for the GPT stage, not a reference transcript:
+```bash
+python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
+    --model pretrained_models/Fun-CosyVoice3-0.5B \
+    --tokenizer pretrained_models/Fun-CosyVoice3-0.5B/CosyVoice-BlankEN \
+    --ref-audio prompt.wav \
+    --prompt-text "You are a helpful assistant.<|endofprompt|>Testing my voices. Why should I not?"
+```
+
+### Notes
+- Stage 0 (`talker`) emits speech tokens; stage 1 (`code2wav`) runs flow matching + HiFiGAN to synthesize waveform.
+- Deploy config auto-loads from `vllm_omni/deploy/cosyvoice3.yaml` based on HF `model_type`. Pass `--deploy-config <path>` to override.
+- `async_chunk: true` is the default; pass `--no-async-chunk` to switch to the legacy synchronous path.
+
+---
+
+## Fish Speech S2 Pro
+
+4B dual-AR text-to-speech model from FishAudio with the DAC codec at 44.1 kHz.
+
+### Prerequisites
+```bash
+pip install fish-speech
+```
+
+### Quick start
+```bash
+python examples/offline_inference/text_to_speech/fish_speech/end2end.py \
+    --text "Hello, this is a test of the Fish Speech text to speech system."
+```
+
+### Voice cloning
+```bash
+python examples/offline_inference/text_to_speech/fish_speech/end2end.py \
+    --text "Hello, this is a cloned voice." \
+    --ref-audio /path/to/reference.wav \
+    --ref-text  "Transcript of the reference audio."
+```
+
+### Streaming
+```bash
+python examples/offline_inference/text_to_speech/fish_speech/end2end.py \
+    --text "Hello, this is a streaming test." \
+    --streaming
+```
+Streaming requires `async_chunk: true` in the stage config.
+
+### Notes
+- Output: 44.1 kHz mono WAV.
+- DAC codec weights (`codec.pth`) are loaded lazily from the model directory.
+
+---
+
+## OmniVoice
+
+Zero-shot multilingual TTS supporting 600+ languages, with three modes (auto / clone / design).
+
+### Prerequisites
+```bash
+huggingface-cli download k2-fsa/OmniVoice
+```
+Voice cloning requires `transformers>=5.3.0`. Auto and design modes work with `transformers>=4.57.0`.
+
+### Quick start (auto voice)
+```bash
+python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
+    --model k2-fsa/OmniVoice \
+    --text "Hello, this is a test."
+```
+
+### Voice cloning
+```bash
+python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
+    --model k2-fsa/OmniVoice \
+    --text "Hello, this is a test." \
+    --ref-audio ref.wav \
+    --ref-text  "This is the reference transcription."
+```
+
+### Voice design
+```bash
+python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
+    --model k2-fsa/OmniVoice \
+    --text "Hello, this is a test." \
+    --instruct "female, low pitch, british accent"
+```
+
+### Language hint
+```bash
+python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
+    --model k2-fsa/OmniVoice \
+    --text "你好，这是一个测试。" \
+    --lang zh
+```
+
+### Notes
+- Stage 0 (Generator): Qwen3-0.6B with 32-step iterative unmasking.
+- Stage 1 (Decoder): HiggsAudioV2 RVQ + DAC at 24 kHz.
+
+---
+
+## Qwen3-TTS
+
+3-task-variant TTS with 24 kHz output. Has its own argparse surface (this script does not follow the common `--text` / `--ref-audio` shape).
+
+### Prerequisites
+For ROCm builds, replace `onnxruntime` with `onnxruntime-rocm`:
+```bash
+pip uninstall onnxruntime
+pip install onnxruntime-rocm
+```
+
+### Task variants
+- `CustomVoice`: predefined speaker (speaker ID) with optional style instruction.
+- `VoiceDesign`: text + descriptive instruction designs a new voice.
+- `Base`: voice cloning from reference audio + transcript.
+
+```bash
+# Single sample
+python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py --query-type CustomVoice
+python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py --query-type VoiceDesign
+python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py --query-type Base
+
+# Base variant has an additional mode flag:
+python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py --query-type Base --mode-tag icl       # default
+python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py --query-type Base --mode-tag xvec_only # x_vector_only_mode
+
+# Batch (multiple prompts in one run)
+python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py --query-type CustomVoice --use-batch-sample
+```
+
+### Streaming
+```bash
+python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
+    --query-type CustomVoice \
+    --streaming \
+    --output-dir /tmp/out_stream
+```
+Streaming requires `async_chunk: true` in the stage config.
+
+### Batched decoding
+The Code2Wav stage supports batched decoding through the SpeechTokenizer. Configure both stages with `max_num_seqs > 1` via `--stage-overrides` and pass multiple prompts via `--txt-prompts`:
+```bash
+python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
+    --query-type CustomVoice \
+    --txt-prompts examples/offline_inference/text_to_speech/qwen3_tts/benchmark_prompts.txt \
+    --batch-size 4 \
+    --stage-overrides '{"0":{"max_num_seqs":4,"gpu_memory_utilization":0.2},"1":{"max_num_seqs":4,"gpu_memory_utilization":0.2}}'
+```
+`--batch-size` must match a CUDA-graph capture size (1, 2, 4, 8, 16…).
+
+### Notes
+- Run `--help` for the full argument surface.
+- See `qwen3_tts/end2end.py` for the prompt-length-estimation logic the Talker uses.
+
+---
+
+## Voxtral TTS
+
+Voxtral-4B-TTS (Mistral). Has its own argparse surface; uses voice presets and the `mistral_common` `SpeechRequest` protocol.
+
+### Prerequisites
+Latest `mistral_common` with `SpeechRequest` support:
+```bash
+pip install -e /path/to/mistral-common  # or upgrade from PyPI when available
+```
+
+### Quick start (voice preset)
+```bash
+python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
+    --write-audio --voice cheerful_female \
+    --model mistralai/Voxtral-4B-TTS-2603 \
+    --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?"
+```
+
+### Voice cloning (capability gated upstream)
+```bash
+python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
+    --write-audio \
+    --model mistralai/Voxtral-4B-TTS-2603 \
+    --text "This is a test message." \
+    --ref-audio path/to/reference_audio.wav
+```
+
+### Streaming + concurrency
+```bash
+python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
+    --num-prompts 32 --concurrency 8 --streaming --write-audio --voice neutral_female \
+    --model mistralai/Voxtral-4B-TTS-2603 \
+    --text "..."
+```
+Available voice presets are listed on the HF model card (`mistralai/Voxtral-4B-TTS-2603`).
+
+### Notes
+- `--num-prompts N` replicates the prompt for performance measurement.
+- `--concurrency M` requires `--streaming` and must evenly divide `--num-prompts`.
+- Run `--help` for the full argument surface.
+
+## Example materials
+
+??? abstract "cosyvoice3/end2end.py"
+    ``````py
+    --8<-- "examples/offline_inference/text_to_speech/cosyvoice3/end2end.py"
+    ``````
+??? abstract "fish_speech/end2end.py"
+    ``````py
+    --8<-- "examples/offline_inference/text_to_speech/fish_speech/end2end.py"
+    ``````
+??? abstract "omnivoice/end2end.py"
+    ``````py
+    --8<-- "examples/offline_inference/text_to_speech/omnivoice/end2end.py"
+    ``````
+??? abstract "qwen3_tts/benchmark_prompts.txt"
+    ``````txt
+    --8<-- "examples/offline_inference/text_to_speech/qwen3_tts/benchmark_prompts.txt"
+    ``````
+??? abstract "qwen3_tts/end2end.py"
+    ``````py
+    --8<-- "examples/offline_inference/text_to_speech/qwen3_tts/end2end.py"
+    ``````
+??? abstract "voxcpm/end2end.py"
+    ``````py
+    --8<-- "examples/offline_inference/text_to_speech/voxcpm/end2end.py"
+    ``````
+??? abstract "voxcpm2/end2end.py"
+    ``````py
+    --8<-- "examples/offline_inference/text_to_speech/voxcpm2/end2end.py"
+    ``````
+??? abstract "voxtral_tts/end2end.py"
+    ``````py
+    --8<-- "examples/offline_inference/text_to_speech/voxtral_tts/end2end.py"
+    ``````

docs/user_guide/examples/offline_inference/voxtral_tts.md

@@ -1,69 +0,0 @@
-# Voxtral TTS Offline Inference
-
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/voxtral_tts>.
-
-
-`end2end.py` runs Voxtral TTS end-to-end offline inference using vLLM. It supports both blocking (`Omni`) and streaming (`AsyncOmni`) generation, batched prompts with configurable concurrency, and voice selection via preset name or reference audio file.
-
-When `mistral_common` has `SpeechRequest` support, prompt token IDs are built via `encode_speech_request`. Otherwise, the script falls back to manual token construction.
-
-## Usage Examples
-
-
-```bash
-# Basic single-prompt with cheerful_female voice preset
-python3 examples/offline_inference/voxtral_tts/end2end.py \
-    --write-audio --voice cheerful_female \
-    --model mistralai/Voxtral-4B-TTS-2603 \
-    --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?"
-
-# 32 replicate prompts with cheerful_female voice preset
-python3 examples/offline_inference/voxtral_tts/end2end.py \
-    --num-prompts 32 --write-audio --voice cheerful_female \
-    --model mistralai/Voxtral-4B-TTS-2603 \
-    --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?"
-
-# Streaming with neutral_female voice preset
-python3 examples/offline_inference/voxtral_tts/end2end.py \
-    --streaming --write-audio --voice neutral_female \
-    --model mistralai/Voxtral-4B-TTS-2603 \
-    --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?"
-
-# 32 prompts, 8 concurrent requests per wave, streaming with neutral_female voice
-python3 examples/offline_inference/voxtral_tts/end2end.py \
-    --num-prompts 32 --concurrency 8 --streaming --write-audio --voice neutral_female \
-    --model mistralai/Voxtral-4B-TTS-2603 \
-    --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?"
-
-# Short debug prompt with reference audio
-# Note: Reference audio capability is not yet released.
-python3 examples/offline_inference/voxtral_tts/end2end.py \
-    --write-audio \
-    --model mistralai/Voxtral-4B-TTS-2603 \
-    --text "This is a test message." \
-    --audio-path path/to/reference_audio.wav
-```
-
-## Arguments
-
-| Argument | Description |
-|---|---|
-| `--model PATH` | HuggingFace repo ID or local directory path (default: `mistralai/Voxtral-4B-TTS-2603`) |
-| `--text TEXT` | Text to synthesize (default: `"This is a test message."`) |
-| `--audio-path PATH` | Path to reference audio file for voice cloning |
-| `--output-dir DIR` | Directory to write output WAV files (default: `output_audio`) |
-| `--deploy-config PATH` | Override the deploy config path. If unset, auto-loads `vllm_omni/deploy/voxtral_tts.yaml` from the HF `model_type`. |
-| `--num-prompts N` | Number of replicate prompts to run for measuring performance (default: 1) |
-| `--streaming` | Use streaming generation via `AsyncOmni` (default: blocking `Omni`) |
-| `--concurrency N` | Max concurrent requests per wave (must be used with `--streaming`, must evenly divide `--num-prompts`) |
-| `--voice NAME` | Voice preset to use instead of reference audio file (e.g., casual_female, casual_male, cheerful_female, neutral_female, neutral_male) |
-| `--write-audio` | Write generated audio to WAV files |
-| `--profiling-mode` | Enable profiling mode (reduces max tokens to 50) |
-| `--log-stats` | Enable detailed statistics logging |
-
-## Example materials
-
-??? abstract "end2end.py"
-    ``````py
-    --8<-- "examples/offline_inference/voxtral_tts/end2end.py"
-    ``````

docs/user_guide/examples/offline_inference/x_to_video_audio.md

@@ -47,6 +47,24 @@ python x_to_video_audio.py \
 ```
 In the current test scenario (2 images + 2 audio inputs), the VRAM requirement is 72GB, regardless of whether cfg-parallel is enabled or disabled.
 The VRAM usage can be reduced by enabling CPU offload via --enable-cpu-offload.
+For multi-GPU memory reduction on the fused DreamID-Omni transformer, you can also enable HSDP:
+
+```python
+python x_to_video_audio.py \
+  --model /path/to/dreamid_omni \
+  --prompt "Two people walking together and singing happily" \
+  --image-path ./example0.png ./example1.png \
+  --audio-path ./example0.wav ./example1.wav \
+  --video-negative-prompt "jitter, bad hands, blur, distortion" \
+  --audio-negative-prompt "robotic, muffled, echo, distorted" \
+  --cfg-parallel-size 2 \
+  --num-inference-steps 45 \
+  --height 704 \
+  --width 1280 \
+  --use-hsdp \
+  --hsdp-shard-size 2 \
+  --output out_dreamid_omni_twoip.mp4
+```
 
 
 You could take reference images/audios from the test cases in the official repo: https://github.com/Guoxu1233/DreamID-Omni
@@ -77,6 +95,9 @@ Key arguments:
 - `--image-path`: path to the input image list.
 - `--audio-path`: path to the input audio list, indicate the timbre of the output video.
 - `--cfg-parallel-size`: number of parallel cfg parallel (defaults 1).
+- `--use-hsdp`: enable HSDP weight sharding for DreamID-Omni fused blocks.
+- `--hsdp-shard-size`: number of GPUs used for HSDP sharding.
+- `--hsdp-replicate-size`: number of HSDP replica groups.
 - `--num-inference-steps`: number of denoising steps (defaults 45).
 - `--video-negative-prompt`: negative prompt for video generation.
 - `--audio-negative-prompt`: negative prompt for audio generation.

docs/user_guide/examples/online_serving/diffusers_pipeline_adapter.md

@@ -1,18 +1,14 @@
-# Diffusers Backend Adapter Example
+# Diffusers Backend Adapter
 
 Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/diffusers_pipeline_adapter>.
 
 
-This example demonstrates how to serve any 🤗 Diffusers pipeline through vLLM-Omni
-using the `diffusers` load format.
-
-## Supported Models
-
-Any model loadable via `DiffusionPipeline.from_pretrained()` should be supported, including text-to-image, image-to-image, text-to-video, image-to-video, and text-to-audio.
+vLLM-Omni supports running diffusion models with the diffusers backend, directly serving any 🤗 Diffusers pipeline online without implementing them natively.
 
 ## Limitations
 
-The diffusers backend is a black-box adapter. The following features are NOT yet supported.
+The diffusers backend is a black-box adapter. Its primary focus is to serve diffusion models online.
+Currently, the following features are NOT yet supported.
 It is not guaranteed whether they will be supported in the future.
 
 - CFG parallel execution
@@ -22,72 +18,87 @@ It is not guaranteed whether they will be supported in the future.
 
 For these features, it is recommended to use natively supported pipelines instead.
 
-## Usage
+## Model Support
 
-### Option 1: CLI arguments
+Any model loadable via `DiffusionPipeline.from_pretrained()` should run, including text-to-image, image-to-image, text-to-video, image-to-video, and text-to-audio.
+
+However, as we strive to ensure output similarity between vLLM-Omni's diffuser backend and plain diffusers library, the following models are particularly verified:
+
+- Qwen/Qwen-Image
+- Tongyi-MAI/Z-Image-Turbo
+- Wan2.2-I2V-A14B-Diffusers
+
+If you find that a model not listed above also produces different outputs from running diffusers model directly.
+Please consider file an issue.
+
+## Usage
 
 ```bash
 vllm serve "stable-diffusion-v1-5/stable-diffusion-v1-5" \
     --omni \
-    --diffusion-load-format diffusers \
-    --diffusers-load-kwargs '{"use_safetensors": true}' \
-    --diffusers-call-kwargs '{"num_inference_steps": 30, "guidance_scale": 7.5}'
+    --diffusion-load-format diffusers
 ```
 
-`--diffusers-load-kwargs` and `--diffusers-call-kwargs` are only valid together with `--diffusion-load-format diffusers`.
+Users turn on the diffusers backend primarily through `--diffusion-load-format diffusers` argument.
+There are two more optional arguments, `--diffusers-load-kwargs` and `--diffusers-call-kwargs`,
+which are only valid together with `--diffusion-load-format diffusers`.
 
-### Option 2: Stage config YAML
-
-```bash
-vllm serve stable-diffusion-v1-5/stable-diffusion-v1-5 --stage-configs-path examples/online_serving/diffusers_pipeline_adapter/stage_config.yaml --omni
-```
-
-The particular fields of interest are `model`, `diffusion_load_format`, `diffusers_load_kwargs`, and `diffusers_call_kwargs` under `engine_args`. They are the same as the CLI arguments.
-
-## Send a Request
-
-```bash
-curl http://localhost:8000/v1/images/generations \
-  -H "Content-Type: application/json" \
-  -d '{
-    "model": "stable-diffusion-v1-5/stable-diffusion-v1-5",
-    "prompt": "a photo of an astronaut riding a horse on mars",
-    "n": 1,
-    "size": "512x512"
-  }'
-```
-
-Or refer to other documentation pages on how to request a particular input/output modality, such as `examples/online_serving/text_to_image/openai_chat_client.py`.
+After launching the model, users send a request as usual. Refer to other documentation pages on how to request a particular input/output modality, such as `examples/online_serving/text_to_image/openai_chat_client.py`.
 
 ## Configuration Reference
 
-For the diffusers adapter, set options under **`engine_args`**:
+### `--diffusers-load-kwargs`
 
-### `diffusion_load_format: "diffusers"`
+Passed as-is to `DiffusionPipeline.from_pretrained()`.
 
-This field selects the Hugging Face diffusers adapter path (see `DiffusersPipelineLoader`).
-
-### `diffusers_load_kwargs`
-
-Passed to `DiffusionPipeline.from_pretrained()`.
-
-This is suitable for model-specific configurations not available through the vLLM-Omni interface (such as `Omni.__init__()`, `vllm serve` CLI arguments, and stage config YAML fields outside `diffusers_load_kwargs`).
+This is suitable for model-specific configurations not available through the vLLM-Omni interface.
+For example: `--diffusers-load-kwargs '{"use_safetensors": true}'`.
 
 When a parameter is available in the vLLM-Omni interface, it will be adapted here.
 But if that parameter is simultaneously set in both the vLLM-Omni interface and `diffusers_load_kwargs`, the **latter** will take precedence.
 
-### `diffusers_call_kwargs`
+### `--diffusers-call-kwargs`
 
 Passed to `pipeline.__call__()`.
 
-This is suitable for sampling parameters not available through the vLLM-Omni interface (such as `Omni.generate()` and online serving payloads).
+This is suitable for sampling parameters not available through the vLLM-Omni interface (such as online serving payloads).
 
 When a parameter is available in the vLLM-Omni interface, it will be adapted here.
 But if that parameter is simultaneously set in both the vLLM-Omni interface and `diffusers_call_kwargs`, the **former** will take precedence (because it is set at request time).
 
-## Example materials
+### Attention Backends
 
-??? abstract "stage_config.yaml"
-    ``````yaml
-    --8<-- "examples/online_serving/diffusers_pipeline_adapter/stage_config.yaml"
-    ``````
+The diffusers backend converts
+[vLLM-Omni standard of attention backend setting](../../../docs/user_guide/diffusion/attention_backends.md)
+to [diffusers standard](https://huggingface.co/docs/diffusers/optimization/attention_backends#available-backends).
+
+Specifically for `FLASH_ATTN`, it will first attempt to use FlashAttention-3 and then FlashAttention-2.
+
+For each attempted version of `FLASH_ATTN` and `SAGE_ATTN`, it will first try to load the attention backend from HuggingFace `kernels` library, then without.
+
+For unsuccessful attention selection or `TORCH_SDPA`, it will use the PyTorch's default attention backend.
+
+The loaded attention backend and the failed attempts (if any) are logged to console.
+
+### Model Specific Settings
+
+The model loading and inference strictly follows the diffusers library, and they may be different from vLLM-Omni's native interface for some specific models.
+Users are encouraged to double-check the model pipeline's interface in [diffusers' official documentation](https://huggingface.co/docs/diffusers/api/pipelines/overview).
+Some particular examples are below.
+
+#### Wan Series
+
+The Wan series video generation models takes `boundary_ratio` and `flow_shift` during model initialization ([ref](https://huggingface.co/docs/diffusers/api/pipelines/wan)), not during inference.
+
+Since our `OmniDiffusionConfig` contains these two values ([source](https://github.com/vllm-project/vllm-omni/blob/main/vllm_omni/diffusion/data.py)), we can directly pass `--boundary-ratio` and `--flow-shift` arguments to `vllm serve` command.
+
+```bash
+vllm serve "Wan2.2-T2V-A14B-Diffusers" \
+    --omni \
+    --boundary-ratio 0.875 \
+    --flow-shift 3 \
+    --diffusion-load-format diffusers
+```
+
+These extra CLI args will be attempted to pass as-is to the `OmniDiffusionConfig` dataclass and being accessible during model loading time.
+Special routines inside the pipeline adapter ensures that they are set properly.

docs/user_guide/examples/online_serving/fish_speech.md

@@ -1,199 +0,0 @@
-# Fish Speech S2 Pro
-
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/fish_speech>.
-
-
-## Model
-
-| Model | Description |
-|-------|-------------|
-| `fishaudio/s2-pro` | Fish Speech S2 Pro -- 4B dual-AR TTS model with DAC codec (44.1 kHz) |
-
-## Gradio Demo
-
-!!! note "Gradio is an optional dependency"
-    The Gradio demo requires the `[demo]` extras. Install them first:
-
-    ```bash
-    pip install 'vllm-omni[demo]'
-    ```
-
-    Or, if installing from source: `pip install -e '.[demo]'`
-
-An interactive Gradio demo is available with text-to-speech synthesis, voice cloning, and streaming support.
-
-```bash
-# Option 1: Launch server + Gradio together
-./run_gradio_demo.sh
-
-# Option 2: If server is already running
-python gradio_demo.py --api-base http://localhost:8091
-```
-
-Then open http://localhost:7860 in your browser.
-
-Features:
-
-- Text-to-speech synthesis
-- Voice cloning from uploaded audio or URL
-- Streaming mode (progressive PCM playback)
-
-## Launch the Server
-
-```bash
-vllm serve fishaudio/s2-pro --omni --port 8091
-```
-
-The deploy config is auto-loaded from `vllm_omni/deploy/fish_qwen3_omni.yaml`.
-
-Or use the convenience script:
-
-```bash
-./run_server.sh
-```
-
-## Send TTS Request
-
-### Using curl
-
-```bash
-# Basic TTS
-curl -X POST http://localhost:8091/v1/audio/speech \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "Hello, how are you?",
-        "voice": "default",
-        "response_format": "wav"
-    }' --output output.wav
-```
-
-### Voice Cloning
-
-Provide a reference audio (URL or base64 data URL) and its transcript:
-
-```bash
-curl -X POST http://localhost:8091/v1/audio/speech \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "Hello, this is a cloned voice.",
-        "voice": "default",
-        "ref_audio": "https://example.com/reference.wav",
-        "ref_text": "Transcript of the reference audio."
-    }' --output cloned.wav
-```
-
-### Using Python
-
-```python
-import httpx
-
-# Basic TTS
-response = httpx.post(
-    "http://localhost:8091/v1/audio/speech",
-    json={
-        "input": "Hello, how are you?",
-        "voice": "default",
-        "response_format": "wav",
-    },
-    timeout=300.0,
-)
-
-with open("output.wav", "wb") as f:
-    f.write(response.content)
-```
-
-### Using the CLI Client
-
-```bash
-cd examples/online_serving/fish_speech
-
-# Basic TTS
-python speech_client.py --text "Hello, how are you?"
-
-# Voice cloning
-python speech_client.py \
-    --text "Hello, this is a cloned voice." \
-    --ref-audio /path/to/reference.wav \
-    --ref-text "Transcript of the reference audio."
-
-# Streaming PCM output
-python speech_client.py --text "Hello world" --stream --output output.pcm
-```
-
-The CLI client supports:
-
-- `--api-base`: API base URL (default: `http://localhost:8091`)
-- `--model` (or `-m`): Model name (default: `fishaudio/s2-pro`)
-- `--text`: Text to synthesize (required)
-- `--ref-audio`: Reference audio for voice cloning (local path or URL)
-- `--ref-text`: Transcript of the reference audio
-- `--stream`: Enable streaming (PCM output)
-- `--response-format`: Audio format: wav, mp3, flac, pcm, aac, opus (default: wav)
-- `--output` (or `-o`): Output file path
-
-## Streaming
-
-Set `stream=true` with `response_format="pcm"` to receive raw PCM audio chunks as they are decoded:
-
-```bash
-curl -X POST http://localhost:8091/v1/audio/speech \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "Hello, how are you?",
-        "voice": "default",
-        "stream": true,
-        "response_format": "pcm"
-    }' --no-buffer | play -t raw -r 44100 -e signed -b 16 -c 1 -
-```
-
-**Note:** Fish Speech outputs at 44.1 kHz (unlike Qwen3-TTS which outputs at 24 kHz).
-
-## API Parameters
-
-Fish Speech uses the same `/v1/audio/speech` endpoint as Qwen3-TTS. See the [Speech API reference](https://docs.vllm.ai/projects/vllm-omni/en/latest/serving/speech_api/) for full parameter documentation.
-
-Key parameters for Fish Speech:
-
-| Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
-| `input` | string | **required** | Text to synthesize |
-| `voice` | string | "default" | Voice name (use "default" for Fish Speech) |
-| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, aac, opus |
-| `ref_audio` | string | null | Reference audio URL or base64 data URL for voice cloning |
-| `ref_text` | string | null | Transcript of reference audio (required for voice cloning) |
-| `max_new_tokens` | int | 4096 | Maximum tokens to generate |
-| `stream` | bool | false | Stream raw PCM chunks |
-
-## Prerequisites
-
-Install the `fish-speech` package for the DAC codec:
-
-```bash
-pip install fish-speech
-```
-
-## Troubleshooting
-
-1. **No audio output**: Make sure the `fish-speech` package is installed for the DAC decoder
-2. **Connection refused**: Ensure the server is running on the correct port
-3. **Flashinfer version mismatch**: Set `FLASHINFER_DISABLE_VERSION_CHECK=1` if you see version warnings
-4. **Out of memory**: Reduce `--gpu-memory-utilization` or use a GPU with more VRAM
-
-## Example materials
-
-??? abstract "gradio_demo.py"
-    ``````py
-    --8<-- "examples/online_serving/fish_speech/gradio_demo.py"
-    ``````
-??? abstract "run_gradio_demo.sh"
-    ``````sh
-    --8<-- "examples/online_serving/fish_speech/run_gradio_demo.sh"
-    ``````
-??? abstract "run_server.sh"
-    ``````sh
-    --8<-- "examples/online_serving/fish_speech/run_server.sh"
-    ``````
-??? abstract "speech_client.py"
-    ``````py
-    --8<-- "examples/online_serving/fish_speech/speech_client.py"
-    ``````

docs/user_guide/examples/online_serving/qwen3_tts.md

@@ -1,451 +0,0 @@
-# Qwen3-TTS
-
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/qwen3_tts>.
-
-
-## 🛠️ Installation
-
-Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/README.md)
-
-## Supported Models
-
-| Model                                  | Task Type   | Description                                           |
-| -------------------------------------- | ----------- | ----------------------------------------------------- |
-| `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | CustomVoice | Predefined speaker voices with optional style control |
-| `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | VoiceDesign | Natural language voice style description              |
-| `Qwen/Qwen3-TTS-12Hz-1.7B-Base`        | Base        | Voice cloning from reference audio                    |
-| `Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice` | CustomVoice | Smaller/faster variant                                |
-| `Qwen/Qwen3-TTS-12Hz-0.6B-Base`        | Base        | Smaller/faster variant for voice cloning              |
-
-## Gradio Demo
-
-!!! note "Gradio is an optional dependency"
-    The Gradio demo requires the `[demo]` extras. Install them first:
-
-    ```bash
-    pip install 'vllm-omni[demo]'
-    ```
-
-    Or, if installing from source: `pip install -e '.[demo]'`
-
-Two interactive Gradio demos are available, both supporting all 3 task types:
-
-| Demo     | File                     | Transport    | Streaming Quality                                  |
-| -------- | ------------------------ | ------------ | -------------------------------------------------- |
-| Standard | `gradio_demo.py`         | HTTP chunked | May have small gaps between chunks                 |
-| FastRTC  | `gradio_fastrtc_demo.py` | WebRTC       | Gapless streaming (requires `pip install fastrtc`) |
-
-```bash
-# Option 1: Launch server + Standard Gradio together
-./run_gradio_demo.sh                                # CustomVoice (default)
-./run_gradio_demo.sh --task-type VoiceDesign        # VoiceDesign
-./run_gradio_demo.sh --task-type Base               # Voice cloning
-
-# Option 2: If server is already running
-python gradio_demo.py --api-base http://localhost:8000
-
-# Option 3: FastRTC demo (gapless streaming)
-pip install fastrtc
-python gradio_fastrtc_demo.py --api-base http://localhost:8000
-```
-
-Then open http://localhost:7860 in your browser.
-
-## Run examples (Qwen3-TTS)
-
-### Launch the Server
-
-```bash
-# CustomVoice model (predefined speakers)
-vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
-    --deploy-config vllm_omni/deploy/qwen3_tts.yaml \
-    --omni \
-    --port 8091 \
-    --trust-remote-code \
-    --enforce-eager
-
-# VoiceDesign model
-vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign \
-    --deploy-config vllm_omni/deploy/qwen3_tts.yaml \
-    --omni \
-    --port 8091 \
-    --trust-remote-code \
-    --enforce-eager
-
-# Base model (voice cloning)
-vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-Base \
-    --deploy-config vllm_omni/deploy/qwen3_tts.yaml \
-    --omni \
-    --port 8091 \
-    --trust-remote-code \
-    --enforce-eager
-```
-
-If you have custom stage configs file, launch the server with command below
-```bash
-vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
-    --stage-configs-path /path/to/stage_configs_file \
-    --omni \
-    --port 8091 \
-    --trust-remote-code \
-    --enforce-eager
-```
-
-Alternatively, use the convenience script:
-```bash
-./run_server.sh                  # Default: CustomVoice model
-./run_server.sh CustomVoice      # CustomVoice model
-./run_server.sh VoiceDesign      # VoiceDesign model
-./run_server.sh Base             # Base (voice clone) model
-```
-
-### Send TTS Request
-
-Get into the example folder
-```bash
-cd examples/online_serving/qwen3_tts
-```
-
-####  Send request via python
-
-```bash
-# CustomVoice: Use predefined speaker
-python openai_speech_client.py \
-    --text "你好，我是通义千问" \
-    --speaker vivian \
-    --language Chinese
-
-# CustomVoice with style instruction
-python openai_speech_client.py \
-    --text "今天天气真好" \
-    --speaker ryan \
-    --instructions "用开心的语气说"
-
-# VoiceDesign: Describe the voice style
-python openai_speech_client.py \
-    --model Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign \
-    --task-type VoiceDesign \
-    --text "哥哥，你回来啦" \
-    --instructions "体现撒娇稚嫩的萝莉女声，音调偏高"
-
-# Base: Voice cloning
-python openai_speech_client.py \
-    --model Qwen/Qwen3-TTS-12Hz-1.7B-Base \
-    --task-type Base \
-    --text "Hello, this is a cloned voice" \
-    --ref-audio /path/to/reference.wav \
-    --ref-text "Original transcript of the reference audio"
-```
-
-The Python client supports the following command-line arguments:
-
-- `--api-base`: API base URL (default: `http://localhost:8091`)
-- `--model` (or `-m`): Model name/path (default: `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice`)
-- `--task-type` (or `-t`): TTS task type. Options: `CustomVoice`, `VoiceDesign`, `Base`
-- `--text`: Text to synthesize (required)
-- `--speaker`: Speaker name (default: `vivian`). Options: `vivian`, `ryan`, `aiden`, etc.
-- `--language`: Language. Options: `Auto`, `Chinese`, `English`, `Japanese`, `Korean`, `German`, `French`, `Russian`, `Portuguese`, `Spanish`, `Italian`
-- `--instructions`: Voice style/emotion instructions
-- `--ref-audio`: Reference audio file path or URL for voice cloning (Base task). Local paths are automatically base64-encoded by the client before sending to the server.
-- `--ref-text`: Reference audio transcript for voice cloning (Base task).
-- `--response-format`: Audio output format (default: `wav`). Options: `wav`, `mp3`, `flac`, `pcm`, `aac`, `opus`
-- `--output` (or `-o`): Output audio file path (default: `tts_output.wav`)
-
-####  Send request via curl
-
-```bash
-# Simple TTS request
-curl -X POST http://localhost:8091/v1/audio/speech \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "Hello, how are you?",
-        "voice": "vivian",
-        "language": "English"
-    }' --output output.wav
-
-# With style instruction
-curl -X POST http://localhost:8091/v1/audio/speech \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "I am so excited!",
-        "voice": "vivian",
-        "instructions": "Speak with great enthusiasm"
-    }' --output excited.wav
-
-# List available voices in CustomVoice models
-curl http://localhost:8091/v1/audio/voices
-```
-
-### Using OpenAI SDK
-
-```python
-from openai import OpenAI
-
-client = OpenAI(base_url="http://localhost:8091/v1", api_key="none")
-
-response = client.audio.speech.create(
-    model="Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
-    voice="vivian",
-    input="Hello, how are you?",
-)
-
-response.stream_to_file("output.wav")
-```
-
-### Using Python httpx
-
-```python
-import httpx
-
-response = httpx.post(
-    "http://localhost:8091/v1/audio/speech",
-    json={
-        "input": "Hello, how are you?",
-        "voice": "vivian",
-        "language": "English",
-    },
-    timeout=300.0,
-)
-
-with open("output.wav", "wb") as f:
-    f.write(response.content)
-```
-
-## API Reference
-
-### Voices Endpoint
-
-#### GET /v1/audio/voices
-
-List all available voices/speakers from the loaded model, including both built-in model voices and uploaded custom voices.
-
-**Response Example:**
-```json
-{
-  "voices": ["vivian", "ryan", "custom_voice_1"],
-  "uploaded_voices": [
-    {
-      "name": "custom_voice_1",
-      "consent": "user_consent_id",
-      "created_at": 1738660000,
-      "file_size": 1024000,
-      "mime_type": "audio/wav",
-      "ref_text": "The exact transcript of the audio sample.",
-      "speaker_description": "warm narrator"
-    }
-  ]
-}
-```
-
-Fields `ref_text` and `speaker_description` are omitted per-entry when not provided at upload time.
-
-#### POST /v1/audio/voices
-
-Upload a new voice sample for voice cloning in Base task TTS requests.
-
-**Form Parameters:**
-- `audio_sample` (required): Audio file (max 10MB, supported formats: wav, mp3, flac, ogg, aac, webm, mp4)
-- `consent` (required): Consent recording ID
-- `name` (required): Name for the new voice
-- `ref_text` (optional): Transcript of the audio. Enables in-context voice cloning (higher quality).
-- `speaker_description` (optional): Free-form description of the voice (e.g. "warm narrator", "energetic presenter"). Stored as metadata.
-
-**Response Example:**
-```json
-{
-  "success": true,
-  "voice": {
-    "name": "custom_voice_1",
-    "consent": "user_consent_id",
-    "created_at": 1738660000,
-    "mime_type": "audio/wav",
-    "file_size": 1024000,
-    "ref_text": "The exact transcript of the audio sample.",
-    "speaker_description": "warm narrator"
-  }
-}
-```
-
-Fields `ref_text` and `speaker_description` are omitted when not provided at upload time.
-
-**Usage Example:**
-```bash
-curl -X POST http://localhost:8091/v1/audio/voices \
-  -F "audio_sample=@/path/to/voice_sample.wav" \
-  -F "consent=user_consent_id" \
-  -F "name=custom_voice_1" \
-  -F "ref_text=The exact transcript of the audio sample." \
-  -F "speaker_description=warm narrator"
-```
-
-### Endpoint
-
-```
-POST /v1/audio/speech
-Content-Type: application/json
-```
-
-This endpoint follows the [OpenAI Audio Speech API](https://platform.openai.com/docs/api-reference/audio/createSpeech) format with additional Qwen3-TTS parameters.
-
-### Request Body
-
-```json
-{
-    "input": "Text to synthesize",
-    "voice": "vivian",
-    "response_format": "wav",
-    "task_type": "CustomVoice",
-    "language": "Auto",
-    "instructions": "Optional style instructions",
-    "ref_audio":  "HTTP URL, base64 data URL, or file:// URI for voice cloning",
-    "ref_text": "Reference audio transcript",
-    "x_vector_only_mode": false,
-    "max_new_tokens": 2048
-}
-```
-
-> **Note:** The `model` field is optional when serving a single model, as the server already knows which model is loaded.
-
-### Response
-
-Returns binary audio data with appropriate `Content-Type` header (e.g., `audio/wav`).
-
-### Voice and language (summary)
-
-- **Speaker**: Use the `voice` request field to select the speaker (e.g., `vivian`, `ryan`, `aiden`). List available speakers with `GET /v1/audio/voices`.
-- **Language**: Use the `language` field for the codec language tag (`Auto`, `Chinese`, `English`, etc.). Default is `Auto` for automatic detection.
-- **CustomVoice**: Requires a valid `voice` from the model’s speaker set. **VoiceDesign**: Use `instructions` to describe the voice. **Base**: Use `ref_audio` and `ref_text` for voice cloning.
-
-## Parameters
-
-### OpenAI Standard Parameters
-
-| Parameter         | Type   | Default        | Description                                                 |
-| ----------------- | ------ | -------------- | ----------------------------------------------------------- |
-| `input`           | string | **required**   | Text to synthesize                                          |
-| `model`           | string | server's model | Model to use (optional, should match server if specified)   |
-| `voice`           | string | "vivian"       | Speaker name (e.g., vivian, ryan, aiden)                    |
-| `response_format` | string | "wav"          | Audio format: wav, mp3, flac, pcm, aac, opus                |
-| `speed`           | float  | 1.0            | Playback speed (0.25-4.0, not supported with `stream=true`) |
-
-### vLLM-Omni Extension Parameters
-
-| Parameter                    | Type   | Default       | Description                                                                                                          |
-| ---------------------------- | ------ | ------------- | -------------------------------------------------------------------------------------------------------------------- |
-| `task_type`                  | string | "CustomVoice" | Task: CustomVoice, VoiceDesign, or Base                                                                              |
-| `language`                   | string | "Auto"        | Language (see supported languages below)                                                                             |
-| `instructions`               | string | ""            | Voice style/emotion instructions                                                                                     |
-| `max_new_tokens`             | int    | 2048          | Maximum tokens to generate                                                                                           |
-| `initial_codec_chunk_frames` | int    | null          | Per-request initial chunk size override for TTFA tuning. When null, IC is computed dynamically based on server load. |
-| `stream`                     | bool   | false         | Stream raw PCM chunks as they are decoded (requires `response_format="pcm"`)                                         |
-
-**Supported languages:** Auto, Chinese, English, Japanese, Korean, German, French, Russian, Portuguese, Spanish, Italian
-
-### Voice Clone Parameters (Base task)
-
-| Parameter            | Type   | Required | Description                                                                                     |
-| -------------------- | ------ | -------- | ----------------------------------------------------------------------------------------------- |
-| `ref_audio`          | string | **Yes**  | Reference audio (HTTP URL, base64 data URL, or `file://` URI with `--allowed-local-media-path`) |
-| `ref_text`           | string | No       | Transcript of reference audio (for ICL mode)                                                    |
-| `x_vector_only_mode` | bool   | No       | Use speaker embedding only (no ICL)                                                             |
-
-## Streaming
-
-Set `stream=true` with `response_format="pcm"` to receive raw PCM audio chunks as they are decoded
-(one chunk per Code2Wav window, default 25 frames; configurable in the stage config):
-
-```bash
-curl -X POST http://localhost:8091/v1/audio/speech \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "Hello, how are you?",
-        "voice": "vivian",
-        "language": "English",
-        "stream": true,
-        "response_format": "pcm"
-    }' --no-buffer | play -t raw -r 24000 -e signed -b 16 -c 1 -
-```
-
-**Constraints:**
-- `stream=true` requires `response_format="pcm"` (raw 16-bit signed PCM, 24 kHz mono).
-- `speed` adjustment is not supported when streaming.
-- Requires the server stage config to have `async_chunk: true` (default in `qwen3_tts.yaml`).
-
-## Streaming Text Input (WebSocket)
-
-The `/v1/audio/speech/stream` WebSocket endpoint accepts text incrementally, buffers and splits it at sentence boundaries, and generates audio per sentence.
-
-When `stream_audio=true`, each sentence is emitted as `audio.start`, one or more binary PCM frames, and `audio.done`.
-
-### Quick Start
-
-```bash
-python streaming_speech_client.py \
-    --text "Hello world. How are you? I am fine."
-
-python streaming_speech_client.py \
-    --text "Hello world. How are you? I am fine." \
-    --simulate-stt --stt-delay 0.1
-```
-
-### WebSocket Protocol
-
-Client -> Server:
-
-```jsonc
-{"type": "session.config", "voice": "Vivian", "task_type": "CustomVoice", "language": "Auto", "split_granularity": "sentence", "stream_audio": true, "response_format": "pcm"}
-{"type": "input.text", "text": "Hello, how are you? "}
-{"type": "input.done"}
-```
-
-Server -> Client:
-
-```jsonc
-{"type": "audio.start", "sentence_index": 0, "sentence_text": "Hello, how are you?", "format": "pcm", "sample_rate": 24000}
-// binary PCM frame(s)
-{"type": "audio.done", "sentence_index": 0, "total_bytes": 96000, "error": false}
-{"type": "session.done", "total_sentences": 1}
-```
-
-## Limitations
-
-- **Single request**: Batch processing is not yet optimized for online serving.
-
-## Troubleshooting
-
-1. **TTS model did not produce audio output**: Ensure you're using the correct model variant for your task type (CustomVoice task → CustomVoice model, etc.)
-2. **Connection refused**: Make sure the server is running on the correct port
-3. **Out of memory**: Use smaller model variant (`Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice`) or reduce `--gpu-memory-utilization`
-4. **Unsupported speaker**: Use `/v1/audio/voices` to list available voices for the loaded model
-5. **Voice clone fails**: Ensure you're using the Base model variant for voice cloning
-
-## Example materials
-
-??? abstract "gradio_demo.py"
-    ``````py
-    --8<-- "examples/online_serving/qwen3_tts/gradio_demo.py"
-    ``````
-??? abstract "gradio_fastrtc_demo.py"
-    ``````py
-    --8<-- "examples/online_serving/qwen3_tts/gradio_fastrtc_demo.py"
-    ``````
-??? abstract "openai_speech_client.py"
-    ``````py
-    --8<-- "examples/online_serving/qwen3_tts/openai_speech_client.py"
-    ``````
-??? abstract "run_gradio_demo.sh"
-    ``````sh
-    --8<-- "examples/online_serving/qwen3_tts/run_gradio_demo.sh"
-    ``````
-??? abstract "run_server.sh"
-    ``````sh
-    --8<-- "examples/online_serving/qwen3_tts/run_server.sh"
-    ``````
-??? abstract "streaming_speech_client.py"
-    ``````py
-    --8<-- "examples/online_serving/qwen3_tts/streaming_speech_client.py"
-    ``````
-??? abstract "tts_common.py"
-    ``````py
-    --8<-- "examples/online_serving/qwen3_tts/tts_common.py"
-    ``````

docs/user_guide/examples/online_serving/text_to_speech.md

@@ -0,0 +1,489 @@
+# Text-To-Speech (Online Serving)
+
+Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/text_to_speech>.
+
+
+vLLM-Omni exposes TTS models through the OpenAI-compatible
+[`POST /v1/audio/speech`](https://github.com/vllm-project/vllm-omni/tree/main/docs/serving/speech_api.md) endpoint,
+launched with `vllm serve <model> --omni`. Each TTS model has its own
+subdirectory containing client snippets, gradio demos, and helper
+scripts; this README is the single doc entry point for all of them.
+
+For offline inference, see [`examples/offline_inference/text_to_speech`](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_speech/README.md).
+For the full list of supported architectures across all modalities, see
+[Supported Models](https://github.com/vllm-project/vllm-omni/tree/main/docs/models/supported_models.md).
+
+## Supported Models
+
+| Model | HuggingFace repo | Voice cloning | Streaming | Voice presets / upload | Gradio demo |
+|---|---|---|---|---|---|
+| Fish Speech S2 Pro | `fishaudio/s2-pro` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | — | ✓ |
+| OmniVoice | `k2-fsa/OmniVoice` | (offline only) | — | — | — |
+| Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | ✓ (Base) | ✓ (PCM + WebSocket) | ✓ (presets + `/v1/audio/voices` upload) | ✓ (standard + FastRTC) |
+| VoxCPM | local model dir | ✓ | ✓ (PCM stream) | — | — |
+| VoxCPM2 | `openbmb/VoxCPM2` | ✓ | ✓ (AudioWorklet via gradio) | — | ✓ |
+| Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✓ (gated upstream) | ✓ | ✓ (presets) | ✓ |
+
+CosyVoice3 is intentionally absent: no online example exists for it yet. See its [offline section](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_speech/README.md#cosyvoice3) instead.
+
+## Common Quick Start
+
+Launch the server (defaults shown — adjust `--port`, `--gpu-memory-utilization`, etc. as needed):
+
+```bash
+vllm serve <hf-repo-or-local-path> --omni --port 8091
+```
+
+Send a TTS request via curl:
+
+```bash
+curl -X POST http://localhost:8091/v1/audio/speech \
+    -H "Content-Type: application/json" \
+    -d '{
+        "input": "Hello, how are you?",
+        "voice": "default",
+        "response_format": "wav"
+    }' --output output.wav
+```
+
+Or via Python httpx:
+
+```python
+import httpx
+
+response = httpx.post(
+    "http://localhost:8091/v1/audio/speech",
+    json={
+        "input": "Hello, how are you?",
+        "voice": "default",
+        "response_format": "wav",
+    },
+    timeout=300.0,
+)
+open("output.wav", "wb").write(response.content)
+```
+
+Or via the OpenAI SDK:
+
+```python
+from openai import OpenAI
+
+client = OpenAI(base_url="http://localhost:8091/v1", api_key="none")
+response = client.audio.speech.create(
+    model="<hf-repo>",
+    voice="default",
+    input="Hello, how are you?",
+)
+response.stream_to_file("output.wav")
+```
+
+Streaming PCM output (where supported) — set `stream=true` with `response_format="pcm"`:
+
+```bash
+curl -X POST http://localhost:8091/v1/audio/speech \
+    -H "Content-Type: application/json" \
+    -d '{
+        "input": "Hello, how are you?",
+        "voice": "default",
+        "stream": true,
+        "response_format": "pcm"
+    }' --no-buffer | play -t raw -r 24000 -e signed -b 16 -c 1 -
+```
+
+Adjust the player's sample rate to match the model (44.1 kHz for Fish Speech, 48 kHz for VoxCPM2, 24 kHz for the others).
+
+For full request-shape documentation (all parameters, response formats, error codes), see the [Speech API reference](https://github.com/vllm-project/vllm-omni/tree/main/docs/serving/speech_api.md).
+
+---
+
+## Fish Speech S2 Pro
+
+4B dual-AR TTS at 44.1 kHz. Server uses the DAC codec.
+
+### Prerequisites
+```bash
+pip install fish-speech
+```
+
+### Launch
+```bash
+vllm serve fishaudio/s2-pro --omni --port 8091
+# or:
+./fish_speech/run_server.sh
+```
+The deploy config auto-loads from `vllm_omni/deploy/fish_qwen3_omni.yaml` (the HF `model_type` on the fishaudio checkpoint is `fish_qwen3_omni`).
+
+### Voice cloning
+```bash
+curl -X POST http://localhost:8091/v1/audio/speech \
+    -H "Content-Type: application/json" \
+    -d '{
+        "input": "Hello, this is a cloned voice.",
+        "voice": "default",
+        "ref_audio": "https://example.com/reference.wav",
+        "ref_text": "Transcript of the reference audio."
+    }' --output cloned.wav
+```
+
+### CLI client
+```bash
+cd examples/online_serving/text_to_speech/fish_speech
+python speech_client.py --text "Hello, how are you?"
+python speech_client.py --text "Hello world" --stream --output output.pcm
+```
+
+### Gradio demo
+```bash
+./fish_speech/run_gradio_demo.sh             # launches server + Gradio
+python fish_speech/gradio_demo.py --api-base http://localhost:8091  # if server already running
+```
+
+### Notes
+- Output: 44.1 kHz mono.
+- Streaming PCM player command must use `-r 44100`.
+
+---
+
+## OmniVoice
+
+Zero-shot multilingual TTS (600+ languages). Online serving currently exposes **auto voice** only; voice cloning and voice design are available offline.
+
+### Prerequisites
+```bash
+huggingface-cli download k2-fsa/OmniVoice
+```
+Voice cloning (offline) needs `transformers>=5.3.0`; auto voice works with `transformers>=4.57.0`.
+
+### Launch
+```bash
+vllm serve k2-fsa/OmniVoice --omni --port 8091 --trust-remote-code
+# or:
+./omnivoice/run_server.sh
+```
+
+### CLI client
+```bash
+cd examples/online_serving/text_to_speech/omnivoice
+python speech_client.py --text "Hello, how are you?"
+python speech_client.py --text "Bonjour, comment allez-vous?" --language French
+```
+
+The client supports `--api-base`, `--model`, `--text`, `--response-format`, `--language`, `--output`.
+
+### Notes
+- Voice cloning and voice design require offline inference; see the [offline OmniVoice section](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_speech/README.md#omnivoice).
+
+---
+
+## Qwen3-TTS
+
+Three model variants exposed via separate checkpoints:
+
+| Variant | HF repo | Use |
+|---|---|---|
+| CustomVoice | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | Predefined speakers (`vivian`, `ryan`, …) with optional style instructions |
+| VoiceDesign | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | Natural-language voice style description |
+| Base | `Qwen/Qwen3-TTS-12Hz-1.7B-Base` | Voice cloning from a reference audio |
+
+Each variant ships smaller `0.6B` companions where available.
+
+### Launch
+```bash
+vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni --port 8091
+# or:
+./qwen3_tts/run_server.sh                # default: CustomVoice
+./qwen3_tts/run_server.sh VoiceDesign
+./qwen3_tts/run_server.sh Base
+```
+
+### Choosing an executor backend (uniproc vs mp)
+Stage configs ship with the chunked-streaming default. To use the uniproc executor (lower IPC overhead for the Base cloning task), pass `--stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_tts_uniproc.yaml`. See [#2603](https://github.com/vllm-project/vllm-omni/issues/2603) and [#2604](https://github.com/vllm-project/vllm-omni/pull/2604) for the full investigation.
+
+To opt out of chunked streaming, pass `--no-async-chunk` instead — the pipeline auto-dispatches to the end-to-end codec processor.
+
+### Sending requests
+```bash
+# CustomVoice with a predefined speaker
+python qwen3_tts/openai_speech_client.py \
+    --model Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
+    --text "今天天气真好" \
+    --voice ryan \
+    --instructions "用开心的语气说"
+
+# VoiceDesign with a style description
+python qwen3_tts/openai_speech_client.py \
+    --model Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign \
+    --task-type VoiceDesign \
+    --text "哥哥，你回来啦" \
+    --instructions "体现撒娇稚嫩的萝莉女声，音调偏高"
+
+# Base voice cloning
+python qwen3_tts/openai_speech_client.py \
+    --model Qwen/Qwen3-TTS-12Hz-1.7B-Base \
+    --task-type Base \
+    --text "Hello, this is a cloned voice" \
+    --ref-audio /path/to/reference.wav \
+    --ref-text "Original transcript of the reference audio"
+```
+
+### Voices endpoint
+List available voices, or upload a custom one for Base cloning:
+```bash
+# List
+curl http://localhost:8091/v1/audio/voices
+
+# Upload
+curl -X POST http://localhost:8091/v1/audio/voices \
+    -F "audio_sample=@/path/to/voice_sample.wav" \
+    -F "consent=user_consent_id" \
+    -F "name=custom_voice_1" \
+    -F "ref_text=The exact transcript of the audio sample." \
+    -F "speaker_description=warm narrator"
+```
+Uploaded voices are then usable as `voice="custom_voice_1"` on subsequent requests.
+
+### Streaming PCM
+```bash
+curl -X POST http://localhost:8091/v1/audio/speech \
+    -H "Content-Type: application/json" \
+    -d '{
+        "input": "Hello, how are you?",
+        "voice": "vivian",
+        "language": "English",
+        "stream": true,
+        "response_format": "pcm"
+    }' --no-buffer | play -t raw -r 24000 -e signed -b 16 -c 1 -
+```
+Streaming requires `response_format="pcm"` and `async_chunk: true` on the stage config (default in `qwen3_tts.yaml`). `speed` is not supported when streaming.
+
+### Streaming WebSocket
+The `/v1/audio/speech/stream` endpoint accepts text incrementally, splits it at sentence boundaries, and emits one PCM stream per sentence:
+```bash
+python qwen3_tts/streaming_speech_client.py --text "Hello world. How are you? I am fine."
+python qwen3_tts/streaming_speech_client.py --text "..." --simulate-stt --stt-delay 0.1
+```
+
+### Gradio demos
+```bash
+./qwen3_tts/run_gradio_demo.sh                              # CustomVoice (default)
+./qwen3_tts/run_gradio_demo.sh --task-type VoiceDesign
+./qwen3_tts/run_gradio_demo.sh --task-type Base
+
+# FastRTC variant (gapless WebRTC streaming):
+pip install fastrtc
+python qwen3_tts/gradio_fastrtc_demo.py --api-base http://localhost:8000
+```
+
+### Speaker embedding interpolation
+`qwen3_tts/speaker_embedding_interpolation.py` blends two predefined speakers' embeddings to produce intermediate voices. See the script for usage.
+
+### Batch client
+`qwen3_tts/batch_speech_client.py` issues many concurrent requests for throughput measurement.
+
+### Notes
+- Base voice cloning has uniproc-vs-mp tradeoffs depending on per-request reference audio cost; see the executor-backend section above.
+- `vllm_omni/deploy/qwen3_tts.yaml` is the default deploy config (loaded by HF `model_type`); per-stage runtime overrides are available via `--stage-N-<field> <value>`.
+
+---
+
+## VoxCPM
+
+Split-stage TTS at 24 kHz.
+
+### Prerequisites
+```bash
+pip install voxcpm
+# or use a local source tree:
+export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/VoxCPM/src
+```
+
+If the native VoxCPM `config.json` lacks HF `model_type`, set up an HF-compatible config dir:
+```bash
+export VOXCPM_MODEL=/path/to/voxcpm-model
+export VLLM_OMNI_VOXCPM_HF_CONFIG_PATH=/tmp/voxcpm_hf_config
+mkdir -p "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"
+cp "$VOXCPM_MODEL/config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/config.json"
+cp "$VOXCPM_MODEL/generation_config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/generation_config.json" 2>/dev/null || true
+python3 -c 'import json, os; p=os.path.join(os.environ["VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"], "config.json"); cfg=json.load(open(p, "r", encoding="utf-8")); cfg["model_type"]="voxcpm"; cfg.setdefault("architectures", ["VoxCPMForConditionalGeneration"]); json.dump(cfg, open(p, "w", encoding="utf-8"), indent=2, ensure_ascii=False)'
+```
+
+### Launch
+```bash
+export VOXCPM_MODEL=/path/to/voxcpm-model
+./voxcpm/run_server.sh                # async-chunk streaming (default)
+./voxcpm/run_server.sh sync           # non-streaming
+```
+Or directly:
+```bash
+vllm serve "$VOXCPM_MODEL" \
+    --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \
+    --trust-remote-code --enforce-eager --omni --port 8091
+```
+
+### Sending requests
+```bash
+# Basic TTS
+python voxcpm/openai_speech_client.py \
+    --model "$VOXCPM_MODEL" \
+    --text "This is a VoxCPM online text-to-speech example."
+
+# Voice cloning
+python voxcpm/openai_speech_client.py \
+    --model "$VOXCPM_MODEL" \
+    --text "This sentence is synthesized with a cloned voice." \
+    --ref-audio /path/to/reference.wav \
+    --ref-text "The exact transcript spoken in reference.wav."
+
+# Streaming PCM
+python voxcpm/openai_speech_client.py \
+    --model "$VOXCPM_MODEL" \
+    --text "This is a streaming VoxCPM request." \
+    --stream --output voxcpm_stream.pcm
+```
+
+### Notes
+- `voxcpm.yaml` for one-shot decode; `voxcpm_async_chunk.yaml` for single-request streaming. Do not use the async-chunk config for concurrent requests or `/v1/audio/speech/batch`.
+- Generic TTS fields not supported by VoxCPM: `voice`, `instructions`, `language`, `speaker_embedding`, `x_vector_only_mode`.
+- For benchmark measurement, see [`benchmarks/voxcpm`](https://github.com/vllm-project/vllm-omni/tree/main/benchmarks/voxcpm/README.md).
+
+---
+
+## VoxCPM2
+
+Single-stage native AR TTS at 48 kHz.
+
+### Launch
+```bash
+vllm serve openbmb/VoxCPM2 --omni --host 0.0.0.0 --port 8000
+```
+Deploy config auto-loads from `vllm_omni/deploy/voxcpm2.yaml`. Pass `--deploy-config <path>` to override or `--stage-N-<field> <value>` for per-stage runtime tweaks.
+
+### Sending requests
+```bash
+# Zero-shot synthesis
+python voxcpm2/openai_speech_client.py --text "Hello, this is VoxCPM2."
+
+# Voice cloning
+python voxcpm2/openai_speech_client.py \
+    --text "This should sound like the reference speaker." \
+    --ref-audio /path/to/reference.wav
+```
+The `ref_audio` field accepts local file paths (auto-base64), HTTP URLs, or `data:audio/wav;base64,...` data URIs.
+
+### Gradio demo (gapless streaming via AudioWorklet)
+```bash
+python voxcpm2/gradio_demo.py
+```
+Uses an AudioWorklet-based player adapted from the Qwen3-TTS demo for gap-free playback. Audio is streamed from the OpenAI Speech endpoint with `stream=true`.
+
+---
+
+## Voxtral TTS
+
+Voxtral-4B-TTS (Mistral). Uses the `mistral_common` `SpeechRequest` protocol; voice presets are model-specific.
+
+### Prerequisites
+Latest `mistral_common` with `SpeechRequest` support:
+```bash
+pip install -e /path/to/mistral-common  # or upgrade from PyPI when available
+```
+
+### Launch
+```bash
+vllm serve mistralai/Voxtral-4B-TTS-2603 --omni --port 8091
+```
+Deploy config auto-loads from `vllm_omni/deploy/voxtral_tts.yaml`.
+
+### Gradio demo
+```bash
+python voxtral_tts/gradio_demo.py
+```
+The demo handles voice-preset selection and reference-audio upload. `voxtral_tts/text_preprocess.py` provides the text-normalization helpers used by the demo (also available for other clients).
+
+### Notes
+- Voice presets are listed on the HF model card (`mistralai/Voxtral-4B-TTS-2603`).
+- Voice cloning is gated upstream and may require a recent `mistral_common`.
+- A standalone CLI client is not yet shipped; the gradio demo is the canonical reference for now.
+
+## Example materials
+
+??? abstract "fish_speech/gradio_demo.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/fish_speech/gradio_demo.py"
+    ``````
+??? abstract "fish_speech/run_gradio_demo.sh"
+    ``````sh
+    --8<-- "examples/online_serving/text_to_speech/fish_speech/run_gradio_demo.sh"
+    ``````
+??? abstract "fish_speech/run_server.sh"
+    ``````sh
+    --8<-- "examples/online_serving/text_to_speech/fish_speech/run_server.sh"
+    ``````
+??? abstract "fish_speech/speech_client.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/fish_speech/speech_client.py"
+    ``````
+??? abstract "omnivoice/run_server.sh"
+    ``````sh
+    --8<-- "examples/online_serving/text_to_speech/omnivoice/run_server.sh"
+    ``````
+??? abstract "omnivoice/speech_client.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/omnivoice/speech_client.py"
+    ``````
+??? abstract "qwen3_tts/batch_speech_client.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/qwen3_tts/batch_speech_client.py"
+    ``````
+??? abstract "qwen3_tts/gradio_demo.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/qwen3_tts/gradio_demo.py"
+    ``````
+??? abstract "qwen3_tts/openai_speech_client.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/qwen3_tts/openai_speech_client.py"
+    ``````
+??? abstract "qwen3_tts/run_gradio_demo.sh"
+    ``````sh
+    --8<-- "examples/online_serving/text_to_speech/qwen3_tts/run_gradio_demo.sh"
+    ``````
+??? abstract "qwen3_tts/run_server.sh"
+    ``````sh
+    --8<-- "examples/online_serving/text_to_speech/qwen3_tts/run_server.sh"
+    ``````
+??? abstract "qwen3_tts/speaker_embedding_interpolation.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/qwen3_tts/speaker_embedding_interpolation.py"
+    ``````
+??? abstract "qwen3_tts/streaming_speech_client.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/qwen3_tts/streaming_speech_client.py"
+    ``````
+??? abstract "qwen3_tts/tts_common.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/qwen3_tts/tts_common.py"
+    ``````
+??? abstract "voxcpm/openai_speech_client.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/voxcpm/openai_speech_client.py"
+    ``````
+??? abstract "voxcpm/run_server.sh"
+    ``````sh
+    --8<-- "examples/online_serving/text_to_speech/voxcpm/run_server.sh"
+    ``````
+??? abstract "voxcpm2/gradio_demo.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/voxcpm2/gradio_demo.py"
+    ``````
+??? abstract "voxcpm2/openai_speech_client.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/voxcpm2/openai_speech_client.py"
+    ``````
+??? abstract "voxtral_tts/gradio_demo.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/voxtral_tts/gradio_demo.py"
+    ``````
+??? abstract "voxtral_tts/text_preprocess.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/voxtral_tts/text_preprocess.py"
+    ``````

docs/user_guide/quantization/fp8.md

@@ -2,11 +2,11 @@
 
 ## Overview
 
-FP8 quantization converts BF16/FP16 weights to FP8 at model load time, or loads
-a checkpoint whose target stage already declares an FP8 quantization config.
-Online activation scaling is the default and does not require calibration.
-Static activation scaling is supported when calibrated scale information is
-available.
+FP8 quantization converts BF16/FP16 weights to FP8 at model load time. Online
+activation scaling is the default and does not require calibration. Static
+activation scaling is supported when calibrated scale information is available.
+For ModelOpt-produced pre-quantized checkpoints, see
+[ModelOpt Quantization](modelopt.md).
 
 Some architectures can quantize all linear layers. Others have
 quality-sensitive layers that should stay in BF16 through `ignored_layers`.
@@ -46,7 +46,7 @@ guide. FP8 on Ampere may use a weight-only path where available.
 
 | Model | Scope | Format | Status |
 |-------|-------|--------|--------|
-| Qwen3-Omni | Thinker language-model stage | ModelOpt `quant_algo=FP8` | Tested for thinker memory reduction |
+| Qwen3-Omni | Thinker language-model stage | [ModelOpt](modelopt.md) `quant_algo=FP8` | Tested for thinker memory reduction |
 | Qwen3-TTS | TTS language-model stage | Checkpoint config | Not validated |
 
 Audio encoder, vision encoder, talker, and code2wav stay in BF16 unless a

docs/user_guide/quantization/modelopt.md

@@ -0,0 +1,138 @@
+# ModelOpt Quantization
+
+## Overview
+
+ModelOpt quantization loads checkpoints produced by NVIDIA ModelOpt. The
+quantized weights and scale tensors are generated before serving, so inference
+does not run online calibration or convert a BF16 checkpoint at startup.
+
+vLLM-Omni currently validates the ModelOpt FP8 checkpoint path for diffusion
+transformers. The loader auto-detects supported ModelOpt FP8 checkpoint configs
+and keeps non-transformer components, such as the tokenizer, scheduler, text
+encoder, and VAE, on the base checkpoint unless a model-specific guide says
+otherwise.
+
+!!! note
+    `--force-cutlass-fp8` is an explicit runtime override for diffusion
+    checkpoints that already carry a supported ModelOpt FP8 config. It does not
+    quantize BF16 checkpoints and it does not apply to online `--quantization
+    fp8`. The flag only takes effect for ModelOpt FP8 diffusion stages on CUDA
+    SM89+ devices; other platforms and non-ModelOpt FP8 paths fall back to the
+    normal vLLM kernel selection.
+
+## Supported ModelOpt Checkpoint Formats
+
+vLLM-Omni treats ModelOpt checkpoints as pre-quantized checkpoints. The
+checkpoint config must identify ModelOpt as the quantization method or producer,
+and the quantization algorithm must be one of the validated FP8 algorithms.
+
+| Checkpoint field | Supported value |
+|------------------|-----------------|
+| `method` / `quant_method` | `modelopt` |
+| `producer.name` | `modelopt` |
+| `quant_algo` | `FP8`, `FP8_PER_CHANNEL_PER_TOKEN` |
+
+Other ModelOpt algorithms, such as NVFP4, are not enabled by this diffusion
+FP8 path until they have separate model and quality validation.
+
+## Hardware Support
+
+| Device | Support |
+|--------|---------|
+| NVIDIA Blackwell GPU (SM 100+) | ✅ |
+| NVIDIA Ada/Hopper GPU (SM 89+) | ✅ |
+| NVIDIA Ampere GPU (SM 80+) | ⭕ |
+| AMD ROCm | ⭕ |
+| Intel XPU | ⭕ |
+| Ascend NPU | ❌ |
+
+Legend: `✅` supported, `❌` unsupported, `⭕` not verified in this guide.
+The optional CUTLASS FP8 runtime override requires CUDA SM89+.
+
+## Model Type Support
+
+### Diffusion Model
+
+| Model | HF checkpoint | Scope | Status |
+|-------|---------------|-------|--------|
+| Qwen-Image 2512 | `feizhai123/qwen-image-2512-modelopt-fp8-dynamic-all` | Diffusion transformer | Validated for ModelOpt FP8 checkpoints |
+| Z-Image | `feizhai123/z-image-modelopt-fp8-conservative` | Diffusion transformer | Validated for ModelOpt FP8 checkpoints |
+| FLUX.2-dev | `feizhai123/flux2-dev-modelopt-fp8` | Diffusion transformer | Validated for ModelOpt FP8 checkpoints |
+| FLUX.2-klein 4B | `feizhai123/flux2-klein-4b-modelopt-fp8` | Diffusion transformer | Validated for ModelOpt FP8 checkpoints |
+| HunyuanImage-3.0 | `feizhai123/hunyuan-image3-modelopt-fp8` | MoE diffusion transformer | Validated for ModelOpt FP8 checkpoints |
+| Wan2.2 | Not available | Diffusion transformer | Not validated |
+
+### Multi-Stage Omni/TTS Model
+
+| Model | Scope | Status |
+|-------|-------|--------|
+| Qwen3-Omni | Thinker language-model stage | ModelOpt FP8 checkpoint path |
+| Qwen3-TTS | TTS language-model stage | Not validated |
+
+Audio encoder, vision encoder, talker, and code2wav stages stay in BF16 unless
+a model-specific guide documents otherwise.
+
+### Multi-Stage Diffusion Model
+
+ModelOpt checkpoints must be routed to the stage whose checkpoint contains the
+ModelOpt `quantization_config`. BAGEL and GLM-Image are not listed as validated
+ModelOpt targets yet.
+
+## Configuration
+
+For pre-quantized ModelOpt FP8 checkpoints, no `--quantization fp8` flag is
+needed. The checkpoint config selects the ModelOpt path.
+
+Online serving:
+
+```bash
+vllm serve <modelopt-fp8-checkpoint> \
+  --omni \
+  --tensor-parallel-size <N> \
+  --force-cutlass-fp8
+```
+
+Offline inference:
+
+```bash
+python examples/offline_inference/text_to_image/text_to_image.py \
+  --model <modelopt-fp8-checkpoint> \
+  --tensor-parallel-size <N> \
+  --prompt "a red ceramic teapot on a wooden table" \
+  --height 1024 \
+  --width 1024 \
+  --num-inference-steps 20 \
+  --seed 42 \
+  --output outputs/modelopt_fp8.png
+```
+
+Python API:
+
+```python
+from vllm_omni import Omni
+
+omni = Omni(
+    model="<modelopt-fp8-checkpoint>",
+    tensor_parallel_size=2,
+    force_cutlass_fp8=True,
+)
+```
+
+## Parameters
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `force_cutlass_fp8` / `--force-cutlass-fp8` | bool | `False` | Force CUTLASS FP8 linear kernels for supported ModelOpt FP8 diffusion stages on CUDA SM89+ |
+
+## Validation and Notes
+
+1. Compare the ModelOpt FP8 checkpoint against the BF16 baseline with the same
+   prompt, resolution, seed, and inference steps.
+2. Use `tests/diffusion/quantization/test_quantization_quality.py` with
+   `VLLM_OMNI_QUALITY_CONFIGS` to validate local baseline and quantized model
+   paths.
+3. Report LPIPS, PSNR, MAE, throughput, latency, and peak memory when adding a
+   new validated ModelOpt diffusion checkpoint.
+4. Keep `--quantization fp8` for online FP8 from BF16 checkpoints; use this
+   ModelOpt path only when the checkpoint already contains ModelOpt FP8 weights
+   and scales.

docs/user_guide/quantization/mxfp8.md

@@ -0,0 +1,187 @@
+# W8A8 MXFP8 Quantization
+
+## Overview
+
+W8A8 MXFP8 (Microscaling FP8) quantizes both weights and activations to FP8
+using the OCP MX format: groups of 32 K-dimension elements share a single
+`float8_e8m0fnu` exponent scale. This gives better accuracy than channel-wise
+FP8 while keeping the same 8-bit weight footprint.
+
+This method supports two modes:
+
+| Mode | Description |
+|------|-------------|
+| **Online** | BF16 weights are quantized to MXFP8 at load time — no pre-processing needed |
+| **Offline** | msModelSlim-exported MXFP8 weights converted to diffusers format via `merge_mxfp8_checkpoint.py` — weights and scales are loaded directly from the preprocessed checkpoint |
+
+## Hardware Support
+
+| Device | Support |
+|--------|---------|
+| NVIDIA Blackwell GPU (SM 100+) | ⭕ |
+| NVIDIA Ada/Hopper GPU (SM 89+) | ⭕ |
+| NVIDIA Ampere GPU (SM 80+) | ⭕ |
+| AMD ROCm | ⭕ |
+| Intel XPU | ⭕ |
+| Ascend NPU (Atlas 950 A5) | ✅ |
+
+Legend: `✅` supported, `❌` unsupported, `⭕` not verified in this
+guide.
+
+## Model Type Support
+
+### Diffusion Model (Wan2.2)
+
+| Model | Mode | Notes |
+|-------|------|-------|
+| Wan2.2-T2V-A14B | Online + Offline | MoE cascade; quantizes two transformers (`transformer` + `transformer_2`) |
+| Wan2.2-I2V-A14B | Online + Offline | MoE cascade; quantizes two transformers (`transformer` + `transformer_2`) |
+| Wan2.2-TI2V-5B | Online + Offline | Single transformer |
+
+### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS)
+
+| Model | Status | Notes |
+|-------|--------|-------|
+| Qwen3-Omni | Not validated | — |
+| Qwen3-TTS | Not validated | — |
+
+### Multi-Stage Diffusion Model (BAGEL, GLM-Image)
+
+| Model | Status | Notes |
+|-------|--------|-------|
+| BAGEL | Not validated | — |
+| GLM-Image | Not validated | — |
+
+## Configuration
+
+### Online Mode
+
+Online mode requires no pre-processing. vLLM-Omni quantizes BF16 weights to
+MXFP8 at load time.
+
+Python API:
+
+```python
+from vllm_omni import Omni
+from vllm_omni.inputs.data import OmniDiffusionSamplingParams
+
+omni = Omni(model="<your-model>", quantization="mxfp8")
+
+outputs = omni.generate(
+    "A cat sitting on a windowsill",
+    OmniDiffusionSamplingParams(num_inference_steps=50),
+)
+```
+
+CLI:
+
+```bash
+python text_to_video.py --model <your-model> --quantization mxfp8
+
+# Online serving
+vllm serve <your-model> --omni --quantization mxfp8
+```
+
+### Offline Mode
+
+Offline mode loads a pre-quantized checkpoint from msModelSlim. A preprocessing
+step converts the raw quantized output to the diffusers format expected by
+vLLM-Omni and injects the quantization config into `transformer/config.json` so
+that vLLM-Omni auto-detects the offline path without a `--quantization` flag.
+
+#### Step 1 — Quantize with msModelSlim
+
+```bash
+msmodelslim quant \
+  --model_path /path/to/Wan2.2-TI2V-5B-Diffusers \
+  --save_path  /path/to/wan2_2_ti2v_quantized_raw \
+  --device npu \
+  --model_type Wan2_2 \
+  --config_path /path/to/wan2_2_w8a8f8_mxfp.yaml \
+  --trust_remote_code True
+```
+
+After this step, `--save_path` contains the raw quantized safetensors files and
+a metadata JSON (`quant_model_description*.json`).
+
+For cascade MoE models (T2V-A14B, I2V-A14B), msModelSlim outputs two
+subdirectories: `high_noise_model/` and `low_noise_model/`.
+
+#### Step 2 — Preprocess with merge_mxfp8_checkpoint.py
+
+The script (`vllm_omni/quantization/tools/merge_mxfp8_checkpoint.py`):
+
+1. Copies the original diffusers model to `--output-path` (VAE, text encoder,
+   scheduler, etc. are preserved).
+2. Remaps tensor names from msModelSlim convention to diffusers convention.
+3. Saves the converted weights as `diffusion_pytorch_model.safetensors`.
+4. Copies the original `transformer/config.json` and injects
+   `quantization_config` so that vLLM-Omni auto-detects offline MXFP8.
+
+For cascade MoE models, steps 2–4 run separately for `high_noise_model/` →
+`transformer/` and `low_noise_model/` → `transformer_2/`.
+
+```bash
+python vllm_omni/quantization/tools/merge_mxfp8_checkpoint.py \
+  --model-type     Wan2.2-TI2V-5B \
+  --original-model /path/to/Wan2.2-TI2V-5B-Diffusers \
+  --quant-path     /path/to/wan2_2_ti2v_quantized_raw \
+  --output-path    /path/to/Wan2.2-TI2V-5B-MXFP8
+```
+
+| Argument | Description |
+|----------|-------------|
+| `--model-type` | Model variant: `Wan2.2-T2V-A14B`, `Wan2.2-I2V-A14B`, or `Wan2.2-TI2V-5B` |
+| `--original-model` | Root directory of the original BF16 diffusers model |
+| `--quant-path` | Root directory of the msModelSlim quantized output |
+| `--output-path` | Output directory for the merged model (created by the script) |
+
+The script outputs a complete diffusers model directory at `--output-path`,
+with each transformer subfolder containing:
+
+- `diffusion_pytorch_model.safetensors` — converted FP8 weights
+- `config.json` — original transformer config with `quantization_config` injected
+- `quant_model_description.json` — renamed quantization metadata (reference only)
+
+#### Step 3 — Serve
+
+```bash
+python text_to_video.py --model /path/to/Wan2.2-TI2V-5B-MXFP8
+
+# Online serving
+vllm serve /path/to/Wan2.2-TI2V-5B-MXFP8 --omni
+```
+
+Python API:
+
+```python
+omni = Omni(model="/path/to/Wan2.2-TI2V-5B-MXFP8")
+```
+
+!!! note
+    No `--quantization` flag is needed for offline mode. The preprocessing
+    script injects `quantization_config` into each `transformer/config.json`,
+    which vLLM-Omni reads automatically to activate the offline MXFP8 method.
+
+## Parameters
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `method` | str | — | Must be `"mxfp8"` |
+| `is_checkpoint_mxfp8_serialized` | bool | `False` | `True` for offline pre-quantized checkpoints; auto-set from `config.json` when using the preprocessing script |
+| `ignored_layers` | list[str] | `[]` | Layer name substrings to keep in BF16 (e.g. `"to_out"` matches `blocks.0.attn1.to_out.0`) |
+
+## Validation and Notes
+
+1. Online mode quantizes BF16 weights at load time using
+   `npu_dynamic_mx_quant`. This adds a one-time overhead on the first load
+   but requires no checkpoint preparation.
+2. Offline mode loads FP8 weights directly from the checkpoint. Scales are
+   stored as `uint8` bytes in safetensors (same bit layout as
+   `float8_e8m0fnu`) and are reinterpreted at load time without a dtype
+   conversion.
+3. If the offline checkpoint was produced with the old `merge_mxfp8_checkpoint.py`
+   interface (arguments `--quant-dir`, `--orig-dir`, `--meta-json`,
+   `--output-dir`), regenerate it with the current script. The old script
+   wrote a separate `quantization_config.json` that is not read by vLLM-Omni;
+   the current script injects the config directly into `transformer/config.json`.

docs/user_guide/quantization/overview.md

@@ -1,4 +1,4 @@
-# Quantization
+# Quantization
 
 vLLM-Omni exposes quantization through the unified `quantization_config`
 path. The same configuration entrypoint is used across diffusion-only models,
@@ -9,19 +9,20 @@ type has a different quantization scope.
 
 | Mode | Guide | Description | Methods |
 |------|-------|-------------|---------|
-| Online quantization | [Online Quantization](online.md) | vLLM-Omni computes quantized weights and scales while loading the model. | FP8 W8A8, Int8 W8A8 |
-| Pre-quantized checkpoints | Method-specific guides | The checkpoint or an offline quantizer provides quantized weights and scales before serving. | GGUF, AutoRound, msModelSlim, serialized Int8 |
+| Online quantization | [Online Quantization](online.md) | vLLM-Omni computes quantized weights and scales while loading the model. | FP8 W8A8, Int8 W8A8, MXFP8 W8A8 |
+| Runtime attention quantization | [Quantized KV Cache](quantized_kvcache.md) | vLLM-Omni dynamically quantizes eligible diffusion Flash Attention tensors during inference. | FP8 FA |
+| Pre-quantized checkpoints | Method-specific guides | The checkpoint or an offline quantizer provides quantized weights and scales before serving. | ModelOpt, GGUF, AutoRound, msModelSlim, serialized Int8, offline MXFP8 |
 
 ## Hardware Support
 
-| Device | FP8 W8A8 | Int8 W8A8 | GGUF | AutoRound | msModelSlim |
-|--------|----------|-----------|------|-----------|-------------|
-| NVIDIA Blackwell GPU (SM 100+) | ✅ | ✅ | ✅ | ✅ | ❌ |
-| NVIDIA Ada/Hopper GPU (SM 89+) | ✅ | ✅ | ✅ | ✅ | ❌ |
-| NVIDIA Ampere GPU (SM 80+) | ✅ | ✅ | ✅ | ✅ | ❌ |
-| AMD ROCm | ⭕ | ⭕ | ⭕ | ⭕ | ❌ |
-| Intel XPU | ⭕ | ⭕ | ⭕ | ✅ | ❌ |
-| Ascend NPU | ❌ | ✅ | ❌ | ❌ | ✅ |
+| Device | FP8 W8A8 | Int8 W8A8 | ModelOpt | MXFP8 W8A8 | GGUF | AutoRound | msModelSlim |
+|--------|----------|-----------|----------|------------|------|-----------|-------------|
+| NVIDIA Blackwell GPU (SM 100+) | ✅ | ✅ | ✅ | ⭕ | ✅ | ✅ | ❌ |
+| NVIDIA Ada/Hopper GPU (SM 89+) | ✅ | ✅ | ✅ | ⭕ | ✅ | ✅ | ❌ |
+| NVIDIA Ampere GPU (SM 80+) | ✅ | ✅ | ⭕ | ⭕ | ✅ | ✅ | ❌ |
+| AMD ROCm | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ | ❌ |
+| Intel XPU | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ | ✅ | ❌ |
+| Ascend NPU | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ✅ |
 
 Legend: `✅` supported, `❌` unsupported, `⭕` not verified in this
 guide. FP8 on Ampere may use a weight-only path where available.
@@ -39,6 +40,8 @@ otherwise.
 |--------|-------|------|----------------|--------|
 | FP8 W8A8 | [FP8](fp8.md) | Online W8A8 or checkpoint FP8 | Qwen-Image; Wan2.2 is not validated | Validated for Qwen-Image family and other DiT models |
 | Int8 W8A8 | [Int8](int8.md) | Online or serialized W8A8 | Qwen-Image; Wan2.2 is not validated | Validated for Qwen-Image and Z-Image |
+| ModelOpt | [ModelOpt](modelopt.md) | Pre-quantized FP8 checkpoints | Qwen-Image, Z-Image, FLUX.2, HunyuanImage-3.0 | Validated for ModelOpt FP8 diffusion checkpoints |
+| MXFP8 W8A8 | [MXFP8](mxfp8.md) | Online W8A8 or offline pre-quantized | Wan2.2-T2V-A14B, I2V-A14B, TI2V-5B | Ascend NPU only; validated for Wan2.2 |
 | GGUF | [GGUF](gguf.md) | Pre-quantized transformer weights | Qwen-Image | Validated where a model-specific GGUF adapter exists |
 | AutoRound | [AutoRound](autoround.md) | Pre-quantized W4A16 checkpoints | FLUX.1-dev; Qwen-Image/Wan2.2 not validated | Checkpoint-driven |
 | msModelSlim | [msModelSlim](msmodelslim.md) | Pre-quantized Ascend checkpoints | Wan2.2 recipe; HunyuanImage-3.0 inference target | Ascend/NPU path |
@@ -52,8 +55,9 @@ in BF16 unless the model guide explicitly adds support.
 
 | Method | Guide | Scope | Example models | Status |
 |--------|-------|-------|----------------|--------|
-| FP8 | [FP8](fp8.md) | Thinker or language-model checkpoint config | Qwen3-Omni thinker | ModelOpt checkpoint path |
+| ModelOpt | [ModelOpt](modelopt.md) | Thinker or language-model checkpoint config | Qwen3-Omni thinker | ModelOpt checkpoint path |
 | Int8 | [Int8](int8.md) | Not currently validated for omni/TTS stages | Qwen3-Omni, Qwen3-TTS | Not validated |
+| MXFP8 | [MXFP8](mxfp8.md) | Not currently validated for omni/TTS stages | Qwen3-Omni, Qwen3-TTS | Not validated |
 | GGUF | [GGUF](gguf.md) | Not currently validated for omni/TTS stages | Qwen3-Omni, Qwen3-TTS | Not validated |
 | AutoRound | [AutoRound](autoround.md) | Thinker or language-model checkpoint config | Qwen2.5-Omni, Qwen3-Omni | Supported through AutoRound checkpoints |
 | msModelSlim | [msModelSlim](msmodelslim.md) | Not currently validated for omni/TTS stages | Qwen3-Omni, Qwen3-TTS | Not validated |
@@ -67,6 +71,8 @@ attached to the intended stage rather than applied globally.
 |--------|-------|-------|----------------|--------|
 | FP8 | [FP8](fp8.md) | Stage-specific DiT or transformer module | BAGEL, GLM-Image | Requires model-specific validation |
 | Int8 | [Int8](int8.md) | Stage-specific DiT or transformer module | BAGEL, GLM-Image | Requires model-specific validation |
+| ModelOpt | [ModelOpt](modelopt.md) | Checkpoint-defined diffusion stage | BAGEL, GLM-Image | Requires model-specific validation |
+| MXFP8 | [MXFP8](mxfp8.md) | Stage-specific DiT or transformer module | BAGEL, GLM-Image | Not validated |
 | GGUF | [GGUF](gguf.md) | Stage-specific transformer weights | BAGEL, GLM-Image | No validated adapter listed |
 | AutoRound | [AutoRound](autoround.md) | Checkpoint-defined stage | BAGEL, GLM-Image | No validated checkpoint listed |
 | msModelSlim | [msModelSlim](msmodelslim.md) | Ascend-generated stage weights | GLM-Image | Requires model-specific adaptation |
@@ -94,7 +100,7 @@ config = build_quant_config({
 
 | Component | Default quantized? | Notes |
 |-----------|--------------------|-------|
-| Diffusion transformer | Yes | Primary target for FP8, Int8, GGUF, AutoRound, and msModelSlim |
+| Diffusion transformer | Yes | Primary target for FP8, Int8, ModelOpt, GGUF, AutoRound, and msModelSlim |
 | Text encoder | No | Keep BF16 unless a method-specific guide documents support |
 | VAE | No | Keep BF16; storage-only paths are method-specific |
 | Scheduler/tokenizer | No | Loaded from the base model repository |

docs/user_guide/quantization/quantized_kvcache.md

@@ -0,0 +1,115 @@
+# Quantized KV Cache
+
+## Overview
+
+In DiT-based image and video generation, Flash Attention can take a large share
+of denoising time, especially for high-resolution or long-frame workloads.
+vLLM-Omni supports online FP8 quantization for eligible diffusion Flash
+Attention (FA) to reduce FA latency while keeping model weights in their
+original dtype.
+
+This feature is configured through `kv_cache_dtype`, matching the option name
+used by vLLM's language-model KV-cache quantization. In vLLM-Omni diffusion
+pipelines, however, it is a runtime FA path: Q/K/V tensors are dynamically
+quantized before the attention operator. It does not quantize model weights and
+is separate from [FP8 W8A8](fp8.md), [Int8 W8A8](int8.md), or pre-quantized
+checkpoint formats.
+
+If `kv_cache_dtype` is not set, behavior is unchanged and attention runs in the
+native dtype.
+
+## Hardware Support
+
+| Device | FP8 FA |
+|--------|--------|
+| Ascend NPU | ✅ |
+| NVIDIA GPU | ❌ |
+| AMD ROCm | ❌ |
+| Intel XPU | ❌ |
+
+Legend: `✅` supported, `❌` unsupported.
+
+FP8 FA is currently implemented only for the NPU Flash Attention backend. Other
+backends do not support `kv_cache_dtype="fp8"` for diffusion attention and fall
+back to native dtype execution.
+
+## Model Type Support
+
+### Diffusion Model
+
+| Model | Scope | Status | Notes |
+|-------|-------|--------|-------|
+| Wan2.2 | Eligible DiT full-attention FA on Ascend NPU | Tested | Compare quality and latency against a BF16 baseline before production use |
+| Other diffusion models | Eligible DiT full-attention FA on Ascend NPU | Not tested | You can try `kv_cache_dtype="fp8"`; tune `kv_cache_skip_steps` and `kv_cache_skip_layers` when higher precision is needed |
+
+### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS)
+
+Not tested for FP8 FA. Treat any use as experimental unless a model-specific
+guide documents support.
+
+### Multi-Stage Diffusion Model (BAGEL, GLM-Image)
+
+Not tested. If the diffusion stage uses the same NPU Flash Attention backend,
+`kv_cache_dtype` may apply in theory; validate quality and latency for each
+stage and model.
+
+## Configuration
+
+Offline diffusion example:
+
+```bash
+python examples/offline_inference/image_to_video/image_to_video.py \
+    --model <your-wan2.2-model> \
+    --prompt "A cat sitting on a surfboard at the beach" \
+    --height 1280 \
+    --width 720 \
+    --num-frames 61 \
+    --num-inference-steps 4 \
+    --ulysses-degree 4 \
+    --vae-patch-parallel-size 4 \
+    --kv-cache-dtype fp8 \
+    --kv-cache-skip-steps "0,1" \
+    --kv-cache-skip-layers "0-2"
+```
+
+Online serving:
+
+```bash
+vllm serve <your-model> --omni --kv-cache-dtype fp8
+```
+
+Stage config:
+
+```yaml
+stage_args:
+  - stage_id: 0
+    stage_type: diffusion
+    engine_args:
+      model_stage: dit
+      kv_cache_dtype: "fp8"
+      kv_cache_skip_steps: "0,1"
+      kv_cache_skip_layers: "0-2"
+```
+
+## Parameters
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `kv_cache_dtype` | str \| None | `None` | Set to `"fp8"` to enable dynamic FP8 FA on supported attention backends |
+| `kv_cache_skip_steps` | str \| None | `None` | Denoising step selector to keep in native dtype, for example `"0,1,4-6"` |
+| `kv_cache_skip_layers` | str \| None | `None` | Transformer layer selector to keep in native dtype, for example `"0-2,10"` |
+
+Selectors use comma-separated integers and inclusive ranges. Listed steps or
+layers skip FP8 FA; all other eligible full-attention forwards use the FP8 path.
+
+## Validation and Notes
+
+1. Compare generated images or videos against a BF16 baseline with the same
+   seed, prompt, resolution, frame count, and denoising steps.
+2. Use `kv_cache_skip_steps` for denoising steps where quality is more
+   sensitive.
+3. Use `kv_cache_skip_layers` for transformer layers that show visible quality
+   regressions.
+4. Report both latency and quality results when enabling this option for a new
+   model. For image or video models, include visual comparison and quantitative
+   metrics when available, such as PSNR or SSIM.

examples/offline_inference/audiox/README.md

@@ -0,0 +1,40 @@
+# AudioX offline inference
+
+Generate audio with the [AudioX](https://zeyuet.github.io/AudioX/) MMDiT diffusion
+pipeline (`AudioXPipeline`). Six tasks: `t2a`, `t2m`, `v2a`, `v2m`, `tv2a`, `tv2m`.
+
+## Prerequisites
+
+Download a vLLM-Omni weight bundle (component-sharded safetensors):
+
+```bash
+huggingface-cli download zhangj1an/AudioX --local-dir ./audiox_weights
+```
+
+The Hugging Face id `zhangj1an/AudioX` also works directly without prefetching.
+
+## Usage
+
+```bash
+# Text-to-audio only (default uses zhangj1an/AudioX from the Hub):
+python end2end.py --tasks t2a
+
+# All six tasks against a local bundle and a sample video for v2*/tv2*:
+python end2end.py \
+  --model ./audiox_weights \
+  --video https://zeyuet.github.io/AudioX/static/samples/V2M/1XeBotOFqHA.mp4
+
+# Subset of tasks, custom seed and steps:
+python end2end.py --tasks t2a tv2a --num-inference-steps 100 --seed 0
+```
+
+## Arguments
+
+- `--model`: HF id or local bundle path (default: `zhangj1an/AudioX`).
+- `--tasks`: any subset of `t2a t2m v2a v2m tv2a tv2m` (default: all).
+- `--video`: video file/URL — required for `v2*` and `tv2*`.
+- `--reference-audio`: optional audio prompt (audio-conditioned generation).
+- `--num-inference-steps`, `--guidance-scale`, `--seed`, `--seconds-total`,
+  `--sample-rate`, `--output-dir`: generation knobs.
+
+Outputs land in `<output-dir>/<task>.wav` as 16-bit stereo WAV.

examples/offline_inference/audiox/end2end.py

@@ -0,0 +1,110 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""End-to-end AudioX offline example covering the 6 t2*/v2*/tv2* tasks.
+
+Provide a directory with the **vLLM-Omni AudioX safetensors bundle** (e.g. from
+``zhangj1an/AudioX`` on Hugging Face)::
+
+    huggingface-cli download zhangj1an/AudioX --local-dir ./audiox_weights
+    python end2end.py --model ./audiox_weights
+    python end2end.py --model ./audiox_weights --tasks t2a tv2a
+"""
+
+from __future__ import annotations
+
+import argparse
+import time
+from pathlib import Path
+
+import soundfile
+import torch
+import torchaudio.functional as TF
+
+from vllm_omni.entrypoints.omni import Omni
+from vllm_omni.inputs.data import OmniDiffusionSamplingParams
+from vllm_omni.platforms import current_omni_platform
+
+ROOT = Path(__file__).resolve().parent
+
+SAMPLE_PROMPTS: dict[str, str] = {
+    "t2a": "Fireworks burst twice, followed by a period of silence before a clock begins ticking.",
+    "t2m": "Uplifting ukulele tune for a travel vlog",
+    "v2a": "",
+    "v2m": "",
+    "tv2a": "drum beating sound and human talking",
+    "tv2m": "uplifting music matching the scene",
+}
+
+ALL_TASKS = ("t2a", "t2m", "v2a", "v2m", "tv2a", "tv2m")
+VIDEO_TASKS = frozenset({"v2a", "v2m", "tv2a", "tv2m"})
+TEXT_TASKS = frozenset({"t2a", "t2m", "tv2a", "tv2m"})
+
+
+def parse_args() -> argparse.Namespace:
+    p = argparse.ArgumentParser(description="AudioX offline end-to-end (6 t2*/v2*/tv2* tasks).")
+    p.add_argument("--model", default="zhangj1an/AudioX", help="HF id or local AudioX bundle path.")
+    p.add_argument("--tasks", nargs="+", default=list(ALL_TASKS), choices=ALL_TASKS)
+    p.add_argument("--video", default="", help="Video path / URL (required for v2*/tv2*).")
+    p.add_argument("--reference-audio", default="", help="Optional audio prompt for audio-conditioned generation.")
+    p.add_argument("--output-dir", default=str(ROOT / "audiox_task_outputs"))
+    p.add_argument("--num-inference-steps", type=int, default=250)
+    p.add_argument("--seconds-total", type=float, default=10.0)
+    p.add_argument("--guidance-scale", type=float, default=6.0)
+    p.add_argument("--seed", type=int, default=42)
+    p.add_argument("--sample-rate", type=int, default=48000, help="Output WAV rate (resampled if != model rate).")
+    return p.parse_args()
+
+
+def save_wav(audio: torch.Tensor, path: Path, sample_rate: int) -> None:
+    """Write 16-bit PCM WAV. ``audio`` is ``[channels, samples]`` float in [-1, 1]."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    soundfile.write(str(path), audio.clamp(-1.0, 1.0).cpu().T.numpy(), sample_rate, subtype="PCM_16")
+
+
+def main() -> None:
+    args = parse_args()
+
+    omni = Omni(model=args.model, model_class_name="AudioXPipeline")
+
+    for task in args.tasks:
+        if task in VIDEO_TASKS and not args.video:
+            raise SystemExit(f"task={task!r} requires --video")
+        prompt = SAMPLE_PROMPTS[task] if task in TEXT_TASKS else ""
+        extra: dict = {"audiox_task": task, "seconds_start": 0.0, "seconds_total": float(args.seconds_total)}
+        if task in VIDEO_TASKS:
+            extra["video_path"] = args.video
+        if args.reference_audio:
+            extra["audio_path"] = args.reference_audio
+
+        generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed)
+        t0 = time.perf_counter()
+        outputs = omni.generate(
+            prompt,
+            OmniDiffusionSamplingParams(
+                generator=generator,
+                guidance_scale=args.guidance_scale,
+                num_inference_steps=args.num_inference_steps,
+                seed=args.seed,
+                extra_args=extra,
+            ),
+        )
+        audio = outputs[0].request_output.multimodal_output.get("audio")
+        if audio is None:
+            raise RuntimeError(f"No audio produced for task {task!r}")
+        audio = torch.as_tensor(audio).detach().cpu().float()
+        if audio.ndim == 3:
+            audio = audio[0]
+
+        model_sr = int(outputs[0].request_output.multimodal_output.get("audio_sample_rate") or 44100)
+        if model_sr != args.sample_rate:
+            audio = TF.resample(audio, model_sr, args.sample_rate)
+
+        out_path = Path(args.output_dir) / f"{task}.wav"
+        save_wav(audio, out_path, args.sample_rate)
+        print(f"[{task}] saved {out_path} ({time.perf_counter() - t0:.2f}s)")
+
+    omni.close()
+
+
+if __name__ == "__main__":
+    main()

examples/offline_inference/bagel/README.md

@@ -195,6 +195,25 @@ stages:
     devices: "0,1"
 ```
 
+#### Hybrid Sharded Data Parallel (HSDP)
+
+For larger Bagel deployments on multiple GPUs, you can enable HSDP (Hybrid Sharded Data Parallel) by modifying the stage configuration (for example, [`bagel.yaml`](../../../vllm_omni/deploy/bagel.yaml)). HSDP shards transformer weights across GPUs to reduce per-GPU memory usage.
+
+1. **Enable HSDP**: Set `use_hsdp: true`.
+2. **Set shard size**: Set `hsdp_shard_size` to the number of GPUs used for sharding (for example, `4`).
+3. **Set replicate size**: Usually keep `hsdp_replicate_size: 1` unless you want replicated HSDP groups.
+4. **Set devices**: Specify the comma-separated GPU IDs used by the diffusion stage (for example, `"0,1,2,3"`).
+
+Example configuration for HSDP across 4 GPUs:
+```yaml
+  - stage_id: 1
+    devices: "0,1,2,3"
+    parallel_config:
+      use_hsdp: true
+      hsdp_shard_size: 4
+      hsdp_replicate_size: 1
+```
+
 Then pass the custom deploy YAML:
 
 ```bash

examples/offline_inference/bagel/end2end.py

@@ -153,7 +153,7 @@ def main():
 
     from vllm_omni.entrypoints.omni import Omni
 
-    omni_kwargs = {}
+    omni_kwargs = vars(args).copy()
     deploy_config = args.deploy_config
     if args.think and deploy_config is None:
         deploy_config = "vllm_omni/deploy/bagel_think.yaml"
@@ -161,22 +161,12 @@ def main():
     if deploy_config:
         omni_kwargs["deploy_config"] = deploy_config
 
-    omni_kwargs.update(
-        {
-            "log_stats": args.log_stats,
-            "init_sleep_seconds": args.init_sleep_seconds,
-            "batch_timeout": args.batch_timeout,
-            "init_timeout": args.init_timeout,
-            "shm_threshold_bytes": args.shm_threshold_bytes,
-            "worker_backend": args.worker_backend,
-            "ray_address": args.ray_address,
-            "enable_diffusion_pipeline_profiler": args.enable_diffusion_pipeline_profiler,
-        }
-    )
     if args.quantization:
         omni_kwargs["quantization_config"] = args.quantization
 
-    omni = Omni.from_cli_args(args, model=model_name, **omni_kwargs)
+    # Override CLI --model with the derived model_name.
+    omni_kwargs["model"] = model_name
+    omni = Omni(**omni_kwargs)
 
     formatted_prompts = []
     for p in prompts:

examples/offline_inference/cosyvoice3/README.md

@@ -1,71 +0,0 @@
-# CosyVoice3
-
-## Setup
-
-Install dependencies:
-```
-uv pip install -e .
-```
-
-> **Note:** This includes required libraries such as `soundfile`,
-> `onnxruntime`, `x-transformers`, and `einops` via
-> `requirements/common.txt` and platform-specific requirements files.
-
-Download the model snapshot:
-```
-from huggingface_hub import snapshot_download
-snapshot_download('FunAudioLLM/Fun-CosyVoice3-0.5B-2512', local_dir='pretrained_models/Fun-CosyVoice3-0.5B')
-```
-
-Add `config.json` in `pretrained_models/Fun-CosyVoice3-0.5B/`:
-```json
-{
-    "model_type": "cosyvoice3",
-    "architectures": [
-        "CosyVoice3Model"
-    ]
-}
-```
-
-> **Why `config.json` is required:**
-> `AutoConfig.register("cosyvoice3", CosyVoice3Config)` only registers a class mapping.
-> The loader still needs `model_type: "cosyvoice3"` from `config.json` to select that class.
-> If no `config.json` is present, model type cannot be inferred automatically.
-> If your downloaded checkpoint already includes a valid `config.json` with
-> `model_type: "cosyvoice3"`, this manual step can be skipped.
-
-Run the offline verification script:
-```
-python examples/offline_inference/cosyvoice3/verify_e2e_cosyvoice.py \
-  --model pretrained_models/Fun-CosyVoice3-0.5B \
-  --tokenizer pretrained_models/Fun-CosyVoice3-0.5B/CosyVoice-BlankEN
-```
-
-## Implementation Overview
-
-CosyVoice3 runs as a **2-stage Omni pipeline**.
-
-- **Stage 0** (`talker`) converts text + prompt audio to speech tokens.
-- **Stage 1** (`code2wav`) converts speech tokens + prompt features to waveform via flow matching and HiFiGAN.
-
-Key components live in `vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py`.
-
-- `CosyVoice3MultiModalProcessor` builds the multimodal inputs:
-    - Tokenizes `prompt` and `prompt_text`.
-    - Extracts speech tokens and mel features from the prompt audio.
-    - Extracts a speaker embedding.
-- `CosyVoice3Model` implements both stages:
-    - Stage 0 uses `CosyVoice3LM` and outputs speech tokens + conditioning features.
-    - Stage 1 runs the flow model (DiT-based CFM) and HiFiGAN to synthesize waveform.
-
-Pipeline topology lives in `vllm_omni/model_executor/models/cosyvoice3/pipeline.py`;
-runtime tunables (batch size, memory limits, sampling) live in
-`vllm_omni/deploy/cosyvoice3.yaml`. The deploy config auto-loads by
-HF `model_type` and defaults to `async_chunk: true` (shared-memory
-streaming). Pass `--no-async-chunk` on `vllm serve` to switch to the
-legacy sync path where stage 1 runs `text2flow` over the full
-speech-token sequence.
-
-- Stage 0 emits latent speech tokens.
-- Stage 1 consumes them via `sync_process_input_func` (sync mode) or the
-  shared-memory connector (async-chunk mode) and outputs audio.

examples/offline_inference/covo_audio/README.md

@@ -0,0 +1,76 @@
+# Covo-Audio-Chat (Offline Inference)
+
+## Setup
+
+Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup.
+
+> **Note**
+> Covo-Audio code2wav requires `torchdiffeq`. Install it with: `pip install torchdiffeq`
+
+## Run examples
+
+Get into the example folder:
+```bash
+cd examples/offline_inference/covo_audio
+```
+
+### Audio input chat
+
+Using the default audio asset:
+```bash
+python end2end.py
+```
+
+Using a custom audio file:
+```bash
+python end2end.py --audio-path /path/to/audio.wav
+```
+
+Using a local model:
+```bash
+python end2end.py -m /path/to/Covo-Audio-Chat --output-dir ./my_output
+```
+
+### Command-line Arguments
+
+| Argument | Short | Default | Description |
+|----------|-------|---------|-------------|
+| `--model-name` | `-m` | `tencent/Covo-Audio-Chat` | Model path or HuggingFace model ID |
+| `--text` | `-t` | `请回答这段音频里的问题。` | Text prompt / question for the audio |
+| `--audio-path` | `-a` | default audio asset | Path to local audio file |
+| `--sampling-rate` | | `16000` | Sampling rate for audio loading (Hz) |
+| `--output-dir` | | `./output_audio` | Output directory for generated files |
+| `--num-prompts` | | `1` | Number of prompts to generate |
+| `--stage-configs-path` | | (auto) | Path to stage configs YAML file |
+| `--log-stats` | | `false` | Enable detailed statistics logging |
+| `--stage-init-timeout` | | `300` | Stage initialization timeout (seconds) |
+| `--batch-timeout` | | `5` | Batching timeout (seconds) |
+| `--init-timeout` | | `300` | Overall initialization timeout (seconds) |
+| `--shm-threshold-bytes` | | `65536` | Shared memory threshold (bytes) |
+
+## Pipeline
+
+Covo-Audio-Chat uses a 2-stage pipeline:
+
+- **Stage 0 (fused_thinker_talker):** The 7B LLM generates interleaved text and audio tokens in a single autoregressive pass.
+- **Stage 1 (code2wav):** A BigVGAN-based vocoder converts the extracted audio codes into a 24kHz WAV waveform.
+
+## Output
+
+The script generates two files per request in the output directory:
+
+- `{request_id}.txt` -- prompt and generated text
+- `{request_id}.wav` -- generated audio (24kHz WAV)
+
+## FAQ
+
+If you encounter `ModuleNotFoundError: No module named 'librosa'`, install it with:
+```bash
+pip install librosa
+```
+
+## Environment
+
+- GPU: 1x A100 (80 GiB)
+- Stage 0 (7B LLM): ~16 GiB VRAM
+- Stage 1 (BigVGAN vocoder): ~2 GiB VRAM

examples/offline_inference/covo_audio/end2end.py

@@ -0,0 +1,222 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""
+This example shows how to use vLLM-Omni for running offline inference
+with the correct prompt format on Covo-Audio-Chat.
+
+Usage:
+    python end2end.py --audio-path /path/to/audio.wav
+"""
+
+import os
+
+import soundfile as sf
+from vllm.assets.audio import AudioAsset
+from vllm.multimodal.media.audio import load_audio
+from vllm.sampling_params import SamplingParams
+from vllm.utils.argparse_utils import FlexibleArgumentParser
+
+from vllm_omni.entrypoints.omni import Omni
+from vllm_omni.model_executor.models.covo_audio.prompt_utils import (
+    COVO_AUDIO_INPUT_PREFIX,
+    build_covo_audio_chat_prompt,
+)
+
+SEED = 42
+
+
+def get_audio_query(
+    question: str | None = None,
+    audio_path: str | None = None,
+    sampling_rate: int = 16000,
+) -> dict:
+    if question is None:
+        question = "请回答这段音频里的问题。"
+    user_content = COVO_AUDIO_INPUT_PREFIX + question
+    prompt = build_covo_audio_chat_prompt(user_content)
+
+    if audio_path is None:
+        audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate
+    else:
+        import numpy as np
+
+        audio_signal, sr = load_audio(audio_path, sr=sampling_rate)
+        audio_data = (audio_signal.astype(np.float32), sr)
+
+    return {
+        "prompt": prompt,
+        "multi_modal_data": {"audio": audio_data},
+        "modalities": ["audio"],
+    }
+
+
+def main(args):
+    query_result = get_audio_query(
+        question=args.text,
+        audio_path=args.audio_path,
+        sampling_rate=args.sampling_rate,
+    )
+
+    omni = Omni(
+        model=args.model_name,
+        stage_configs_path=args.stage_configs_path,
+        log_stats=args.log_stats,
+        stage_init_timeout=args.stage_init_timeout,
+        batch_timeout=args.batch_timeout,
+        init_timeout=args.init_timeout,
+        shm_threshold_bytes=args.shm_threshold_bytes,
+    )
+
+    # Stage 0: fused_thinker_talker
+    # stop_token_ids=[151645] (<|im_end|>) and ignore_eos=True are required
+    # so the model generates interleaved text+audio tokens before stopping.
+    thinker_sampling_params = SamplingParams(
+        temperature=0.0,
+        top_p=1.0,
+        top_k=-1,
+        max_tokens=2048,
+        seed=SEED,
+        detokenize=True,
+        repetition_penalty=1.05,
+        stop_token_ids=[151645],
+        ignore_eos=True,
+    )
+    # Stage 1: code2wav (audio codes, not real token IDs — skip detokenize)
+    code2wav_sampling_params = SamplingParams(
+        temperature=0.0,
+        top_p=1.0,
+        top_k=-1,
+        max_tokens=2048,
+        seed=SEED,
+        detokenize=False,
+        repetition_penalty=1.1,
+    )
+
+    sampling_params_list = [
+        thinker_sampling_params,
+        code2wav_sampling_params,
+    ]
+
+    prompts = [query_result for _ in range(args.num_prompts)]
+
+    omni_outputs = omni.generate(prompts, sampling_params_list)
+
+    output_dir = args.output_dir
+    os.makedirs(output_dir, exist_ok=True)
+
+    for stage_outputs in omni_outputs:
+        output = stage_outputs.request_output
+        if stage_outputs.final_output_type == "text":
+            request_id = output.request_id
+            text_output = output.outputs[0].text
+            prompt_text = output.prompt
+            out_txt = os.path.join(output_dir, f"{request_id}.txt")
+            lines = [
+                "Prompt:\n",
+                str(prompt_text) + "\n",
+                "vllm_text_output:\n",
+                str(text_output).strip() + "\n",
+            ]
+            try:
+                with open(out_txt, "w", encoding="utf-8") as f:
+                    f.writelines(lines)
+            except Exception as e:
+                print(f"[Warn] Failed writing text file {out_txt}: {e}")
+            print(f"Request ID: {request_id}, Text saved to {out_txt}")
+        elif stage_outputs.final_output_type == "audio":
+            request_id = output.request_id
+            audio_tensor = output.outputs[0].multimodal_output.get("audio")
+            if audio_tensor is None:
+                continue
+            output_wav = os.path.join(output_dir, f"{request_id}.wav")
+            audio_numpy = audio_tensor.float().detach().cpu().numpy()
+            if audio_numpy.ndim > 1:
+                audio_numpy = audio_numpy.flatten()
+            sf.write(output_wav, audio_numpy, samplerate=24000, format="WAV")
+            print(f"Request ID: {request_id}, Audio saved to {output_wav}")
+
+    omni.close()
+
+
+def parse_args():
+    parser = FlexibleArgumentParser(description="Offline inference demo for Covo-Audio-Chat")
+    parser.add_argument(
+        "--model-name",
+        "-m",
+        type=str,
+        default="tencent/Covo-Audio-Chat",
+        help="Model path or HuggingFace model ID.",
+    )
+    parser.add_argument(
+        "--text",
+        "-t",
+        type=str,
+        default=None,
+        help="Text prompt / question for the audio.",
+    )
+    parser.add_argument(
+        "--audio-path",
+        "-a",
+        type=str,
+        default=None,
+        help="Path to local audio file. Uses default asset if not provided.",
+    )
+    parser.add_argument(
+        "--sampling-rate",
+        type=int,
+        default=16000,
+        help="Sampling rate for audio loading (default: 16000).",
+    )
+    parser.add_argument(
+        "--stage-configs-path",
+        type=str,
+        default=None,
+        help="Path to stage configs YAML file.",
+    )
+    parser.add_argument(
+        "--log-stats",
+        action="store_true",
+        default=False,
+        help="Enable writing detailed statistics.",
+    )
+    parser.add_argument(
+        "--stage-init-timeout",
+        type=int,
+        default=300,
+        help="Timeout for initializing a single stage in seconds.",
+    )
+    parser.add_argument(
+        "--batch-timeout",
+        type=int,
+        default=5,
+        help="Timeout for batching in seconds.",
+    )
+    parser.add_argument(
+        "--init-timeout",
+        type=int,
+        default=300,
+        help="Timeout for initializing stages in seconds.",
+    )
+    parser.add_argument(
+        "--shm-threshold-bytes",
+        type=int,
+        default=65536,
+        help="Threshold for using shared memory in bytes.",
+    )
+    parser.add_argument(
+        "--output-dir",
+        default="./output_audio",
+        help="Output directory for generated files.",
+    )
+    parser.add_argument(
+        "--num-prompts",
+        type=int,
+        default=1,
+        help="Number of prompts to generate.",
+    )
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    main(args)

examples/offline_inference/custom_pipeline/image_to_image/image_edit.py

@@ -118,9 +118,6 @@ def parse_args() -> argparse.Namespace:
         help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
     )
 
-    from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
-
-    nullify_stage_engine_defaults(parser)
     return parser.parse_args()
 
 

examples/offline_inference/dynin_omni/end2end.py

@@ -18,6 +18,8 @@ import numpy as np
 import torch
 from PIL import Image
 
+from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
+
 TASK_CHOICES = ("t2t", "t2i", "t2s", "i2i", "i2t", "s2t", "v2t")
 
 TASK_DEFAULT_RUNTIME = {
@@ -970,7 +972,6 @@ def parse_args(repo_root: Path) -> argparse.Namespace:
     parser.add_argument("--vq-model-audio-local-files-only", action=argparse.BooleanOptionalAction, default=None)
 
     parser.add_argument("--disable-hf-xet", action=argparse.BooleanOptionalAction, default=True)
-    from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
 
     nullify_stage_engine_defaults(parser)
     return parser.parse_args()

examples/offline_inference/fish_speech/README.md

@@ -1,75 +0,0 @@
-# Fish Speech S2 Pro
-
-This directory contains an offline demo for running Fish Speech S2 Pro (`fishaudio/s2-pro`) with vLLM Omni. It supports text-only synthesis and voice cloning with reference audio.
-
-## Model Overview
-
-[Fish Speech S2 Pro](https://huggingface.co/fishaudio/s2-pro) is a 4B dual-AR text-to-speech model by FishAudio. It uses:
-
-- **Slow AR**: Generates semantic tokens autoregressively (Qwen3-based backbone)
-- **Fast AR**: Predicts residual codebook tokens from semantic tokens
-- **DAC Decoder**: Converts codec codes to 44.1 kHz audio waveform
-
-The model produces high-quality speech with voice cloning capabilities.
-
-## Prerequisites
-
-Install the `fish-speech` package for the DAC codec:
-
-```bash
-pip install fish-speech
-```
-
-## Setup
-
-Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup.
-
-## Quick Start
-
-### Text-Only Synthesis
-
-```bash
-python end2end.py --text "Hello, this is a test of the Fish Speech text to speech system."
-```
-
-Generated audio files are saved to `output_audio/` by default.
-
-### Voice Cloning
-
-Provide a reference audio file and its transcript to clone a voice:
-
-```bash
-python end2end.py \
-    --text "Hello, this is a cloned voice." \
-    --ref-audio /path/to/reference.wav \
-    --ref-text "Transcript of the reference audio."
-```
-
-## Streaming Mode
-
-Add `--streaming` to stream audio chunks progressively via `AsyncOmni` (requires `async_chunk: true` in the stage config):
-
-```bash
-python end2end.py --text "Hello, this is a test." --streaming
-```
-
-Each DAC decoder chunk is logged as it arrives. The final WAV file is written once generation completes.
-
-## Configuration
-
-| Argument | Default | Description |
-|----------|---------|-------------|
-| `--text` | `"Hello, this is a test..."` | Text to synthesize |
-| `--ref-audio` | None | Path to reference audio for voice cloning |
-| `--ref-text` | None | Transcript of the reference audio |
-| `--model` | `fishaudio/s2-pro` | HuggingFace model path |
-| `--stage-configs-path` | `fish_speech_s2_pro.yaml` | Path to stage configs YAML |
-| `--output-dir` | `output_audio` | Output directory for WAV files |
-| `--streaming` | False | Enable streaming via AsyncOmni |
-| `--stage-init-timeout` | 600 | Stage initialization timeout (seconds) |
-
-## Notes
-
-- Output audio is 44.1 kHz mono WAV.
-- The DAC codec weights (`codec.pth`) are loaded lazily from the model directory.
-- Voice cloning encodes the reference audio using the DAC codec to extract semantic codes, then prepends them as a system message.

examples/offline_inference/helios/end2end.py

@@ -196,9 +196,6 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--cfg-parallel-size", type=int, default=1, choices=[1, 2], help="CFG parallel size.")
     parser.add_argument("--tensor-parallel-size", type=int, default=1, help="Tensor parallelism size.")
 
-    from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
-
-    nullify_stage_engine_defaults(parser)
     return parser.parse_args()
 
 

examples/offline_inference/hunyuan_image3/README.md

@@ -1,161 +1,169 @@
 # HunyuanImage-3.0-Instruct
 
-## Set up
+This example runs HunyuanImage-3.0-Instruct offline with the unified deploy
+YAMLs under `vllm_omni/deploy/`.
 
-Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup.
+## Deploy Configs
 
-## Run examples
+| File | Topology | Default use |
+| :--- | :--- | :--- |
+| `vllm_omni/deploy/hunyuan_image3.yaml` | AR + DiT | Default for `text2img` and `img2img`. |
+| `vllm_omni/deploy/hunyuan_image3_ar.yaml` | AR only | Default for `img2text` and `text2text`. |
+| `vllm_omni/deploy/hunyuan_image3_dit.yaml` | DiT only | Standalone diffusion stage. Pass it explicitly with `--deploy-config`. |
 
-**Note**: These examples work with the default configuration on **8x NVIDIA L40S (48GB)**. For different GPU setups, modify the stage configuration to adjust device allocation and memory utilization.
+The example chooses a deploy config automatically when `--deploy-config` and
+`--stage-configs-path` are both omitted:
 
-Get into the hunyuan_image3 folder:
+| `--modality` | `mode` passed to Omni | Default deploy |
+| :--- | :--- | :--- |
+| `text2img` | `text-to-image` | `hunyuan_image3.yaml` |
+| `img2img` | `image-editing` | `hunyuan_image3.yaml` |
+| `img2text` | `image-to-text` | `hunyuan_image3_ar.yaml` |
+| `text2text` | `text-to-text` | `hunyuan_image3_ar.yaml` |
+
+`--modality` is an offline example convenience flag. It maps to the internal
+`mode` argument passed to `Omni(...)` by this script. HunyuanImage3 uses
+separate deploy YAMLs for AR + DiT, AR-only, and DiT-only topologies, so the
+stage topology is selected by the deploy file rather than by YAML mode
+overrides.
+
+Online serving does not expose a `--modality` flag or accept `mode` as an API
+request field. Choose the deploy topology when starting the server with
+`--deploy-config`, then use the OpenAI-compatible endpoint and request shape for
+the scenario. The `modalities` request field is used by the chat completions
+path; the image endpoints infer the image task from the endpoint and payload.
+
+| Online scenario | Server deploy | Request |
+| :--- | :--- | :--- |
+| Text to image | `--deploy-config vllm_omni/deploy/hunyuan_image3.yaml` | `POST /v1/images/generations`, or `POST /v1/chat/completions` with `"modalities": ["image"]`. |
+| Image editing | `--deploy-config vllm_omni/deploy/hunyuan_image3.yaml` | `POST /v1/images/edits`. |
+| Image/text to text | `--deploy-config vllm_omni/deploy/hunyuan_image3_ar.yaml` | `POST /v1/chat/completions` for text output, for example with `"modalities": ["text"]`. |
+| DiT-only image generation | `--deploy-config vllm_omni/deploy/hunyuan_image3_dit.yaml` | `POST /v1/images/generations`. |
+
+## Run Examples
+
+Text to image, using the default AR + DiT deploy:
 
 ```bash
-cd examples/offline_inference/hunyuan_image3
+python examples/offline_inference/hunyuan_image3/end2end.py \
+  --model tencent/HunyuanImage-3.0-Instruct \
+  --modality text2img \
+  --prompts "A cute cat sitting on a windowsill watching the sunset"
 ```
 
-### Modality Control
-
-HunyuanImage-3.0-Instruct supports multiple modality modes. You can control the mode using the `--modality` argument:
-
-#### Text to Image (text2img)
-
-- **Pipeline**: Text → AR (CoT + latent tokens) → DiT (denoise) → VAE Decode → Image
-- **Stages Used**: Stage 0 (AR) + Stage 1 (DiT)
-- **KV Transfer**: AR sends KV cache to DiT for conditioned generation
-- **Default Config**: `hunyuan_image3_t2i.yaml`
+Image editing, using the default AR + DiT deploy:
 
 ```bash
-python end2end.py --model tencent/HunyuanImage-3.0-Instruct \
-                  --modality text2img \
-                  --prompts "A cute cat sitting on a windowsill watching the sunset"
+python examples/offline_inference/hunyuan_image3/end2end.py \
+  --model tencent/HunyuanImage-3.0-Instruct \
+  --modality img2img \
+  --image-path /path/to/image.png \
+  --prompts "Make the petals neon pink"
 ```
 
-#### Image to Image (img2img)
-
-- **Pipeline**: Image + Text → AR (CoT + recaption + latent) → DiT → Edited Image
-- **Stages Used**: Stage 0 (AR) + Stage 1 (DiT)
-- **KV Transfer**: AR sends KV cache to DiT
-- **Default Config**: `hunyuan_image3_it2i.yaml`
+Image to text, using the AR-only deploy:
 
 ```bash
-python end2end.py --model tencent/HunyuanImage-3.0-Instruct \
-                  --modality img2img \
-                  --image-path /path/to/image.png \
-                  --prompts "Make the petals neon pink"
+python examples/offline_inference/hunyuan_image3/end2end.py \
+  --model tencent/HunyuanImage-3.0-Instruct \
+  --modality img2text \
+  --image-path /path/to/image.jpg \
+  --prompts "Describe the content of the picture."
 ```
 
-#### Image to Text (img2text)
-
-- **Pipeline**: Image + Question → AR → Text description
-- **Stages Used**: Stage 0 (AR) only
-- **Default Config**: `hunyuan_image3_i2t.yaml`
+Text to text, using the AR-only deploy:
 
 ```bash
-python end2end.py --model tencent/HunyuanImage-3.0-Instruct \
-                  --modality img2text \
-                  --image-path /path/to/image.jpg \
-                  --prompts "Describe the content of the picture."
+python examples/offline_inference/hunyuan_image3/end2end.py \
+  --model tencent/HunyuanImage-3.0-Instruct \
+  --modality text2text \
+  --prompts "What is the capital of France?"
 ```
 
-#### Text to Text (text2text)
-
-- **Pipeline**: Text → AR → Text
-- **Stages Used**: Stage 0 (AR) only
-- **Default Config**: `hunyuan_image3_t2t.yaml`
+Standalone DiT, using the DiT-only deploy explicitly:
 
 ```bash
-python end2end.py --model tencent/HunyuanImage-3.0-Instruct \
-                  --modality text2text \
-                  --prompts "What is the capital of France?"
+python examples/offline_inference/hunyuan_image3/end2end.py \
+  --model tencent/HunyuanImage-3.0-Instruct \
+  --modality text2img \
+  --deploy-config vllm_omni/deploy/hunyuan_image3_dit.yaml \
+  --prompts "A cinematic portrait of an astronaut in a greenhouse"
 ```
 
-### Inference Steps & Guidance
+Override the default full AR + DiT deploy explicitly:
 
-Control generation quality for image modalities:
+```bash
+python examples/offline_inference/hunyuan_image3/end2end.py \
+  --model tencent/HunyuanImage-3.0-Instruct \
+  --modality text2img \
+  --deploy-config vllm_omni/deploy/hunyuan_image3.yaml \
+  --prompts "A cute cat"
+```
+
+## Additional Config
+
+You can pass diffusion worker `additional_config` from the offline example as a JSON object.
+This maps to the upstream vLLM `VllmConfig.additional_config` platform extension field:
+https://docs.vllm.ai/en/stable/api/vllm/config/#vllm.config.VllmConfig.additional_config
 
 ```bash
 python end2end.py --modality text2img \
-                  --steps 50 \
-                  --guidance-scale 5.0 \
-                  --height 1024 --width 1024 \
-                  --prompts "A photo-realistic sunset over the ocean"
+                  --prompts "A cute cat" \
+                  --additional-config '{"torchair_graph_config":{"enabled":true}}'
 ```
 
-### Key Arguments
+## Key Arguments
 
-#### 📌 Command Line Arguments (end2end.py)
+| Argument | Description |
+| :--- | :--- |
+| `--deploy-config` | Preferred config path for unified deploy YAMLs. |
+| `--stage-configs-path` | Legacy stage config path, kept only for compatibility. Prefer `--deploy-config`. |
+| `--additional-config` | JSON object forwarded to diffusion worker `additional_config`. |
+| `--modality` | Offline-only convenience flag. One of `text2img`, `img2img`, `img2text`, `text2text`. It selects prompt formatting, internal `mode`, and default deploy config for this script. Online serving uses `--deploy-config` plus the endpoint and, for chat completions, request `modalities` instead. |
+| `--steps` | Number of diffusion inference steps for image generation. |
+| `--guidance-scale` | Classifier-free guidance scale for image generation. |
+| `--height`, `--width` | Output image size for `text2img`. |
+| `--bot-task` | Prompt behavior. `auto` selects the default from `--modality`; `think` adds `<think>`; `recaption` adds `<recaption>`; `vanilla` uses the text-to-image pretrain template. |
+| `--sys-type` | Override the system prompt type, for example `en_unified` or `en_vanilla`. |
+| `--vae-use-tiling` | Enable VAE tiling for memory reduction. |
 
-| Argument               | Type   | Default                              | Description                                                  |
-| :--------------------- | :----- | :----------------------------------- | :----------------------------------------------------------- |
-| `--model`              | string | `tencent/HunyuanImage-3.0-Instruct` | Model path or name                                           |
-| `--modality`           | choice | `text2img`                           | Modality: `text2img`, `img2img`, `img2text`, `text2text`     |
-| `--prompts`            | list   | `None`                               | Input text prompts                                           |
-| `--image-path`         | string | `None`                               | Input image path (for `img2img`/`img2text`)                  |
-| `--output`             | string | `.`                                  | Output directory for saved images                            |
-| `--steps`              | int    | `50`                                 | Number of inference steps                                    |
-| `--guidance-scale`     | float  | `5.0`                                | Classifier-free guidance scale                               |
-| `--seed`               | int    | `42`                                 | Random seed                                                  |
-| `--height`             | int    | `1024`                               | Output image height                                          |
-| `--width`              | int    | `1024`                               | Output image width                                           |
-| `--bot-task`           | string | auto                                 | Override prompt task (e.g. `it2i_think`, `t2i_recaption`)    |
-| `--sys-type`           | string | auto                                 | Override system prompt type (e.g. `en_unified`, `en_vanilla`) |
-| `--stage-configs-path` | string | auto                                 | Custom stage config YAML path                                |
-| `--enforce-eager`      | flag   | `False`                              | Disable torch.compile                                        |
-| `--init-timeout`       | int    | `300`                                | Initialization timeout (seconds)                             |
+## Notes
 
-------
-
-#### ⚙️ Stage Configurations
-
-| Config YAML                         | Modality  | Stages | GPUs   | Description                           |
-| :---------------------------------- | :-------- | :----- | :----- | :------------------------------------ |
-| `hunyuan_image3_t2i.yaml`           | text2img  | 2      | 8      | T2I with AR→DiT, 4 GPU each          |
-| `hunyuan_image3_it2i.yaml`          | img2img   | 2      | 8      | IT2I with AR→DiT, 4 GPU each         |
-| `hunyuan_image3_i2t.yaml`           | img2text  | 1      | 4      | I2T (AR only)                         |
-| `hunyuan_image3_t2t.yaml`           | text2text | 1      | 4      | T2T (AR only)                         |
-| `hunyuan_image3_t2i_2gpu.yaml`      | text2img  | 2      | 2      | T2I for 2-GPU setups                  |
-| `hunyuan_image3_moe.yaml`           | text2img  | 2      | 8      | T2I with MoE AR→DiT KV reuse          |
-| `hunyuan_image3_moe_dit_2gpu_fp8.yaml` | text2img | 2   | 2      | T2I with FP8 quantization             |
-
-------
-
-## Using MoE Config
-
-The `hunyuan_image3_moe.yaml` config enables AR→DiT KV cache reuse with 8 GPUs (4 for AR + 4 for DiT).
-
-```bash
-python end2end.py --model tencent/HunyuanImage-3.0-Instruct \
-                  --modality text2img \
-                  --stage-configs-path hunyuan_image3_moe.yaml \
-                  --prompts "A cute cat"
-```
-
-------
+- `hunyuan_image3_ar.yaml` is a 4-card AR-only text/comprehension deploy. It sets `engine_output_type: text`, `final_output_type: text`, and text sampling defaults.
+- `hunyuan_image3_dit.yaml` is a single-stage DiT deploy with `stage_id: 0`; it does not require stage 1 or a running AR stage.
+- The old HunyuanImage3 YAMLs under `model_executor/stage_configs/` and `platforms/*/stage_configs/` have been folded into the deploy YAMLs.
+- This PR does not keep the HunyuanImage3 AR-to-DiT KV reuse wiring. The deploy YAMLs describe the topology and platform settings only.
 
 ## Prompt Format
 
-HunyuanImage-3.0 uses a pretrain template format:
+HunyuanImage-3.0-Instruct uses an instruct chat template:
 
-```
-<|startoftext|>{system_prompt}{<img>}{trigger_tag}{user_prompt}
+```text
+<|startoftext|>{system_prompt}
+
+User: {<img>?}{user_prompt}
+
+Assistant: {trigger_tag?}
 ```
 
-- `<img>`: Placeholder for each input image (auto-inserted by `prompt_utils.py`)
-- Trigger tags: `<think>` (CoT), `<recaption>` (recaptioning)
-- System prompt: Auto-selected based on task
+- `<img>`: Placeholder for each input image (single token; expanded by the multimodal pipeline).
+- Trigger tags: `<think>` for CoT and `<recaption>` for recaptioning, placed after `Assistant: `.
+- System prompt: Auto-selected based on task.
+- `t2i_vanilla` is the only task that uses the bare pretrain template without chat structure.
+- The example composes the internal prompt task from `--modality` and `--bot-task`
+  before calling `prompt_utils`; for example, `img2text + think` becomes
+  `i2t_think` for prompt and stop-token lookup.
 
-The `prompt_utils.build_prompt()` handles this formatting automatically.
-
-------
+The shared `vllm_omni.diffusion.models.hunyuan_image3.prompt_utils.build_prompt_tokens()`
+helper handles segment-by-segment tokenization and matches HF `apply_chat_template`.
 
 ## FAQ
 
-- **OOM errors**: Decrease `gpu_memory_utilization` in the YAML stage config, or use a smaller `max_num_batched_tokens`.
+- **OOM errors**: Decrease `gpu_memory_utilization` in the deploy YAML, use a smaller `max_num_batched_tokens`, or enable VAE tiling with `--vae-use-tiling`.
 - **Custom image sizes**: Use `--height` and `--width` flags (multiples of 16 recommended).
 
-| Stage             | VRAM (approx)        |
-| :---------------- | :------------------- |
-| Stage 0 (AR)      | ~15 GiB + KV Cache   |
-| Stage 1 (DiT)     | ~30 GiB              |
-| Total (8-GPU)     | ~45 GiB + KV Cache   |
+| Stage | VRAM (approx) |
+| :--- | :--- |
+| Stage 0 (AR) | ~15 GiB + KV Cache |
+| Stage 1 (DiT) | ~30 GiB |
+| Total (8-GPU) | ~45 GiB + KV Cache |

examples/offline_inference/hunyuan_image3/end2end.py

@@ -14,73 +14,45 @@ Usage:
 """
 
 import argparse
+import json
 import os
+from pathlib import Path
 
-from vllm_omni.diffusion.models.hunyuan_image3.system_prompt import (
-    get_system_prompt,
+from vllm_omni.diffusion.models.hunyuan_image3.prompt_utils import (
+    _TASK_PRESETS,
+    build_prompt_tokens,
+    resolve_stop_token_ids,
 )
 from vllm_omni.entrypoints.omni import Omni
 from vllm_omni.inputs.data import OmniPromptType
 
-# task → (sys_type, bot_task, trigger_tag)
-_TASK_PRESETS: dict[str, tuple[str, str | None, str | None]] = {
-    "t2t": ("en_unified", None, None),
-    "i2t": ("en_unified", None, None),
-    "it2i_think": ("en_unified", "think", "<think>"),
-    "it2i_recaption": ("en_unified", "recaption", "<recaption>"),
-    "t2i_think": ("en_unified", "think", "<think>"),
-    "t2i_recaption": ("en_unified", "recaption", "<recaption>"),
-    "t2i_vanilla": ("en_vanilla", "image", None),
+# Default deploy configs are absolute so this example works from any cwd.
+_REPO_ROOT = Path(__file__).resolve().parents[3]
+_DEFAULT_DEPLOY_CONFIG = str(_REPO_ROOT / "vllm_omni" / "deploy" / "hunyuan_image3.yaml")
+_DEFAULT_AR_DEPLOY_CONFIG = str(_REPO_ROOT / "vllm_omni" / "deploy" / "hunyuan_image3_ar.yaml")
+
+_MODALITY_DEFAULT_DEPLOY_CONFIG = {
+    "text2img": _DEFAULT_DEPLOY_CONFIG,
+    "img2img": _DEFAULT_DEPLOY_CONFIG,
+    "img2text": _DEFAULT_AR_DEPLOY_CONFIG,
+    "text2text": _DEFAULT_AR_DEPLOY_CONFIG,
+}
+
+_MODALITY_MODE = {
+    "text2img": "text-to-image",
+    "img2img": "image-editing",
+    "img2text": "image-to-text",
+    "text2text": "text-to-text",
 }
 
-# Modality → prompt_utils task mapping
 _MODALITY_TASK_MAP = {
-    "text2img": "t2i_think",
-    "img2img": "it2i_think",
+    "text2img": "t2i",
+    "img2img": "it2i",
     "img2text": "i2t",
     "text2text": "t2t",
 }
 
 
-def build_prompt(
-    user_prompt: str,
-    task: str = "it2i_think",
-    sys_type: str | None = None,
-    custom_system_prompt: str | None = None,
-) -> str:
-    """Build a HunyuanImage-3.0 prompt using pretrain template format."""
-    if task not in _TASK_PRESETS:
-        raise ValueError(f"Unknown task {task!r}. Choose from: {sorted(_TASK_PRESETS)}")
-
-    preset_sys_type, preset_bot_task, trigger_tag = _TASK_PRESETS[task]
-    effective_sys_type = sys_type or preset_sys_type
-
-    system_prompt = get_system_prompt(effective_sys_type, preset_bot_task, custom_system_prompt)
-    sys_text = system_prompt.strip() if system_prompt else ""
-
-    has_image_input = task.startswith("i2t") or task.startswith("it2i")
-
-    parts = ["<|startoftext|>"]
-    if sys_text:
-        parts.append(sys_text)
-    if has_image_input:
-        parts.append("<img>")
-    if trigger_tag:
-        parts.append(trigger_tag)
-    parts.append(user_prompt)
-
-    return "".join(parts)
-
-
-# Modality → default stage config
-_MODALITY_DEFAULT_CONFIG = {
-    "text2img": "hunyuan_image3_t2i.yaml",
-    "img2img": "hunyuan_image3_it2i.yaml",
-    "img2text": "hunyuan_image3_i2t.yaml",
-    "text2text": "hunyuan_image3_t2t.yaml",
-}
-
-
 def parse_args():
     parser = argparse.ArgumentParser(description="HunyuanImage-3.0-Instruct end-to-end inference.")
     parser.add_argument(
@@ -114,13 +86,23 @@ def parse_args():
     parser.add_argument("--seed", type=int, default=42, help="Random seed.")
     parser.add_argument("--height", type=int, default=1024, help="Output image height.")
     parser.add_argument("--width", type=int, default=1024, help="Output image width.")
+    parser.add_argument(
+        "--vae-use-tiling",
+        action="store_true",
+        help="Enable VAE tiling for memory optimization.",
+    )
 
     # Prompt configuration
     parser.add_argument(
         "--bot-task",
         type=str,
-        default=None,
-        help="Override prompt task (e.g. it2i_think, t2i_recaption). Default: auto from modality.",
+        default="auto",
+        choices=["auto", "think", "recaption", "think_recaption", "vanilla"],
+        help=(
+            "Prompt behavior. 'auto' selects the default for the modality; "
+            "'think' adds <think>; 'recaption' adds <recaption>; "
+            "'vanilla' uses the t2i pretrain template."
+        ),
     )
     parser.add_argument(
         "--sys-type",
@@ -130,10 +112,22 @@ def parse_args():
     )
 
     # Omni init args
-    parser.add_argument("--stage-configs-path", type=str, default=None, help="Custom stage config YAML path.")
+    parser.add_argument("--deploy-config", type=str, default=None, help="Custom deploy YAML path.")
+    parser.add_argument("--stage-configs-path", type=str, default=None, help="Custom legacy stage config YAML path.")
     parser.add_argument("--log-stats", action="store_true", default=False)
     parser.add_argument("--init-timeout", type=int, default=300, help="Initialization timeout in seconds.")
     parser.add_argument("--enforce-eager", action="store_true", help="Disable torch.compile.")
+    parser.add_argument(
+        "--additional-config",
+        type=str,
+        default=None,
+        help=(
+            "JSON object forwarded to Omni/additional_config, for example "
+            '\'{"torchair_graph_config":{"enabled":true}}\'. Different platforms may support different '
+            "configs. Make sure the configs are valid for the platform you are using. "
+            "Contents must be hashable."
+        ),
+    )
 
     from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
 
@@ -141,26 +135,69 @@ def parse_args():
     return parser.parse_args()
 
 
+def parse_additional_config(raw_value: str | None) -> dict | None:
+    """Parse a JSON string into an additional_config mapping."""
+    if raw_value is None:
+        return None
+
+    try:
+        additional_config = json.loads(raw_value)
+    except json.JSONDecodeError as exc:
+        raise ValueError(f"Invalid --additional-config JSON: {exc}") from exc
+
+    if additional_config is None:
+        return None
+    if not isinstance(additional_config, dict):
+        raise ValueError(f"--additional-config must decode to a JSON object, got {type(additional_config).__name__}")
+    return additional_config
+
+
 def main():
     args = parse_args()
     os.makedirs(args.output, exist_ok=True)
+    additional_config = parse_additional_config(args.additional_config)
 
-    # Determine task for prompt formatting
-    task = args.bot_task or _MODALITY_TASK_MAP[args.modality]
+    # Determine task for prompt formatting from modality + bot behavior.
+    task = _MODALITY_TASK_MAP[args.modality]
+    assert task is not None
+    bot_task = args.bot_task
+    if bot_task != "auto":
+        task = task + "_" + bot_task
+    if task not in _TASK_PRESETS:
+        valid_bot_tasks = {
+            "text2img": ["think", "recaption", "vanilla"],
+            "img2img": ["think", "recaption", "think_recaption"],
+            "img2text": ["auto"],
+            "text2text": ["auto"],
+        }[args.modality]
+        raise ValueError(
+            f"--bot-task {bot_task!r} is not supported for {args.modality}. Choose from: {valid_bot_tasks}"
+        )
 
-    # Determine stage config
-    stage_configs_path = args.stage_configs_path or _MODALITY_DEFAULT_CONFIG[args.modality]
+    if args.deploy_config is not None and args.stage_configs_path is not None:
+        raise ValueError("--deploy-config and --stage-configs-path are mutually exclusive.")
+
+    deploy_config = args.deploy_config
+    stage_configs_path = args.stage_configs_path
+    if deploy_config is None and stage_configs_path is None:
+        deploy_config = _MODALITY_DEFAULT_DEPLOY_CONFIG[args.modality]
 
     # Build Omni
     omni_kwargs = {
         "model": args.model,
-        "stage_configs_path": stage_configs_path,
+        "vae_use_tiling": args.vae_use_tiling,
         "log_stats": args.log_stats,
         "init_timeout": args.init_timeout,
         "enforce_eager": args.enforce_eager,
     }
-    if args.modality in ("text2img", "img2img"):
-        omni_kwargs["mode"] = "text-to-image"
+
+    if additional_config is not None:
+        omni_kwargs["additional_config"] = additional_config
+    if deploy_config is not None:
+        omni_kwargs["deploy_config"] = deploy_config
+    else:
+        omni_kwargs["stage_configs_path"] = stage_configs_path
+    omni_kwargs["mode"] = _MODALITY_MODE[args.modality]
 
     omni = Omni(**omni_kwargs)
 
@@ -179,12 +216,28 @@ def main():
 
         input_image = Image.open(args.image_path).convert("RGB")
 
+    # Load tokenizer for segment-wise prompt tokenization (matches HF
+    # apply_chat_template byte-for-byte; see build_prompt_tokens docstring).
+    from transformers import AutoTokenizer
+
+    tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
+
     # Format prompts
     formatted_prompts: list[OmniPromptType] = []
     for p in prompts:
-        formatted_text = build_prompt(p, task=task, sys_type=args.sys_type)
+        result = build_prompt_tokens(p, tokenizer, task=task, sys_type=args.sys_type)
+        token_ids = result.token_ids
+        effective_sys_type = result.system_prompt_type
 
-        prompt_dict: dict = {"prompt": formatted_text}
+        # `prompt_token_ids` drives the AR stage (matches HF byte-for-byte).
+        # `prompt` and `use_system_prompt` are forwarded by ar2diffusion to
+        # the DiT stage so the diffusion pipeline can rebuild the same
+        # system prefix when constructing its model inputs.
+        prompt_dict: dict = {
+            "prompt_token_ids": token_ids,
+            "prompt": p,
+            "use_system_prompt": effective_sys_type,
+        }
 
         if args.modality == "text2img":
             prompt_dict["modalities"] = ["image"]
@@ -207,22 +260,32 @@ def main():
     # Override diffusion params if applicable
     from vllm_omni.inputs.data import OmniDiffusionSamplingParams
 
-    for i, sp in enumerate(params_list):
+    ar_stop_token_ids = resolve_stop_token_ids(task=task, bot_task=bot_task, tokenizer=tokenizer)
+    assert ar_stop_token_ids is not None
+    for sp in params_list:
         if isinstance(sp, OmniDiffusionSamplingParams):
             sp.num_inference_steps = args.steps
             sp.guidance_scale = args.guidance_scale
+            sp.guidance_scale_provided = True
             if args.seed is not None:
                 sp.seed = args.seed
             if args.modality in ("text2img",):
                 sp.height = args.height
                 sp.width = args.width
+        elif hasattr(sp, "stop_token_ids"):
+            sp.stop_token_ids = ar_stop_token_ids
 
     # Print configuration
     print(f"\n{'=' * 60}")
     print("HunyuanImage-3.0 Generation Configuration:")
     print(f"  Model: {args.model}")
     print(f"  Modality: {args.modality}")
-    print(f"  Stage config: {stage_configs_path}")
+    print(f"  Prompt task: {task}")
+    print(f"  Bot task: {bot_task}")
+    if deploy_config is not None:
+        print(f"  Deploy config: {deploy_config}")
+    else:
+        print(f"  Stage config: {stage_configs_path}")
     print(f"  Num stages: {omni.num_stages}")
     if args.modality in ("text2img", "img2img"):
         print(f"  Inference steps: {args.steps}")
@@ -232,6 +295,8 @@ def main():
         print(f"  Output size: {args.width}x{args.height}")
     if args.image_path:
         print(f"  Input image: {args.image_path}")
+    if additional_config is not None:
+        print(f"  Additional config: {additional_config}")
     print(f"  Prompts: {prompts}")
     print(f"{'=' * 60}\n")
 
@@ -243,10 +308,17 @@ def main():
     for req_output in omni_outputs:
         # Text output (AR stage or text-only)
         ro = getattr(req_output, "request_output", None)
+        txt = ""
         if ro and getattr(ro, "outputs", None):
             txt = "".join(getattr(o, "text", "") or "" for o in ro.outputs)
-            if txt:
-                print(f"[Output] Text:\n{txt}")
+        if not txt:
+            ar_text = getattr(req_output, "custom_output", {}).get("ar_generated_text")
+            if isinstance(ar_text, list):
+                txt = "\n".join(text for text in ar_text if text)
+            else:
+                txt = ar_text or ""
+        if txt:
+            print(f"[Output] Text:\n{txt}")
 
         # Image output (DiT stage)
         images = getattr(req_output, "images", None)

examples/offline_inference/image_to_image/image_edit.py

@@ -378,9 +378,6 @@ def parse_args() -> argparse.Namespace:
         default=None,
         help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
     )
-    from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
-
-    nullify_stage_engine_defaults(parser)
     return parser.parse_args()
 
 

examples/offline_inference/image_to_video/image_to_video.py

@@ -102,6 +102,24 @@ def parse_args() -> argparse.Namespace:
         choices=["unipc", "euler"],
         help="Sampling solver for Wan2.2 pipelines. Use 'euler' for Lightning/Distill setups.",
     )
+    parser.add_argument(
+        "--kv-cache-dtype",
+        type=str,
+        default=None,
+        help="Config-level KV cache dtype (e.g. float8_e4m3fn).",
+    )
+    parser.add_argument(
+        "--kv-cache-skip-steps",
+        type=str,
+        default=None,
+        help="Config-level KV-cache quantization skip-step selector, e.g. '0-9,20,25-30'.",
+    )
+    parser.add_argument(
+        "--kv-cache-skip-layers",
+        type=str,
+        default=None,
+        help="Config-level KV-cache quantization skip-layer selector, e.g. '0,1,4-8'.",
+    )
     parser.add_argument("--output", type=str, default="i2v_output.mp4", help="Path to save the video (mp4).")
     parser.add_argument("--fps", type=int, default=None, help="Frames per second for the output video.")
     parser.add_argument(
@@ -200,6 +218,13 @@ def parse_args() -> argparse.Namespace:
             "Default 1 means pure sharding (no replication). "
         ),
     )
+    parser.add_argument(
+        "--quantization",
+        type=str,
+        default=None,
+        choices=["fp8", "mxfp8", "int8", "gguf"],
+        help="Quantization method for the transformer. mxfp8: W8A8 MXFP8 online quant (NPU). fp8: online FP8 (GPU).",
+    )
     parser.add_argument(
         "--enable-diffusion-pipeline-profiler",
         action="store_true",
@@ -211,9 +236,6 @@ def parse_args() -> argparse.Namespace:
         default=None,
         help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
     )
-    from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
-
-    nullify_stage_engine_defaults(parser)
     return parser.parse_args()
 
 
@@ -305,13 +327,16 @@ def main():
         hsdp_shard_size=args.hsdp_shard_size,
         hsdp_replicate_size=args.hsdp_replicate_size,
     )
-    omni = Omni(
+    omni_kwargs = dict(
         model=args.model,
         enable_layerwise_offload=args.enable_layerwise_offload,
         vae_use_slicing=args.vae_use_slicing,
         vae_use_tiling=args.vae_use_tiling,
         boundary_ratio=args.boundary_ratio,
         flow_shift=args.flow_shift,
+        kv_cache_dtype=args.kv_cache_dtype,
+        kv_cache_skip_steps=args.kv_cache_skip_steps,
+        kv_cache_skip_layers=args.kv_cache_skip_layers,
         enable_cpu_offload=args.enable_cpu_offload,
         parallel_config=parallel_config,
         enforce_eager=args.enforce_eager,
@@ -321,6 +346,9 @@ def main():
         enable_diffusion_pipeline_profiler=args.enable_diffusion_pipeline_profiler,
         profiler_config=args.profiler_config,
     )
+    if args.quantization is not None:
+        omni_kwargs["quantization"] = args.quantization
+    omni = Omni(**omni_kwargs)
 
     if profiler_enabled:
         print("[Profiler] Starting profiling...")
@@ -333,6 +361,9 @@ def main():
     print(f"  Inference steps: {args.num_inference_steps}")
     print(f"  Frames: {args.num_frames}")
     print(f"  Solver: {args.sample_solver}")
+    print(f"  kv_cache_dtype(config): {args.kv_cache_dtype}")
+    print(f"  kv_cache_skip_steps(config): {args.kv_cache_skip_steps}")
+    print(f"  kv_cache_skip_layers(config): {args.kv_cache_skip_layers}")
     print(
         f"  Parallel configuration: cfg_parallel_size={args.cfg_parallel_size},"
         f" tensor_parallel_size={args.tensor_parallel_size}, vae_patch_parallel_size={args.vae_patch_parallel_size}"

examples/offline_inference/internvla_a1/end2end.py

@@ -172,7 +172,7 @@ def run_pipeline_forward(
 
 def _synchronize(device: str) -> None:
     if device.startswith("cuda") and torch.cuda.is_available():
-        torch.cuda.synchronize()
+        torch.accelerator.synchronize()
 
 
 def _latency_summary(values_ms: list[float]) -> dict[str, float]:

examples/offline_inference/magi_human/end2end.py

@@ -24,9 +24,6 @@ def parse_args():
     parser.add_argument("--width", type=int, default=448, help="Video width.")
     parser.add_argument("--num-inference-steps", type=int, default=8, help="Number of denoising steps.")
     parser.add_argument("--seed", type=int, default=52, help="Random seed for generation.")
-    from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
-
-    nullify_stage_engine_defaults(parser)
     return parser.parse_args()
 
 

examples/offline_inference/mimo_audio/end2end.py

@@ -23,6 +23,7 @@ from message_convert import (
 from vllm import SamplingParams
 from vllm.utils.argparse_utils import FlexibleArgumentParser
 
+from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
 from vllm_omni.entrypoints.omni import Omni
 from vllm_omni.inputs.data import OmniTokensPrompt
 
@@ -438,6 +439,7 @@ def parse_args():
         "vllm_omni/deploy/mimo_audio.yaml based on the HF model_type.",
     )
 
+    nullify_stage_engine_defaults(parser)
     return parser.parse_args()
 
 

examples/offline_inference/ming_flash_omni/README.md

@@ -4,41 +4,41 @@
 
 vLLM-Omni supports two deployment modes:
 
-| Mode | Stage config | Output |
-|------|-------------|--------|
-| Thinker only (multimodal understanding) | `ming_flash_omni_thinker.yaml` (default `--omni`) | Text |
-| Thinker + Talker (omni-speech) | `ming_flash_omni.yaml` | Text + Audio |
+| Mode | Deploy config | Output |
+|------|--------------|--------|
+| Thinker + Talker (omni-speech, default) | `vllm_omni/deploy/ming_flash_omni.yaml` | Text + Audio |
+| Thinker only (multimodal understanding) | `vllm_omni/deploy/ming_flash_omni_thinker_only.yaml` | Text |
 
-For standalone TTS (talker only), see [`examples/offline_inference/ming_flash_omni_tts/`](../ming_flash_omni_tts/).
+For standalone TTS (talker only), see the [Ming-flash-omni-TTS section in the Text-To-Speech hub](../text_to_speech/README.md#ming-flash-omni-tts).
 
 ## Setup
 
 Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup.
 
-The default `--omni` flag runs thinker only.  For omni-speech, pass the two-stage config explicitly:
+When no `--deploy-config` is passed, the model registry auto-loads the full thinker+talker `vllm_omni/deploy/ming_flash_omni.yaml` (See [Omni-Speech](#omni-speech-thinker--talker)).
+
+For text-only output without spinning up the talker, pass:
 
 ```bash
---stage-configs-path vllm_omni/model_executor/stage_configs/ming_flash_omni.yaml
+--deploy-config vllm_omni/deploy/ming_flash_omni_thinker_only.yaml
 ```
 
 ## Run examples
 
-The end-to-end script defaults to built-in assets; pass `--image-path`,
-`--audio-path`, or `--video-path` to override.
+The end-to-end script defaults to built-in assets; pass `--image-path`, `--audio-path`, or `--video-path` to override.
+
+### Multi-Modality Understanding (Standalone Thinker)
+
+Here we pass thinker-only deploy yaml:
 
 ```bash
-# Text-only
-python examples/offline_inference/ming_flash_omni/end2end.py --query-type text
-
-# Image / audio / video / mixed understanding
-python examples/offline_inference/ming_flash_omni/end2end.py --query-type use_image
-python examples/offline_inference/ming_flash_omni/end2end.py --query-type use_audio
-python examples/offline_inference/ming_flash_omni/end2end.py --query-type use_video --num-frames 16
-python examples/offline_inference/ming_flash_omni/end2end.py --query-type use_mixed_modalities \
-    --image-path /path/to/image.jpg --audio-path /path/to/audio.wav
+python examples/offline_inference/ming_flash_omni/end2end.py --deploy-config vllm_omni/deploy/ming_flash_omni_thinker_only.yaml --query-type text
+python examples/offline_inference/ming_flash_omni/end2end.py --deploy-config vllm_omni/deploy/ming_flash_omni_thinker_only.yaml --query-type use_image
+python examples/offline_inference/ming_flash_omni/end2end.py --deploy-config vllm_omni/deploy/ming_flash_omni_thinker_only.yaml --query-type use_audio
+python examples/offline_inference/ming_flash_omni/end2end.py --deploy-config vllm_omni/deploy/ming_flash_omni_thinker_only.yaml --query-type use_video --num-frames 16
 ```
 
-#### Reasoning (Thinking Mode)
+### Reasoning (Thinking Mode)
 
 Reasoning ("detailed thinking on") is applied by the script when
 `--query-type reasoning` is set. The default prompt matches Ming's cookbook
@@ -46,19 +46,21 @@ and expects the reference figure from the upstream repo — see
 `get_reasoning_query` in `end2end.py`.
 
 ```bash
-python examples/offline_inference/ming_flash_omni/end2end.py -q reasoning --image-path ./3_0.png
+python examples/offline_inference/ming_flash_omni/end2end.py \
+    --deploy-config vllm_omni/deploy/ming_flash_omni_thinker_only.yaml \
+    --query-type reasoning \
+    --image-path ./3_0.png
 ```
 
-### Omni-speech (thinker + talker)
+### Omni-Speech (Thinker + Talker)
 
-To enable spoken output, use the two-stage config and request `audio` (or `text,audio`) modalities.
+The default deploy YAML already runs thinker+talker, so spoken output only requires requesting `audio` (or `text,audio`) modalities.
 The thinker processes your multimodal input, generates text, then the talker synthesises the response as speech.
 
 **Audio-only output** (speech response, no text):
 ```bash
 python examples/offline_inference/ming_flash_omni/end2end.py \
     --query-type text \
-    --stage-configs-path vllm_omni/model_executor/stage_configs/ming_flash_omni.yaml \
     --modalities audio \
     --output-dir output_ming_omni_speech
 ```
@@ -67,14 +69,13 @@ python examples/offline_inference/ming_flash_omni/end2end.py \
 ```bash
 python examples/offline_inference/ming_flash_omni/end2end.py \
     --query-type use_audio \
-    --stage-configs-path vllm_omni/model_executor/stage_configs/ming_flash_omni.yaml \
     --modalities text,audio \
     --output-dir output_ming_omni_speech
 ```
 
 Generated `.wav` files are saved to `--output-dir` (default `output_ming`), one per request.
 
-The stage config allocates thinker on GPUs 0–3 and talker on GPU 3 by default. Adjust `devices` in the YAML to match your hardware.
+The default deploy YAML allocates thinker on GPUs 0–3 and talker on GPU 3 for a common device topology (4 rather than 5 devices that talker on its own device). Adjust `devices` in a copied YAML and pass it via `--deploy-config` to match your hardware or requirements.
 
 ### Modality control
 
@@ -84,8 +85,7 @@ The stage config allocates thinker on GPUs 0–3 and talker on GPU 3 by default.
 | `audio` | Text (internal) | Runs | `<id>.wav` |
 | `text,audio` | Text | Runs | `<id>.txt` + `<id>.wav` |
 
-Pass `--stage-configs-path /path/to/your_config.yaml` to any of the commands
-above to override the stage config.
+Pass `--deploy-config /path/to/your_deploy.yaml` to any of the commands above to override the bundled deploy config.
 
 ## Online serving
 

examples/offline_inference/ming_flash_omni/end2end.py

@@ -20,6 +20,7 @@ from vllm.multimodal.media.audio import load_audio
 from vllm.utils.argparse_utils import FlexibleArgumentParser
 
 import vllm_omni
+from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
 from vllm_omni.entrypoints.omni import Omni
 
 # Imports the processor also registers itself
@@ -302,14 +303,10 @@ def main(args):
     else:
         query_result = query_func(processor)
 
-    # Initialize Omni (with thinker-only stage config)
-    omni = Omni(
-        model=MODEL_NAME,
-        stage_configs_path=args.stage_configs_path,
-        log_stats=args.log_stats,
-        init_timeout=args.init_timeout,
-        stage_init_timeout=args.stage_init_timeout,
-    )
+    omni_kwargs = vars(args).copy()
+    # override CLI --model with derived model_name
+    omni_kwargs["model"] = MODEL_NAME
+    omni = Omni(**omni_kwargs)
 
     # Thinker sampling params
     thinker_sampling_params = SamplingParams(
@@ -410,10 +407,10 @@ def parse_args():
         help="Query type.",
     )
     parser.add_argument(
-        "--stage-configs-path",
+        "--deploy-config",
         type=str,
         default=None,
-        help="Path to a stage configs YAML file.",
+        help="Path to a deploy YAML; leave unset to auto-load full thinker+talker. Pass custom for text-only",
     )
     parser.add_argument(
         "--log-stats",
@@ -499,6 +496,7 @@ def parse_args():
         help="Output directory for results.",
     )
 
+    nullify_stage_engine_defaults(parser)
     return parser.parse_args()
 
 

examples/offline_inference/ming_flash_omni_tts/README.md

@@ -1,47 +0,0 @@
-# Ming-flash-omni Standalone TTS (Offline)
-
-This example runs **Ming-flash-omni-2.0 talker-only** offline inference with:
-
-- `model`: `Jonathan1909/Ming-flash-omni-2.0`
-- `stage config`: `vllm_omni/model_executor/stage_configs/ming_flash_omni_tts.yaml`
-
-It follows the Ming cookbook parameter style:
-
-- `prompt`: `"Please generate speech based on the following description.\n"`
-- `max_decode_steps`: `200`
-- `cfg`: `2.0`
-- `sigma`: `0.25`
-- `temperature`: `0.0`
-
-## Quick Start
-
-```bash
-python examples/offline_inference/ming_flash_omni_tts/end2end.py --case style
-```
-
-## Cases
-
-```bash
-# Style
-python examples/offline_inference/ming_flash_omni_tts/end2end.py --case style
-
-# IP
-python examples/offline_inference/ming_flash_omni_tts/end2end.py --case ip
-
-# Basic (speed/pitch/volume control)
-python examples/offline_inference/ming_flash_omni_tts/end2end.py --case basic
-```
-
-## Useful Arguments
-
-- `--text`: override default text in the selected case
-- `--output`: custom output wav path
-- `--model`: local model path or HF repo id
-- `--stage-configs-path`: custom talker stage config path
-- `--log-stats`: enable runtime stats logs
-
-## Notes
-
-- This directory is for **standalone talker deployment (TTS)**.
-- For Ming thinker multimodal understanding examples, see:
-  `examples/offline_inference/ming_flash_omni/`.

examples/offline_inference/moss_tts_nano/README.md

@@ -1,101 +0,0 @@
-# MOSS-TTS-Nano Offline Inference
-
-## Overview
-
-Single-stage offline TTS pipeline using the 0.1B MOSS-TTS-Nano AR LM and MOSS-Audio-Tokenizer-Nano codec. Outputs 48 kHz stereo WAV.
-
-## Quick Start
-
-```bash
-python end2end.py --text "Hello, this is MOSS-TTS-Nano."
-```
-
-The first run downloads `OpenMOSS-Team/MOSS-TTS-Nano` and `OpenMOSS-Team/MOSS-Audio-Tokenizer-Nano` from Hugging Face.
-
-## Usage
-
-```
-python end2end.py [OPTIONS]
-
-Options:
-  --text TEXT               Text to synthesize (default: "Hello, this is MOSS-TTS-Nano speaking.")
-  --voice VOICE             Built-in voice preset (default: Junhao). See voice table below.
-  --mode MODE               voice_clone (default) or continuation
-  --prompt-audio PATH       Reference WAV/MP3 for custom voice cloning
-  --prompt-text TEXT        Reference transcript (continuation mode)
-  --max-new-frames N        Max AR frames, default 375 (~14 s audio)
-  --seed INT                Random seed for reproducibility
-  --audio-temperature F     Audio sampling temperature (default: 0.8)
-  --audio-top-k N           Audio top-k sampling (default: 25)
-  --audio-top-p F           Audio top-p sampling (default: 0.95)
-  --text-temperature F      Text layer temperature (default: 1.0)
-  --batch                   Run a built-in batch of diverse samples (ZH/EN/FR)
-  --output-dir DIR          Directory for WAV outputs (default: /tmp/moss_tts_nano_output)
-  --deploy-config PATH      Override deploy YAML (defaults to vllm_omni/deploy/moss_tts_nano.yaml)
-  --stage-init-timeout INT  Timeout in seconds for stage init (default: 120)
-```
-
-## Examples
-
-```bash
-# Built-in Chinese voice
-python end2end.py --text "你好，这是MOSS-TTS-Nano的语音合成演示。" --voice Junhao
-
-# Built-in English voice
-python end2end.py --text "Hello from MOSS-TTS-Nano." --voice Ava
-
-# Custom voice clone
-python end2end.py \
-    --text "Hello, this is a cloned voice." \
-    --prompt-audio /path/to/reference.wav \
-    --prompt-text "Exact transcript of the reference audio."
-
-# Batch synthesis (ZH/EN/FR)
-python end2end.py --batch --output-dir /tmp/batch_output
-
-# Reproducible output
-python end2end.py --text "Deterministic test." --seed 42
-```
-
-## Built-in Voice Presets
-
-| Voice | Language |
-|-------|----------|
-| `Junhao` | ZH |
-| `Zhiming` | ZH |
-| `Weiguo` | ZH |
-| `Xiaoyu` | ZH |
-| `Yuewen` | ZH |
-| `Lingyu` | ZH |
-| `Ava` | EN |
-| `Bella` | EN |
-| `Adam` | EN |
-| `Nathan` | EN |
-| `Sakura` | JA |
-| `Yui` | JA |
-| `Aoi` | JA |
-| `Hina` | JA |
-| `Mei` | JA |
-
-## Deploy Config
-
-Runtime knobs live in `vllm_omni/deploy/moss_tts_nano.yaml` (auto-loaded;
-override with `--deploy-config PATH`). Key stage-level settings:
-
-```yaml
-stages:
-  - stage_id: 0
-    gpu_memory_utilization: 0.3   # ~2 GB VRAM; increase for faster init
-    max_num_seqs: 4               # concurrent requests
-    max_model_len: 4096
-```
-
-## Output Format
-
-WAV files, 48 kHz, stereo (2-channel). The codec interleaves stereo as `[L, R, L, R, ...]` in the flat tensor returned by the model.
-
-## Troubleshooting
-
-- **`libnvrtc.so.13: cannot open shared object file`**: torchaudio 2.10+ torchcodec backend requires NVRTC. The model patches `torchaudio.load/save` automatically at load time to fall back to soundfile.
-- **`flash_attn not installed`**: The model falls back to `sdpa` attention automatically.
-- **Empty audio**: Check that `--text` is non-empty and the model loaded successfully (look for "MOSS-TTS-Nano LM loaded" in logs).

examples/offline_inference/omnivoice/README.md

@@ -1,73 +0,0 @@
-# OmniVoice
-
-This directory contains an offline demo for running OmniVoice TTS models with vLLM Omni. It generates speech from text and saves WAV files locally.
-
-## Model Overview
-
-[OmniVoice](https://huggingface.co/k2-fsa/OmniVoice) is a zero-shot multilingual TTS model supporting 600+ languages. It uses a diffusion language model (Qwen3-0.6B backbone) with iterative masked unmasking to generate speech.
-
-Three inference modes are supported:
-
-- **Auto Voice**: Generate speech without any reference — the model picks a voice automatically.
-- **Voice Clone**: Clone a voice from a reference audio + transcription.
-- **Voice Design**: Control voice style via natural language instruction (e.g., "female, low pitch, british accent").
-
-## Setup
-
-Ensure the model is downloaded:
-
-```bash
-huggingface-cli download k2-fsa/OmniVoice
-```
-
-> **Note:** Voice cloning requires `transformers>=5.3.0` for `HiggsAudioV2TokenizerModel`. Auto voice and voice design modes work with `transformers>=4.57.0`.
-
-## Quick Start
-
-Auto voice (text only):
-
-```bash
-python end2end.py --model k2-fsa/OmniVoice --text "Hello, this is a test."
-```
-
-Voice design (with style instruction):
-
-```bash
-python end2end.py --model k2-fsa/OmniVoice \
-    --text "Hello, this is a test." \
-    --instruct "female, low pitch, british accent"
-```
-
-Voice clone (with reference audio):
-
-```bash
-python end2end.py --model k2-fsa/OmniVoice \
-    --text "Hello, this is a test." \
-    --ref-audio ref.wav \
-    --ref-text "This is the reference transcription."
-```
-
-## Language Support
-
-Specify a language for improved quality:
-
-```bash
-python end2end.py --model k2-fsa/OmniVoice \
-    --text "你好，这是一个测试。" \
-    --lang zh
-```
-
-## Architecture
-
-OmniVoice uses a two-stage pipeline:
-
-- **Stage 0 (Generator)**: Qwen3-0.6B transformer with 32-step iterative unmasking and classifier-free guidance. Generates 8-codebook audio tokens from text.
-- **Stage 1 (Decoder)**: HiggsAudioV2 RVQ quantizer + DAC acoustic decoder. Converts tokens to 24kHz waveform.
-
-Both stages use `GPUGenerationWorker` with `OmniGenerationScheduler`.
-
-## Notes
-
-- Output audio is saved to `output.wav` by default. Use `--output` to change the path.
-- The model estimates duration from text automatically via `RuleDurationEstimator`.
-- Use `--stage-init-timeout` to increase the stage initialization timeout for first-time model downloads.

examples/offline_inference/qwen2_5_omni/end2end.py

@@ -21,6 +21,7 @@ from vllm.multimodal.media.audio import load_audio
 from vllm.sampling_params import SamplingParams
 from vllm.utils.argparse_utils import FlexibleArgumentParser
 
+from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
 from vllm_omni.entrypoints.omni import Omni
 
 SEED = 42
@@ -325,7 +326,10 @@ def main(args):
     else:
         query_result = query_func()
     args.quantization_config = quantization_config
-    omni = Omni.from_cli_args(args, model=model_name)
+    omni_kwargs = vars(args).copy()
+    # Override CLI --model with the derived model_name.
+    omni_kwargs["model"] = model_name
+    omni = Omni(**omni_kwargs)
     thinker_sampling_params = SamplingParams(
         temperature=0.0,  # Deterministic - no randomness
         top_p=1.0,  # Disable nucleus sampling
@@ -550,6 +554,7 @@ def parse_args():
         default=False,
         help="Use py_generator mode. The returned type of Omni.generate() is a Python Generator object.",
     )
+    nullify_stage_engine_defaults(parser)
     return parser.parse_args()
 
 

examples/offline_inference/qwen3_omni/end2end.py

@@ -295,7 +295,10 @@ def main(args):
     else:
         query_result = query_func()
 
-    omni = Omni.from_cli_args(args, model=model_name)
+    omni_kwargs = vars(args).copy()
+    # Override CLI --model with the derived model_name.
+    omni_kwargs["model"] = model_name
+    omni = Omni(**omni_kwargs)
 
     thinker_sampling_params = SamplingParams(
         temperature=0.9,
@@ -389,6 +392,13 @@ def main(args):
             output_wav = os.path.join(output_dir, f"output_{request_id}.wav")
 
             # Convert to numpy array and ensure correct format
+            # In async_chunk mode, audio may arrive as a list of chunks
+            if isinstance(audio_tensor, list):
+                import torch
+
+                audio_tensor = torch.cat(
+                    [(t if isinstance(t, torch.Tensor) else torch.tensor(t)).flatten() for t in audio_tensor]
+                )
             audio_numpy = audio_tensor.float().detach().cpu().numpy()
 
             # Ensure audio is 1D (flatten if needed)

examples/offline_inference/qwen3_omni/end2end_async_chunk.py

@@ -41,6 +41,7 @@ from vllm.multimodal.image import convert_image_mode
 from vllm.multimodal.media.audio import load_audio
 from vllm.utils.argparse_utils import FlexibleArgumentParser
 
+from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
 from vllm_omni.entrypoints.async_omni import AsyncOmni
 
 logger = logging.getLogger(__name__)
@@ -382,13 +383,7 @@ async def run_all(args):
     print(f"[Info] Creating AsyncOmni with deploy_config={args.deploy_config}")
     async_omni = None
     try:
-        # ``from_cli_args`` expands vars(args) into kwargs and auto-captures
-        # ``_cli_explicit_keys`` from ``sys.argv[1:]`` so argparse defaults
-        # do not silently override deploy YAML values. Mirrors the
-        # ``EngineArgs.from_cli_args`` pattern used throughout vllm /
-        # vllm-omni. ``deploy_config=None`` (the default) falls through to
-        # the bundled ``vllm_omni/deploy/qwen3_omni_moe.yaml``.
-        async_omni = AsyncOmni.from_cli_args(args)
+        async_omni = AsyncOmni(**vars(args))
 
         # Use default sampling params from stage config (they are pre-configured
         # in the YAML for each stage).
@@ -594,6 +589,7 @@ def parse_args():
         default=16000,
         help="Sampling rate for audio loading.",
     )
+    nullify_stage_engine_defaults(parser)
     return parser.parse_args()
 
 

examples/offline_inference/qwen3_tts/README.md

@@ -1,121 +0,0 @@
-# Qwen3-TTS
-
-This directory contains an offline demo for running Qwen3 TTS models with vLLM Omni. It builds task-specific inputs and generates WAV files locally.
-
-## Model Overview
-
-Qwen3 TTS provides multiple task variants for speech generation:
-
-- **CustomVoice**: Generate speech with a known speaker identity (speaker ID) and optional instruction.
-- **VoiceDesign**: Generate speech from text plus a descriptive instruction that designs a new voice.
-- **Base**: Voice cloning using a reference audio + reference transcript, with optional mode selection. The `ref_audio` field accepts a local file path, HTTP URL, or base64 data URL.
-
-## Setup
-Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup.
-
-### ROCm Dependencies
-
-You will need to install the dependency `onnxruntime-rocm`.
-
-```
-pip uninstall onnxruntime # should be removed before we can install onnxruntime-rocm
-pip install onnxruntime-rocm
-```
-
-## Quick Start
-
-Run a single sample for a task:
-
-```
-python end2end.py --query-type CustomVoice
-```
-
-Generated audio files are saved to `output_audio/` by default.
-
-## Task Usage
-
-### CustomVoice
-
-Single sample:
-
-```
-python end2end.py --query-type CustomVoice
-```
-
-Batch sample (multiple prompts in one run):
-
-```
-python end2end.py --query-type CustomVoice --use-batch-sample
-```
-
-### VoiceDesign
-
-Single sample:
-
-```
-python end2end.py --query-type VoiceDesign
-```
-
-Batch sample:
-
-```
-python end2end.py --query-type VoiceDesign --use-batch-sample
-```
-
-### Base (Voice Clone)
-
-Single sample:
-
-```
-python end2end.py --query-type Base
-```
-
-Batch sample:
-
-```
-python end2end.py --query-type Base --use-batch-sample
-```
-
-Mode selection for Base:
-
-- `--mode-tag icl` (default): standard mode
-- `--mode-tag xvec_only`: enable `x_vector_only_mode` in the request
-
-Examples:
-
-```
-python end2end.py --query-type Base --mode-tag icl
-```
-
-## Streaming Mode
-
-Add `--streaming` to stream audio chunks progressively via `AsyncOmni` (requires `async_chunk: true` in the stage config):
-
-```bash
-python end2end.py --query-type CustomVoice --streaming --output-dir /tmp/out_stream
-```
-
-Each Code2Wav chunk is logged as it arrives (default 25 frames; configurable via `codec_chunk_frames`
-in the stage config). The initial chunk size is dynamically selected based on server load for reduced
-TTFA, and can be overridden per-request via the `initial_codec_chunk_frames` API field. The final WAV file is written once generation
-completes. This demonstrates that audio data is available progressively rather than only at the end.
-
-> **Note:** Streaming uses `AsyncOmni` internally. The non-streaming path (`Omni`) is unchanged.
-
-## Batched Decoding
-
-The Code2Wav stage (stage 1) supports batched decoding, where multiple requests are decoded in a single forward pass through the SpeechTokenizer. To use it, set `max_num_seqs > 1` on both stages via `--stage-overrides` and pass multiple prompts via `--txt-prompts` with a matching `--batch-size`.
-
-```
-python end2end.py --query-type CustomVoice \
-    --txt-prompts benchmark_prompts.txt \
-    --batch-size 4 \
-    --stage-overrides '{"0":{"max_num_seqs":4,"gpu_memory_utilization":0.2},"1":{"max_num_seqs":4,"gpu_memory_utilization":0.2}}'
-```
-
-**Important:** `--batch-size` must match a CUDA graph capture size (1, 2, 4, 8, 16...) because the Talker's code predictor KV cache is sized to `max_num_seqs`, and CUDA graphs pad the batch to the next capture size. Both stages need `max_num_seqs >= batch_size` in the stage config for batching to take effect. If only stage 1 has a higher `max_num_seqs`, it won't help — stage 1 can only batch chunks from requests that are in-flight simultaneously, which requires stage 0 to also process multiple requests concurrently.
-
-## Notes
-
-- The script uses the model paths embedded in `end2end.py`. Update them if your local cache path differs.
-- Use `--output-dir` to change the output folder.