Address review feedback on precheck-pr skill

- Fix base-branch detection: use BASE_SHA, fallback to origin/main ref
- Add union instruction for multi-category PRs in Step 3
- Enforce [Model] prefix includes model identifier
- Scope Cache-DiT check: blocking for top-level pipelines, warning for subclass
- Recognize deferred accuracy/perf claims in PR body
- Add import hygiene carve-out for closures/factory functions

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
Signed-off-by: hsliu <liuhongsheng4@huawei.com>

.buildkite/bootstrap-amd-omni.sh

@@ -56,6 +56,88 @@ check_run_all_label() {
     fi
 }
 
+is_docs_only_change() {
+    local file_path
+    local has_any=0
+
+    while IFS= read -r file_path; do
+        [[ -z "${file_path}" ]] && continue
+        has_any=1
+
+        if [[ "${file_path}" == docs/* ]]; then
+            continue
+        fi
+        if [[ "${file_path}" == *.md ]]; then
+            continue
+        fi
+        if [[ "${file_path}" == "mkdocs.yml" ]]; then
+            continue
+        fi
+        return 1
+    done
+
+    [[ "${has_any}" -eq 1 ]]
+}
+
+resolve_skip_ci() {
+    local is_pr_build=0
+    local files
+    local base_branch base_ref
+
+    if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" && -n "${BUILDKITE_PULL_REQUEST:-}" ]]; then
+        is_pr_build=1
+    fi
+
+    if [[ "${is_pr_build}" -eq 1 ]]; then
+        base_branch="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}"
+        if ! git rev-parse --verify "origin/${base_branch}" >/dev/null 2>&1; then
+            echo "resolve_skip_ci: origin/${base_branch} not found locally; trying fetch" >&2
+            git fetch --depth=200 origin "${base_branch}" >/dev/null 2>&1 || true
+        fi
+
+        base_ref=""
+        if git rev-parse --verify "origin/${base_branch}" >/dev/null 2>&1; then
+            base_ref="origin/${base_branch}"
+        elif git rev-parse --verify "${base_branch}" >/dev/null 2>&1; then
+            base_ref="${base_branch}"
+        else
+            echo "resolve_skip_ci: cannot resolve PR base ${base_branch}; skip-ci=0" >&2
+            echo -n 0
+            return 0
+        fi
+
+        if ! files="$(git diff --name-only "${base_ref}...${BUILDKITE_COMMIT}" 2>/dev/null)"; then
+            echo "resolve_skip_ci: failed to compute PR changed files; skip-ci=0" >&2
+            echo -n 0
+            return 0
+        fi
+    elif [[ "${BUILDKITE_BRANCH:-}" == "main" ]]; then
+        if ! git rev-parse --verify "${BUILDKITE_COMMIT}^" >/dev/null 2>&1; then
+            echo "resolve_skip_ci: commit has no parent on main; skip-ci=0" >&2
+            echo -n 0
+            return 0
+        fi
+        if ! files="$(git diff --name-only "${BUILDKITE_COMMIT}^..${BUILDKITE_COMMIT}" 2>/dev/null)"; then
+            echo "resolve_skip_ci: failed to compute main changed files; skip-ci=0" >&2
+            echo -n 0
+            return 0
+        fi
+    else
+        echo "resolve_skip_ci: not PR/main build; skip-ci=0" >&2
+        echo -n 0
+        return 0
+    fi
+
+    if is_docs_only_change <<< "${files}"; then
+        echo "resolve_skip_ci: docs-only change detected; skip-ci=1" >&2
+        echo -n 1
+        return 0
+    fi
+
+    echo "resolve_skip_ci: non-doc changes detected; skip-ci=0" >&2
+    echo -n 0
+}
+
 if [[ -z "${COV_ENABLED:-}" ]]; then
     COV_ENABLED=0
 fi
@@ -91,8 +173,24 @@ upload_pipeline() {
 
     cd .buildkite
 
-    # Select test definition file: merge suite for main, ready suite for PRs
-    if [[ $BUILDKITE_BRANCH == "main" ]]; then
+    # Select test definition file: merge suite for main, ready suite for PRs.
+    # For debugging, DEBUG_TEST_YAML can override the selection — accepts
+    # "merge" or "ready" (case-insensitive).
+    if [[ -n "${DEBUG_TEST_YAML:-}" ]]; then
+        case "${DEBUG_TEST_YAML,,}" in
+            merge)
+                TEST_YAML="test-amd-merge.yml"
+                ;;
+            ready)
+                TEST_YAML="test-amd-ready.yaml"
+                ;;
+            *)
+                echo "ERROR: DEBUG_TEST_YAML must be 'merge' or 'ready', got '$DEBUG_TEST_YAML'" >&2
+                exit 1
+                ;;
+        esac
+        echo "DEBUG_TEST_YAML override: using $TEST_YAML"
+    elif [[ $BUILDKITE_BRANCH == "main" ]]; then
         TEST_YAML="test-amd-merge.yml"
     else
         TEST_YAML="test-amd-ready.yaml"
@@ -140,30 +238,13 @@ fi
 # Early exit start: skip pipeline if conditions are met
 # ----------------------------------------------------------------------
 
-# skip pipeline if all changed files are under docs/
+# Match CUDA Buildkite skip-ci behavior for docs/markdown-only changes.
 if [[ "${DOCS_ONLY_DISABLE}" != "1" ]]; then
-  if [[ -n "${file_diff:-}" ]]; then
-    docs_only=1
-    # Robust iteration over newline-separated file_diff
-    while IFS= read -r f; do
-      [[ -z "$f" ]] && continue
-      # **Policy:** only skip if *every* path starts with docs/
-      if [[ "$f" != docs/* ]]; then
-        docs_only=0
-        break
-      fi
-    done < <(printf '%s\n' "$file_diff" | tr ' ' '\n' | tr -d '\r')
-
-    if [[ "$docs_only" -eq 1 ]]; then
-      buildkite-agent annotate ":memo: CI skipped — docs/** only changes detected
-
-\`\`\`
-${file_diff}
-\`\`\`" --style "info" || true
-      echo "[docs-only] All changes are under docs/. Exiting before pipeline upload."
-      exit 0
+    SKIP_CI="$(resolve_skip_ci)"
+    if [[ "${SKIP_CI}" == "1" ]]; then
+        echo "[docs-only] Docs/markdown-only changes detected. Exiting before pipeline upload."
+        exit 0
     fi
-  fi
 fi
 
 # ----------------------------------------------------------------------

.buildkite/nightly-release-pipeline.yaml

@@ -1,20 +0,0 @@
-steps:
-  - label: "Build and upload wheel"
-    key: "build-wheel"
-    agents:
-      queue: cpu_queue_release
-    commands:
-      - "curl -LsSf https://astral.sh/uv/install.sh | sh"
-      - 'export PATH="$HOME/.local/bin:$PATH"'
-      - "uv venv --python=3.12 && source .venv/bin/activate"
-      - "uv pip install --upgrade build"
-      - "python3 -m build"
-      - "bash .buildkite/scripts/upload-nightly-wheels.sh"
-
-  - label: "Generate and upload wheel indices"
-    depends_on: "build-wheel"
-    allow_dependency_failure: true
-    agents:
-      queue: small_cpu_queue_release
-    commands:
-      - "bash .buildkite/scripts/generate-and-upload-nightly-index.sh"

.buildkite/pipeline-intel.yaml

@@ -10,7 +10,7 @@ steps:
           DOCKER_BUILDKIT: "1"
           # Buildkite will automatically replace this with the actual commit hash
           VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
-          VLLM_VERSION: "v0.19.0"
+          VLLM_VERSION: "v0.20.0"
         priority: 100
         timeout_in_minutes: 60
-        soft_fail: true
+        soft_fail: false

.buildkite/pipeline-npu-a3.yaml

@@ -0,0 +1,110 @@
+# steps:
+#   - label: ":buildkit: Build and Push NPU Test Image - A3"
+#     key: image-build-a3
+#     agents:
+#       queue: "ascend-a3"
+#       resource_class: "npu-2"
+#     plugins:
+#       - kubernetes:
+#           metadata:
+#             annotations:
+#               vault.hashicorp.com/agent-init-first: "true"
+#               vault.hashicorp.com/agent-inject: "true"
+#               vault.hashicorp.com/agent-inject-perms-ca.pem: "0400"
+#               vault.hashicorp.com/agent-inject-perms-cert.pem: "0400"
+#               vault.hashicorp.com/agent-inject-perms-config.json: "0400"
+#               vault.hashicorp.com/agent-inject-perms-key.pem: "0400"
+#               vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd
+#               vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd
+#               vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd
+#               vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd
+#               vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}"
+#               vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}"
+#               vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}"
+#               vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}"
+#               vault.hashicorp.com/agent-pre-populate-only: "true"
+#               vault.hashicorp.com/agent-run-as-group: "1000"
+#               vault.hashicorp.com/agent-run-as-user: "1000"
+#               vault.hashicorp.com/agent-service-account-token-volume-name: token-vol
+#               vault.hashicorp.com/role: ascend-gha-runners
+#               vault.hashicorp.com/secret-volume-path: /home/user/.docker/
+#               vault.hashicorp.com/tls-skip-verify: "true"
+#           podSpecPatch:
+#             volumes:
+#             - name: token-vol
+#               projected:
+#                 defaultMode: 420
+#                 sources:
+#                 - serviceAccountToken:
+#                     audience: api
+#                     expirationSeconds: 600
+#                     path: token
+#     env:
+#       VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
+#       IMAGE_NAME: "vllm-omni-ci-npu"
+#       IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
+#       BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
+#     command: |
+#       set -ex
+
+#       echo "--- Building and pushing NPU Test Image"
+#       echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
+#       echo "buildkitd address: $${BUILDKITD_ADDR}"
+
+#       git config --global url."https://gh-proxy.test.osinfra.cn/https://github.com/".insteadOf "https://github.com/"
+
+#       if ! command -v buildctl &> /dev/null; then
+#         echo "Installing buildctl..."
+#         mkdir -p /tmp/buildkit
+#         BUILDKIT_VERSION="v0.29.0"
+#         wget -q "https://gh-proxy.test.osinfra.cn/https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz
+#         tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
+#         cp /tmp/buildkit/bin/buildctl /usr/local/bin/
+#       fi
+
+#       export DOCKER_CONFIG=/home/user/.docker
+#       buildctl \
+#         --addr="$${BUILDKITD_ADDR}" \
+#         --tlscacert=/home/user/.docker/ca.pem \
+#         --tlscert=/home/user/.docker/cert.pem \
+#         --tlskey=/home/user/.docker/key.pem \
+#         build \
+#         --frontend dockerfile.v0 \
+#         --local context=. \
+#         --local dockerfile=./docker \
+#         --opt filename=Dockerfile.npu.ci.a3 \
+#         --secret id=dockerconfig,src=/home/user/.docker/config.json \
+#         --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \
+#         --progress=plain
+
+#       echo "--- Image pushed successfully"
+#       echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
+
+#   # - label: "Upload Ready Pipeline - A3"
+#   #   depends_on: image-build-a3
+#   #   key: upload-ready-pipeline-a3
+#   #   if: build.branch != "main" && build.pull_request.labels includes "npu-test"
+#   #   agents:
+#   #     queue: "ascend-a3"
+#   #     resource_class: "npu-2"
+#   #   command: |
+#   #     buildkite-agent pipeline upload .buildkite/test-npu-ready.yaml
+
+  # # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
+  # - label: "Upload Nightly Pipeline"
+  #   depends_on: image-build
+  #   key: upload-nightly-pipeline
+  #   if: >-
+  #     (build.branch == "main" && build.env("NIGHTLY") == "1") ||
+  #     (build.branch != "main" && (
+  #       build.pull_request.labels includes "nightly-test" ||
+  #       build.pull_request.labels includes "omni-test" ||
+  #       build.pull_request.labels includes "tts-test" ||
+  #       build.pull_request.labels includes "diffusion-x2iat-test" ||
+  #       build.pull_request.labels includes "diffusion-x2v-test"
+  #     ))
+  #   commands:
+  #     - buildkite-agent pipeline upload .buildkite/test-npu-nightly.yaml
+  #   agents:
+  #     queue: "ascend-a3"
+  #     resource_class: "npu-2"

.buildkite/pipeline-npu.yaml

@@ -0,0 +1,112 @@
+steps:
+  - label: ":buildkit: Build and Push NPU Test Image - A2"
+    key: image-build
+    agents:
+      queue: "ascend-a2b3"
+      resource_class: "npu-1"
+    plugins:
+      - kubernetes:
+          metadata:
+            annotations:
+              vault.hashicorp.com/agent-init-first: "true"
+              vault.hashicorp.com/agent-inject: "true"
+              vault.hashicorp.com/agent-inject-perms-ca.pem: "0400"
+              vault.hashicorp.com/agent-inject-perms-cert.pem: "0400"
+              vault.hashicorp.com/agent-inject-perms-config.json: "0400"
+              vault.hashicorp.com/agent-inject-perms-key.pem: "0400"
+              vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd
+              vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd
+              vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd
+              vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd
+              vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}"
+              vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}"
+              vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}"
+              vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}"
+              vault.hashicorp.com/agent-pre-populate-only: "true"
+              vault.hashicorp.com/agent-run-as-group: "1000"
+              vault.hashicorp.com/agent-run-as-user: "1000"
+              vault.hashicorp.com/agent-service-account-token-volume-name: token-vol
+              vault.hashicorp.com/role: ascend-gha-runners
+              vault.hashicorp.com/secret-volume-path: /home/user/.docker/
+              vault.hashicorp.com/tls-skip-verify: "true"
+          podSpecPatch:
+            volumes:
+            - name: token-vol
+              projected:
+                defaultMode: 420
+                sources:
+                - serviceAccountToken:
+                    audience: api
+                    expirationSeconds: 600
+                    path: token
+    env:
+      VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
+      IMAGE_NAME: "vllm-omni-ci-npu"
+      IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
+      BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
+    command: |
+      set -ex
+
+      echo "--- Building and pushing NPU Test Image"
+      echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
+      echo "buildkitd address: $${BUILDKITD_ADDR}"
+
+      if ! command -v buildctl &> /dev/null; then
+        echo "Installing buildctl..."
+        mkdir -p /tmp/buildkit
+        BUILDKIT_VERSION="v0.29.0"
+        wget -q "https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz
+        tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
+        cp /tmp/buildkit/bin/buildctl /usr/local/bin/
+      fi
+
+      export DOCKER_CONFIG=/home/user/.docker
+      buildctl \
+        --addr="$${BUILDKITD_ADDR}" \
+        --tlscacert=/home/user/.docker/ca.pem \
+        --tlscert=/home/user/.docker/cert.pem \
+        --tlskey=/home/user/.docker/key.pem \
+        build \
+        --frontend dockerfile.v0 \
+        --local context=. \
+        --local dockerfile=./docker \
+        --opt filename=Dockerfile.npu.ci \
+        --secret id=dockerconfig,src=/home/user/.docker/config.json \
+        --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \
+        --progress=plain
+
+      echo "--- Image pushed successfully"
+      echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
+
+  - label: "Upload Ready Pipeline"
+    depends_on: image-build
+    key: upload-ready-pipeline
+    if: >-
+      build.branch != "main" && (
+        build.pull_request.labels includes "npu-test" ||
+        build.pull_request.labels includes "ready"
+      )
+    agents:
+      queue: "ascend-a2b3"
+      resource_class: "npu-1"
+    command: |
+      buildkite-agent pipeline upload .buildkite/test-npu-ready.yaml
+
+  # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
+  - label: "Upload Nightly Pipeline"
+    depends_on: image-build
+    key: upload-nightly-pipeline
+    if: >-
+      (build.branch == "main" && build.env("NIGHTLY") == "1") ||
+      (build.branch != "main" && (
+        build.pull_request.labels includes "nightly-test" ||
+        build.pull_request.labels includes "omni-test" ||
+        build.pull_request.labels includes "tts-test" ||
+        build.pull_request.labels includes "diffusion-x2iat-test" ||
+        build.pull_request.labels includes "diffusion-x2v-test"
+      ))
+    commands:
+      - buildkite-agent pipeline upload .buildkite/test-npu-nightly.yaml
+    agents:
+      queue: "ascend-a2b3"
+      resource_class: "npu-1"

.buildkite/rebase-pipeline.yaml

@@ -0,0 +1,38 @@
+steps:
+  - label: ":docker: Build image"
+    key: image-build
+    commands:
+      - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
+      - "docker build --progress=plain --file docker/Dockerfile.ci -t vllm-omni-ci ."
+      - "docker tag vllm-omni-ci public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
+      - "docker push public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
+    agents:
+      queue: "cpu_queue_premerge"
+
+  # L2 Test
+  - label: "Upload Ready Pipeline"
+    depends_on: image-build
+    key: upload-ready-pipeline
+    commands:
+      - buildkite-agent pipeline upload .buildkite/test-ready.yml
+    agents:
+      queue: "cpu_queue_premerge"
+
+  # L3 Test
+  - label: "Upload Merge Pipeline"
+    depends_on: image-build
+    key: upload-merge-pipeline
+    commands:
+      - buildkite-agent pipeline upload .buildkite/test-merge.yml
+    agents:
+      queue: "cpu_queue_premerge"
+
+  # L4 Test
+  - label: "Upload Nightly Pipeline"
+    depends_on: image-build
+    key: upload-nightly-pipeline
+    if: build.env("NIGHTLY") == "1"
+    commands:
+      - buildkite-agent pipeline upload .buildkite/test-nightly.yml
+    agents:
+      queue: "cpu_queue_premerge"

.buildkite/release-pipeline.yaml

@@ -1,38 +1,110 @@
 steps:
-  - label: ":docker: Build image"
-    key: image-build
-    commands:
-      - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
-      - "docker build --progress=plain --file docker/Dockerfile.ci -t vllm-omni-ci ."
-      - "docker tag vllm-omni-ci public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
-      - "docker push public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
-    agents:
-      queue: "cpu_queue_premerge"
+  - input: "Provide Release version here"
+    id: input-release-version
+    fields:
+      - text: "What is the release version?"
+        key: release-version
 
-  # L2 Test
-  - label: "Upload Ready Pipeline"
-    depends_on: image-build
-    key: upload-ready-pipeline
-    commands:
-      - buildkite-agent pipeline upload .buildkite/test-ready.yml
+  - label: "Build and upload wheel"
+    depends_on: ~
+    key: "build-wheel"
     agents:
-      queue: "cpu_queue_premerge"
+      queue: cpu_queue_release
+    commands:
+      - "curl -LsSf https://astral.sh/uv/install.sh | sh"
+      - 'export PATH="$HOME/.local/bin:$PATH"'
+      - "uv venv --python=3.12 && source .venv/bin/activate"
+      - "uv pip install --upgrade build"
+      - "python3 -m build"
+      - "bash .buildkite/scripts/upload-nightly-wheels.sh"
 
-  # L3 Test
-  - label: "Upload Merge Pipeline"
-    depends_on: image-build
-    key: upload-merge-pipeline
-    commands:
-      - buildkite-agent pipeline upload .buildkite/test-merge.yml
+  - label: "Generate and upload wheel indices"
+    depends_on: "build-wheel"
+    allow_dependency_failure: true
     agents:
-      queue: "cpu_queue_premerge"
+      queue: small_cpu_queue_release
+    commands:
+      - "bash .buildkite/scripts/generate-and-upload-nightly-index.sh"
 
-  # L4 Test
-  - label: "Upload Nightly Pipeline"
-    depends_on: image-build
-    key: upload-nightly-pipeline
-    if: build.env("NIGHTLY") == "1"
-    commands:
-      - buildkite-agent pipeline upload .buildkite/test-nightly.yml
+  - block: "Upload release wheels to PyPI"
+    depends_on: "build-wheel"
+    key: block-upload-release-wheels
+
+  - label: "Upload release wheels to PyPI"
+    depends_on:
+      - block-upload-release-wheels
+      - input-release-version
+    key: upload-release-wheels
     agents:
-      queue: "cpu_queue_premerge"
+      queue: small_cpu_queue_release
+    commands:
+      - "bash .buildkite/scripts/publish-release-wheels-pypi.sh"
+
+  - block: "Unblock to build release Docker images"
+    depends_on: ~
+    key: block-build-release-images
+
+  - group: "Build release Docker images"
+    key: "build-release-images"
+    depends_on: block-build-release-images
+    allow_dependency_failure: true
+    steps:
+      - label: "Build release image - x86_64"
+        key: build-release-image-x86
+        depends_on: ~
+        agents:
+          queue: cpu_queue_release
+        commands:
+          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
+          - |
+            DOCKER_BUILDKIT=1 docker build \
+              --build-arg max_jobs=16 \
+              --tag public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m) \
+              --progress plain \
+              -f docker/Dockerfile.cuda .
+          - "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m)"
+          # re-tag to default image tag and push, just in case arm64 build fails
+          - "docker tag public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m) public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT"
+          - "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT"
+        env:
+          DOCKER_BUILDKIT: "1"
+
+      - label: "Build release image - aarch64"
+        key: build-release-image-arm64
+        depends_on: ~
+        agents:
+          queue: arm64_cpu_queue_release
+        commands:
+          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
+          - |
+            DOCKER_BUILDKIT=1 docker build \
+              --build-arg max_jobs=16 \
+              --tag public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m) \
+              --progress plain \
+              -f docker/Dockerfile.cuda .
+          - "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m)"
+        env:
+          DOCKER_BUILDKIT: "1"
+
+  - block: "Publish release images to DockerHub"
+    key: block-publish-release-images
+    depends_on:
+      - build-release-image-x86
+      - build-release-image-arm64
+      - input-release-version
+
+  - label: "Publish release images to DockerHub"
+    depends_on:
+      - block-publish-release-images
+    key: publish-release-images-dockerhub
+    agents:
+      queue: small_cpu_queue_release
+    commands:
+      - "bash .buildkite/scripts/publish-release-images.sh"
+    plugins:
+      - docker-login#v3.0.0:
+          username: vllmbot
+          password-env: DOCKERHUB_TOKEN
+    env:
+      DOCKER_BUILDKIT: "1"
+      DOCKERHUB_USERNAME: "vllmbot"

.buildkite/scripts/publish-release-images.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+# SPDX-License-Identifier: Apache-2.0
+#
+# Publish release Docker images from ECR to DockerHub.
+# Pulls per-arch images, tags with `latest` and versioned tags, pushes them,
+# then creates and pushes a multi-arch manifest.
+
+set -euo pipefail
+
+RELEASE_VERSION=$(buildkite-agent meta-data get release-version --default "" | sed 's/^v//')
+if [ -z "${RELEASE_VERSION}" ]; then
+  echo "ERROR: release-version metadata not set"
+  exit 1
+fi
+
+COMMIT="$BUILDKITE_COMMIT"
+DOCKERHUB_REPO="vllm/vllm-omni"
+ECR_REPO="public.ecr.aws/q9t5s3a7/vllm-omni-release-repo"
+
+echo "========================================"
+echo "Publishing release images v${RELEASE_VERSION}"
+echo "  Commit: ${COMMIT}"
+echo "========================================"
+
+# Login to ECR to pull staging images
+aws ecr-public get-login-password --region us-east-1 | \
+  docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
+
+docker pull "${ECR_REPO}:${COMMIT}-x86_64"
+docker pull "${ECR_REPO}:${COMMIT}-aarch64"
+
+docker tag "${ECR_REPO}:${COMMIT}-x86_64" "${DOCKERHUB_REPO}:latest-x86_64"
+docker tag "${ECR_REPO}:${COMMIT}-x86_64" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-x86_64"
+docker push "${DOCKERHUB_REPO}:latest-x86_64"
+docker push "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-x86_64"
+
+docker tag "${ECR_REPO}:${COMMIT}-aarch64" "${DOCKERHUB_REPO}:latest-aarch64"
+docker tag "${ECR_REPO}:${COMMIT}-aarch64" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-aarch64"
+docker push "${DOCKERHUB_REPO}:latest-aarch64"
+docker push "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-aarch64"
+
+docker manifest rm "${DOCKERHUB_REPO}:latest" || true
+docker manifest rm "${DOCKERHUB_REPO}:v${RELEASE_VERSION}" || true
+docker manifest create "${DOCKERHUB_REPO}:latest" "${DOCKERHUB_REPO}:latest-x86_64" "${DOCKERHUB_REPO}:latest-aarch64"
+docker manifest create "${DOCKERHUB_REPO}:v${RELEASE_VERSION}" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-x86_64" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-aarch64"
+docker manifest push "${DOCKERHUB_REPO}:latest"
+docker manifest push "${DOCKERHUB_REPO}:v${RELEASE_VERSION}"
+
+echo ""
+echo "Successfully published release images for v${RELEASE_VERSION}"

.buildkite/scripts/publish-release-wheels-pypi.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: Apache-2.0
+
+set -e
+
+BUCKET="vllm-wheels"
+SUBPATH="omni/$BUILDKITE_COMMIT"
+S3_COMMIT_PREFIX="s3://$BUCKET/$SUBPATH/"
+
+RELEASE_VERSION=$(buildkite-agent meta-data get release-version | sed 's/^v//')
+if [[ -z "$RELEASE_VERSION" ]]; then
+  echo "[FATAL] release-version metadata not set."
+  exit 1
+fi
+echo "Release version: $RELEASE_VERSION"
+
+if [[ -z "$PYPI_TOKEN" ]]; then
+  echo "[FATAL] PYPI_TOKEN is not set."
+  exit 1
+else
+  export TWINE_USERNAME="__token__"
+  export TWINE_PASSWORD="$PYPI_TOKEN"
+fi
+
+set -x
+
+if ! command -v uv &> /dev/null; then
+  curl -LsSf https://astral.sh/uv/install.sh | sh
+  export PATH="$HOME/.local/bin:$PATH"
+fi
+
+uv venv --python 3.12 /tmp/vllm-omni-release-env
+source /tmp/vllm-omni-release-env/bin/activate
+uv pip install twine
+
+DIST_DIR=/tmp/vllm-omni-release-dist
+mkdir -p "$DIST_DIR"
+
+echo "Downloading wheels from S3:"
+aws s3 ls "$S3_COMMIT_PREFIX"
+aws s3 cp --recursive --exclude "*" --include "vllm_omni-${RELEASE_VERSION}*.whl" --exclude "*dev*" "$S3_COMMIT_PREFIX" "$DIST_DIR"
+ls -la "$DIST_DIR"
+
+PYPI_WHEEL_FILES=$(find "$DIST_DIR" -name "vllm_omni-${RELEASE_VERSION}*.whl" -not -name "*+*")
+if [[ -z "$PYPI_WHEEL_FILES" ]]; then
+  echo "[FATAL] No wheels found for version ${RELEASE_VERSION}"
+  exit 1
+fi
+
+python3 -m twine check $PYPI_WHEEL_FILES
+python3 -m twine upload --non-interactive --verbose $PYPI_WHEEL_FILES
+echo "Wheels uploaded to PyPI"

.buildkite/scripts/upload_pipeline_with_skip_ci.sh

@@ -1,10 +1,18 @@
 #!/usr/bin/env bash
-# Evaluate docs-only skip-ci and upload continuation steps from the same `.buildkite/pipeline.yml`
+# Evaluate docs-only skip-ci and upload continuation steps from the given pipeline YAML
 # (YAML document after the first `---`). Buildkite `if` is evaluated at upload time.
+#
+# Usage: upload_pipeline_with_skip_ci.sh [pipeline_yaml]
+#   pipeline_yaml: path relative to repo root or absolute (default: .buildkite/pipeline.yml)
 set -euo pipefail
 
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
-PIPELINE_YML="${ROOT}/.buildkite/pipeline.yml"
+PIPELINE_ARG="${1:-.buildkite/pipeline.yml}"
+if [[ "${PIPELINE_ARG}" = /* ]]; then
+  PIPELINE_YML="${PIPELINE_ARG}"
+else
+  PIPELINE_YML="${ROOT}/${PIPELINE_ARG}"
+fi
 
 # Prints a single digit to stdout: 1 = skip image CI, 0 = run. Logs go to stderr.
 is_docs_only_change() {
@@ -21,7 +29,7 @@ is_docs_only_change() {
     if [[ "${file_path}" == *.md ]]; then
       continue
     fi
-    if [[ "${file_path}" == "mkdocs.yaml" ]]; then
+    if [[ "${file_path}" == "mkdocs.yml" ]]; then
       continue
     fi
     return 1
@@ -104,12 +112,13 @@ import pathlib
 path = pathlib.Path(os.environ["PIPELINE_YML"])
 text = path.read_text(encoding="utf-8")
 sep = "\n---\n"
-if sep not in text:
-    raise SystemExit(
-        "upload_pipeline_with_skip_ci: .buildkite/pipeline.yml must contain a '\\n---\\n' separator "
-        "(document 1 = bootstrap, document 2 = uploaded steps)"
-    )
-_, continuation = text.split(sep, 1)
+# Two supported layouts:
+#   - multi-doc: doc 1 = bootstrap (loaded by Buildkite), doc 2 (after `---`) = uploaded steps.
+#   - single-doc: whole file is the uploaded steps; caller (UI init step) is the bootstrap.
+if sep in text:
+    _, continuation = text.split(sep, 1)
+else:
+    continuation = text
 
 skip = os.environ.get("SKIP_CI") == "1"
 # When docs-only skip-ci: skip default CI image, but still build for L4 nightly (PR label nightly-test or
@@ -126,7 +135,7 @@ if skip:
 else:
     rep = "'true'"
     ready_rep = "'build.branch != \"main\" && build.pull_request.labels includes \"ready\"'"
-    merge_rep = "'(build.branch == \"main\" && build.env(\"NIGHTLY\") != \"1\") || (build.branch != \"main\" && build.pull_request.labels includes \"merge-test\")'"
+    merge_rep = "'(build.branch == \"main\" && build.env(\"NIGHTLY\") != \"1\" && build.env(\"WEEKLY\") != \"1\") || (build.branch != \"main\" && build.pull_request.labels includes \"merge-test\")'"
 rendered = (
     continuation
     .replace("__IMAGE_BUILD_IF__", rep)

.buildkite/test-amd-merge.yml

@@ -7,16 +7,9 @@ steps:
   grade: Blocking
   commands:
     - export VLLM_ROCM_USE_AITER=0
-    - "timeout 20m pytest -v -s -m 'core_model and cpu' --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml"
-
-- label: "Diffusion Model Test"
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - timeout 20m pytest -s -v tests/e2e/offline_inference/test_t2i_model.py -m "advanced_model and diffusion" --run-level "advanced_model"
+    # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
+    # that is not supported on CPU
+    - "timeout 60m pytest -v -s -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
 
 - label: "Diffusion Images API LoRA E2E"
   agent_pool: mi325_1
@@ -24,7 +17,7 @@ steps:
   mirror_hardwares: [amdproduction]
   grade: Blocking
   commands:
-    - timeout 20m pytest -s -v tests/e2e/online_serving/test_images_generations_lora.py
+    - timeout 20m pytest -s -v tests/diffusion/lora/test_images_generations_lora.py
 
 - label: "Diffusion Model CPU offloading Test"
   agent_pool: mi325_1
@@ -37,9 +30,9 @@ steps:
     - |
       timeout 20m bash -c '
         set +e
-        pytest -s -v tests/e2e/offline_inference/test_diffusion_cpu_offload.py
+        pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
         EXIT1=\$?
-        pytest -s -v tests/e2e/offline_inference/test_diffusion_layerwise_offload.py
+        pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
         EXIT2=\$?
         exit \$((EXIT1 | EXIT2))
       '
@@ -74,7 +67,7 @@ steps:
   commands:
     - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - timeout 20m pytest -s -v tests/e2e/offline_inference/test_sequence_parallel.py
+    - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py
     - timeout 20m pytest -s -v tests/diffusion/distributed/test_ulysses_uaa_perf.py
 
 # merge-only tests
@@ -84,7 +77,15 @@ steps:
   mirror_hardwares: [amdproduction]
   grade: Blocking
   commands:
-    - pytest -s -v tests/e2e/offline_inference/test_zimage_parallelism.py
+    - pytest -s -v tests/diffusion/distributed/test_zimage_parallelism.py
+
+- label: "Diffusion Model Test"
+  agent_pool: mi325_1
+  depends_on: amd-build
+  mirror_hardwares: [amdproduction]
+  grade: Blocking
+  commands:
+    - timeout 30m pytest -s -v tests/e2e/offline_inference/test_z_image.py
 
 - label: "Diffusion GPU Worker Test"
   agent_pool: mi325_2
@@ -103,16 +104,17 @@ steps:
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
     - timeout 20m pytest -s -v tests/engine/test_async_omni_engine_abort.py
 
+# This is from test-nightly.yml.
+# This model seems to be important, so I am trying to run it here.
+# Since this is from test-nightly.yml,
+# I keep this non-blocking.
 - label: "Omni Model Test Qwen2-5-Omni"
-  agent_pool: mi325_2
+  agent_pool: mi325_4
   depends_on: amd-build
   mirror_hardwares: [amdproduction]
-  grade: Blocking
   commands:
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - timeout 20m pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py
-    - timeout 20m pytest -s -v tests/e2e/online_serving/test_qwen2_5_omni_expansion.py -m "advanced_model" --run-level "core_model"
+    - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+    - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model"
 
 - label: "Omni Model Test Qwen3-Omni"
   agent_pool: mi325_2
@@ -120,10 +122,8 @@ steps:
   mirror_hardwares: [amdproduction]
   grade: Blocking
   commands:
-    - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - export VLLM_TEST_CLEAN_GPU_MEMORY=1
-    - timeout 30m pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/online_serving/test_mimo_audio.py -m "advanced_model" --run-level "advanced_model"
+    - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/online_serving/test_mimo_audio.py -m "advanced_model" --run-level "advanced_model"
 
 - label: "Qwen3-TTS CustomVoice E2E Test"
   agent_pool: mi325_1
@@ -153,17 +153,6 @@ steps:
         pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m "advanced_model" --run-level "advanced_model"
       '
 
-- label: "Diffusion Image Edit Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - timeout 20m pytest -s -v tests/e2e/online_serving/test_image_gen_edit.py
-
 # TODO: Bagel test on ROCm is very unstable. @tjtanaa
 # Need to debug before reneable numerical changes across large PRs
 # # split Bagel Model Test with H100 (Real Weights) into three tests
@@ -174,11 +163,10 @@ steps:
 #   grade: Blocking
 #   commands:
 #     - export GPU_ARCHS=gfx942
-#     - export VLLM_TEST_CLEAN_GPU_MEMORY=1
 #     - export VLLM_LOGGING_LEVEL=DEBUG
 #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
 #     - export VLLM_ROCM_USE_AITER_RMSNORM=0
-#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel_text2img.py -m "advanced_model" --run-level "advanced_model" -k "shared_memory" -k "rocm"
+#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "advanced_model" --run-level "advanced_model" -k "shared_memory" -k "rocm"
 
 # - label: "Bagel Img2Img Model Test (2/3)"
 #   agent_pool: mi325_1
@@ -187,11 +175,10 @@ steps:
 #   grade: Blocking
 #   commands:
 #     - export GPU_ARCHS=gfx942
-#     - export VLLM_TEST_CLEAN_GPU_MEMORY=1
 #     - export VLLM_LOGGING_LEVEL=DEBUG
 #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
 #     - export VLLM_ROCM_USE_AITER_RMSNORM=0
-#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel_img2img.py -m "advanced_model" --run-level "advanced_model" -k "rocm"
+#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "advanced_model" --run-level "advanced_model" -k "rocm"
 
 # - label: "Bagel Online Serving Test (3/3)"
 #   agent_pool: mi325_1
@@ -200,12 +187,23 @@ steps:
 #   grade: Blocking
 #   commands:
 #     - export GPU_ARCHS=gfx942
-#     - export VLLM_TEST_CLEAN_GPU_MEMORY=1
 #     - export VLLM_IMAGE_FETCH_TIMEOUT=60
 #     - export VLLM_LOGGING_LEVEL=DEBUG
 #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
 #     - export VLLM_ROCM_USE_AITER_RMSNORM=0
-#     - timeout 40m pytest -s -v tests/e2e/online_serving/test_bagel_online.py -m "advanced_model" --run-level "advanced_model" -k "rocm"
+#     - timeout 40m pytest -s -v tests/e2e/online_serving/test_bagel.py -m "advanced_model" --run-level "advanced_model" -k "rocm"
+
+- label: "Diffusion · Wan22 Test"
+  agent_pool: mi325_1
+  depends_on: amd-build
+  mirror_hardwares: [amdproduction]
+  grade: Blocking
+  commands:
+    - |
+      timeout 40m bash -c "
+        export VLLM_IMAGE_FETCH_TIMEOUT=60
+        pytest -s -v tests/e2e/offline_inference/test_wan22_t2v.py tests/e2e/online_serving/test_wan22_t2v.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
+      "
 
 - label: "Voxtral-TTS E2E Test"
   agent_pool: mi325_1

.buildkite/test-amd-ready.yaml

@@ -7,7 +7,9 @@ steps:
   grade: Blocking
   commands:
     - export VLLM_ROCM_USE_AITER=0
-    - "timeout 20m pytest -v -s -m 'core_model and cpu' --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml"
+    # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
+    # that is not supported on CPU
+    - "timeout 60m pytest -v -s -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
 
 - label: "Voxtral TTS CUDA Unit Test"
   agent_pool: mi325_1
@@ -17,21 +19,13 @@ steps:
   commands:
     - timeout 10m pytest -s -v tests/model_executor/models/voxtral_tts/test_cuda_graph_acoustic_transformer.py
 
-- label: "Diffusion Model Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - timeout 30m pytest -s -v tests/e2e/offline_inference/test_t2i_model.py -m "core_model and diffusion" --run-level "core_model"
-
 - label: "Diffusion Batching Test"
   agent_pool: mi325_1
   depends_on: amd-build
   mirror_hardwares: [amdproduction]
   grade: Blocking
   commands:
-    - timeout 20m pytest -s -v tests/e2e/offline_inference/test_qwen_image_diffusion_batching.py -m "core_model and diffusion" --run-level "core_model"
+    - timeout 20m pytest -s -v tests/diffusion/batching/test_diffusion_batching.py -m "core_model and diffusion" --run-level "core_model"
 
 - label: "Custom Pipeline Test"
   agent_pool: mi325_1
@@ -52,9 +46,9 @@ steps:
     - |
       timeout 20m bash -c '
         set +e
-        pytest -s -v tests/e2e/offline_inference/test_diffusion_cpu_offload.py
+        pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
         EXIT1=\$?
-        pytest -s -v tests/e2e/offline_inference/test_diffusion_layerwise_offload.py
+        pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
         EXIT2=\$?
         exit \$((EXIT1 | EXIT2))
       '
@@ -90,7 +84,7 @@ steps:
   commands:
     - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - timeout 20m pytest -s -v tests/e2e/offline_inference/test_sequence_parallel.py -m core_model
+    - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py -m core_model
 
 - label: "Diffusion GPU Worker Test"
   agent_pool: mi325_2
@@ -100,6 +94,27 @@ steps:
   commands:
     - timeout 20m pytest -s -v tests/diffusion/test_diffusion_worker.py
 
+- label: "Diffusion · Model Test"
+  agent_pool: mi325_1
+  depends_on: amd-build
+  mirror_hardwares: [amdproduction]
+  grade: Blocking
+  commands:
+    - timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model"
+
+- label: "Diffusion · Wan22 Test"
+  agent_pool: mi325_1
+  depends_on: amd-build
+  mirror_hardwares: [amdproduction]
+  grade: Blocking
+  commands:
+    - |
+      timeout 40m bash -c "
+        export VLLM_TEST_CLEAN_GPU_MEMORY=1
+        export VLLM_IMAGE_FETCH_TIMEOUT=60
+        pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model' --run-level 'core_model'
+      "
+
 - label: "Engine Test"
   agent_pool: mi325_1
   depends_on: amd-build
@@ -120,7 +135,6 @@ steps:
 #   commands:
 #     - export VLLM_LOGGING_LEVEL=DEBUG
 #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-#     - export VLLM_TEST_CLEAN_GPU_MEMORY=1
 #     - timeout 10m pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py
 #     - timeout 20m pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m "core_model" --run-level "core_model"
 
@@ -162,17 +176,6 @@ steps:
         pytest -s -v tests/e2e/offline_inference/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model"
       '
 
-- label: "Diffusion Image Edit Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - timeout 20m pytest -s -v tests/e2e/online_serving/test_image_gen_edit.py
-
 # TODO: Bagel test on ROCm is very unstable. @tjtanaa
 # Need to debug before reneable numerical changes across large PRs
 # - label: "Bagel Text2Img Model Test"
@@ -182,11 +185,10 @@ steps:
 #   grade: Blocking
 #   commands:
 #     - export GPU_ARCHS=gfx942
-#     - export VLLM_TEST_CLEAN_GPU_MEMORY=1
 #     - export VLLM_LOGGING_LEVEL=DEBUG
 #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
 #     - export VLLM_ROCM_USE_AITER_RMSNORM=0
-#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel_text2img.py -m "core_model" --run-level "core_model" -k "rocm"
+#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"
 
 # - label: "Bagel Img2Img Model Test"
 #   agent_pool: mi325_1
@@ -195,11 +197,10 @@ steps:
 #   grade: Blocking
 #   commands:
 #     - export GPU_ARCHS=gfx942
-#     - export VLLM_TEST_CLEAN_GPU_MEMORY=1
 #     - export VLLM_LOGGING_LEVEL=DEBUG
 #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
 #     - export VLLM_ROCM_USE_AITER_RMSNORM=0
-#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel_img2img.py -m "core_model" --run-level "core_model" -k "rocm"
+#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"
 
 # - label: "Bagel Online Serving Test"
 #   agent_pool: mi325_1
@@ -208,12 +209,11 @@ steps:
 #   grade: Blocking
 #   commands:
 #     - export GPU_ARCHS=gfx942
-#     - export VLLM_TEST_CLEAN_GPU_MEMORY=1
 #     - export VLLM_IMAGE_FETCH_TIMEOUT=60
 #     - export VLLM_LOGGING_LEVEL=DEBUG
 #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
 #     - export VLLM_ROCM_USE_AITER_RMSNORM=0
-#     - timeout 40m pytest -s -v tests/e2e/online_serving/test_bagel_online.py -m "core_model" --run-level "core_model" -k "rocm"
+#     - timeout 40m pytest -s -v tests/e2e/online_serving/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"
 
 - label: "CosyVoice3-TTS E2E Test"
   agent_pool: mi325_1
@@ -223,5 +223,5 @@ steps:
   commands:
     - |
       timeout 20m bash -c '
-        pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts.py -m "core_model" --run-level "core_model"
+        pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "full_model" --run-level "core_model"
       '

.buildkite/test-amd.yaml

@@ -8,7 +8,7 @@ steps:
   grade: Blocking
   commands:
     - export GPU_ARCHS=gfx942
-    - pytest -s -v tests/e2e/offline_inference/test_t2i_model.py
+    - pytest -s -v tests/e2e/offline_inference/test_z_image.py
 
 - label: "Diffusion Images API LoRA E2E"
   timeout_in_minutes: 30
@@ -20,7 +20,7 @@ steps:
     - export GPU_ARCHS=gfx942
     - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/online_serving/test_images_generations_lora.py
+    - pytest -s -v tests/diffusion/lora/test_images_generations_lora.py
 
 - label: "Diffusion Model CPU offloading Test"
   timeout_in_minutes: 20
@@ -32,8 +32,8 @@ steps:
     - export GPU_ARCHS=gfx942
     - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/offline_inference/test_diffusion_cpu_offload.py
-    - pytest -s -v tests/e2e/offline_inference/test_diffusion_layerwise_offload.py
+    - pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
+    - pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
 
 - label: "Diffusion Cache Backend Test"
   timeout_in_minutes: 15
@@ -57,7 +57,7 @@ steps:
     - export GPU_ARCHS=gfx942
     - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/offline_inference/test_sequence_parallel.py -m core_model
+    - pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py -m core_model
 
 - label: "Diffusion Tensor Parallelism Test"
   timeout_in_minutes: 20
@@ -67,7 +67,7 @@ steps:
     - export GPU_ARCHS=gfx942
     - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/offline_inference/test_zimage_parallelism.py
+    - pytest -s -v tests/diffusion/distributed/test_zimage_parallelism.py
 
 - label: "Diffusion GPU Worker Test"
   timeout_in_minutes: 20
@@ -100,25 +100,11 @@ steps:
   commands:
     - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - export VLLM_TEST_CLEAN_GPU_MEMORY="1"
     - export VLLM_ROCM_USE_AITER=0
     - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py
     - pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py
 
 
-- label: "Diffusion Image Edit Test"
-  timeout_in_minutes: 15
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/online_serving/test_image_gen_edit.py
-
-
 - label: "Omni Sleep Mode Test"
   timeout_in_minutes: 40
   agent_pool: mi325_2
@@ -129,5 +115,4 @@ steps:
     - export GPU_ARCHS=gfx942
     - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - export VLLM_TEST_CLEAN_GPU_MEMORY="1"
-    - pytest -s -v tests/e2e/offline_inference/test_omni_sleep_mode.py -m "advanced_model and omni and MI325" --run-level "advanced_model"
+    - pytest -s -v tests/entrypoints/test_omni_sleep_mode.py -m "advanced_model and omni and MI325" --run-level "advanced_model"

.buildkite/test-nightly.yml

@@ -1,5 +1,6 @@
 env:
   VLLM_WORKER_MULTIPROC_METHOD: spawn
+  VLLM_USE_FLASHINFER_MOE_FP16: 0
   HF_HUB_DOWNLOAD_TIMEOUT: 300
   HF_HUB_ETAG_TIMEOUT: 60
 
@@ -13,7 +14,7 @@ steps:
       - label: ":full_moon: Omni · Function Test with H100"
         timeout_in_minutes: 90
         commands:
-          - pytest -s -v tests/e2e/ -m "full_model and H100 and omni" --run-level "full_model" --ignore=tests/e2e/accuracy
+          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py
         agents:
           queue: "mithril-h100-pool"
         plugins:
@@ -169,14 +170,104 @@ steps:
                       path: /mnt/hf-cache
                       type: DirectoryOrCreate
 
-      - label: ":full_moon: Omni · Perf Test"
-        key: nightly-omni-performance
+      - label: ":full_moon: Omni · Perf Test · No Async Chunk"
+        key: nightly-omni-performance-no-async-chunk
         timeout_in_minutes: 180
         commands:
           - export BENCHMARK_DIR=tests/dfx/perf/results
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_omni.json
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
+            exit $$EXIT
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          - kubernetes:
+              podSpec:
+                containers:
+                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                    resources:
+                      limits:
+                        nvidia.com/gpu: 2
+                    volumeMounts:
+                      - name: devshm
+                        mountPath: /dev/shm
+                      - name: hf-cache
+                        mountPath: /root/.cache/huggingface
+                    env:
+                      - name: HF_HOME
+                        value: /root/.cache/huggingface
+                      - name: HF_TOKEN
+                        valueFrom:
+                          secretKeyRef:
+                            name: hf-token-secret
+                            key: token
+                nodeSelector:
+                  node.kubernetes.io/instance-type: gpu-h100-sxm
+                volumes:
+                  - name: devshm
+                    emptyDir:
+                      medium: Memory
+                  - name: hf-cache
+                    hostPath:
+                      path: /mnt/hf-cache
+                      type: DirectoryOrCreate
+
+      - label: ":full_moon: Omni · Perf Test · Async Chunk"
+        key: nightly-omni-performance-async-chunk
+        timeout_in_minutes: 180
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
+            exit $$EXIT
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          - kubernetes:
+              podSpec:
+                containers:
+                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                    resources:
+                      limits:
+                        nvidia.com/gpu: 2
+                    volumeMounts:
+                      - name: devshm
+                        mountPath: /dev/shm
+                      - name: hf-cache
+                        mountPath: /root/.cache/huggingface
+                    env:
+                      - name: HF_HOME
+                        value: /root/.cache/huggingface
+                      - name: HF_TOKEN
+                        valueFrom:
+                          secretKeyRef:
+                            name: hf-token-secret
+                            key: token
+                nodeSelector:
+                  node.kubernetes.io/instance-type: gpu-h100-sxm
+                volumes:
+                  - name: devshm
+                    emptyDir:
+                      medium: Memory
+                  - name: hf-cache
+                    hostPath:
+                      path: /mnt/hf-cache
+                      type: DirectoryOrCreate
+
+      - label: ":full_moon: Omni · Perf Test · vLLM Text"
+        key: nightly-omni-performance-vllm-text
+        timeout_in_minutes: 180
+        commands:
+          - export BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
@@ -215,6 +306,44 @@ steps:
                       type: DirectoryOrCreate
 
 
+      - label: ":full_moon: Omni · Multi-Replica Startup Test with 4x H100"
+        timeout_in_minutes: 45
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py -m "full_model" --run-level "core_model"
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          - kubernetes:
+              podSpec:
+                containers:
+                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                    resources:
+                      limits:
+                        nvidia.com/gpu: 4
+                    volumeMounts:
+                      - name: devshm
+                        mountPath: /dev/shm
+                      - name: hf-cache
+                        mountPath: /root/.cache/huggingface
+                    env:
+                      - name: HF_HOME
+                        value: /root/.cache/huggingface
+                      - name: HF_TOKEN
+                        valueFrom:
+                          secretKeyRef:
+                            name: hf-token-secret
+                            key: token
+                nodeSelector:
+                  node.kubernetes.io/instance-type: gpu-h100-sxm
+                volumes:
+                  - name: devshm
+                    emptyDir:
+                      medium: Memory
+                  - name: hf-cache
+                    hostPath:
+                      path: /mnt/hf-cache
+                      type: DirectoryOrCreate
+
   - group: ":card_index_dividers: TTS Model Test"
     key: nightly-tts-test-group
     depends_on: upload-nightly-pipeline
@@ -224,7 +353,7 @@ steps:
         timeout_in_minutes: 90
         commands:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-          - pytest -s -v tests/e2e/ -m "full_model and L4 and omni" -k "not test_qwen2_5" --run-level "full_model" --ignore=tests/e2e/accuracy
+          - pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy
         agents:
           queue: "gpu_1_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
         plugins:
@@ -260,7 +389,7 @@ steps:
                   - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                     resources:
                       limits:
-                        nvidia.com/gpu: 1
+                        nvidia.com/gpu: 2
                     volumeMounts:
                       - name: devshm
                         mountPath: /dev/shm
@@ -294,10 +423,65 @@ steps:
       build.pull_request.labels includes "nightly-test" ||
       build.pull_request.labels includes "diffusion-x2iat-test"
     steps:
-      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100"
+      # Use explicit file shards so single-card cases do not reserve 2+ GPUs,
+      # and Bagel feature coverage is not duplicated via a broad tests/e2e sweep.
+      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Single-GPU"
         timeout_in_minutes: 120
         commands:
-          - pytest -sv tests/e2e/ -k "not test_wan and not hunyuan" -m "full_model and diffusion and H100" --run-level "full_model" --ignore=tests/e2e/accuracy
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_qwen_image_expansion.py
+            tests/e2e/online_serving/test_qwen_image_edit_expansion.py
+            tests/e2e/online_serving/test_qwen_image_layered_expansion.py
+            tests/e2e/online_serving/test_longcat_image_expansion.py
+            tests/e2e/online_serving/test_longcat_image_edit_expansion.py
+            tests/e2e/online_serving/test_flux_2_dev_expansion.py
+            tests/e2e/online_serving/test_bagel_expansion.py
+            -m "full_model and diffusion and H100 and not distributed_cuda"
+            --run-level "full_model"
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          - kubernetes:
+              podSpec:
+                containers:
+                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                    resources:
+                      limits:
+                        nvidia.com/gpu: 1
+                    volumeMounts:
+                      - name: devshm
+                        mountPath: /dev/shm
+                      - name: hf-cache
+                        mountPath: /root/.cache/huggingface
+                    env:
+                      - name: HF_HOME
+                        value: /root/.cache/huggingface
+                      - name: HF_TOKEN
+                        valueFrom:
+                          secretKeyRef:
+                            name: hf-token-secret
+                            key: token
+                nodeSelector:
+                  node.kubernetes.io/instance-type: gpu-h100-sxm
+                volumes:
+                  - name: devshm
+                    emptyDir:
+                      medium: Memory
+                  - name: hf-cache
+                    hostPath:
+                      path: /mnt/hf-cache
+                      type: DirectoryOrCreate
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Qwen-Image"
+        timeout_in_minutes: 120
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_qwen_image_expansion.py
+            tests/e2e/online_serving/test_qwen_image_edit_expansion.py
+            -m "full_model and diffusion and H100 and distributed_cuda"
+            --run-level "full_model"
         agents:
           queue: "mithril-h100-pool"
         plugins:
@@ -332,10 +516,98 @@ steps:
                       path: /mnt/hf-cache
                       type: DirectoryOrCreate
 
+      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Layered/LongCat/FLUX"
+        timeout_in_minutes: 90
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_qwen_image_layered_expansion.py
+            tests/e2e/online_serving/test_longcat_image_expansion.py
+            tests/e2e/online_serving/test_longcat_image_edit_expansion.py
+            tests/e2e/online_serving/test_flux_2_dev_expansion.py
+            -m "full_model and diffusion and H100 and distributed_cuda"
+            --run-level "full_model"
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          - kubernetes:
+              podSpec:
+                containers:
+                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                    resources:
+                      limits:
+                        nvidia.com/gpu: 2
+                    volumeMounts:
+                      - name: devshm
+                        mountPath: /dev/shm
+                      - name: hf-cache
+                        mountPath: /root/.cache/huggingface
+                    env:
+                      - name: HF_HOME
+                        value: /root/.cache/huggingface
+                      - name: HF_TOKEN
+                        valueFrom:
+                          secretKeyRef:
+                            name: hf-token-secret
+                            key: token
+                nodeSelector:
+                  node.kubernetes.io/instance-type: gpu-h100-sxm
+                volumes:
+                  - name: devshm
+                    emptyDir:
+                      medium: Memory
+                  - name: hf-cache
+                    hostPath:
+                      path: /mnt/hf-cache
+                      type: DirectoryOrCreate
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Bagel"
+        timeout_in_minutes: 120
+        commands:
+          - >-
+            pytest -sv
+            tests/e2e/online_serving/test_bagel_expansion.py
+            -m "full_model and diffusion and H100 and distributed_cuda"
+            --run-level "full_model"
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          - kubernetes:
+              podSpec:
+                containers:
+                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                    resources:
+                      limits:
+                        nvidia.com/gpu: 3
+                    volumeMounts:
+                      - name: devshm
+                        mountPath: /dev/shm
+                      - name: hf-cache
+                        mountPath: /root/.cache/huggingface
+                    env:
+                      - name: HF_HOME
+                        value: /root/.cache/huggingface
+                      - name: HF_TOKEN
+                        valueFrom:
+                          secretKeyRef:
+                            name: hf-token-secret
+                            key: token
+                nodeSelector:
+                  node.kubernetes.io/instance-type: gpu-h100-sxm
+                volumes:
+                  - name: devshm
+                    emptyDir:
+                      medium: Memory
+                  - name: hf-cache
+                    hostPath:
+                      path: /mnt/hf-cache
+                      type: DirectoryOrCreate
+
+
       - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4"
         timeout_in_minutes: 60
         commands:
-          - pytest -sv tests/e2e/ -k "not test_wan and not hunyuan" -m "full_model and diffusion and L4" --run-level "full_model" --ignore=tests/e2e/accuracy
+          - pytest -sv tests/e2e/ -k "not test_wan and not test_bagel_expansion and not hunyuan" -m "full_model and diffusion and L4" --run-level "full_model" --ignore=tests/e2e/accuracy
         agents:
           queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
         plugins:
@@ -353,7 +625,6 @@ steps:
       - label: ":full_moon: Diffusion X2I(&A&T) · Doc Test"
         timeout_in_minutes: 60
         commands:
-          - export VLLM_TEST_CLEAN_GPU_MEMORY="1"
           - pytest -s -v tests/examples/*/test_text_to_image.py -m "full_model and example and H100" --run-level "full_model"
         agents:
           queue: "mithril-h100-pool"
@@ -389,91 +660,11 @@ steps:
                       path: /mnt/hf-cache
                       type: DirectoryOrCreate
 
-      - label: ":full_moon: Diffusion X2I(&A&T) · GEBench Accuracy Test"
-        timeout_in_minutes: 60
-        commands:
-          - pytest -s -v tests/e2e/accuracy/test_gebench_h100_smoke.py --run-level full_model --gebench-model Qwen/Qwen-Image-2512 --accuracy-judge-model QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ --accuracy-gpu 0 --gebench-port 8093 --accuracy-workers 1
-          - buildkite-agent artifact upload "tests/e2e/accuracy/artifacts/gebench_qwen-image-2512/summary*.json"
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: ":full_moon: Diffusion X2I(&A&T) · GEdit-Bench Accuracy Test"
-        timeout_in_minutes: 60
-        commands:
-          - pytest -s -v tests/e2e/accuracy/test_gedit_bench_h100_smoke.py --run-level full_model --gedit-model Qwen/Qwen-Image-Edit --accuracy-judge-model QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ --accuracy-gpu 0 --gedit-port 8093 --gedit-samples-per-group 20 --accuracy-workers 1
-          - buildkite-agent artifact upload "tests/e2e/accuracy/artifacts/gedit_scores_qwen-image-edit/qwen-image-edit_all_all_vie_score_*.csv"
-          - buildkite-agent artifact upload "tests/e2e/accuracy/artifacts/gedit_scores_qwen-image-edit/qwen-image-edit_all_all_summary_*.json"
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: VLLM_HTTP_TIMEOUT_KEEP_ALIVE
-                        value: "120"
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
       - label: ":full_moon: Diffusion X2I(&A&T) · Accuracy Test"
         timeout_in_minutes: 180
         commands:
           - pytest -s -v tests/e2e/accuracy/test_qwen_image*.py --run-level full_model
+          - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2i_matches_diffusers' -m full_model --run-level full_model
         agents:
           queue: "mithril-h100-pool"
         plugins:
@@ -510,28 +701,218 @@ steps:
                       path: /mnt/hf-cache
                       type: DirectoryOrCreate
 
-      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test"
-        key: nightly-diffusion-x2iat-performance
+      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image"
+        key: nightly-diffusion-x2iat-performance-qwen-image
         timeout_in_minutes: 180
         commands:
           - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
           - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
           - export CACHE_DIT_VERSION=1.3.0
-          # [HACK]: run upload in the same command block as pytest.
-          # Because `exit` aborts the entire commands list.
           - |
             set +e
             pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
-            EXIT1=$$?
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json
-            EXIT2=$$?
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2509_vllm_omni.json
-            EXIT3=$$?
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json
-            EXIT4=$$?
+            EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
             buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
-            exit $$((EXIT1 | EXIT2 | EXIT3 | EXIT4))
+            exit $$EXIT
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          - kubernetes:
+              podSpec:
+                containers:
+                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                    resources:
+                      limits:
+                        nvidia.com/gpu: 4
+                    volumeMounts:
+                      - name: devshm
+                        mountPath: /dev/shm
+                      - name: hf-cache
+                        mountPath: /root/.cache/huggingface
+                    env:
+                      - name: HF_HOME
+                        value: /root/.cache/huggingface
+                      - name: HF_TOKEN
+                        valueFrom:
+                          secretKeyRef:
+                            name: hf-token-secret
+                            key: token
+                nodeSelector:
+                  node.kubernetes.io/instance-type: gpu-h100-sxm
+                volumes:
+                  - name: devshm
+                    emptyDir:
+                      medium: Memory
+                  - name: hf-cache
+                    hostPath:
+                      path: /mnt/hf-cache
+                      type: DirectoryOrCreate
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit"
+        key: nightly-diffusion-x2iat-performance-qwen-image-edit
+        timeout_in_minutes: 180
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
+          - export CACHE_DIT_VERSION=1.3.0
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          - kubernetes:
+              podSpec:
+                containers:
+                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                    resources:
+                      limits:
+                        nvidia.com/gpu: 4
+                    volumeMounts:
+                      - name: devshm
+                        mountPath: /dev/shm
+                      - name: hf-cache
+                        mountPath: /root/.cache/huggingface
+                    env:
+                      - name: HF_HOME
+                        value: /root/.cache/huggingface
+                      - name: HF_TOKEN
+                        valueFrom:
+                          secretKeyRef:
+                            name: hf-token-secret
+                            key: token
+                nodeSelector:
+                  node.kubernetes.io/instance-type: gpu-h100-sxm
+                volumes:
+                  - name: devshm
+                    emptyDir:
+                      medium: Memory
+                  - name: hf-cache
+                    hostPath:
+                      path: /mnt/hf-cache
+                      type: DirectoryOrCreate
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit-2509"
+        key: nightly-diffusion-x2iat-performance-qwen-image-edit-2509
+        timeout_in_minutes: 180
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
+          - export CACHE_DIT_VERSION=1.3.0
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2509_vllm_omni.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          - kubernetes:
+              podSpec:
+                containers:
+                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                    resources:
+                      limits:
+                        nvidia.com/gpu: 4
+                    volumeMounts:
+                      - name: devshm
+                        mountPath: /dev/shm
+                      - name: hf-cache
+                        mountPath: /root/.cache/huggingface
+                    env:
+                      - name: HF_HOME
+                        value: /root/.cache/huggingface
+                      - name: HF_TOKEN
+                        valueFrom:
+                          secretKeyRef:
+                            name: hf-token-secret
+                            key: token
+                nodeSelector:
+                  node.kubernetes.io/instance-type: gpu-h100-sxm
+                volumes:
+                  - name: devshm
+                    emptyDir:
+                      medium: Memory
+                  - name: hf-cache
+                    hostPath:
+                      path: /mnt/hf-cache
+                      type: DirectoryOrCreate
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Layered"
+        key: nightly-diffusion-x2iat-performance-qwen-image-layered
+        timeout_in_minutes: 120
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
+          - export CACHE_DIT_VERSION=1.3.0
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          - kubernetes:
+              podSpec:
+                containers:
+                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                    resources:
+                      limits:
+                        nvidia.com/gpu: 1
+                    volumeMounts:
+                      - name: devshm
+                        mountPath: /dev/shm
+                      - name: hf-cache
+                        mountPath: /root/.cache/huggingface
+                    env:
+                      - name: HF_HOME
+                        value: /root/.cache/huggingface
+                      - name: HF_TOKEN
+                        valueFrom:
+                          secretKeyRef:
+                            name: hf-token-secret
+                            key: token
+                nodeSelector:
+                  node.kubernetes.io/instance-type: gpu-h100-sxm
+                volumes:
+                  - name: devshm
+                    emptyDir:
+                      medium: Memory
+                  - name: hf-cache
+                    hostPath:
+                      path: /mnt/hf-cache
+                      type: DirectoryOrCreate
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · HunyuanImage3 · DiT Perf Test"
+        key: nightly-hunyuan-image3-performance
+        soft_fail: true
+        timeout_in_minutes: 120
+        if: build.env("RUN_HUNYUAN_IMAGE3_PERF") == "1"
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
+          - export CACHE_DIT_VERSION=1.3.0
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp4_fp8.json
+            EXIT1=$$?
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp2_fp8_sp2.json
+            EXIT2=$$?
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp2_fp8_cfgp2.json
+            EXIT3=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$((EXIT1 | EXIT2 | EXIT3))
         agents:
           queue: "mithril-h100-pool"
         plugins:
@@ -578,7 +959,7 @@ steps:
       - label: ":full_moon: Diffusion X2V · Function Test"
         timeout_in_minutes: 90
         commands:
-          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_hunyuan_video_15_expansion.py  --run-level "full_model"
+          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_hunyuan_video_15_expansion.py  -m "full_model and cuda" --run-level "full_model"
         agents:
           queue: "mithril-h100-pool"
         plugins:
@@ -617,7 +998,8 @@ steps:
         timeout_in_minutes: 180
         commands:
           - pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py -m full_model --run-level full_model
-          - pytest -s -v tests/e2e/accuracy/test_ltx2_3_video_similarity.py -m advanced_model --run-level advanced_model
+          - pytest -s -v tests/e2e/accuracy/test_ltx2_3_video_similarity.py -m full_model --run-level full_model
+          - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m full_model --run-level full_model
         agents:
           queue: "mithril-h100-pool"
         plugins:
@@ -699,6 +1081,65 @@ steps:
                       path: /mnt/hf-cache
                       type: DirectoryOrCreate
 
+  - group: ":card_index_dividers: Diffusion · Quantization Test"
+    depends_on: upload-nightly-pipeline
+    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "diffusion-x2iat-test"
+    steps:
+      - label: ":full_moon: Quantization Test with H100"
+        timeout_in_minutes: 90
+        commands:
+          - timeout 30m pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100' --run-level "full_model"
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          - kubernetes:
+              podSpec:
+                containers:
+                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                    resources:
+                      limits:
+                        nvidia.com/gpu: 1
+                    volumeMounts:
+                      - name: devshm
+                        mountPath: /dev/shm
+                      - name: hf-cache
+                        mountPath: /root/.cache/huggingface
+                    env:
+                      - name: HF_HOME
+                        value: /root/.cache/huggingface
+                      - name: HF_TOKEN
+                        valueFrom:
+                          secretKeyRef:
+                            name: hf-token-secret
+                            key: token
+                nodeSelector:
+                  node.kubernetes.io/instance-type: gpu-h100-sxm
+                volumes:
+                  - name: devshm
+                    emptyDir:
+                      medium: Memory
+                  - name: hf-cache
+                    hostPath:
+                      path: /mnt/hf-cache
+                      type: DirectoryOrCreate
+
+      - label: ":full_moon: Quantization Test with L4"
+        timeout_in_minutes: 60
+        commands:
+          - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4' --run-level "full_model"
+        agents:
+          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
+        plugins:
+          - docker#v5.2.0:
+              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+              always-pull: true
+              propagate-environment: true
+              environment:
+                - "HF_HOME=/fsx/hf_cache"
+                - "HF_TOKEN"
+              volumes:
+                - "/fsx/hf_cache:/fsx/hf_cache"
+
   - label: ":bar_chart: Testcase Statistics"
     key: nightly-testcase-statistics
     timeout_in_minutes: 120
@@ -716,7 +1157,7 @@ steps:
               - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                 resources:
                   limits:
-                    nvidia.com/gpu: 2
+                    nvidia.com/gpu: 1
                 volumeMounts:
                   - name: devshm
                     mountPath: /dev/shm
@@ -745,9 +1186,14 @@ steps:
   - label: ":email: Nightly Collection & Email"
     key: nightly-perf-distribution
     depends_on:
-      - nightly-omni-performance
+      - nightly-omni-performance-no-async-chunk
+      - nightly-omni-performance-async-chunk
+      - nightly-omni-performance-vllm-text
       - nightly-tts-performance
-      - nightly-diffusion-x2iat-performance
+      - nightly-diffusion-x2iat-performance-qwen-image
+      - nightly-diffusion-x2iat-performance-qwen-image-edit
+      - nightly-diffusion-x2iat-performance-qwen-image-edit-2509
+      - nightly-diffusion-x2iat-performance-qwen-image-layered
       - nightly-diffusion-x2v-performance
       - nightly-testcase-statistics
     if: build.env("NIGHTLY") == "1"
@@ -756,8 +1202,13 @@ steps:
       - export DEFAULT_INPUT_DIR=tests/dfx/perf/results
       - export DEFAULT_OUTPUT_DIR=tests/dfx/perf/results
       - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance
-      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance
-      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-no-async-chunk
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-async-chunk
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-vllm-text
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit-2509
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-layered
       - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance
       - buildkite-agent artifact download "tests/dfx/perf/results/*.html" . --step nightly-testcase-statistics
       - python tools/nightly/generate_nightly_perf_excel.py

.buildkite/test-npu-nightly.yaml

@@ -0,0 +1,24 @@
+steps:
+  - group: ":card_index_dividers: Diffusion X2V Model Test"
+    key: nightly-diffusion-x2v-group
+    depends_on: upload-nightly-pipeline
+    if: >-
+      build.env("NIGHTLY") == "1" ||
+      build.pull_request.labels includes "nightly-test" ||
+      build.pull_request.labels includes "diffusion-x2v-test"
+    steps:
+      - label: ":full_moon: Diffusion X2V · Function Test"
+        agents:
+          queue: "ascend-a2b3"
+          resource_class: "npu-4"
+        image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}"
+        plugins:
+          - kubernetes:
+              podSpecPatch:
+                imagePullSecrets:
+                  - name: swr-secret
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          HF_TOKEN: "${HF_TOKEN}"
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k npu_i2v --run-level "full_model"

.buildkite/test-npu-ready.yaml

@@ -0,0 +1,21 @@
+steps:
+  - group: ":card_index_dividers: TTS Model Test"
+    key: ready-tts-group
+    depends_on: upload-ready-pipeline
+    steps:
+      - label: ":sound: TTS · Qwen3-TTS CustomVoice Test"
+        agents:
+          queue: "ascend-a2b3"
+          resource_class: "npu-1"
+        image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}"
+        plugins:
+          - kubernetes:
+              podSpecPatch:
+                imagePullSecrets:
+                  - name: swr-secret
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'

.buildkite/test-weekly.yml

@@ -1,5 +1,6 @@
 env:
   VLLM_WORKER_MULTIPROC_METHOD: spawn
+  VLLM_USE_FLASHINFER_MOE_FP16: 0
   HF_HUB_DOWNLOAD_TIMEOUT: 300
   HF_HUB_ETAG_TIMEOUT: 60
 

.claude/skills/add-diffusion-model/SKILL.md

@@ -515,15 +515,15 @@ After adding parallelism support, update:
 
 ### Step 11: Add CPU Offload Support
 
-Implement `SupportsModuleOffload` on your pipeline class to enable
+Implement `SupportsComponentDiscovery` on your pipeline class to enable
 `--enable-cpu-offload` and `--enable-layerwise-offload`. The protocol
 declares which submodules the offloader should manage:
 
 ```python
 from typing import ClassVar
-from vllm_omni.diffusion.models.interface import SupportsModuleOffload
+from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery
 
-class YourPipeline(nn.Module, SupportsModuleOffload):
+class YourPipeline(nn.Module, SupportsComponentDiscovery):
     _dit_modules: ClassVar[list[str]] = ["transformer"]
     _encoder_modules: ClassVar[list[str]] = ["text_encoder"]
     _vae_modules: ClassVar[list[str]] = ["vae"]
@@ -540,7 +540,7 @@ class YourPipeline(nn.Module, SupportsModuleOffload):
 All attribute names support dotted paths for nested submodules
 (e.g. `"pipe.transformer"`, `"bagel.time_embedder"`).
 
-Pipelines without `SupportsModuleOffload` fall back to scanning
+Pipelines without `SupportsComponentDiscovery` fall back to scanning
 well-known attribute names (`transformer`, `text_encoder`, `vae`,
 etc.), which fails for non-standard names.
 

.claude/skills/add-diffusion-model/references/parallelism-patterns.md

@@ -92,6 +92,7 @@ class TPSelfAttention(nn.Module):
             softmax_scale=1.0 / (self.head_dim ** 0.5),
             causal=False,
             num_kv_heads=self.to_qkv.num_kv_heads,  # Local KV heads per GPU
+            role="self",
         )
 
     def forward(self, x):

.claude/skills/add-diffusion-model/references/transformer-adaptation.md

@@ -61,6 +61,7 @@ class SelfAttentionBlock(nn.Module):
             softmax_scale=1.0 / (self.head_dim ** 0.5),
             causal=False,
             num_kv_heads=num_heads,
+            role="self",
         )
 
     def forward(self, x, attn_mask=None):
@@ -75,6 +76,24 @@ class SelfAttentionBlock(nn.Module):
         return self.to_out(out)
 ```
 
+**Declaring the attention role.** Always pass a `role` string to `Attention(...)`. It is the lookup key users target with `--diffusion-attention-config.per_role.<role>.backend` to swap kernels per site without touching model code.
+
+| Convention | When to use |
+|---|---|
+| `role="self"` | Q/K/V come from the same hidden state |
+| `role="cross"` | K/V come from a separate `encoder_hidden_states`; pair with `skip_sequence_parallel=True` if K/V is replicated across SP ranks |
+
+For multi-modal sites that don't fit `self` / `cross`, use a dot-namespaced role and provide `role_category` so user config can fall back to the generic category:
+
+```python
+self.audio_to_video_attn = Attention(
+    num_heads=num_heads, head_size=self.head_dim,
+    softmax_scale=1.0 / (self.head_dim ** 0.5), causal=False,
+    role="mymodel.audio_to_video",
+    role_category="cross",
+)
+```
+
 #### Fused QKV with TP (Advanced)
 
 For tensor parallelism, use vLLM's parallel linear layers:
@@ -104,6 +123,7 @@ class TPSelfAttention(nn.Module):
             softmax_scale=1.0 / (self.head_dim ** 0.5),
             causal=False,
             num_kv_heads=num_heads,
+            role="self",
         )
 ```
 

.claude/skills/add-diffusion-model/references/troubleshooting.md

@@ -68,7 +68,7 @@ out = out.reshape(batch, seq_len, -1)
 ### Different Output vs Diffusers Reference
 
 **Common causes**:
-1. **Attention backend difference**: FlashAttention vs SDPA may produce slightly different results. Set `DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA` to match diffusers
+1. **Attention backend difference**: FlashAttention vs SDPA may produce slightly different results. Force SDPA via `--diffusion-attention-backend TORCH_SDPA` (or `DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA`) to match diffusers; for finer-grained per-role overrides see `--diffusion-attention-config.per_role.<role>.backend`
 2. **Float precision**: vLLM-Omni may use bfloat16 where diffusers uses float32 for some operations
 3. **Missing normalization**: Check all LayerNorm/RMSNorm are preserved
 4. **Scheduler rounding**: Some schedulers have numerical sensitivity

.claude/skills/add-tts-model/SKILL.md

@@ -211,8 +211,8 @@ See `plan/voxcpm2_native_ar_design.md`.
 
 - Model files in `vllm_omni/model_executor/models/<model_name>/`
 - Stage config YAML
-- Working `end2end.py` with correct audio output
-- README.md in the example directory
+- Working `end2end.py` at `examples/offline_inference/text_to_speech/<model>/end2end.py`
+- New section in `examples/offline_inference/text_to_speech/README.md` (table row + per-model section). Do **not** create a top-level `examples/offline_inference/<model>/` dir or a per-model `README.md` inside `text_to_speech/<model>/` — the hub README is the documented surface and the mkdocs `generate_examples` hook only descends one level into `examples/<category>/`.
 
 ## Phase 3: Online Serving
 
@@ -308,11 +308,11 @@ def build_voice_clone_prompt(ref_audio_path: str, text: str, codec) -> list:
 ### Deliverables
 
 - Updated `serving_speech.py` with all 5 integration points (single commit)
-- Client scripts and server launcher
-- Gradio demo with streaming and voice cloning UI
+- Client scripts and server launcher under `examples/online_serving/text_to_speech/<model>/`
+- Gradio demo with streaming and voice cloning UI in the same dir
 - E2E online serving test (`tests/e2e/online_serving/test_<model>.py`)
 - Buildkite CI entry in `.buildkite/test-merge.yml`
-- Documentation (offline + online serving docs)
+- New section in `examples/online_serving/text_to_speech/README.md` (table row + per-model section). Do **not** create a top-level `examples/online_serving/<model>/` dir or a per-model `README.md` inside `text_to_speech/<model>/`.
 
 ### E2E test pitfalls to avoid
 

.claude/skills/precheck-pr/SKILL.md

@@ -0,0 +1,97 @@
+---
+name: precheck-pr
+description: Self-check your branch before creating a PR — catch dead code, verify accuracy/perf claims, validate PR title format, and confirm merge readiness. Use when the user says "precheck", "self review", "pre-submit check", or "check my PR before I open it." Never posts to GitHub.
+---
+
+# PR Pre-Check
+
+Self-review your branch before creating a PR against `vllm-project/vllm-omni`. Two modes: **quick** catches showstoppers, **full** does a thorough maintainer-grade review. Never posts to GitHub; the report is for the contributor's terminal only.
+
+## Mode Selection
+
+| Mode | When | Time |
+|------|------|------|
+| **Quick** | About to push, final sanity check | ~3 min |
+| **Full** | Ready for review, want maintainer-level scan | ~10 min |
+
+Default to quick if unsure. Run full before marking a PR "ready for review."
+
+## Workflow
+
+### Step 1: Detect Base Branch
+
+```bash
+BASE_SHA=$(git merge-base HEAD origin/main 2>/dev/null \
+         || git merge-base HEAD main 2>/dev/null \
+         || echo origin/main)
+echo "diffing against ${BASE_SHA}"
+git diff --name-only ${BASE_SHA}...HEAD
+```
+
+### Step 2: Validate PR Title
+
+Check the most recent commit message (or branch name if no commit yet) against the project convention. Valid prefixes:
+
+| Prefix | Applies to |
+|--------|-----------|
+| `[Bugfix]` | Bug fixes |
+| `[CI/Build]` | Build or CI improvements |
+| `[Doc]` | Documentation changes |
+| `[Model]` | New/improved models (include model name) |
+| `[Frontend]` | Frontend changes (API server, OmniLLM class, etc.) |
+| `[Kernel]` | CUDA/kernel changes |
+| `[Core]` | Core logic changes (OmniProcessor, OmniARScheduler, etc.) |
+| `[Hardware][Vendor]` | Hardware-specific (e.g., `[Hardware][Ascend]`) |
+| `[Misc]` | Other changes (use sparingly) |
+
+✗ if: missing prefix, wrong case (`[bugfix]`), or WIP/Draft in title.
+⚠ if: `[Model]` prefix without the model identifier (e.g., `[Model] Add new model` — should be `[Model] Add <ModelName> ...`).
+
+### Step 3: Categorize the PR
+
+| Diff contains | PR type |
+|---------------|---------|
+| New files under `vllm_omni/model_executor/models/<name>/` | **New Model** |
+| Changes to `vllm_omni/diffusion/` | **Diffusion Model** |
+| `[Bugfix]` prefix or single-file fix | **Bug Fix** |
+| Perf/benchmark/throughput claims in commit msg or diff | **Performance** |
+| Everything else | **General** |
+
+If multiple rows apply (e.g., a diffusion model is also a new model), union the checklists.
+
+### Step 4: Run Checklist
+
+Ask: "Quick mode or full mode?" Then walk the checklist for the detected PR type from [references/checklists.md](references/checklists.md). Each item produces ✓, ✗, or ⚠.
+
+### Step 5: Print Report
+
+```
+Pre-check report for <branch>
+
+  Mode: quick | full
+  Type: <new-model | diffusion-model | bug-fix | perf | general>
+
+  Dimension          Result
+  ─────────────────  ──────
+  PR title format    ✓
+  PR desc integrity  ✓
+  Registry/config    ✓
+  Dead code          ⚠ 2 warnings
+  Accuracy           ✓
+  Benchmark          ✗ missing software versions
+
+  Verdict: 1 blocking | 2 warnings | recommend fixing ✗ before PR
+```
+
+**Severity:**
+
+| Mark | Meaning |
+|------|---------|
+| ✗ | Blocking — fix before opening PR |
+| ⚠ | Warning — consider fixing |
+| ✓ | Pass |
+| — | Skipped (not applicable) |
+
+### Stop Here
+
+Do not post comments, open PRs, or modify files. The report is for the contributor's terminal only.

.claude/skills/precheck-pr/references/checklists.md

@@ -0,0 +1,135 @@
+# Pre-Check Checklists
+
+Each item is a concrete check to run against the diff. ✗ = fix before PR. ⚠ = consider fixing.
+
+---
+
+## All PRs: Title Format (check first)
+
+PR title must follow the project convention documented in `docs/contributing/README.md`:
+
+| Prefix | Applies to |
+|--------|-----------|
+| `[Bugfix]` | Bug fixes |
+| `[CI/Build]` | Build or CI improvements |
+| `[Doc]` | Documentation changes |
+| `[Model]` | New/improved models (include model name) |
+| `[Frontend]` | Frontend changes (API server, OmniLLM class, etc.) |
+| `[Kernel]` | CUDA/kernel changes |
+| `[Core]` | Core logic changes (OmniProcessor, OmniARScheduler, etc.) |
+| `[Hardware][Vendor]` | Hardware-specific (e.g., `[Hardware][Ascend]`) |
+| `[Misc]` | Other changes (use sparingly) |
+
+**Examples:**
+```
+✓ [Model] Add MiniCPM-o 4.5 support
+✓ [Bugfix] Fix Ovis image text encoder dtype
+✓ [Core] Qwen3-Omni performance optimization
+✓ [Hardware][Ascend] Fix VoxCPM2 on Ascend
+✗ Fix Ovis image text encoder dtype              ← missing prefix
+✗ [bugfix] Fix Ovis image text encoder           ← wrong case
+✗ [WIP] Add model                                ← WIP in title
+```
+
+- [ ] **Title has valid prefix** — ✗ if missing prefix, wrong case, or WIP/Draft in title. For `[Model]` PRs, the title MUST also include the model identifier (e.g., `[Model] Add <ModelName> ...`). ⚠ if missing the model name.
+
+---
+
+## Bug Fix PRs
+
+### Quick
+
+- [ ] **PR body has 4 sections:** what broke (error message), repro steps, root cause, fix description
+- [ ] **Fix is minimal:** diff touches only files directly related to the bug. No unrelated refactors or style changes.
+- [ ] **Repro is runnable:** the PR body includes a copy-paste command that reproduces the bug
+- [ ] **Branch is rebased:** `git merge-base HEAD origin/main` is recent, no merge conflicts
+
+### Full (adds)
+
+- [ ] **Regression test exists:** `git diff --name-only` includes at least one `tests/` file
+- [ ] **Regression test reproduces the original error:** the test would fail on main, pass on this branch
+- [ ] **Fix matches root cause exactly:** no "fix the symptom + something else" — if the root cause is dtype, the fix is dtype, not dtype + reformatting
+- [ ] **No silent failure risk:** no bare `except: pass`, no `try: ... except: return None`, no empty fallback added
+- [ ] **Upstream pattern match:** the fix follows the same pattern as existing code for similar cases (grep for analogous `from_pretrained` calls, etc.)
+- [ ] **Environment documented:** torch/transformers/vllm versions listed if the bug is version-dependent
+
+---
+
+## Performance PRs
+
+### Quick
+
+- [ ] **Before/after numbers in PR body:** at minimum, one metric (RTF, RPS, latency, VRAM) with main vs PR values
+- [ ] **Hardware specified:** GPU model, count, VRAM. "L4" or "H20" is sufficient. "tested on GPU" is not.
+- [ ] **Concurrency scaling:** if the optimization claims better throughput, numbers at 2+ concurrency levels
+- [ ] **No unexplained regressions:** if any metric goes down at any concurrency level, explain why
+- [ ] **Branch is rebased:** `git merge-base HEAD origin/main` is recent, no merge conflicts
+
+### Full (adds)
+
+- [ ] **Software versions:** torch, CUDA, vllm, vllm-omni versions in PR body
+- [ ] **Warmup stated:** "1 warmup + 3 measured" or equivalent
+- [ ] **Stddev or range:** not just a single best run. "average of 2 rounds" is minimum.
+- [ ] **VRAM measured:** peak GPU memory before and after
+- [ ] **Benchmark script checked in:** under `tests/` or `examples/`, runnable by reviewer
+- [ ] **Exact command line in PR:** not pseudocode, not "run the benchmark"
+- [ ] **Pytest summary included:** `36 passed, 3 skipped, 17 warnings in 2897.66s` — proves the whole suite passes
+- [ ] **Config changes documented:** if `max_num_seqs`, `gpu_memory_utilization`, or deploy YAML changed, old and new values stated with reasoning
+
+---
+
+## New Model PRs
+
+### Quick
+
+- [ ] **PR body file list matches `git diff --name-only`:** every file path in the description exists in the diff
+- [ ] **Registry entries match claimed count:** count `_OMNI_MODELS` or `_DIFFUSION_MODELS` entries added vs claimed in body
+- [ ] **Pipeline config `custom_process_input_func` paths resolve:** each referenced module.function exists
+- [ ] **`__init__.py` exports match registry:** each registered class appears in `__init__.py` and `__all__`
+- [ ] **Shell script modes match README:** `run_curl.sh` supports every mode the README documents
+- [ ] **Branch is rebased:** `git merge-base HEAD origin/main` is recent, no merge conflicts
+
+### Full (adds)
+
+- [ ] **Dead code scan (5 patterns):**
+  1. Dead `forward()` — defined but never called through the pipeline path
+  2. Dead factory/builder functions — instantiated nowhere
+  3. Dead wrappers — thin wrappers with no callers
+  4. Dead branch guards — `if version > X` where X is always satisfied
+  5. Unused parameters — `__init__` args never accessed
+- [ ] **Copy-paste detection:** no string constant defined in 3+ files, no cross-module validation duplication, no near-identical shape coercion fns
+- [ ] **Import hygiene:** no re-export-only imports, no module-level side effects without docs, no `import os` in function body when already at top. Function-body imports are acceptable when the import is only reachable from a closure or factory return value (i.e., `import` would otherwise execute at module load even though the user may never trigger that path). Either lift to top-of-file or annotate with `# noqa: PLC0415`.
+- [ ] **Accuracy:** at least one test compares output against reference; audio = valid WAV at correct sample rate; determinism verified if seed claimed. ⚠ if missing and not explicitly deferred to a follow-up PR or RFC section; — if explicitly deferred (the deferral target SHOULD be linked in the PR body).
+- [ ] **Performance:** hardware, software versions, warmup, and metrics (RTF, VRAM, RPS, latency) stated. ⚠ if missing and not explicitly deferred; — if explicitly deferred.
+- [ ] **Benchmark settings:** model config (TP, PP, max_model_len, enforce_eager, quant), runtime config (batch, input/output spec), environment versions
+- [ ] **Benchmark script checked in + exact command line + pytest summary**
+
+---
+
+## Diffusion Model PRs
+
+Diffusion models live under `vllm_omni/diffusion/`. In addition to the New Model checklist above, verify:
+
+- [ ] **Transformer adapter follows contract:** inherits from `nn.Module`, implements `load_weights()`, uses `vllm_omni.diffusion.attention.layer.Attention`
+- [ ] **Pipeline has both offline and online paths:** `forward()` for batch generation, streaming path for online serving
+- [ ] **Cache-DiT integration:** at least one acceleration (TeaCache, FBCache, etc.) is wired up. ✗ for new **top-level** pipelines. For **subclass** pipelines that inherit `__call__` from a parent where Cache-DiT is already absent, downgrade to ⚠ and recommend filing a follow-up issue against the parent pipeline family.
+- [ ] **Parallelism config:** TP/SP/USP/CFG-Parallel options exposed in pipeline config
+- [ ] **Docs table updated:** model added to `docs/models/supported_models.md` with correct metadata
+- [ ] **E2E test exists:** at least one test exercises the full generate path
+
+---
+
+## General PRs
+
+### Quick
+
+- [ ] **PR body matches diff:** description doesn't claim changes not in the diff
+- [ ] **Branch is rebased:** `git merge-base HEAD origin/main` is recent, no merge conflicts
+- [ ] **CI gates passing:** DCO, pre-commit, build — check `gh pr view --json statusCheckRollup` if a PR already exists
+
+### Full (adds)
+
+- [ ] **No new dead code in changed files:** scan for unreachable paths, unused imports
+- [ ] **Test coverage:** if core code changed (`engine/`, `stages/`, `connectors/`), tests are added or existing tests cover the path
+- [ ] **Import hygiene:** no redundant imports added
+- [ ] **No unrelated changes:** diff scope matches PR description

.claude/skills/readme.md

@@ -22,7 +22,12 @@ include:
 - `add-tts-model`: covers integration of new TTS models and related serving
   workflows
 - `generate-release-note`: helps prepare release notes for repository changes
+- `precheck-pr`: self-check a branch before creating a PR — validates PR title
+  format, catches dead code, verifies accuracy/perf claims, and confirms merge
+  readiness
 - `review-pr`: provides a structured workflow for reviewing pull requests
+- `vllm-omni-npu-model-runner-upgrade`: upgrades NPU model runners to align with the
+  latest vllm-ascend NPUModelRunner
 
 ## Maintenance Guidelines
 

.gitattributes

@@ -27,6 +27,7 @@
 *.mov binary
 *.mp4 binary
 *.mp3 binary
+*.wav binary
 *.flv binary
 *.fla binary
 *.swf binary

.github/CODEOWNERS

@@ -3,15 +3,64 @@
 # More info: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners
 
 # Default owners (fallback reviewers for all files)
-# later, we can add more reviewers here
-* @hsliuustc0106
+* @Gaohan123 @hsliuustc0106 @ywang96
 
-# Assign reviewers by directory
-/.github/ @owner1
-/tests/ @owner1
-/docs/ @owner1
+# Repository operations
+/.github/ @hsliuustc0106
 
-# Example: reviewers for specific directories
-# /examples/offline_inference/ @offline-inference-team
-# /vllm_omni/core/ @core-team
-# /vllm_omni/model_executor/ @model-team
+# Documentation and Recipes
+/docs/ @hsliuustc0106 @Gaohan123 @david6666666
+/recipes @hsliuustc0106 @Gaohan123 @david6666666
+
+# Skills
+/.claude/ @hsliuustc0106
+
+# Tests
+/tests/ @yenuo26
+
+# CI
+/.buildkite/ @yenuo26 @congw729
+
+# Engine and model_executor
+/vllm_omni/core/ @tzhouam @yinpeiqi @fake0fan
+/vllm_omni/engine/ @tzhouam @yinpeiqi @fake0fan
+/vllm_omni/entrypoints/ @tzhouam @yinpeiqi @fake0fan
+/vllm_omni/model_executor/ @tzhouam @gcanlin
+/vllm_omni/worker/ @tzhouam @gcanlin
+
+# Diffusion
+/vllm_omni/diffusion/ @Isotr0py @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa @david6666666
+/vllm_omni/lora/ @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa @yuanheng-zhao
+
+# Diffusion CPU offloading
+/vllm_omni/diffusion/offloader/ @Isotr0py @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa @yuanheng-zhao
+
+# Omni and TTS support
+/vllm_omni/transformers_utils/ @tzhouam @yuanheng-zhao
+/vllm_omni/model_executor/models/ @linyueqian @ZeldaHuang @princepride @yuanheng-zhao @amy-why-3459 @Bounty-hunter
+
+# Benchmarks
+/benchmarks/accuracy/ @david6666666
+/benchmarks/build_dataset/ @linyueqian @yuanheng-zhao
+/benchmarks/diffusion/ @Isotr0py @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa
+/benchmarks/distributed/ @princepride
+/benchmarks/glm_image/ @princepride @JaredforReal
+/benchmarks/tts/ @linyueqian @ZeldaHuang @princepride
+
+# Usability and Observability
+/vllm_omni/deploy/ @tzhouam @lishunyang12
+/vllm_omni/metrics/ @lishunyang12
+/vllm_omni/profiler/ @gcanlin
+
+# ComfyUI
+/apps/ @fhfuih
+
+# Large Scale Serving
+/vllm_omni/distributed/ @princepride @yuanheng-zhao @xuechendi @natureofnature @fake0fan
+
+# Quantization and configuration
+/vllm_omni/config/ @lishunyang12
+/vllm_omni/quantization/ @david6666666 @Isotr0py @lishunyang12
+
+# Hardware plugins and accelerators
+/vllm_omni/platforms/ @gcanlin @tjtanaa @xuechendi

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,4 +1,6 @@
 <!-- markdownlint-disable -->
+**Before submitting:** run the precheck-pr skill with code agent for a self-check against project conventions — catches dead code, missing benchmarks, and title format issues.
+
 PLEASE FILL IN THE PR DESCRIPTION HERE ENSURING ALL CHECKLIST ITEMS (AT THE BOTTOM) HAVE BEEN CONSIDERED.
 
 ## Purpose
@@ -18,4 +20,5 @@ PLEASE FILL IN THE PR DESCRIPTION HERE ENSURING ALL CHECKLIST ITEMS (AT THE BOTT
 - [ ] (Optional) Release notes update. If your change is user-facing, please update the release notes draft.
 </details>
 
-**BEFORE SUBMITTING, PLEASE READ <https://github.com/vllm-project/vllm-omni/blob/main/CONTRIBUTING.md>** (anything written below this line will be removed by GitHub Actions)
+**BEFORE SUBMITTING,** please read <https://github.com/vllm-project/vllm-omni/blob/main/CONTRIBUTING.md>
+(anything written below this line will be removed by GitHub Actions)

.github/mergify.yml

@@ -0,0 +1,223 @@
+pull_request_rules:
+- name: label-documentation
+  description: Automatically apply documentation label
+  conditions:
+    - label != stale
+    - or:
+      - files~=^[^/]+\.md$
+      - files~=^docs/
+      - files~=^examples/
+  actions:
+    label:
+      add:
+        - documentation
+    comment:
+      message: "Documentation preview: https://vllm--{{number}}.org.readthedocs.build/en/{{number}}/"
+
+- name: comment-pre-commit-failure
+  description: Comment on PR when pre-commit check fails
+  conditions:
+    - status-failure=pre-commit
+    - -closed
+    - -draft
+  actions:
+    comment:
+      message: |
+        Hi @{{author}}, the pre-commit checks have failed. Please run:
+
+        ```bash
+        uv pip install pre-commit
+        pre-commit install
+        pre-commit run --all-files
+        ```
+
+        Then, commit the changes and push to your branch.
+
+        For future commits, `pre-commit` will run automatically on changed files before each commit.
+
+        > [!TIP]
+        > <details>
+        > <summary>Is <code>mypy</code> or <code>markdownlint</code> failing?</summary>
+        > <br/>
+        > <code>mypy</code> and <code>markdownlint</code> are run differently in CI. If the failure is related to either of these checks, please use the following commands to run them locally:
+        >
+        > ```bash
+        > # For mypy (substitute "3.10" with the failing version if needed)
+        > pre-commit run --hook-stage manual mypy-3.10
+        > # For markdownlint
+        > pre-commit run --hook-stage manual markdownlint
+        > ```
+        > </details>
+
+- name: comment-dco-failure
+  description: Comment on PR when DCO check fails
+  conditions:
+    - status-failure=dco
+    - -closed
+    - -draft
+  actions:
+    comment:
+      message: |
+        Hi @{{author}}, the DCO check has failed. Please click on DCO in the Checks section for instructions on how to resolve this.
+
+- name: label-ci-build
+  description: Automatically apply ci/build label
+  conditions:
+    - label != stale
+    - or:
+      - files~=^\.github/
+      - files~=\.buildkite/
+      - files~=^cmake/
+      - files=CMakeLists.txt
+      - files~=^docker/Dockerfile
+      - files~=^requirements.*\.txt
+      - files=setup.py
+  actions:
+    label:
+      add:
+        - ci/build
+
+- name: label-frontend
+  description: Automatically apply frontend label
+  conditions:
+    - label != stale
+    - files~=^vllm_omni/entrypoints/
+  actions:
+    label:
+      add:
+        - frontend
+
+- name: label-diffusion
+  description: Automatically apply diffusion label
+  conditions:
+    - label != stale
+    - files~=^vllm_omni/diffusion/
+  actions:
+    label:
+      add:
+        - diffusion
+
+- name: label-new-model
+  description: Automatically apply new-model label
+  conditions:
+    - label != stale
+    - or:
+        - files=vllm_omni/model_executor/models/registry.py
+        - files=vllm_omni/diffusion/registry.py
+  actions:
+    label:
+      add:
+        - new-model
+
+- name: label-npu
+  description: Automatically apply npu label
+  conditions:
+    - label != stale
+    - or:
+      - files~=^vllm_omni/platforms/npu/
+      - title~=(?i)NPU
+      - title~=(?i)Ascend
+  actions:
+    label:
+      add:
+        - npu
+    assign:
+      users:
+        - "gcanlin"
+
+- name: label-rocm
+  description: Automatically apply rocm label
+  conditions:
+    - label != stale
+    - or:
+      - files~=^vllm_omni/platforms/rocm/
+      - title~=(?i)AMD
+      - title~=(?i)ROCm
+  actions:
+    label:
+      add:
+        - rocm
+    assign:
+      users:
+        - "tjtanaa"
+
+- name: label-xpu
+  description: Automatically apply xpu label
+  conditions:
+    - label != stale
+    - or:
+      - files~=^vllm_omni/platforms/xpu/
+      - title~=(?i)Intel
+      - title~=(?i)XPU
+  actions:
+    label:
+      add:
+        - xpu
+    assign:
+      users:
+        - "yma11"
+
+- name: label-modelrunner
+  description: Automatically apply modelrunner label
+  conditions:
+    - label != stale
+    - or:
+      - files~=^vllm_omni/worker/
+      - title~=(?i)ModelRunner
+      - title~=(?i)worker
+  actions:
+    label:
+      add:
+        - ModelRunner
+    assign:
+      users:
+        - "gcanlin" # To check whether need to sync to NPU
+
+- name: auto-rebase if approved, ready, and 30 commits behind main
+  conditions:
+    - base = main
+    - label=ready
+    - "#approved-reviews-by >= 1"
+    - "#commits-behind >= 30"
+    - -closed
+    - -draft
+    - -conflict
+  actions:
+    rebase: {}
+
+- name: ping author on conflicts and add 'needs-rebase' label
+  conditions:
+    - label != stale
+    - conflict
+    - -closed
+  actions:
+    label:
+      add:
+        - needs-rebase
+    comment:
+      message: |
+       This pull request has merge conflicts that must be resolved before it can be
+       merged. Please rebase the PR, @{{author}}.
+
+       https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/syncing-a-fork
+
+- name: remove 'needs-rebase' label when conflict is resolved
+  conditions:
+    - -conflict
+    - -closed
+  actions:
+    label:
+      remove:
+        - needs-rebase
+
+- name: label-bug
+  description: Automatically apply bug label
+  conditions:
+    - label != stale
+    - or:
+      - title~=(?i)\bbug\b
+      - title~=(?i)\bbugfix\b
+  actions:
+    label:
+      add:
+        - bug

.gitignore

@@ -239,6 +239,7 @@ data/
 datasets/
 *.csv
 *.json
+!tests/dfx/perf/tests/*.json
 !apps/ComfyUI-vLLM-Omni/example_workflows/*.json
 *.jsonl
 *.parquet
@@ -264,6 +265,8 @@ tmp_test
 vllm_omni/_version.py
 # output files
 *.wav
+# Vendored test reference audio (small, intentionally checked in)
+!tests/assets/**/*.wav
 .worktrees/
 # CI overlay yamls materialized from tests/utils.py:_CI_OVERLAYS at test time
 tests/.ci_generated/

README.md

@@ -9,19 +9,19 @@ Easy, fast, and cheap omni-modality model serving for everyone
 </h3>
 
 <p align="center">
-| <a href="https://vllm-omni.readthedocs.io/en/latest/"><b>Documentation</b></a> | <a href="https://discuss.vllm.ai"><b>User Forum</b></a> | <a href="https://slack.vllm.ai"><b>Developer Slack</b></a> | <a href="docs/assets/WeChat.jpg"><b>WeChat</b></a> | <a href="https://arxiv.org/abs/2602.02204"><b>Paper</b></a> | <a href="https://docs.google.com/presentation/d/1XJWgv79lORl8rbaVvp2d5Sqs6ZEBgAgj/edit?slide=id.p1#slide=id.p1"><b>Slides</b></a> |
+| <a href="https://vllm-omni.readthedocs.io/en/latest/"><b>Documentation</b></a> | <a href="https://discuss.vllm.ai"><b>User Forum</b></a> | <a href="https://slack.vllm.ai"><b>Developer Slack</b></a> | <a href="docs/assets/WeChat.jpg"><b>WeChat</b></a> | <a href="https://arxiv.org/abs/2602.02204"><b>Paper</b></a> | <a href="https://docs.google.com/presentation/d/111-L8zF7A1j_YI_cR8JsblofdScdRr2f/edit?usp=sharing&ouid=110473603432222024453&rtpof=true&sd=true"><b>Slides</b></a> |
 </p>
 
 
 ---
 
 *Latest News* 🔥
+- [2026/05] We released [0.20.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.20.0) - refreshes the serving/runtime stack for large-scale omni workloads, and improves diffusion model performance, quantization, and hardware readiness across CUDA, ROCm, MUSA, NPU, and XPU backends.
 - [2026/03] We released [0.18.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.18.0) - strengthens the core runtime through a large entrypoint refactor and scheduler/runtime cleanups, expands unified quantization and diffusion execution, broadens multimodal model coverage, and improves production readiness across audio, omni, image, video, RL, and multi-platform deployments.
 - [2026/03] Check out our first public [project deepdive](https://youtu.be/sgwNfsNnR9I) at the vLLM Hong Kong Meetup!
 - [2026/03] **[vllm-omni-skills](https://github.com/hsliuustc0106/vllm-omni-skills)** is a community-driven collection of AI assistant skills that help developers work with vLLM-Omni more effectively. These skills can be used with popular agentic AI coding assistants like **Cursor IDE**, **Claude**, **Codex**, and more.
 - [2026/02] We released [0.16.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.16.0) - A major alignment + capability release that rebases onto **upstream vLLM v0.16.0** and significantly expands performance, distributed execution, and production readiness across **Qwen3-Omni / Qwen3-TTS**, **Bagel**, **MiMo-Audio**, **GLM-Image** and the **Diffusion (DiT) image/video stack**—while also improving platform coverage (CUDA / ROCm / NPU / XPU), CI quality, and documentation.
 - [2026/02] We released [0.14.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.14.0) - This is the first **stable release** of vLLM-Omni that expands Omni’s diffusion / image-video generation and audio / TTS stack, improves distributed execution and memory efficiency, and broadens platform/backend coverage (GPU/ROCm/NPU/XPU). It also brings meaningful upgrades to serving APIs, profiling & benchmarking, and overall stability. Please check our latest [paper](https://arxiv.org/abs/2602.02204) for architecture design and performance results.
-- [2026/01] We released [0.12.0rc1](https://github.com/vllm-project/vllm-omni/releases/tag/v0.12.0rc1) - a major RC milestone focused on maturing the diffusion stack, strengthening OpenAI-compatible serving, expanding omni-model coverage, and improving stability across platforms (GPU/NPU/ROCm).
 - [2025/11] vLLM community officially released [vllm-project/vllm-omni](https://github.com/vllm-project/vllm-omni) in order to support omni-modality models serving.
 
 ---

benchmarks/build_dataset/seed_tts_design/en/meta.lst

@@ -18,3 +18,13 @@ vd017|||The quarterly results exceed expectations across all major metrics.|A sh
 vd018|||Chapter one. The morning sun filtered gently through the forest canopy.|A smooth, rich male audiobook narrator voice, expressive and engaging.
 vd019|||To be or not to be, that is the question.|A theatrical female voice, dramatic and expressive, stage projection quality.
 vd020|||And that is all for tonight. Stay well out there, everyone.|A warm, velvety male late-night radio DJ voice, smooth and intimate.
+vd021|||Please fasten your seatbelts; we will be experiencing some turbulence shortly.|A calm, professional female flight attendant voice, reassuring and clear.
+vd022|||The fossil record suggests these creatures lived more than sixty million years ago.|A measured male documentary narrator voice, scientific and curious.
+vd023|||Hey kiddo, did you finish your homework before dinner today?|A gentle, caring father voice, warm and slightly tired after a long day.
+vd024|||Live from courtside, the home team has just taken a commanding lead.|A high-energy male sports broadcaster, urgent and engaged.
+vd025|||Please remain on the line while I transfer you to the next available agent.|A polite, neutral female call-center voice, professional and patient.
+vd026|||I have a dream that one day this nation will rise up and live out the true meaning of its creed.|A powerful, resonant male orator voice, slow and impassioned.
+vd027|||Mama always said life was like a box of chocolates; you never know what you're gonna get.|A folksy male voice with a Southern American accent, warm and reflective.
+vd028|||Ahoy there, matey! Hoist the colors and prepare to set sail.|A theatrical pirate male voice, gruff and boisterous.
+vd029|||Ladies and gentlemen, please welcome to the stage our keynote speaker.|A polished female event MC voice, gracious and articulate.
+vd030|||Once you've completed the form, click submit and we'll review it within two business days.|A friendly female product-onboarding voice, encouraging and clear.

benchmarks/build_dataset/seed_tts_smoke/en/meta.lst

@@ -18,3 +18,13 @@ smoke017|||In the distance, the mountains were shrouded in thick morning fog.
 smoke018|||Our company reported record revenue for the fourth quarter of the fiscal year.
 smoke019|||She explained the new policy in detail during the staff meeting this morning.
 smoke020|||The children laughed and played in the garden until the sun began to set.
+smoke021|||A long line of cars stretched along the coastal highway after the lighthouse opened.
+smoke022|||The chef recommended the seared scallops with a lemon butter sauce as the special.
+smoke023|||Our quarterly engineering review begins promptly at two in the afternoon on Thursday.
+smoke024|||Despite the heavy rain, the marathon runners kept a steady pace through downtown.
+smoke025|||She finished the crossword puzzle in under fifteen minutes while sipping her tea.
+smoke026|||The new release adds streaming output, faster startup, and better error reporting.
+smoke027|||His grandfather taught him how to whittle small wooden birds from cedar offcuts.
+smoke028|||Travelers were advised to check the timetable since trains were running on a holiday schedule.
+smoke029|||The startup launched a public beta after eighteen months of closed development.
+smoke030|||At night the desert sky filled with stars so bright the Milky Way was clearly visible.

benchmarks/diffusion/README.md

@@ -130,9 +130,21 @@ Warmup flags:
 
 - `--warmup-requests`: Number of warmup requests.
 - `--warmup-num-inference-steps`: Steps used during warmup.
+- `--warmup-concurrency`: Maximum concurrent warmup requests. Use this to warm
+  the same batch shape as the measured run instead of warming only batch=`1`.
 - For `--task t2v`: warmup requests are forced to use `num_frames=1` to make warmup faster and less noisy.
 
 Traffic / concurrency flags:
 
 - `--request-rate`: Target request rate (requests/second). If set to `inf`, the script sends all requests immediately.
 - `--max-concurrency`: Max number of in-flight requests (default: `1`). This can hard-cap the achieved QPS: if it is too small, requests will queue behind the semaphore, and both achieved throughput and observed SLO attainment can be skewed.
+
+### Batched warmup note
+
+For batched serving runs, warm the same in-flight shape you plan to measure.
+For example, a run with `--max-concurrency 8` should usually also use
+`--warmup-requests 8 --warmup-concurrency 8`; otherwise the first measured
+batch may still pay compile or CUDA-graph capture cost.
+
+For a Qwen-Image continuous-batching replay example, see
+[`performance_dashboard/qwen_image_serving_performance.md`](./performance_dashboard/qwen_image_serving_performance.md).

benchmarks/diffusion/diffusion_benchmark_serving.py

@@ -751,6 +751,52 @@ async def iter_requests(
         yield req
 
 
+def _make_warmup_request(
+    requests_list: list[RequestFuncInput],
+    index: int,
+    args,
+) -> RequestFuncInput:
+    warm_req = requests_list[index % len(requests_list)]
+    if args.warmup_num_inference_steps is not None:
+        warm_req = replace(
+            warm_req,
+            num_inference_steps=args.warmup_num_inference_steps,
+        )
+    if args.task == "t2v":
+        warm_req = replace(warm_req, num_frames=1)
+    return warm_req
+
+
+async def _run_warmups(
+    requests_list: list[RequestFuncInput],
+    args,
+    session: aiohttp.ClientSession,
+    request_func,
+) -> list[tuple[RequestFuncInput, RequestFuncOutput]]:
+    if not args.warmup_requests or not requests_list:
+        return []
+
+    warmup_requests = [_make_warmup_request(requests_list, i, args) for i in range(args.warmup_requests)]
+    warmup_concurrency = min(int(args.warmup_concurrency), len(warmup_requests))
+    warmup_semaphore = asyncio.Semaphore(warmup_concurrency)
+
+    print(
+        f"Running {len(warmup_requests)} warmup request(s) "
+        f"with num_inference_steps={args.warmup_num_inference_steps} "
+        f"and warmup_concurrency={warmup_concurrency}..."
+    )
+
+    async def limited_warmup_request_func(
+        req: RequestFuncInput,
+    ) -> RequestFuncOutput:
+        async with warmup_semaphore:
+            return await request_func(req, session, None)
+
+    warmup_tasks = [asyncio.create_task(limited_warmup_request_func(req)) for req in warmup_requests]
+    warmup_outputs = await asyncio.gather(*warmup_tasks)
+    return list(zip(warmup_requests, warmup_outputs))
+
+
 def calculate_metrics(
     outputs: list[RequestFuncOutput],
     total_duration: float,
@@ -900,23 +946,12 @@ async def benchmark(args):
     pbar = tqdm(total=len(requests_list), disable=args.disable_tqdm)
 
     async with aiohttp.ClientSession() as session:
-        warmup_pairs: list[tuple[RequestFuncInput, RequestFuncOutput]] = []
-        if args.warmup_requests and requests_list:
-            print(
-                f"Running {args.warmup_requests} warmup request(s) \
-                with num_inference_steps={args.warmup_num_inference_steps}..."
-            )
-            for i in range(args.warmup_requests):
-                warm_req = requests_list[i % len(requests_list)]
-                if args.warmup_num_inference_steps is not None:
-                    warm_req = replace(
-                        warm_req,
-                        num_inference_steps=args.warmup_num_inference_steps,
-                    )
-                if args.task == "t2v":
-                    warm_req = replace(warm_req, num_frames=1)
-                warm_out = await limited_request_func(warm_req, session, None)
-                warmup_pairs.append((warm_req, warm_out))
+        warmup_pairs = await _run_warmups(
+            requests_list=requests_list,
+            args=args,
+            session=session,
+            request_func=request_func,
+        )
 
         if args.slo:
             # Prefer trace-provided per-request slo_ms. Only populate when missing.
@@ -1072,6 +1107,13 @@ if __name__ == "__main__":
         default=1,
         help="num_inference_steps used for warmup requests.",
     )
+    parser.add_argument(
+        "--warmup-concurrency",
+        type=int,
+        default=1,
+        help="Maximum number of warmup requests to run concurrently. "
+        "Set this to match the real batch shape when warming up torch.compile or CUDA graphs.",
+    )
     parser.add_argument("--width", type=int, default=None, help="Image/Video width.")
     parser.add_argument("--height", type=int, default=None, help="Image/Video height.")
     parser.add_argument("--num-frames", type=int, default=None, help="Number of frames (for video).")
@@ -1135,5 +1177,4 @@ if __name__ == "__main__":
     )
 
     args = parser.parse_args()
-
     asyncio.run(benchmark(args))

benchmarks/diffusion/performance_dashboard/qwen_image_serving_performance.md

@@ -33,6 +33,12 @@ vllm serve Qwen/Qwen-Image --omni \
     --port 8091
 ```
 
+To replay step-wise continuous batching, compare
+`vllm serve Qwen/Qwen-Image --omni --port 8089 --step-execution --max-num-seqs 1`
+against the same command with `--max-num-seqs 8`. `--step-execution` is the
+feature gate, and increasing `--max-num-seqs` above `1` lets the scheduler
+keep more compatible requests active at once.
+
 ## 3.2 Key Parameters
 
 | Parameter             | Description              |
@@ -70,6 +76,7 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \
 | `--task`               | Task type (e.g., `t2i`)           |
 | `--num-prompts`        | Total number of requests          |
 | `--max-concurrency`    | Client-side concurrency           |
+| `--warmup-concurrency` | Warmup concurrency for batch-shape prewarming |
 | `--random-request-config`| JSON string defining random request |
 
 ---
@@ -133,6 +140,12 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \
     ]'
 ```
 
+For a continuous-batching A/B replay, run the same benchmark command twice
+against the two serving configs above and keep traffic plus warmup settings
+identical across both runs. If the measured run uses `--max-concurrency 8`,
+warm with `--warmup-requests 8 --warmup-concurrency 8` so the first measured
+batch does not include compile or CUDA-graph capture overhead.
+
 ---
 
 # 6. Performance Metrics
@@ -163,6 +176,7 @@ To ensure consistent and comparable benchmark results:
 * Record parallel configuration
 * Record benchmark parameters (resolution, concurrency, number of prompts)
 * Ensure no background workload on GPUs during testing
+* When comparing continuous batching, keep warmup settings the same across runs
 
 ---
 

benchmarks/diffusion/quantization_quality.py

@@ -151,7 +151,7 @@ def _generate_image(omni, args, prompt, seed):
     from vllm_omni.platforms import current_omni_platform
 
     generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(seed)
-    torch.cuda.reset_peak_memory_stats()
+    torch.accelerator.reset_peak_memory_stats()
     start = time.perf_counter()
     outputs = omni.generate(
         {"prompt": prompt},
@@ -163,7 +163,7 @@ def _generate_image(omni, args, prompt, seed):
         ),
     )
     elapsed = time.perf_counter() - start
-    peak_mem = torch.cuda.max_memory_allocated() / (1024**3)
+    peak_mem = torch.accelerator.max_memory_allocated() / (1024**3)
 
     first = outputs[0]
     req_out = first.request_output[0] if hasattr(first, "request_output") else first
@@ -178,7 +178,7 @@ def _generate_video(omni, args, prompt, seed):
     from vllm_omni.platforms import current_omni_platform
 
     generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(seed)
-    torch.cuda.reset_peak_memory_stats()
+    torch.accelerator.reset_peak_memory_stats()
     start = time.perf_counter()
     outputs = omni.generate(
         {"prompt": prompt, "negative_prompt": ""},
@@ -192,7 +192,7 @@ def _generate_video(omni, args, prompt, seed):
         ),
     )
     elapsed = time.perf_counter() - start
-    peak_mem = torch.cuda.max_memory_allocated() / (1024**3)
+    peak_mem = torch.accelerator.max_memory_allocated() / (1024**3)
 
     first = outputs[0]
     if hasattr(first, "request_output") and isinstance(first.request_output, list):
@@ -228,8 +228,8 @@ def _unload_omni(omni):
     del omni
     gc.collect()
     if torch.cuda.is_available():
-        torch.cuda.empty_cache()
-        torch.cuda.synchronize()
+        torch.accelerator.empty_cache()
+        torch.accelerator.synchronize()
 
 
 def run_benchmark(args):

benchmarks/distributed/omni_connectors/cross_node_mooncake_transfer_engine.py

@@ -614,8 +614,8 @@ Examples:
         if not torch.cuda.is_available():
             print("[ERROR] CUDA is not available but GPU mode was requested.")
             sys.exit(1)
-        if args.gpu_id >= torch.cuda.device_count():
-            print(f"[ERROR] GPU {args.gpu_id} not available. Found {torch.cuda.device_count()} GPUs.")
+        if args.gpu_id >= torch.accelerator.device_count():
+            print(f"[ERROR] GPU {args.gpu_id} not available. Found {torch.accelerator.device_count()} GPUs.")
             sys.exit(1)
         print(f"[INFO] Using GPU {args.gpu_id}: {torch.cuda.get_device_name(args.gpu_id)}")
 

benchmarks/fish-speech/bench_speaker_cache.py

@@ -3,11 +3,11 @@
 Measures TTFP improvement from DAC-code caching when using uploaded voices.
 
 Setup:
-  1. Start vllm-omni with Fish Speech S2 Pro (use our feat branch)
+  1. Start vllm-omni with Fish Speech S2 Pro
   2. Provide a reference audio file for voice cloning
 
 Usage:
-    python bench_voice_cache.py \
+    python bench_speaker_cache.py \
         --ref-audio /path/to/reference.wav \
         --ref-text "Transcript of the reference audio." \
         --num-prompts 20 \

benchmarks/tts/bench_voxcpm_offline.py

@@ -63,7 +63,7 @@ def _find_repo_root(start: Path) -> Path:
 
 REPO_ROOT = _find_repo_root(Path(__file__).resolve())
 DEFAULT_STAGE_ASYNC = REPO_ROOT / "vllm_omni" / "model_executor" / "stage_configs" / "voxcpm_async_chunk.yaml"
-DEFAULT_STAGE_SYNC = REPO_ROOT / "vllm_omni" / "model_executor" / "stage_configs" / "voxcpm.yaml"
+DEFAULT_STAGE_SYNC = REPO_ROOT / "vllm_omni" / "deploy" / "voxcpm.yaml"
 
 logger = logging.getLogger(__name__)
 

docker/Dockerfile.ci

@@ -1,5 +1,5 @@
 ARG VLLM_BASE_IMAGE=vllm/vllm-openai
-ARG VLLM_BASE_TAG=v0.19.0
+ARG VLLM_BASE_TAG=v0.21.0
 FROM ${VLLM_BASE_IMAGE}:${VLLM_BASE_TAG}
 ARG APP_DIR=/workspace/vllm-omni
 WORKDIR ${APP_DIR}

docker/Dockerfile.cuda

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=vllm/vllm-openai:v0.19.0
+ARG BASE_IMAGE=vllm/vllm-openai:v0.20.0
 FROM ${BASE_IMAGE}
 
 ARG COMMON_WORKDIR=/app

docker/Dockerfile.npu.ci

@@ -0,0 +1,48 @@
+ARG VLLM_ASCEND_IMAGE=quay.io/ascend/vllm-ascend
+ARG VLLM_ASCEND_TAG=nightly-main
+FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
+
+ARG APP_DIR=/vllm-workspace/vllm-omni
+WORKDIR ${APP_DIR}
+COPY . .
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        ffmpeg \
+        libgl1 \
+        libglib2.0-0 \
+        libgomp1 \
+        libsm6 \
+        libxext6 \
+        libxrender1 \
+        libxcb1 \
+        espeak-ng && \
+    rm -rf /var/cache/apt/* && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
+    source /usr/local/Ascend/nnal/atb/set_env.sh && \
+    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
+    python3 -m pip install -U pip && \
+    python3 -m pip install -v \
+      "pytest>=7.0.0" \
+      "pytest-asyncio>=0.21.0" \
+      "pytest-cov>=4.0.0" \
+      "pytest-mock>=3.10.0" \
+      "openpyxl>=3.0.0" \
+      "psutil>=7.2.0" \
+      "soundfile>=0.13.1" \
+      "pyttsx3>=2.99" \
+      "opencv-python>=4.12.0.88" \
+      "imageio[ffmpeg]>=0.6.0" \
+      --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
+      --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
+      --default-timeout=60 && \
+    python3 -m pip install -v -e /vllm-workspace/vllm-omni/ --no-build-isolation \
+      --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
+      --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
+      --default-timeout=60
+
+ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
+
+ENTRYPOINT []

docker/Dockerfile.npu.ci.a3

@@ -0,0 +1,48 @@
+ARG VLLM_ASCEND_IMAGE=quay.io/ascend/vllm-ascend
+ARG VLLM_ASCEND_TAG=nightly-main-a3
+FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
+
+ARG APP_DIR=/vllm-workspace/vllm-omni
+WORKDIR ${APP_DIR}
+COPY . .
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        ffmpeg \
+        libgl1 \
+        libglib2.0-0 \
+        libgomp1 \
+        libsm6 \
+        libxext6 \
+        libxrender1 \
+        libxcb1 \
+        espeak-ng && \
+    rm -rf /var/cache/apt/* && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
+    source /usr/local/Ascend/nnal/atb/set_env.sh && \
+    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
+    python3 -m pip install -U pip && \
+    python3 -m pip install -v \
+      "pytest>=7.0.0" \
+      "pytest-asyncio>=0.21.0" \
+      "pytest-cov>=4.0.0" \
+      "pytest-mock>=3.10.0" \
+      "openpyxl>=3.0.0" \
+      "psutil>=7.2.0" \
+      "soundfile>=0.13.1" \
+      "pyttsx3>=2.99" \
+      "opencv-python>=4.12.0.88" \
+      "imageio[ffmpeg]>=0.6.0" \
+      --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
+      --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
+      --default-timeout=60 && \
+    python3 -m pip install -v -e /vllm-workspace/vllm-omni/ --no-build-isolation \
+      --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
+      --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
+      --default-timeout=60
+
+ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
+
+ENTRYPOINT []

docker/Dockerfile.rocm

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.19.0
+ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.21.0
 FROM ${BASE_IMAGE} AS base
 
 # Declare a variable to know if we want to use the nightly build or the stable build.

docker/Dockerfile.xpu

@@ -14,8 +14,9 @@ RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRO
 RUN apt clean && apt-get update -y && \
     apt-get install -y --no-install-recommends --fix-missing \
     curl \
-    espeak-ng \
+    ffmpeg \
     git \
+    libsndfile1 \
     libsm6 \
     libxext6 \
     libgl1 \
@@ -28,8 +29,20 @@ RUN apt clean && apt-get update -y && \
     python3.12-dev \
     python3-pip
 
-RUN apt update && apt upgrade -y && \
-    apt install -y intel-oneapi-compiler-dpcpp-cpp-2025.3
+# Add oneAPI repo, pin oneAPI to 2025.3, then install pinned packages in one layer.
+RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \
+    echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list && \
+    printf '%s\n' \
+    'Package: intel-oneapi-* intel-deep-learning-essentials* intel-pti*' \
+    'Pin: version 2025.3*' \
+    'Pin-Priority: 1001' \
+    > /etc/apt/preferences.d/oneapi-2025.3.pref && \
+    apt-get update -y && \
+    apt-get install -y --no-install-recommends \
+    intel-oneapi-compiler-dpcpp-cpp-2025.3 \
+    intel-oneapi-mkl-devel-2025.3 \
+    intel-oneapi-dnnl-devel-2025.3 && \
+    rm -rf /var/lib/apt/lists/*
 
 # Install UMD with fixed version
 RUN mkdir neo && \
@@ -52,9 +65,9 @@ RUN curl -LsSf https://astral.sh/uv/install.sh | sh
 RUN uv venv --python ${PYTHON_VERSION} --seed ${VIRTUAL_ENV}
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
-# This oneccl contains the BMG support which is not the case for default version of oneapi 2025.2.
-ARG ONECCL_INSTALLER="intel-oneccl-2021.15.7.8_offline.sh"
-RUN wget "https://github.com/uxlfoundation/oneCCL/releases/download/2021.15.7/${ONECCL_INSTALLER}" && \
+# This oneccl contains the BMG support which is not the case for default version of oneapi 2025.3.
+ARG ONECCL_INSTALLER="intel-oneccl-2021.15.9.14_offline.sh"
+RUN wget "https://github.com/uxlfoundation/oneCCL/releases/download/2021.15.9/${ONECCL_INSTALLER}" && \
     bash "${ONECCL_INSTALLER}" -a --silent --eula accept && \
     rm "${ONECCL_INSTALLER}" && \
     echo "source /opt/intel/oneapi/setvars.sh --force" >> /root/.bashrc && \
@@ -66,6 +79,7 @@ SHELL ["bash", "-c"]
 CMD ["bash", "-c", "source /root/.bashrc && exec bash"]
 
 WORKDIR /workspace/
+
 ENV UV_HTTP_TIMEOUT=500
 
 # Configure package index for XPU
@@ -74,24 +88,18 @@ ENV UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
 ENV UV_INDEX_STRATEGY="unsafe-best-match"
 ENV UV_LINK_MODE="copy"
 
-ARG VLLM_VERSION=v0.19.0
+ARG VLLM_VERSION=v0.20.0
 RUN git clone -b ${VLLM_VERSION} https://github.com/vllm-project/vllm
 WORKDIR /workspace/vllm
 
 RUN --mount=type=cache,target=/root/.cache/uv \
     uv pip install --upgrade pip && \
-    uv pip install -r requirements/xpu.txt
-
- # used for suffix method speculative decoding
- # build deps for proto + nanobind-based extensions to set up the build environment
-RUN --mount=type=cache,target=/root/.cache/uv \
-    uv pip install grpcio-tools protobuf nanobind
- # arctic-inference is built from source which needs torch-xpu properly installed first
-RUN --mount=type=cache,target=/root/.cache/uv \
+    uv pip install -r requirements/xpu.txt && \
+    uv pip install setuptools_scm &&\
+    uv pip install grpcio-tools protobuf nanobind && \
     source /opt/intel/oneapi/setvars.sh --force && \
     source /opt/intel/oneapi/ccl/2021.15/env/vars.sh --force && \
-    export CMAKE_PREFIX_PATH="$(python -c 'import site; print(site.getsitepackages()[0])'):${CMAKE_PREFIX_PATH}" && \
-    uv pip install --no-build-isolation arctic-inference==0.1.1
+    export CMAKE_PREFIX_PATH="$(python3 -c 'import site; print(site.getsitepackages()[0])'):${CMAKE_PREFIX_PATH}"
 
 ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib/"
 
@@ -105,6 +113,9 @@ CMD ["/bin/bash"]
 
 FROM vllm-base AS vllm-openai
 
+# install nixl from source code
+RUN uv pip install nixl
+
 # install additional dependencies for openai api server
 RUN --mount=type=cache,target=/root/.cache/uv \
     uv pip install accelerate hf_transfer pytest pytest_asyncio lm_eval[api] modelscope
@@ -112,10 +123,6 @@ RUN --mount=type=cache,target=/root/.cache/uv \
 # install development dependencies (for testing)
 RUN uv pip install -e tests/vllm_test_utils
 
-# install nixl from source code
-ENV NIXL_VERSION=0.7.0
-RUN python /workspace/vllm/tools/install_nixl_from_source_ubuntu.py
-
 # ensure vllm is properly installed
 RUN python -c "import vllm, inspect; print(vllm.__file__)"
 RUN uv pip show vllm
@@ -135,7 +142,7 @@ RUN uv pip install --no-cache-dir ".[dev]" --no-build-isolation
 # FIX triton
 RUN --mount=type=cache,target=/root/.cache/uv \
     uv pip uninstall triton triton-xpu && \
-    uv pip install triton-xpu==3.6.0
+    uv pip install triton-xpu==3.7.0
 
 # remove torch bundled oneccl to avoid conflicts
 RUN --mount=type=cache,target=/root/.cache/uv \

docs/.nav.yml

@@ -10,13 +10,12 @@ nav:
       - Image Generation: serving/image_generation_api.md
       - Image Edit: serving/image_edit_api.md
       - Text to Speech: serving/speech_api.md
+      - Audio Generation: serving/audio_generate_api.md
       - Streaming Video Input: serving/video_stream_api.md
   - Examples:
     - examples/README.md
     - Offline Inference:
       - BAGEL-7B-MoT: user_guide/examples/offline_inference/bagel.md
-      - CosyVoice3: user_guide/examples/offline_inference/cosyvoice3.md
-      - Fish Speech S2 Pro: user_guide/examples/offline_inference/fish_speech.md
       - GLM-Image Multistage End-to-End Inference: user_guide/examples/offline_inference/glm_image.md
       - Helios Video Generation: user_guide/examples/offline_inference/helios.md
       - HunyuanImage-3.0 Image-to-Text Inference: user_guide/examples/offline_inference/hunyuan_image3.md
@@ -27,24 +26,22 @@ nav:
       - MiMo-Audio Offline Inference: user_guide/examples/offline_inference/mimo_audio.md
       - Qwen2.5-Omni: user_guide/examples/offline_inference/qwen2_5_omni.md
       - Qwen3-Omni: user_guide/examples/offline_inference/qwen3_omni.md
-      - Qwen3-TTS: user_guide/examples/offline_inference/qwen3_tts.md
       - Text-To-Audio: user_guide/examples/offline_inference/text_to_audio.md
       - Text-To-Image: user_guide/examples/offline_inference/text_to_image.md
+      - Text-To-Speech: user_guide/examples/offline_inference/text_to_speech.md
       - Text-To-Video: user_guide/examples/offline_inference/text_to_video.md
-      - Voxtral TTS Offline Inference: user_guide/examples/offline_inference/voxtral_tts.md
       - X-To-Video-Audio: user_guide/examples/offline_inference/x_to_video_audio.md
     - Online Serving:
       - BAGEL-7B-MoT: user_guide/examples/online_serving/bagel.md
       - vLLM-Omni Helm Chart: user_guide/examples/online_serving/chart-helm.md
-      - Diffusers Backend Adapter Example: user_guide/examples/online_serving/diffusers_pipeline_adapter.md
-      - Fish Speech S2 Pro: user_guide/examples/online_serving/fish_speech.md
+      - Diffusers Backend Adapter: user_guide/examples/online_serving/diffusers_pipeline_adapter.md
       - GLM-Image Online Serving: user_guide/examples/online_serving/glm_image.md
       - Image-To-Image: user_guide/examples/online_serving/image_to_image.md
       - Image-To-Video: user_guide/examples/online_serving/image_to_video.md
       - Online serving Example of vLLM-Omni for MiMo-Audio: user_guide/examples/online_serving/mimo_audio.md
       - Qwen2.5-Omni: user_guide/examples/online_serving/qwen2_5_omni.md
       - Qwen3-Omni: user_guide/examples/online_serving/qwen3_omni.md
-      - Qwen3-TTS: user_guide/examples/online_serving/qwen3_tts.md
+      - Text-To-Speech: user_guide/examples/online_serving/text_to_speech.md
       - Text-To-Image: user_guide/examples/online_serving/text_to_image.md
       - Text-To-Video: user_guide/examples/online_serving/text_to_video.md
   - General:
@@ -62,11 +59,6 @@ nav:
       - Cache Acceleration:
         - TeaCache: user_guide/diffusion/cache_acceleration/teacache.md
         - Cache-DiT: user_guide/diffusion/cache_acceleration/cache_dit.md
-      - Quantization:
-        - Overview: user_guide/diffusion/quantization/overview.md
-        - FP8: user_guide/diffusion/quantization/fp8.md
-        - Int8: user_guide/diffusion/quantization/int8.md
-        - GGUF: user_guide/diffusion/quantization/gguf.md
       - Attention Backends: user_guide/diffusion/attention_backends.md
       - Frame Interpolation: user_guide/diffusion/frame_interpolation.md
       - Parallelism:
@@ -81,6 +73,16 @@ nav:
       - LoRA: user_guide/diffusion/lora.md
       - Custom Pipeline: features/custom_pipeline.md
       - Step Execution: user_guide/diffusion/step_execution.md
+    - Quantization:
+      - Overview: user_guide/quantization/overview.md
+      - Online Quantization: user_guide/quantization/online.md
+      - Quantized KV Cache: user_guide/quantization/quantized_kvcache.md
+      - FP8 W8A8: user_guide/quantization/fp8.md
+      - Int8 W8A8: user_guide/quantization/int8.md
+      - ModelOpt: user_guide/quantization/modelopt.md
+      - GGUF: user_guide/quantization/gguf.md
+      - AutoRound: user_guide/quantization/autoround.md
+      - msModelSlim: user_guide/quantization/msmodelslim.md
     - ComfyUI: features/comfyui.md
 - Developer Guide:
   - General:
@@ -92,7 +94,6 @@ nav:
     - contributing/model/adding_omni_model.md
     - contributing/model/adding_tts_model.md
     - contributing/model/adding_diffusion_model.md
-    - contributing/model/adding_quantization_model.md
   - CI: contributing/ci
   - Design Documents:
     - design/index.md
@@ -113,6 +114,7 @@ nav:
       - design/feature/async_chunk.md
       - design/feature/vae_parallel.md
       - design/feature/diffusion_step_execution.md
+      - design/feature/diffusion_continuous_batching.md
     - Module Design:
       - design/module/ar_module.md
       - design/module/dit_module.md

docs/api/README.md

@@ -99,7 +99,6 @@ Configuration classes.
 - [vllm_omni.diffusion.cache.teacache.config.TeaCacheConfig][]
 - [vllm_omni.distributed.omni_connectors.utils.config.ConnectorSpec][]
 - [vllm_omni.distributed.omni_connectors.utils.config.OmniTransferConfig][]
-- [vllm_omni.model_executor.models.cosyvoice3.config.CosyVoice3Config][]
 - [vllm_omni.model_executor.models.fish_speech.configuration_fish_speech.FishSpeechConfig][]
 - [vllm_omni.model_executor.models.fish_speech.configuration_fish_speech.FishSpeechFastARConfig][]
 - [vllm_omni.model_executor.models.fish_speech.configuration_fish_speech.FishSpeechSlowARConfig][]
@@ -116,10 +115,14 @@ Configuration classes.
 - [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1DecoderConfig][]
 - [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1DecoderDiTConfig][]
 - [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1EncoderConfig][]
+- [vllm_omni.transformers_utils.configs.cosyvoice3.CosyVoice3Config][]
 - [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Config][]
 - [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Qwen2_5_VLConfig][]
 - [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Qwen2_5_VLTextConfig][]
 - [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Qwen2_5_VLVisionConfig][]
+- [vllm_omni.transformers_utils.configs.omnivoice.OmniVoiceConfig][]
+- [vllm_omni.transformers_utils.configs.voxcpm.VoxCPMConfig][]
+- [vllm_omni.transformers_utils.configs.voxcpm2.VoxCPM2Config][]
 
 ## Workers
 

docs/contributing/README.md

@@ -116,6 +116,20 @@ Only specific types of PRs will be reviewed. The PR title is prefixed appropriat
 !!! note
     If the PR spans more than one category, please include all relevant prefixes.
 
+### Pre-Check Before Submitting
+
+Before submitting a PR, run the precheck-pr skill with code agent for a self-review against project conventions:
+
+```
+/precheck-pr
+```
+
+The skill offers two modes:
+- **Quick (~3 min):** catches showstoppers — PR title format, missing benchmark claims, rebase status
+- **Full (~10 min):** thorough maintainer-grade review — dead code scan, copy-paste detection, import hygiene
+
+The precheck covers five PR types: Bug Fix, Performance, New Model, Diffusion Model, and General. Each type has a tailored checklist that validates evidence quality (repro steps, A/B benchmarks, registry entries, etc.). See the skill in `.claude/skills/precheck-pr/` for the full checklist.
+
 ### Local Test
 Please run the L1 and L2 test cases locally first and attach the results before contacting us to add the "ready" label. Please refer to the [test instructions](./ci/test_guide.md) for running the test cases.
 

docs/contributing/ci/CI_5levels.md

@@ -152,6 +152,8 @@ vllm_omni/                                    tests/
 │   │                                            │   │   └── test_lora_manager.py        ✅
 │   ├── models/                                 │   ├── models/
 │   │   ├── qwen_image/                         │   │   ├── qwen_image/                 (e2e coverage)
+│   │   ├── ovis_image/                         │   │   ├── ovis_image/
+│   │   │   └── ...                             │   │   │   └── test_ovis_image.py     ✅
 │   │   ├── z_image/                            │   │   └── z_image/
 │   │   └── ...                                 │   │       └── test_zimage_tp_constraints.py  ✅
 │   └── worker/                                 │   └── worker/
@@ -232,15 +234,13 @@ vllm_omni/                                    tests/
                                                │   ├── test_qwen3_omni.py
                                                │   ├── test_qwen3_omni_expansion.py
                                                │   ├── test_mimo_audio.py
-                                               │   ├── test_image_gen_edit.py
                                                │   └── test_images_generations_lora.py
                                                └── offline_inference/                  ✅
                                                    ├── test_qwen2_5_omni.py
                                                    ├── test_qwen3_omni.py
                                                    ├── test_bagel_text2img.py
-                                                   ├── test_t2i_model.py
-                                                   ├── test_t2v_model.py
-                                                   ├── test_ovis_image.py
+                                                   ├── test_z_image.py
+                                                   ├── test_wan22.py
                                                    ├── test_zimage_tensor_parallel.py
                                                    ├── test_cache_dit.py
                                                    ├── test_teacache.py

docs/contributing/ci/test_examples/l4_functionality_tests.inc.md

@@ -1,6 +1,6 @@
 **Scope**
 
-For diffusion models, the L4 functionality test covers all or common *diffusion features* that are supported by this model, including several [parallelism acceleration methods](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/parallelism_acceleration/), [CPU offloading](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/cpu_offload_diffusion/), [TeaCache](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/teacache/) and [Cache-DiT](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/cache_dit_acceleration/) cache backends, [quantization methods](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/quantization/overview/).
+For diffusion models, the L4 functionality test covers all or common *diffusion features* that are supported by this model, including several [parallelism acceleration methods](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/parallelism_acceleration/), [CPU offloading](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/cpu_offload_diffusion/), [TeaCache](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/teacache/) and [Cache-DiT](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/cache_dit_acceleration/) cache backends, [quantization methods](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/quantization/overview/).
 
 **Test Case Design**
 

docs/contributing/ci/test_guide.md

@@ -80,7 +80,7 @@ Our test scripts use the pytest framework. First, please use `git clone https://
 
     ```bash
     cd tests
-    pytest -s -v -m "full_model and omni and H100" --run-level=full_model
+    pytest -s -v -m "full_model and (omni or tts) and H100" --run-level=full_model
     ```
 
 === "L5 level"
@@ -91,10 +91,16 @@ Our test scripts use the pytest framework. First, please use `git clone https://
     cd tests
 
     # Stability: Qwen3-Omni
-    pytest -s -v dfx/stability/scripts/test_stability_qwen3_omni.py
+    pytest -s -v dfx/stability/scripts/test_stability_qwen3_omni.py -m slow
 
     # Stability: Wan2.2 (v1/videos diffusion benchmark loop)
-    pytest -s -v dfx/stability/scripts/test_stability_wan22.py
+    pytest -s -v dfx/stability/scripts/test_stability_wan22.py -m slow
+
+    # Reliability: Qwen3-Omni
+    pytest -s -v dfx/reliability/test_reliability_qwen3_omni.py -m slow
+
+    # Reliability: Wan2.2
+    pytest -s -v dfx/reliability/test_reliability_wan22.py -m slow
 
     ```
 

docs/contributing/ci/tests_markers.md

@@ -11,7 +11,8 @@ Defined in `pyproject.toml`:
 | `advanced_model`   | L3 tests (run on each merge to main)                 |
 | `full_model`       | L4 tests (run nightly) |
 | `diffusion`        | Diffusion model tests                                     |
-| `omni`             | Omni model tests                                          |
+| `omni`             | Omni multimodal model tests |
+| `tts`              | TTS model tests     |
 | `cache`            | Cache backend tests                                       |
 | `parallel`         | Parallelism/distributed tests                             |
 | `cpu`              | Tests that run on CPU                                     |

docs/contributing/ci/tests_style.md

@@ -54,6 +54,8 @@ vllm_omni/                                    tests/
 │   │                                            │   │   └── test_lora_manager.py        ✅
 │   ├── models/                                 │   ├── models/
 │   │   ├── qwen_image/                         │   │   ├── qwen_image/                 (e2e coverage)
+│   │   ├── ovis_image/                         │   │   ├── ovis_image/
+│   │   │   └── ...                             │   │   │   └── test_ovis_image.py     ✅
 │   │   ├── z_image/                            │   │   └── z_image/
 │   │   └── ...                                 │   │       └── test_zimage_tp_constraints.py  ✅
 │   └── worker/                                 │   └── worker/
@@ -134,15 +136,13 @@ vllm_omni/                                    tests/
                                                │   ├── test_qwen3_omni.py
                                                │   ├── test_qwen3_omni_expansion.py
                                                │   ├── test_mimo_audio.py
-                                               │   ├── test_image_gen_edit.py
                                                │   └── test_images_generations_lora.py
                                                └── offline_inference/                  ✅
                                                    ├── test_qwen2_5_omni.py
                                                    ├── test_qwen3_omni.py
                                                    ├── test_bagel_text2img.py
-                                                   ├── test_t2i_model.py
-                                                   ├── test_t2v_model.py
-                                                   ├── test_ovis_image.py
+                                                   ├── test_z_image.py
+                                                   ├── test_wan22.py
                                                    ├── test_zimage_tensor_parallel.py
                                                    ├── test_cache_dit.py
                                                    ├── test_teacache.py
@@ -213,7 +213,6 @@ E2E Online tests for Qwen3-Omni model with mix input and audio+text output.
 import os
 
 os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
-os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
 
 import threading
 from pathlib import Path

docs/contributing/model/adding_diffusion_model.md

@@ -115,17 +115,20 @@ class YourAttentionBlock(nn.Module):
 from vllm_omni.diffusion.attention.layer import Attention
 from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
 
-class YourAttentionBlock(nn.Module):
+class YourSelfAttentionBlock(nn.Module):
     def __init__(self, ...):
         super().__init__()
 
-        # Initialize vLLM-Omni's Attention layer
+        # Initialize vLLM-Omni's Attention layer.
+        # `role` lets users target this site with --diffusion-attention-config
+        # (e.g. --diffusion-attention-config.per_role.self.backend SAGE_ATTN).
         self.attn = Attention(
             num_heads=self.num_heads,
             head_size=self.head_dim,
             softmax_scale=1.0 / (self.head_dim ** 0.5),
             causal=False,  # Diffusion models typically use bidirectional attention
             num_kv_heads=self.num_kv_heads,
+            role="self",
         )
 
     def forward(self, hidden_states, encoder_hidden_states=None, attention_mask=None, ...):
@@ -133,7 +136,6 @@ class YourAttentionBlock(nn.Module):
         # Create attention metadata
         attn_metadata = AttentionMetadata(attn_mask=attention_mask)
         hidden_states = self.attn(query, key, value, attn_metadata=attn_metadata)
-
 ```
 
 **Key Points:**
@@ -141,8 +143,35 @@ class YourAttentionBlock(nn.Module):
 - **Attention layer initialization:** Done in `__init__`, not per-forward
 - **Tensor shapes:** vLLM-Omni `Attention` expects QKV to have `[B, seq, num_heads, head_dim]` shape
 - **AttentionMetadata:** Wraps attention mask and other metadata
+- **Role:** Tag every `Attention` site with a `role` string so users can configure backends per role (see below)
 
-**Attention backends:** vLLM-Omni automatically selects the attention backend given the environmental variable `DIFFUSION_ATTENTION_BACKEND`. The default attention backend is `FLASH_ATTN` for diffusion models.
+**Declaring attention roles**
+
+The `role` argument is a free-form string that identifies this attention site. Users can match it from `--diffusion-attention-config.per_role.<role>.*` to swap backends without touching model code. Two conventions cover the common cases:
+
+| Convention | When to use | Example |
+|---|---|---|
+| `"self"` | Q/K/V come from the same hidden state | DiT self-attention block |
+| `"cross"` | K/V come from a separate `encoder_hidden_states` | Text-conditioned cross-attention |
+
+For multi-modal or unusual sites, use a dot-namespaced role and pair it with `role_category` so it can fall back to the generic config when nothing model-specific is set:
+
+```python
+# A model-specific cross-attention site that user config can target
+# either as 'mymodel.audio_to_video' (exact) or as 'cross' (category fallback).
+self.audio_to_video_attn = Attention(
+    num_heads=self.num_heads,
+    head_size=self.head_dim,
+    softmax_scale=1.0 / (self.head_dim ** 0.5),
+    causal=False,
+    role="mymodel.audio_to_video",
+    role_category="cross",
+)
+```
+
+For cross-attention sites whose K/V are replicated across ranks (e.g. text encoder output), pass `skip_sequence_parallel=True` to opt this layer out of sequence-parallel sharding.
+
+**Attention backends:** When the user does not configure a backend, vLLM-Omni asks the current platform for its default (typically `FLASH_ATTN` on CUDA when available). Users override the default via `--diffusion-attention-backend`, the `DIFFUSION_ATTENTION_BACKEND` env var, or finer-grained `--diffusion-attention-config.per_role.*` flags. See [Diffusion Attention Backends](../../user_guide/diffusion/attention_backends.md) for the full configuration surface.
 
 #### 1.3: Replace Imports and Utilities
 
@@ -737,6 +766,16 @@ step-scoped phases. The reference implementation is
 Do not enable `step_execution=True` until those four methods are implemented
 and validated against the request-level path.
 
+If you want the pipeline to work with the experimental batched step-wise path
+(`max_num_seqs > 1`), also see:
+[Continuous Batching for Step-Wise Diffusion](../../design/feature/diffusion_continuous_batching.md).
+
+If you expose this in example scripts or recipes, keep it opt-in. Surface
+runtime features like `step_execution` as optional flags instead of silently
+turning them on. For Qwen-Image-style serving examples, document
+`--step-execution` as the feature gate and `--max-num-seqs N` as the
+companion batching knob.
+
 ### Cache Acceleration
 
 #### TeaCache

docs/contributing/model/adding_quantization_model.md

@@ -1,478 +0,0 @@
-# Support FP8 Quantization
-
-This section describes how to add FP8 quantization support to a diffusion model. We use the Qwen-Image pipeline as the reference implementation.
-
----
-
-## Table of Contents
-
-- [Overview](#overview)
-- [Step-by-Step Implementation](#step-by-step-implementation)
-- [Testing](#testing)
-- [Troubleshooting](#troubleshooting)
-- [Reference Implementations](#reference-implementations)
-- [Summary](#summary)
-
----
-
-## Overview
-
-### What is FP8 Quantization?
-
-FP8 quantization reduces model memory footprint by storing weights in 8-bit floating point (`float8_e4m3fn`) instead of 16-bit (`bfloat16`/`float16`). This saves ~50% of weight memory.
-
-A diffusion pipeline typically has three quantizable components:
-
-| Component | Layer Types | Quantization Approach |
-|-----------|-----------|----------------------|
-| **DiT (transformer)** | `nn.Linear` | Replace with vLLM quantized linear layers |
-| **Text encoder** | `nn.Linear` | FP8 weight storage via hooks |
-| **VAE** | `nn.Conv2d`, `nn.Conv3d` | FP8 weight storage via hooks |
-
-### Architecture
-
-The quantization system has two mechanisms:
-
-**1. vLLM Quantized Linear Layers (for DiT)**
-
-vLLM provides drop-in replacements for `nn.Linear` that perform FP8 W8A8 computation on supported hardware. These are configured via `quant_config` passed to layer constructors.
-
-Key classes:
-
-| Class | Location | Purpose |
-|-------|----------|---------|
-| `build_quant_config` | `vllm_omni/quantization/__init__.py` | Build quantization config from string/dict/per-component spec |
-| `ComponentQuantizationConfig` | `vllm_omni/quantization/component_config.py` | Per-component routing by layer prefix |
-
-**2. FP8 Weight Storage via Hooks (for text encoder and VAE)**
-
-For models loaded via `from_pretrained()` (outside vLLM's weight pipeline), we use forward hooks to store weights in FP8 and dequantize on-the-fly:
-
-```
-At rest:     weight.data = fp8_tensor (half memory)
-Pre-hook:    weight.data = fp8_weight.to(bf16) * scale
-Forward:     Normal computation in BF16
-Post-hook:   weight.data = fp8_weight (back to FP8)
-```
-
-Key function:
-
-| Function | Location | Purpose |
-|----------|----------|---------|
-| `apply_fp8_weight_storage` | `vllm_omni/diffusion/models/utils.py` | Apply FP8 hooks to Linear/Conv layers (introduced in [#1412](https://github.com/vllm-project/vllm-omni/pull/1412)) |
-
----
-
-## Step-by-Step Implementation
-
-### Step 1: Add `quant_config` to the Transformer
-
-Pass `quant_config` to vLLM parallel linear layers in the transformer. This enables FP8 W8A8 for DiT linear layers.
-
-#### 1.1: Accept `quant_config` in Constructor
-
-```python
-from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
-
-class YourTransformer2DModel(nn.Module):
-    def __init__(
-        self,
-        *,
-        od_config: OmniDiffusionConfig,
-        quant_config: QuantizationConfig | None = None,
-        # ... other params
-    ):
-        self.quant_config = quant_config
-        # Pass to sub-modules
-        self.blocks = nn.ModuleList([
-            YourTransformerBlock(quant_config=quant_config, ...)
-            for _ in range(num_layers)
-        ])
-```
-
-#### 1.2: Pass `quant_config` to vLLM Linear Layers
-
-Replace `nn.Linear` with vLLM parallel linear layers that accept `quant_config`:
-
-```python
-from vllm.model_executor.layers.linear import (
-    ColumnParallelLinear,
-    QKVParallelLinear,
-    RowParallelLinear,
-)
-
-class YourAttentionBlock(nn.Module):
-    def __init__(self, hidden_size, num_heads, quant_config=None, prefix=""):
-        # QKV projection
-        self.to_qkv = QKVParallelLinear(
-            hidden_size=hidden_size,
-            head_size=hidden_size // num_heads,
-            total_num_heads=num_heads,
-            quant_config=quant_config,
-            prefix=f"{prefix}.to_qkv",
-        )
-        # Output projection
-        self.to_out = RowParallelLinear(
-            input_size=hidden_size,
-            output_size=hidden_size,
-            quant_config=quant_config,
-            prefix=f"{prefix}.to_out",
-        )
-
-class YourFeedForward(nn.Module):
-    def __init__(self, hidden_size, intermediate_size, quant_config=None, prefix=""):
-        self.gate_up_proj = ColumnParallelLinear(
-            input_size=hidden_size,
-            output_size=intermediate_size * 2,
-            quant_config=quant_config,
-            prefix=f"{prefix}.gate_up_proj",
-        )
-        self.down_proj = RowParallelLinear(
-            input_size=intermediate_size,
-            output_size=hidden_size,
-            quant_config=quant_config,
-            prefix=f"{prefix}.down_proj",
-        )
-```
-
-!!! important
-    Always pass the `prefix` parameter. It is used by `ignored_layers` matching to skip sensitive layers.
-
-#### 1.3: Identify Sensitive Layers
-
-Some layers degrade output quality when quantized. Common sensitive layers:
-
-| Layer | Why Sensitive | Typical Models | Quant Methods Affected |
-|-------|-------------|---------------|----------------------|
-| `img_mlp` | Processes denoising latents with shifting dynamic range | Qwen-Image | FP8, Int8 |
-| `feed_forward` | FFN layers in DiT blocks, large dynamic range | Z-Image | Int8 |
-| `proj_out` | Final output projection, small quantization errors amplified | Various | FP8, Int8 |
-| `lm_head` | Output vocabulary projection, precision-critical | Omni (thinker) | NVFP4 |
-| `mlp.gate` | MoE router gates, routing decisions are precision-critical | MoE models | NVFP4 |
-
-**Quantization quality benchmarks** (LPIPS: lower = better, <0.01 imperceptible, >0.1 noticeable):
-
-| Model | Method | All Layers | With `ignored_layers` | Recommendation |
-|-------|--------|-----------|----------------------|----------------|
-| Qwen-Image-2512 | Int8 | LPIPS 0.0197 | 0.0027 (skip `img_mlp`) | Skip `img_mlp` |
-| Z-Image-Turbo | Int8 | LPIPS 0.1597 | 0.0290 (skip `feed_forward`) | Skip `feed_forward` |
-| Z-Image-Turbo | FP8 | LPIPS ~0.005 | — | All layers OK |
-
-To identify sensitive layers for a new model:
-
-1. Run inference with all layers quantized
-2. Compare output quality with BF16 baseline using LPIPS (see `benchmarks/diffusion/quantization_quality.py`)
-3. Selectively disable quantization per layer (via `ignored_layers`) until quality matches
-4. Document the recommended `ignored_layers` in the model's supported models table
-
-### Step 2: Wire Up Quantization in the Pipeline
-
-#### 2.1: Create `quant_config` and Pass to Transformer
-
-In the pipeline's `__init__`, extract the vLLM quant config and pass it to the transformer:
-
-```python
-from vllm_omni.quantization import build_quant_config
-
-class YourPipeline(nn.Module):
-    def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = ""):
-        super().__init__()
-        # ... load scheduler, text_encoder, vae ...
-
-        # Build quant config from OmniDiffusionConfig
-        quant_config = build_quant_config(od_config.quantization_config)
-
-        # Pass to transformer
-        self.transformer = YourTransformer2DModel(
-            od_config=od_config,
-            quant_config=quant_config,
-            # ... other kwargs
-        )
-```
-
-#### 2.2: Apply FP8 Weight Storage to Text Encoder and VAE
-
-For models loaded via `from_pretrained()`, apply hook-based FP8 weight storage.
-
-!!! note
-    The `apply_fp8_weight_storage` function is provided by `vllm_omni/diffusion/models/utils.py`, introduced in [PR #1412](https://github.com/vllm-project/vllm-omni/pull/1412).
-
-```python
-from vllm_omni.diffusion.models.utils import apply_fp8_weight_storage
-
-class YourPipeline(nn.Module):
-    def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = ""):
-        super().__init__()
-
-        self.text_encoder = SomeTextEncoder.from_pretrained(...)
-        self.vae = SomeVAE.from_pretrained(...).to(self.device)
-
-        # Apply FP8 weight storage to text encoder and VAE
-        if (
-            od_config.quantization_config is not None
-            and getattr(od_config.quantization_config, "quant_method", None) == "fp8"
-        ):
-            apply_fp8_weight_storage(self.vae)
-            apply_fp8_weight_storage(self.text_encoder)
-
-        # ... rest of init
-```
-
-`apply_fp8_weight_storage` walks all `nn.Linear`, `nn.Conv2d`, and `nn.Conv3d` modules in the model and applies FP8 hooks to each.
-
-#### 2.3: Fix `load_weights` to Include Pre-loaded Components
-
-Since text encoders and VAEs are loaded via `from_pretrained()` (not through vLLM's weight pipeline), their parameters won't appear in the `loaded_weights` set. Mark them explicitly:
-
-```python
-def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
-    loader = AutoWeightsLoader(self)
-    loaded_weights = loader.load_weights(weights)
-    # Mark pre-loaded component weights as loaded
-    loaded_weights |= {f"vae.{name}" for name, _ in self.vae.named_parameters()}
-    loaded_weights |= {f"text_encoder.{name}" for name, _ in self.text_encoder.named_parameters()}
-    return loaded_weights
-```
-
-### Step 3: Add to Supported Models Documentation
-
-Update the FP8 supported models table in `docs/user_guide/diffusion/quantization/fp8.md`:
-
-```markdown
-| Your-Model | `org/Your-Model` | ✅ | ✅ | ✅ | `sensitive_layer` |
-```
-
----
-
-## Testing
-
-### Quick Verification
-
-```python
-from vllm_omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-
-# Test with FP8 enabled
-omni = Omni(model="your-model", quantization="fp8")
-
-outputs = omni.generate(
-    "A cat sitting on a windowsill",
-    OmniDiffusionSamplingParams(num_inference_steps=50),
-)
-```
-
-### Quality Gate Test (LPIPS)
-
-We provide a pytest-based quality gate at `tests/diffusion/quantization/test_quantization_quality.py`.
-It generates outputs with both BF16 and your quantized method using the same seed, computes
-[LPIPS](https://github.com/richzhang/PerceptualSimilarity) perceptual distance, and **fails if it
-exceeds a threshold**. This is the recommended way to validate that a quantization method does not
-unacceptably degrade output quality.
-
-**Step 1: Add your test config** — edit `QUALITY_CONFIGS` in the test file:
-
-```python
-QualityTestConfig(
-    id="int8_z_image",              # pytest ID
-    model="Tongyi-MAI/Z-Image-Turbo",
-    quantization="int8",            # your quantization method
-    task="t2i",                     # "t2i" or "t2v"
-    prompt="a cup of coffee on a wooden table, morning light",
-    max_lpips=0.15,                 # fail threshold (see guide below)
-    num_inference_steps=20,
-),
-```
-
-**Step 2: Run the test:**
-
-```bash
-# Install dependency
-pip install lpips
-
-# Run all quality tests
-pytest tests/diffusion/quantization/test_quantization_quality.py -v -m ""
-
-# Run only your method
-pytest tests/diffusion/quantization/test_quantization_quality.py -v -m "" -k "int8"
-
-# Run a specific model
-pytest tests/diffusion/quantization/test_quantization_quality.py -v -m "" -k "z_image"
-```
-
-**Step 3: Read the output:**
-
-```
-Quantization Quality: int8_z_image
-  Model:         Tongyi-MAI/Z-Image-Turbo
-  Method:        int8
-  LPIPS:         0.0290  (threshold: 0.15)
-  BF16 memory:   19.15 GiB
-  Quant memory:  14.80 GiB  (23% reduction)
-  Result:        PASS
-```
-
-**LPIPS threshold guide:**
-
-| LPIPS Range | Meaning | Recommendation |
-|-------------|---------|----------------|
-| < 0.01 | Imperceptible | Ideal — no visible difference |
-| 0.01 – 0.05 | Minor differences | Good — acceptable for most use cases |
-| 0.05 – 0.15 | Noticeable on close inspection | Acceptable with `ignored_layers` tuning |
-| 0.15 – 0.30 | Clearly visible | Investigate — likely needs `ignored_layers` |
-| > 0.30 | Significant degradation | Quantization method may not suit this model |
-
-!!! tip
-    If your test fails, don't raise the threshold — instead, identify sensitive layers and add
-    them to `ignored_layers` (see next section). Then re-run the test with the lower LPIPS.
-
-### Quality Benchmark (Detailed)
-
-For a full multi-prompt benchmark with Markdown output (suitable for PR descriptions), use the
-standalone benchmark script:
-
-```bash
-python benchmarks/diffusion/quantization_quality.py \
-    --model Tongyi-MAI/Z-Image-Turbo \
-    --quantization fp8 \
-    --prompts \
-        "a cup of coffee on the table" \
-        "an aerial view of a coral reef" \
-    --height 1024 --width 1024 \
-    --num-inference-steps 50 --seed 42
-```
-
-This generates BF16 and quantized outputs, computes per-prompt LPIPS, and prints a Markdown
-summary table ready to paste into your PR.
-
-### Identifying Sensitive Layers with `ignored_layers`
-
-Not all layers tolerate quantization equally. **`ignored_layers`** lets you keep specific layers
-in BF16 while quantizing everything else. This is critical for achieving good quality with
-aggressive quantization methods (Int8, NVFP4).
-
-**Why some layers are sensitive:**
-
-| Layer Type | Why Sensitive | Typical Impact |
-|------------|--------------|----------------|
-| `img_mlp` | Processes denoising latents with shifting dynamic range across timesteps. No built-in normalization to absorb quantization error. | Color shifts, blurriness |
-| `feed_forward` | FFN layers with large dynamic range in DiT blocks. | Artifacts, loss of detail |
-| `proj_out` | Final output projection — small errors get amplified since there are no subsequent layers to correct them. | Overall quality loss |
-| `lm_head` | Vocabulary projection in LLMs — precision-critical for token selection. | Garbage text output |
-| `mlp.gate` | MoE router gates — routing decisions are binary and precision-critical. | Wrong expert selection, quality collapse |
-
-**How to find sensitive layers for your model:**
-
-1. Run the quality gate test with all layers quantized:
-    ```bash
-    pytest tests/diffusion/quantization/test_quantization_quality.py -v -m "" -k "your_model"
-    ```
-
-2. If LPIPS exceeds your threshold, try skipping common sensitive layers one at a time:
-    ```python
-    QualityTestConfig(
-        id="fp8_your_model_skip_mlp",
-        model="your-org/Your-Model",
-        quantization="fp8",
-        task="t2i",
-        prompt="a cup of coffee on a wooden table",
-        max_lpips=0.05,
-    ),
-    ```
-
-    And in the test, modify the Omni call:
-    ```python
-    omni_qt = Omni(
-        model=config.model,
-        quantization_config={
-            "method": config.quantization,
-            "ignored_layers": ["img_mlp"],
-        },
-    )
-    ```
-
-3. Compare LPIPS with and without `ignored_layers` to isolate the problematic layers.
-
-4. Document the recommended `ignored_layers` in the supported models table.
-
-**Real-world examples:**
-
-| Model | Method | All Layers (LPIPS) | With `ignored_layers` | Which Layers to Skip |
-|-------|--------|-------------------|----------------------|---------------------|
-| Qwen-Image-2512 | Int8 | 0.0197 | 0.0027 | `img_mlp` |
-| Z-Image-Turbo | Int8 | 0.1597 | 0.0290 | `feed_forward` |
-| Z-Image-Turbo | FP8 | ~0.005 | — | None needed |
-
-### What to Verify (Checklist)
-
-1. **Model loads without errors** — Check logs for quantization initialization messages
-2. **Quality gate passes** — LPIPS within threshold vs BF16 baseline
-3. **Memory reduction** — Expect ~30-50% savings on quantized components
-4. **Sensitive layers identified** — Document which layers (if any) need `ignored_layers`
-5. **`ignored_layers` pattern works** — Verify that the layer name prefix matches your transformer's module names
-
----
-
-## Troubleshooting
-
-### Issue: Output Quality Degradation
-
-**Symptoms:** Generated images have visible artifacts, color shifts, or blurriness compared to BF16 baseline.
-
-**Causes & Solutions:**
-
-- **Sensitive layers being quantized:**
-
-    **Solution:** Identify and skip sensitive layers:
-    ```python
-    omni = Omni(
-        model="your-model",
-        quantization_config={
-            "method": "fp8",
-            "ignored_layers": ["img_mlp"],
-        },
-    )
-    ```
-
-- **All layers sensitive (rare):**
-
-    **Solution:** The model may not be suitable for FP8 quantization. Consider weight-only quantization or other methods.
-
-### Issue: Missing Weights Warning
-
-**Symptoms:** Warnings about unloaded weights for `vae.*` or `text_encoder.*` parameters.
-
-**Solution:** Ensure `load_weights` marks pre-loaded component weights. See [Step 2.3](#23-fix-load_weights-to-include-pre-loaded-components).
-
-### Issue: CUDA Out of Memory
-
-**Symptoms:** OOM errors during model loading despite FP8 being enabled.
-
-**Causes & Solutions:**
-
-- **Text encoder/VAE loaded in BF16 before FP8 conversion:**
-
-    FP8 weight storage is applied after `from_pretrained()`, so peak load memory is still BF16. This is a one-time cost during initialization.
-
----
-
-## Reference Implementations
-
-| Model | Pipeline File | Transformer File | Notes |
-|-------|-------------|-----------------|-------|
-| **Z-Image** | `models/z_image/pipeline_z_image.py` | `models/z_image/z_image_transformer.py` | DiT + text encoder FP8 |
-| **Qwen-Image** | `models/qwen_image/pipeline_qwen_image.py` | `models/qwen_image/qwen_image_transformer.py` | DiT + text encoder + VAE FP8 |
-
-All files are under `vllm_omni/diffusion/`.
-
----
-
-## Summary
-
-Adding FP8 quantization support to a new model:
-
-1. ✅ **Add `quant_config` to transformer** — Accept and pass to vLLM parallel linear layers with proper `prefix`
-2. ✅ **Identify sensitive layers** — Profile which layers need `ignored_layers` for quality
-3. ✅ **Apply FP8 weight storage** — Call `apply_fp8_weight_storage()` on text encoder and VAE in pipeline `__init__`
-4. ✅ **Fix `load_weights`** — Mark pre-loaded component weights as loaded
-5. ✅ **Update docs** — Add model to the supported models table in `docs/user_guide/diffusion/quantization/fp8.md`
-6. ✅ **Test** — Verify quality, memory savings, and sensitive layer behavior

docs/contributing/model/adding_tts_model.md

@@ -186,6 +186,17 @@ vllm_omni/model_executor/stage_configs/
   your_model_name_async_chunk.yaml   # Streaming mode config
 ```
 
+### Example placement
+
+TTS examples live in the consolidated text-to-speech hub, **not** in their
+own top-level directory. Place per-model scripts under
+`examples/offline_inference/text_to_speech/<your_model>/` and
+`examples/online_serving/text_to_speech/<your_model>/`, and add a section
+to the hub `README.md` files (table row + per-model section) instead of a
+new per-model `README.md`. The mkdocs `generate_examples` hook treats the
+`text_to_speech/` parent as a single example, so per-model READMEs inside
+it would not be picked up — the hub README is the documented surface.
+
 **Qwen3-TTS reference files:**
 
 | File | Purpose |

docs/contributing/profiling.md

@@ -37,6 +37,8 @@ Supported fields:
 | `warmup_iterations` | Torch-profiler warmup iterations. |
 | `active_iterations` | Torch-profiler active iterations. |
 
+For detailed explanations of the fields, please refer to upstream vLLM implementation [vllm/config/profiler.py](https://github.com/vllm-project/vllm/blob/v0.20.1/vllm/config/profiler.py)
+
 ### Minimal configurations by output
 
 Only collect trace output:
@@ -124,7 +126,7 @@ profiler_config = {
     "max_iterations": 0,
     "wait_iterations": 0,
     "warmup_iterations": 0,
-    "active_iterations": 0,
+    "active_iterations": 1,
 }
 ```
 
@@ -220,7 +222,7 @@ vllm serve Wan-AI/Wan2.2-I2V-A14B-Diffusers \
     "max_iterations": 0,
     "wait_iterations": 0,
     "warmup_iterations": 0,
-    "active_iterations": 0
+    "active_iterations": 1
   }'
 ```
 

docs/design/feature/diffusion_continuous_batching.md

@@ -0,0 +1,156 @@
+# Continuous Batching for Step-Wise Diffusion
+
+!!! warning "Experimental Feature"
+    This feature is experimental. It currently applies only to native
+    diffusion pipelines running with `step_execution=True`.
+
+This document describes the batching extension built on top of
+[Diffusion Step Execution](diffusion_step_execution.md). The base
+step-execution contract is unchanged. The batching work is mainly in the
+scheduler and runner layers.
+
+## Why It Helps
+
+Step-wise execution breaks a long denoise loop into scheduler-visible units.
+That gives the runtime a place to admit other compatible requests between
+steps instead of waiting for an entire request to finish.
+
+This matters most in low-MFU or bursty serving scenarios:
+
+- one request's denoise step may not fully saturate the GPU
+- several compatible requests can share the same denoise forward pass
+- throughput and device utilization can improve without changing request-local
+  scheduler state
+
+This is **not** a guaranteed single-request latency win. The main benefit is
+usually higher utilization and better throughput when the workload contains
+multiple in-flight compatible requests.
+
+## Overview
+
+With continuous batching enabled:
+
+- the scheduler may keep multiple compatible requests active at the same time
+- the runner packs request-local step state into one `InputBatch`
+- `denoise_step()` runs on that batch
+- `step_scheduler()` and `post_decode()` still run per request
+
+The current implementation is conservative:
+
+- only compatible requests are batched together
+- request-mode diffusion still runs with `max_num_seqs=1`
+- per-request progress and completion remain independent
+
+## Enablement
+
+Use `--step-execution` as the feature gate, then increase `--max-num-seqs`
+above `1` if you want batching:
+
+```bash
+vllm serve Qwen/Qwen-Image --omni \
+  --port 8091 \
+  --step-execution \
+  --max-num-seqs 8
+```
+
+`--max-num-seqs 1` keeps the step-wise path without enabling batching.
+
+For a reproducible replay flow using the bundled serving benchmark, see the
+Qwen-Image replay commands in
+[`benchmarks/diffusion/README.md`](gh-file:benchmarks/diffusion/README.md)
+and
+[`benchmarks/diffusion/performance_dashboard/qwen_image_serving_performance.md`](gh-file:benchmarks/diffusion/performance_dashboard/qwen_image_serving_performance.md).
+
+## Scheduler
+
+The scheduler derives its batch capacity from `max_num_seqs` through
+`max_num_running_reqs`.
+
+Batch admission is gated by
+[`SamplingParamsKey`](gh-file:vllm_omni/diffusion/sched/interface.py),
+which is built from shape-sensitive and CFG-sensitive sampling fields. This is
+the core correctness rule for batching: requests are only co-batched when they
+share the same denoise tensor contract.
+
+There are two important details:
+
+- `num_inference_steps` is not part of the key, so requests with different
+  total step counts can still share a batch
+- requests also do not need to be at the same current denoise progress; active
+  requests can continue batching even when their current step indices diverge
+- admission is still FIFO, so an incompatible request at the head of the
+  waiting queue blocks later compatible requests
+
+Today that compatibility rule is still shape-sensitive. `height`, `width`,
+`num_frames`, and CFG-related fields remain part of the key, so different
+resolutions or incompatible guidance settings do **not** co-batch yet.
+
+The current batching unit is one `OmniDiffusionRequest`. Requests with
+multiple prompts do not participate in batching today.
+
+## Runner
+
+The runner keeps persistent per-request execution state in
+[`DiffusionRequestState`](gh-file:vllm_omni/diffusion/worker/utils.py),
+while the scheduler owns a separate lightweight request state for queueing and
+lifecycle tracking.
+
+For each step, the runner builds an
+[`InputBatch`](gh-file:vllm_omni/diffusion/worker/input_batch.py) from the
+active request states:
+
+- prompt embeddings and masks are normalized and padded
+- dynamic tensors such as `latents` and `timesteps` are gathered each step
+- buffers are reused when batch composition stays the same
+
+The step-wise batched path is:
+
+1. Run `prepare_encode()` for newly admitted requests.
+2. Build or refresh `InputBatch`.
+3. Run one batched `denoise_step(input_batch)`.
+4. Slice the batched `noise_pred` back per request.
+5. Run per-request `step_scheduler()`.
+6. Run `post_decode()` only for requests that finished denoising.
+7. Scatter updated latents back into persistent request state with
+   [`scatter_latents()`](gh-file:vllm_omni/diffusion/worker/input_batch.py).
+
+This keeps the shared work limited to the denoise forward pass while preserving
+request-local scheduler state and outputs.
+
+## Engine
+
+[`DiffusionEngine`](gh-file:vllm_omni/diffusion/diffusion_engine.py) provides
+the background loop and async add-request path needed for multiple requests to
+accumulate in the scheduler.
+
+This is supporting infrastructure, not the main design point. The batching
+behavior is defined by scheduler-side compatibility gating and runner-side
+batch packing.
+
+## Current Limitations
+
+- Experimental feature; use `max_num_seqs=1` for the older conservative path.
+- Only native pipelines that already support `step_execution=True`.
+- Request-mode diffusion still clamps `max_num_seqs` back to `1`.
+- Only homogeneous batches keyed by `SamplingParamsKey` are supported.
+- Multi-prompt requests are not batched.
+- `cache_backend`, KV transfer, and other request-mode extras are not wired
+  into the batched step-wise path yet.
+- Future work can relax the current same-shape restriction with richer
+  heterogeneous batching policies such as bucketing or padded execution for
+  different resolutions.
+
+## Related Files
+
+- Scheduler base:
+  [`vllm_omni/diffusion/sched/base_scheduler.py`](gh-file:vllm_omni/diffusion/sched/base_scheduler.py)
+- Scheduler interface:
+  [`vllm_omni/diffusion/sched/interface.py`](gh-file:vllm_omni/diffusion/sched/interface.py)
+- Step scheduler:
+  [`vllm_omni/diffusion/sched/step_scheduler.py`](gh-file:vllm_omni/diffusion/sched/step_scheduler.py)
+- Runner:
+  [`vllm_omni/diffusion/worker/diffusion_model_runner.py`](gh-file:vllm_omni/diffusion/worker/diffusion_model_runner.py)
+- Input batch:
+  [`vllm_omni/diffusion/worker/input_batch.py`](gh-file:vllm_omni/diffusion/worker/input_batch.py)
+- Tests:
+  [`tests/diffusion/test_diffusion_scheduler.py`](gh-file:tests/diffusion/test_diffusion_scheduler.py)

docs/design/feature/diffusion_step_execution.md

@@ -1,4 +1,4 @@
-# Diffusion Step Execution
+# Adding Step Execution Support for Diffusion Pipelines
 
 This guide documents vLLM-Omni's stepwise diffusion contract for model authors
 and contributors implementing `step_execution=True` support for a diffusion
@@ -7,12 +7,20 @@ pipeline.
 For end-user enablement, supported models, and current limitations, see
 [Step Execution](../../user_guide/diffusion/step_execution.md).
 
+This document describes the base step-execution contract only. For the
+experimental batching policy layered on top of the step-wise path, see
+[Continuous Batching for Step-Wise Diffusion](diffusion_continuous_batching.md).
+
 ## Current Support Scope
 
 `step_execution` is **not** a generic diffusion toggle. It only works for
 pipelines that implement the segmented stateful contract in
 [`vllm_omni/diffusion/models/interface.py`](gh-file:vllm_omni/diffusion/models/interface.py).
 
+This page is intentionally author-facing. Treat runtime enablement
+(`step_execution=True` in Python or `--step-execution` in serving) as an
+opt-in user knob layered on top of the implementation contract below.
+
 Current in-tree support:
 
 | Pipeline | Example models | Step execution |
@@ -22,7 +30,9 @@ Current in-tree support:
 
 Current engine/runtime limitations:
 
-- `StepScheduler` only schedules `batch_size=1`.
+- Continuous batching with `max_num_seqs > 1` is experimental and documented in
+  [Continuous Batching for Step-Wise Diffusion](diffusion_continuous_batching.md).
+  Keep `max_num_seqs=1` if you want the older conservative behavior.
 - `cache_backend` is not supported in step mode.
 - Request-mode extras such as KV transfer are not wired into step mode yet.
 - Unsupported pipelines now fail early during model loading instead of failing on the first request.

docs/design/index.md

@@ -10,6 +10,8 @@ This section contains design documents and architecture specifications for vLLM-
 
 - [Disaggregated Inference](feature/disaggregated_inference.md)
 - [Ray-based Execution](feature/ray_based_execution.md)
+- [Adding Step Execution Support for Diffusion Pipelines](feature/diffusion_step_execution.md)
+- [Continuous Batching for Step-Wise Diffusion](feature/diffusion_continuous_batching.md)
 
 ## Module Design Documents
 

docs/design/module/dit_module.md

@@ -201,7 +201,7 @@ class _BaseScheduler(SchedulerInterface):
 
 - **Shared cleanup logic**: Request-id registration, finish handling, and state removal are centralized instead of duplicated in each policy.
 
-- **Current constraint**: `max_num_running_reqs` remains `1` because the current engine path is still synchronous request-mode execution.
+- **Current constraint boundary**: `_BaseScheduler` derives `max_num_running_reqs` from `max_num_seqs`, but request-mode diffusion is still clamped back to `1` by the engine. The step-wise path can keep this above `1` for compatible-request batching.
 
 #### 2.4 Current `RequestScheduler` Policy
 
@@ -217,7 +217,7 @@ class RequestScheduler(_BaseScheduler):
 
 - **FIFO request scheduling**: Waiting requests are promoted in queue order.
 
-- **Single-request admission**: The current policy only admits one active request at a time.
+- **Single-request admission**: `RequestScheduler` still admits one active request at a time because request-mode execution completes a whole request per dispatch.
 
 - **Executor result feedback**: `update_from_output()` converts executor output into `FINISHED_COMPLETED` or `FINISHED_ERROR` and returns finished scheduler ids.
 
@@ -237,7 +237,7 @@ while True:
 
 - **No scheduler-owned IPC**: Scheduler no longer talks to workers directly.
 
-- **Conservative concurrency**: The current request-mode implementation still allows only one active request at a time.
+- **Split concurrency model**: Request-mode diffusion remains single-active-request, while the step-wise path can keep multiple compatible requests running and advance them independently between denoise steps.
 
 ---
 
@@ -455,7 +455,7 @@ To learn more about the diffusion pipeline and how to add a new diffusion pipeli
 
 #### Architecture
 
-The attention system uses a **backend selector pattern** that automatically chooses the optimal attention implementation based on hardware and model configuration.
+The attention system uses a **role-aware backend selector pattern**. Each `Attention` site declares a semantic `role` (`"self"`, `"cross"`, or a model-specific string). The selector consults the user's `AttentionConfig` to pick a backend per role and falls back to the platform default when nothing is configured.
 
 #### Backend Selection
 
@@ -463,92 +463,90 @@ The attention system uses a **backend selector pattern** that automatically choo
 
 ```python
 class Attention(nn.Module):
-    def __init__(self, num_heads, head_size, causal, softmax_scale, ...):
-        # Auto-select backend
-        self.attn_backend = get_attn_backend(-1)
-        self.attn_impl_cls = self.attn_backend.get_impl_cls()
-        self.attention = self.attn_impl_cls(...)
+    def __init__(self, num_heads, head_size, causal, softmax_scale, *,
+                 role="self", role_category=None, ...):
+        # Resolve backend for this role from the active OmniDiffusionConfig
+        config = get_current_diffusion_config_or_none()
+        attention_config = config.attention_config if config is not None else None
+        attn_backend_cls, spec = get_attn_backend_for_role(
+            role=role,
+            head_size=head_size,
+            attention_config=attention_config,
+            role_category=role_category,
+        )
+        self.attn_impl_cls = attn_backend_cls.get_impl_cls()
+        self.attention = self.attn_impl_cls(..., backend_kwargs=spec.extra if spec else None)
 ```
 
 **Available Backends**:
 
-- **FlashAttention**: Optimized CUDA kernel (FA2/FA3) - memory efficient via tiling
+- **FlashAttention** (`FLASH_ATTN`): Memory-efficient kernel that dispatches to FA2/FA3 on CUDA, AITER on ROCm, and `mindiesd` on Ascend NPU.
 
-- **SDPA**: PyTorch's scaled dot-product attention - default, cross-platform
+- **SDPA** (`TORCH_SDPA`): PyTorch's scaled dot-product attention — cross-platform fallback.
 
-- **SageAttention**: Sparse attention implementation from SageAttention library
-
-- **AscendAttention**: NPU-optimized attention for Ascend hardware
+- **SageAttention** (`SAGE_ATTN`): Quantized attention implementation from the SageAttention library.
 
 These backends provide the **kernel implementations** for attention computation. For attention-level sequence parallelism strategies (Ring Attention, Ulysses), see [Parallel Attention](#52-parallel-attention).
 
 #### Backend Selection Mechanism
 
+Selection is driven by `AttentionConfig` on `OmniDiffusionConfig`, which carries a global `default` plus a `per_role` map of `AttentionSpec` entries. A role string is resolved in this order:
+
 ```python
-def get_attn_backend(head_size: int) -> type[AttentionBackend]:
-    # Check environment variable
-    backend_name = os.environ.get("DIFFUSION_ATTENTION_BACKEND")
-
-    if backend_name:
-        return load_backend(backend_name.upper())
-
-    # Default to SDPA
-    return SDPABackend
+def get_attn_backend_for_role(role, head_size, attention_config=None, role_category=None):
+    # 1. attention_config.per_role[role]            — exact match
+    # 2. attention_config.per_role[role_category]   — category fallback
+    # 3. attention_config.default                   — global default
+    # 4. Platform default                           — hardware-specific
+    if attention_config is not None:
+        spec, source = attention_config.resolve_with_source(
+            role=role, role_category=role_category,
+        )
+        if spec is not None:
+            return load_backend(spec.backend), spec
+    return current_omni_platform.get_diffusion_attn_backend_cls(...), None
 ```
 
 **Selection Priority**:
 
-1. **Environment Variable**: `DIFFUSION_ATTENTION_BACKEND` for manual override
+1. **Per-role override** (`--diffusion-attention-config.per_role.<role>.backend`) — finest control; matched on the layer's exact `role` string.
 
-    - Valid values: `FLASH_ATTN`, `TORCH_SDPA`, `SAGE_ATTN`, `ASCEND`
+2. **Role category fallback** (`--diffusion-attention-config.per_role.<category>.backend`) — used when an exact match is missing and the layer declared `role_category` (e.g. a `"mymodel.audio_to_video"` site can fall back to `"cross"`).
 
-    - Example: `export DIFFUSION_ATTENTION_BACKEND=SAGE_ATTN`
+3. **Global default** (`--diffusion-attention-backend FLASH_ATTN`, or `--diffusion-attention-config.default.backend`, or `DIFFUSION_ATTENTION_BACKEND` env var).
 
-2. **Automatic Fallback**: Falls back to SDPA if selected backend unavailable
+4. **Platform default** — `current_omni_platform.get_diffusion_attn_backend_cls(...)` picks the best-available kernel for the hardware.
 
-3. **Hardware Detection**: Can select based on device type (NPU, CUDA, etc.)
+`AttentionSpec.extra` is forwarded to the backend constructor as `backend_kwargs`, so backend-specific parameters (e.g. SparseBlock `block_size`) can be set without changing model code.
+
+For the user-facing CLI surface, see [Diffusion Attention Backends](../../user_guide/diffusion/attention_backends.md). For the role declaration contract on the model side, see [Adding a Diffusion Model](../../contributing/model/adding_diffusion_model.md).
 
 **Backend Availability**:
 
 - **SDPA**: Always available (PyTorch built-in)
 
-- **FlashAttention**: Requires `flash-attn` package installed
+- **FlashAttention**: Requires `flash-attn` on CUDA, `aiter` on ROCm, or `mindiesd` on Ascend NPU
 
 - **SageAttention**: Requires `sage-attention` package (from THU-ML GitHub)
 
-- **AscendAttention**: Only available on Ascend NPU hardware
-
 #### Attention Backend Registry
 
-**Location**: `vllm_omni/diffusion/attention/selector.py`
+**Location**: `vllm_omni/diffusion/attention/selector.py` + `vllm_omni/platforms/<device>/platform.py`
 
-The attention system uses a **registry pattern** to manage and dynamically load attention backends. This allows for easy extension and runtime selection of backends.
-
-
-**Registry Structure**:
+Backend resolution is delegated to the active platform: `current_omni_platform.get_diffusion_attn_backend_cls(selected_backend, head_size)` returns a fully-qualified class path that `_load_backend_cls` imports lazily, with the result cached by `(backend_name, head_size)`.
 
 ```python
-# Registry mapping backend names to their module paths and class names
-_BACKEND_CONFIG = {
-    "FLASH_ATTN": {
-        "module": "vllm_omni.diffusion.attention.backends.flash_attn",
-        "class": "FlashAttentionBackend",
-    },
-    "TORCH_SDPA": {
-        "module": "vllm_omni.diffusion.attention.backends.sdpa",
-        "class": "SDPABackend",
-    },
-    "SAGE_ATTN": {
-        "module": "vllm_omni.diffusion.attention.backends.sage_attn",
-        "class": "SageAttentionBackend",
-    },
-    "ASCEND": {
-        "module": "vllm_omni.diffusion.attention.backends.ascend_attn",
-        "class": "AscendAttentionBackend",
-    },
-}
+@cache
+def _cached_get_backend_cls(backend_name: str | None, head_size: int) -> type[AttentionBackend]:
+    backend_cls_path = current_omni_platform.get_diffusion_attn_backend_cls(
+        selected_backend=backend_name,
+        head_size=head_size,
+    )
+    return _load_backend_cls(backend_cls_path)
 ```
 
+Each platform (`cuda`, `rocm`, `xpu`, `musa`, `npu`) maps backend names like `"FLASH_ATTN"`, `"TORCH_SDPA"`, `"SAGE_ATTN"` to the right backend class path for that hardware, including head-size compatibility checks. Passing `selected_backend=None` lets the platform pick its own default.
+
 #### Attention Backend Integration
 
 The `Attention` layer integrates backends through a unified interface. Here's how **FlashAttentionBackend** is integrated as an example:

docs/getting_started/installation/gpu.md

@@ -34,7 +34,7 @@ vLLM-Omni is a Python library that supports the following GPU variants. The libr
 
 ### Pre-built wheels
 
-Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, and 0.18.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).
+Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, 0.18.0, 0.20.0, and 0.21.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).
 
 === "NVIDIA CUDA"
 

docs/getting_started/installation/gpu/cuda.inc.md

@@ -20,7 +20,7 @@ Therefore, it is recommended to install vLLM and vLLM-Omni with a **fresh new**
 
 vLLM-Omni is built based on vLLM. Please install it with command below.
 ```bash
-uv pip install vllm --torch-backend=auto
+uv pip install vllm==0.21.0 --torch-backend=auto
 ```
 
 #### Installation of vLLM-Omni
@@ -39,13 +39,13 @@ uv pip install 'vllm-omni[demo]'
 # --8<-- [start:build-wheel-from-source]
 
 #### Installation of vLLM
-If you do not need to modify source code of vLLM, you can directly install the stable 0.19.0 release version of the library
+If you do not need to modify source code of vLLM, you can directly install the stable 0.21.0 release version of the library
 
 ```bash
-uv pip install vllm==0.19.0 --torch-backend=auto
+uv pip install vllm==0.21.0 --torch-backend=auto
 ```
 
-The 0.19.0 release of vLLM ships CUDA 13.0-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.
+The 0.21.0 release of vLLM ships CUDA 13.0-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.
 
 #### Installation of vLLM-Omni
 Since vllm-omni is rapidly evolving, it's recommended to install it from source
@@ -66,12 +66,12 @@ If you want to check, modify or debug with source code of vLLM, install the libr
 ```bash
 git clone https://github.com/vllm-project/vllm.git
 cd vllm
-git checkout v0.19.0
+git checkout v0.21.0
 ```
 Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set `VLLM_PRECOMPILED_WHEEL_LOCATION` as your local absolute path of whl file.
 ```bash
-#For CUDA 13.0
-export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.19.0/vllm-0.19.0+cu130-cp38-abi3-manylinux_2_35_x86_64.whl
+#For CUDA 13.0 (the default for v0.21.0; the wheel filename has no `+cu130` suffix)
+export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.21.0/vllm-0.21.0-cp38-abi3-manylinux_2_35_x86_64.whl
 ```
 Install vllm with command below (If you have no existing PyTorch).
 ```bash
@@ -80,7 +80,7 @@ uv pip install --editable .
 Install vllm with command below (If you already have PyTorch).
 ```bash
 python use_existing_torch.py
-uv pip install -r requirements/build.txt
+uv pip install -r requirements/build/cuda.txt
 uv pip install --no-build-isolation --editable .
 ```
 </details>
@@ -102,7 +102,7 @@ docker run --runtime nvidia --gpus 2 \
     --env "HF_TOKEN=$HF_TOKEN" \
     -p 8091:8091 \
     --ipc=host \
-    vllm/vllm-omni:v0.18.0 \
+    vllm/vllm-omni:v0.21.0 \
     --model Qwen/Qwen3-Omni-30B-A3B-Instruct --port 8091
 ```
 

docs/getting_started/installation/gpu/rocm.inc.md

@@ -13,7 +13,7 @@ vLLM-Omni current recommends the steps in under setup through Docker Images.
 
 vLLM-Omni is built based on vLLM. Please install it with command below.
 ```bash
-uv pip install vllm==0.19.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.19.0/rocm700
+uv pip install vllm==0.21.0+rocm722 --extra-index-url https://wheels.vllm.ai/rocm/0.21.0/rocm722
 ```
 
 #### Installation of vLLM-Omni
@@ -34,13 +34,13 @@ uv pip install onnxruntime-rocm
 # --8<-- [start:build-wheel-from-source]
 
 #### Installation of vLLM
-If you do not need to modify source code of vLLM, you can directly install the stable 0.19.0 release version of the library
+If you do not need to modify source code of vLLM, you can directly install the stable 0.21.0 release version of the library
 
 ```bash
-uv pip install vllm==0.19.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.19.0/rocm700
+uv pip install vllm==0.21.0+rocm722 --extra-index-url https://wheels.vllm.ai/rocm/0.21.0/rocm722
 ```
 
-The pre-built 0.19.0 vLLM wheel targets ROCm 7.0. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
+The pre-built 0.21.0 vLLM wheel targets ROCm 7.2.2. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
 
 #### Installation of vLLM-Omni
 Since vllm-omni is rapidly evolving, it's recommended to install it from source
@@ -58,7 +58,7 @@ If you want to check, modify or debug with source code of vLLM, install the libr
 ```bash
 git clone https://github.com/vllm-project/vllm.git
 cd vllm
-git checkout v0.19.0
+git checkout v0.21.0
 python3 -m pip install -r requirements/rocm.txt
 python3 setup.py develop
 ```
@@ -130,7 +130,7 @@ docker run --rm \
   -v ~/.cache/huggingface:/root/.cache/huggingface \
   --env "HF_TOKEN=$HF_TOKEN" \
   -p 8091:8091 \
-  vllm/vllm-omni-rocm:v0.19.0 \
+  vllm/vllm-omni-rocm:v0.21.0 \
   --model Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
 ```
 
@@ -149,7 +149,7 @@ docker run --rm -it \
   -v ~/.cache/huggingface:/root/.cache/huggingface \
   --env "HF_TOKEN=$HF_TOKEN" \
   --entrypoint bash \
-  vllm/vllm-omni-rocm:v0.19.0
+  vllm/vllm-omni-rocm:v0.21.0
 ```
 
 # --8<-- [end:pre-built-images]

docs/getting_started/quickstart.md

@@ -19,10 +19,10 @@ uv venv --python 3.12 --seed
 source .venv/bin/activate
 
 # On CUDA
-uv pip install vllm==0.19.0 --torch-backend=auto
+uv pip install vllm==0.21.0 --torch-backend=auto
 
 # On ROCm
-uv pip install vllm==0.19.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.19.0/rocm700
+uv pip install vllm==0.21.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.21.0/rocm721
 
 git clone https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni
@@ -31,6 +31,12 @@ uv pip install -e .
 
 For additional installation methods — please see the [installation guide](installation/README.md).
 
+
+!!! note
+    It is important to install the same major & minor version of vLLM and vLLM Omni, otherwise things may not work as expected. If the versions are misaligned, you will see a warning when you import vLLM Omni.
+
+    If you are seeing strange behavior with the `vllm` command not handling the `--omni` flag correctly, you most likely have a version mismatch with vLLM < `0.21.0` and vLLM Omni `0.21.0`, as vLLM Omni no longer hijacks the vLLM entrypoint. Updating vLLM should resolve this issue.
+
 ## Offline Inference
 
 Text-to-image generation quickstart with vLLM-Omni:

docs/models/supported_models.md

@@ -32,6 +32,7 @@ th {
 | `ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `WanPipeline` | Wan2.1-T2V, Wan2.2-T2V, Wan2.2-TI2V | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`, `Wan-AI/Wan2.1-T2V-14B-Diffusers`, `Wan-AI/Wan2.2-T2V-A14B-Diffusers`, `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `WanImageToVideoPipeline` | Wan2.2-I2V | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
+| `WanSpeechToVideoPipeline` | Wan2.2-S2V | `Wan-AI/Wan2.2-S2V-14B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `Wan22VACEPipeline` | Wan2.1-VACE | `Wan-AI/Wan2.1-VACE-1.3B-diffusers`, `Wan-AI/Wan2.1-VACE-14B-diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `LTX2Pipeline` | LTX-2-T2V | `Lightricks/LTX-2` | ✅︎ | ✅︎ | | |
 | `LTX2ImageToVideoPipeline` | LTX-2-I2V | `Lightricks/LTX-2` | ✅︎ | ✅︎ | | |
@@ -53,6 +54,7 @@ th {
 | `FluxPipeline` | FLUX.1-schnell | `black-forest-labs/FLUX.1-schnell` | ✅︎ | ✅︎ | | ✅︎ |
 | `OmniGen2Pipeline` | OmniGen2 | `OmniGen2/OmniGen2` | ✅︎ | ✅︎ | | ✅︎ |
 | `StableAudioPipeline` | Stable-Audio-Open | `stabilityai/stable-audio-open-1.0` | ✅︎ | ✅︎ | | ✅︎ |
+| `AudioXPipeline` | AudioX | `zhangj1an/AudioX` | ✅︎ | ✅︎ | | |
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-CustomVoice | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-VoiceDesign | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-Base | `Qwen/Qwen3-TTS-12Hz-0.6B-Base` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
@@ -62,9 +64,11 @@ th {
 | `Flux2Pipeline` | FLUX.2-dev | `black-forest-labs/FLUX.2-dev` | ✅︎ | ✅︎ | | |
 | `FishSpeechSlowARForConditionalGeneration` | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✅︎ | ✅︎ | | |
 | `DreamIDOmniPipeline` | DreamID-Omni | `XuGuo699/DreamID-Omni` | ✅︎ | ✅︎ | | |
+| `SenseNovaU1Pipeline` | SenseNova-U1 (DiT-only) | `SenseNova/SenseNova-U1-8B-MoT` | ✅︎ | | | |
 | `HunyuanVideo15Pipeline` | HunyuanVideo-1.5-T2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | ✅︎ | ✅︎ | | |
 | `HunyuanVideo15ImageToVideoPipeline` | HunyuanVideo-1.5-I2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_i2v` | ✅︎ | ✅︎ | | |
 | `VoxtralTTSForConditionalGeneration` | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✅︎ | ✅︎ | | |
+| `CovoAudioForConditionalGeneration` | Covo-Audio-Chat | `tencent/Covo-Audio-Chat` | ✅︎ | | | |
 |`DyninOmniForConditionalGeneration` | Dynin-Omni | `snu-aidas/Dynin-Omni` | ✅︎ | | | |
-
+| `ErnieImagePipeline` | ERNIE-Image | `baidu/ERNIE-Image`, `baidu/ERNIE-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 ✅︎ indicates the model is supported on that backend. Empty cells mean not listed as supported on that backend.

docs/serving/audio_generate_api.md

@@ -0,0 +1,338 @@
+# Audio Generate API
+
+vLLM-Omni provides an API for text-to-audio generation using diffusion-based models such as Stable Audio.
+
+Unlike the [Speech API](speech_api.md) which targets text-to-speech synthesis, the Audio Generate API is designed for general-purpose audio generation from text descriptions (sound effects, music, ambient soundscapes, etc.).
+
+Each server instance runs a single model (specified at startup via `vllm-omni serve <model> --omni`).
+
+## Quick Start
+
+### Start the Server
+
+```bash
+vllm-omni serve stabilityai/stable-audio-open-1.0 \
+    --host 0.0.0.0 \
+    --port 8091 \
+    --gpu-memory-utilization 0.9 \
+    --trust-remote-code \
+    --enforce-eager \
+    --omni
+```
+
+### Generate Audio
+
+**Using curl:**
+
+```bash
+curl -X POST http://localhost:8091/v1/audio/generate \
+    -H "Content-Type: application/json" \
+    -d '{
+        "input": "The sound of a cat purring",
+        "audio_length": 10.0
+    }' --output cat.wav
+```
+
+**Using Python:**
+
+```python
+import httpx
+
+response = httpx.post(
+    "http://localhost:8091/v1/audio/generate",
+    json={
+        "input": "The sound of a cat purring",
+        "audio_length": 10.0,
+    },
+    timeout=300.0,
+)
+
+with open("cat.wav", "wb") as f:
+    f.write(response.content)
+```
+
+## API Reference
+
+### Endpoint
+
+```
+POST /v1/audio/generate
+Content-Type: application/json
+```
+
+### Request Parameters
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `input` | string | **required** | Text prompt describing the audio to generate |
+| `model` | string | server's model | Model to use (optional, should match server if specified) |
+| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, aac, opus |
+| `speed` | float | 1.0 | Playback speed (0.25 - 4.0) |
+
+#### Diffusion Parameters
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `audio_length` | float | null | Audio duration in seconds (default value is the max ~47s for `stable-audio-open-1.0`) |
+| `audio_start` | float | 0.0 | Audio start time in seconds |
+| `negative_prompt` | string | null | Text describing what to avoid in generation |
+| `guidance_scale` | float | model default | Classifier-free guidance scale (higher = more adherence to prompt) |
+| `num_inference_steps` | int | model default | Number of denoising steps (higher = better quality, slower) |
+| `seed` | int | null | Random seed for reproducible generation |
+
+### Response Format
+
+Returns binary audio data with the appropriate `Content-Type` header:
+
+| `response_format` | Content-Type |
+|--------------------|--------------|
+| `wav` | `audio/wav` |
+| `mp3` | `audio/mpeg` |
+| `flac` | `audio/flac` |
+| `pcm` | `audio/pcm` |
+| `aac` | `audio/aac` |
+| `opus` | `audio/opus` |
+
+## Examples
+
+### Basic Generation
+
+Generate audio with only a text prompt (model defaults for all other parameters):
+
+```bash
+curl -X POST http://localhost:8091/v1/audio/generate \
+    -H "Content-Type: application/json" \
+    -d '{
+        "input": "The sound of ocean waves crashing on a beach"
+    }' --output ocean.wav
+```
+
+### Custom Duration
+
+Specify an explicit audio length in seconds:
+
+```bash
+curl -X POST http://localhost:8091/v1/audio/generate \
+    -H "Content-Type: application/json" \
+    -d '{
+        "input": "A dog barking",
+        "audio_length": 5.0
+    }' --output dog_5s.wav
+```
+
+### High Quality with Negative Prompt
+
+Use a negative prompt to steer generation away from undesired characteristics, and increase inference steps for higher quality:
+
+```bash
+curl -X POST http://localhost:8091/v1/audio/generate \
+    -H "Content-Type: application/json" \
+    -d '{
+        "input": "A piano playing a gentle melody",
+        "audio_length": 10.0,
+        "negative_prompt": "Low quality, distorted, noisy",
+        "guidance_scale": 8.0,
+        "num_inference_steps": 150
+    }' --output piano_hq.wav
+```
+
+### Reproducible Generation
+
+Set a `seed` to get deterministic results across runs:
+
+```bash
+curl -X POST http://localhost:8091/v1/audio/generate \
+    -H "Content-Type: application/json" \
+    -d '{
+        "input": "Thunder and rain sounds",
+        "audio_length": 15.0,
+        "seed": 42
+    }' --output thunder.wav
+```
+
+### Full Control
+
+Combine all parameters for precise control over generation:
+
+```bash
+curl -X POST http://localhost:8091/v1/audio/generate \
+    -H "Content-Type: application/json" \
+    -d '{
+        "input": "Thunder and rain sounds",
+        "audio_length": 15.0,
+        "negative_prompt": "Low quality",
+        "guidance_scale": 7.0,
+        "num_inference_steps": 100,
+        "seed": 42
+    }' --output thunder_rain.wav
+```
+
+### Quick Generation (Fewer Steps)
+
+For faster generation with slightly lower quality:
+
+```bash
+curl -X POST http://localhost:8091/v1/audio/generate \
+    -H "Content-Type: application/json" \
+    -d '{
+        "input": "Birds chirping in a forest",
+        "audio_length": 8.0,
+        "num_inference_steps": 50
+    }' --output birds_quick.wav
+```
+
+### Python Client
+
+```python
+import httpx
+
+response = httpx.post(
+    "http://localhost:8091/v1/audio/generate",
+    json={
+        "input": "Thunder and rain",
+        "audio_length": 15.0,
+        "negative_prompt": "Low quality",
+        "guidance_scale": 7.0,
+        "num_inference_steps": 100,
+        "seed": 42,
+        "response_format": "wav",
+    },
+    timeout=300.0,
+)
+
+with open("thunder.wav", "wb") as f:
+    f.write(response.content)
+```
+
+## Parameter Tuning Guide
+
+### `guidance_scale`
+
+Controls how closely the generated audio follows the text prompt.
+
+| Range | Behaviour |
+|-------|-----------|
+| 3 - 5 | More creative / varied output |
+| 7 (default) | Balanced adherence |
+| 10+ | Strict adherence to the prompt |
+
+### `num_inference_steps`
+
+Controls the number of denoising steps in the diffusion process.
+
+| Steps | Quality | Speed | Use Case |
+|-------|---------|-------|----------|
+| 50 | Good | Fast | Quick previews |
+| 100 | Very Good | Medium | General purpose |
+| 150+ | Excellent | Slow | Final / critical audio |
+
+### `audio_length`
+
+Duration of the generated audio clip. For `stable-audio-open-1.0`, the maximum is approximately 47 seconds. If omitted, the model uses its own default length.
+
+### `negative_prompt`
+
+Describes characteristics to avoid. Common negative prompts include:
+
+- `"Low quality, distorted, noisy"`
+- `"Silence, static"`
+- `"Music"` (when generating sound effects only)
+
+## Supported Models
+
+| Model | Description |
+|-------|-------------|
+| `stabilityai/stable-audio-open-1.0` | Open-source audio generation model, up to ~47 seconds, 44.1 kHz stereo |
+
+## Error Responses
+
+### 400 Bad Request
+
+Invalid or missing parameters:
+
+```json
+{
+    "error": {
+        "message": "Audio generation model did not produce audio output.",
+        "type": "BadRequestError",
+        "param": null,
+        "code": 400
+    }
+}
+```
+
+### 404 Not Found
+
+Model mismatch:
+
+```json
+{
+    "error": {
+        "message": "The model `xxx` does not exist.",
+        "type": "NotFoundError",
+        "param": "model",
+        "code": 404
+    }
+}
+```
+
+### 422 Unprocessable Entity
+
+Pydantic validation failure (e.g. invalid `response_format`, `speed` out of range):
+
+```json
+{
+    "detail": [
+        {
+            "type": "literal_error",
+            "msg": "Input should be 'wav', 'pcm', 'flac', 'mp3', 'aac' or 'opus'",
+            ...
+        }
+    ]
+}
+```
+
+## Troubleshooting
+
+### "Audio generation model did not produce audio output"
+
+The model finished but returned no audio data. Verify the server started successfully and the model loaded without errors.
+
+### Server Not Responding
+
+```bash
+# Check if the server is healthy
+curl http://localhost:8091/health
+```
+
+### Audio Quality Issues
+
+- Increase `num_inference_steps` (e.g. 150).
+- Add a negative prompt: `"Low quality, distorted, noisy"`.
+- Increase `guidance_scale` for stronger prompt adherence.
+
+### Generation Timeout
+
+- Reduce `num_inference_steps`.
+- Reduce `audio_length`.
+- Check GPU memory with `nvidia-smi`.
+
+### Out of Memory
+
+- Lower `--gpu-memory-utilization` (e.g. 0.8).
+- Reduce `audio_length`.
+
+## Development
+
+Enable debug logging:
+
+```bash
+vllm-omni serve stabilityai/stable-audio-open-1.0 \
+    --host 0.0.0.0 \
+    --port 8091 \
+    --gpu-memory-utilization 0.9 \
+    --trust-remote-code \
+    --enforce-eager \
+    --omni \
+    --uvicorn-log-level debug
+```

docs/serving/speech_api.md

@@ -5,7 +5,6 @@ vLLM-Omni provides an OpenAI-compatible API for text-to-speech (TTS) generation.
 - **Qwen3-TTS** (`Qwen/Qwen3-TTS-12Hz-*`) -- Qwen3-based TTS with CustomVoice, VoiceDesign, and Base (voice cloning) task types. Output: 24 kHz.
 - **Fish Speech S2 Pro** (`fishaudio/s2-pro`) -- Dual-AR TTS with DAC codec. Supports text-to-speech and voice cloning via reference audio. Output: 44.1 kHz.
 - **Voxtral TTS** (`mistralai/Voxtral-4B-TTS-2603`) -- AR + FlowMatching TTS with preset voices. Output: 24 kHz.
-- **VoxCPM2** (`openbmb/VoxCPM2`) -- Single-stage AR + DiT/CFM TTS. Three synthesis modes: Voice Design (text-only voice description), Controllable Cloning (reference audio + style instructions), and Hi-Fi Cloning (reference audio + transcript for tightest identity match; aka "Ultimate Cloning" in the Gradio demo). Output: 48 kHz.
 
 Each server instance runs a single model (specified at startup via `vllm serve <model> --omni`).
 
@@ -27,9 +26,6 @@ vllm serve fishaudio/s2-pro --omni --port 8091
 
 # Voxtral TTS
 vllm serve mistralai/Voxtral-4B-TTS-2603 --omni --port 8091
-
-# VoxCPM2
-vllm serve openbmb/VoxCPM2 --omni --port 8091
 ```
 
 ### Generate Speech
@@ -123,16 +119,6 @@ Content-Type: application/json
 | `ref_text` | string | null | Transcript of reference audio |
 | `x_vector_only_mode` | bool | null | Use speaker embedding only (no ICL) |
 
-#### VoxCPM2-specific Parameters
-
-VoxCPM2 reuses the shared `extra_params: dict` field (introduced in #2338 for Voxtral TTS) for model-specific knobs rather than adding new top-level fields:
-
-| Key (inside `extra_params`) | Type | Default | Description |
-|-----------------------------|------|---------|-------------|
-| `cfg_value` | float | 2.0 | Classifier-free guidance scale for the CFM decoder. Range `0.1`–`10.0`; typical useful range is `1.5`–`3.0`. Higher values track the text more strictly at the cost of naturalness. |
-
-Send as e.g. `{"input": "...", "extra_params": {"cfg_value": 2.5}}`. VoxCPM2 also reuses the standard `instructions`, `ref_audio`, and `ref_text` fields, see the mode table below for how they combine.
-
 ### Response Format
 
 Returns binary audio data with appropriate `Content-Type` header (e.g., `audio/wav`).
@@ -372,6 +358,31 @@ curl -X POST http://localhost:8091/v1/audio/speech \
     }' --output cloned.wav
 ```
 
+### Voice Storage & Caching
+
+Uploaded voices are persisted to disk as a single `.safetensors` file per voice
+(audio samples + metadata — name, consent, ref_text, sample_rate, created_at —
+in the file header). On server restart the directory is scanned and all
+previously uploaded voices are restored automatically, so uploads survive
+process restarts.
+
+Uploading an existing name overwrites the previous entry (a warning is logged).
+
+Feature extraction artifacts (ref_code, speaker_embedding, DAC codes, etc.)
+are cached in-process with a shared LRU so repeated requests with the same
+`voice=...` skip the extraction pipeline. The cache is a true singleton across
+all TTS model types; deleting a voice invalidates every model-type slot at
+once.
+
+**Configuration (environment variables):**
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `SPEAKER_SAMPLES_DIR` | `~/.cache/vllm-omni/speakers` | Directory for persisted uploaded speakers (`.safetensors` files). |
+| `SPEAKER_MAX_UPLOADED` | `1000` | Maximum number of uploaded speakers kept on disk. Upload requests past the cap return 400. |
+
+The in-memory LRU has a fixed 512 MiB byte budget.
+
 ## Batch Speech Generation
 
 The batch endpoint synthesizes multiple texts in a single request, returning all results as JSON with base64-encoded audio.
@@ -520,16 +531,18 @@ for result in response.json()["results"]:
 
 All items are fanned out to `generate()` concurrently. The engine's stage worker automatically batches them up to the configured `max_batch_size` and queues the rest — no client-side throttling needed.
 
-For best throughput, set both stages' `max_num_seqs` to ≥4 via `--stage-overrides`:
+For best throughput, set both stages' `max_num_seqs` above 1 via `--stage-overrides`. On the current Qwen3-TTS CustomVoice benchmark, stage 1 performed best at `max_num_seqs: 10`:
 
 ```bash
 vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
     --omni --port 8091 --trust-remote-code --enforce-eager \
-    --stage-overrides '{"0":{"max_num_seqs":4,"gpu_memory_utilization":0.2},
-                        "1":{"max_num_seqs":4,"gpu_memory_utilization":0.2}}'
+    --stage-overrides '{"0":{"max_num_seqs":10,"gpu_memory_utilization":0.2},
+                        "1":{"max_num_seqs":10,"gpu_memory_utilization":0.2}}'
 ```
 
-The bundled `qwen3_tts.yaml` uses `max_num_seqs: 1` (single request) on both stages. Bumping to 4 yields roughly 4× throughput on the talker and lets stage 1 batch chunks across in-flight requests.
+The bundled `qwen3_tts.yaml` uses a multi-request default and lets stage 1 batch chunks across in-flight requests. For latency-sensitive deployments, avoid forcing stage 1 back to `max_num_seqs: 1`; benchmark before reducing it below `10`.
+
+The bundled config also sets `initial_codec_chunk_frames: 1`. This emits only the first audio chunk early for lower TTFA, then returns to the normal `codec_chunk_frames` window so Code2Wav does not repeatedly decode tiny overlapping chunks.
 
 ## Supported Models
 
@@ -549,7 +562,7 @@ The bundled `qwen3_tts.yaml` uses `max_num_seqs: 1` (single request) on both sta
 |-------|-------------|
 | `fishaudio/s2-pro` | 4B dual-AR TTS with DAC codec (44.1 kHz). Supports text-to-speech and voice cloning. |
 
-Fish Speech uses `ref_audio` and `ref_text` for voice cloning (no `task_type` needed). The `voice` field should be set to `"default"`. See the [Fish Speech online serving example](../user_guide/examples/online_serving/fish_speech.md) for details.
+Fish Speech uses `ref_audio` and `ref_text` for voice cloning (no `task_type` needed). The `voice` field should be set to `"default"`. See the [Fish Speech section of the online TTS hub](../user_guide/examples/online_serving/text_to_speech.md#fish-speech-s2-pro) for details.
 
 ### Voxtral TTS
 
@@ -557,63 +570,29 @@ Fish Speech uses `ref_audio` and `ref_text` for voice cloning (no `task_type` ne
 |-------|-------------|
 | `mistralai/Voxtral-4B-TTS-2603` | 3B AR + FlowMatching TTS. Supports text-to-speech with preset voices. |
 
+### CosyVoice3
+
+| Model | Description |
+|-------|-------------|
+| `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | Voice cloning from `ref_audio` + `ref_text`. No built-in voice presets — upload a voice or pass `ref_audio`/`ref_text` per request. |
+
+### OmniVoice
+
+| Model | Description |
+|-------|-------------|
+| `k2-fsa/OmniVoice` | Pure-diffusion TTS. Supports voice cloning via `ref_audio` (with optional `ref_text`); no built-in voice presets. |
+
 ### VoxCPM2
 
 | Model | Description |
 |-------|-------------|
-| `openbmb/VoxCPM2` | Single-stage AR + DiT/CFM TTS. 48 kHz output, multilingual (primary EN/ZH, usable cross-lingual identity transfer via Ultimate Cloning). |
+| `openbmb/VoxCPM2` | TTS + voice cloning with built-in speaker presets and uploaded-voice support. Accepts `voice` (preset or uploaded) or `ref_audio` + optional `ref_text`. |
 
-VoxCPM2 exposes three synthesis modes through the same `/v1/audio/speech` endpoint — the mode is inferred from which fields are present in the request.
+### MOSS-TTS-Nano
 
-| Mode | Required fields | Optional fields | Notes |
-|------|-----------------|-----------------|-------|
-| **Voice Design** | `input` | `instructions`, `extra_params.cfg_value` | Synthesize a voice from a text description alone. `instructions` is prepended to the target text as `(instructions)input` following `voxcpm.cli.build_final_text`. Both English and Chinese instructions are supported. |
-| **Controllable Cloning** | `input`, `ref_audio` | `instructions`, `extra_params.cfg_value` | Clone the reference speaker's timbre; use `instructions` to steer emotion, pace, or other style attributes while preserving identity. |
-| **Hi-Fi Cloning** | `input`, `ref_audio`, `ref_text` | `extra_params.cfg_value` | The model treats `ref_audio` + `ref_text` as a spoken prefix and continues from it. Reproduces every vocal nuance of the reference. Per the canonical VoxCPM2 docs, when Hi-Fi mode is enabled the control instruction is ignored; the server drops `instructions` in this mode automatically if present. |
-
-Reference audio guidelines: 16 kHz mono, at least 5 seconds recommended for a good clone (1 second hard minimum, 30 seconds hard maximum). Clean, non-reverberant speech; the same `ref_audio` can be reused across Controllable and Hi-Fi modes.
-
-Inline non-verbal tags in `input` are passed through as regular text and recognized by the model when present (for example `[laughing]`, `[sigh]`, `[Uhm]`, `[Question-ah]`). See the upstream [VoxCPM2 cookbook](https://voxcpm.readthedocs.io/en/latest/cookbook.html) for the full tag list and usage tips.
-
-**Examples**
-
-Voice Design (no reference, pure text description):
-
-```bash
-curl -X POST http://localhost:8091/v1/audio/speech \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "Hello everyone. Today I want to talk about open source AI.",
-        "instructions": "A warm young woman with a gentle, friendly voice",
-        "extra_params": {"cfg_value": 2.5}
-    }' --output voice_design.wav
-```
-
-Controllable Cloning (reference audio + style instructions):
-
-```bash
-curl -X POST http://localhost:8091/v1/audio/speech \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "Good morning, let me tell you a short story.",
-        "ref_audio": "https://example.com/reference.wav",
-        "instructions": "Warm storyteller, slow cadence, dramatic pauses",
-        "extra_params": {"cfg_value": 2.5}
-    }' --output controllable_cloning.wav
-```
-
-Hi-Fi Cloning (reference audio + transcript; tightest identity):
-
-```bash
-curl -X POST http://localhost:8091/v1/audio/speech \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "Welcome everyone. I will give a brief update on the project today.",
-        "ref_audio": "https://example.com/reference.wav",
-        "ref_text": "Transcript of what the reference speaker is saying in the reference clip.",
-        "extra_params": {"cfg_value": 2.5}
-    }' --output ultimate_cloning.wav
-```
+| Model | Description |
+|-------|-------------|
+| `OpenMOSS-Team/MOSS-TTS-Nano` | Voice cloning only. Requires `ref_audio` (or an uploaded `voice`); no built-in voice presets. `ref_text` is accepted but ignored — upstream's `voice_clone` mode does not consume a transcript. |
 
 ## Error Responses
 

docs/user_guide/diffusion/attention_backends.md

@@ -1,10 +1,10 @@
 # Diffusion Attention Backends
 
-This document describes the diffusion attention backends available in vLLM-Omni, how to select them, and how to use SageAttention.
+This document describes the diffusion attention backends available in vLLM-Omni, how to select them globally and per-role, and how to use SageAttention.
 
 ## Overview
 
-Diffusion attention backend selection is controlled by the `DIFFUSION_ATTENTION_BACKEND` environment variable and resolved in `vllm_omni.diffusion.attention.selector`.
+Diffusion attention backend selection is resolved in `vllm_omni.diffusion.attention.selector`. It looks up the backend from a structured `AttentionConfig` carried on `OmniDiffusionConfig` and falls back to the platform default when nothing is configured.
 
 This backend is used by diffusion attention layers such as the DiT attention in video and image generation models.
 
@@ -14,7 +14,7 @@ On CUDA, the practical choices today are:
 - `TORCH_SDPA`: PyTorch `scaled_dot_product_attention`.
 - `SAGE_ATTN`: SageAttention backend, if `sageattention` is installed.
 
-If `DIFFUSION_ATTENTION_BACKEND` is unset, vLLM-Omni asks the current platform to choose the default backend. On CUDA, that normally means `FLASH_ATTN` when available, otherwise `TORCH_SDPA`.
+If no attention backend is configured, vLLM-Omni asks the current platform to choose the default. On CUDA, that normally means `FLASH_ATTN` when available, otherwise `TORCH_SDPA`.
 
 ## Backend Options
 
@@ -24,19 +24,77 @@ If `DIFFUSION_ATTENTION_BACKEND` is unset, vLLM-Omni asks the current platform t
 | `TORCH_SDPA` | Most conservative fallback. Useful for debugging or compatibility. |
 | `SAGE_ATTN` | Requires `sageattention`. Can improve performance on some workloads, but output quality must be validated model-by-model. |
 
-## Selection Priority
+## Configuration
 
-Diffusion attention backend selection follows this order:
+Diffusion attention backends can be configured three ways, in priority order:
 
-1. `DIFFUSION_ATTENTION_BACKEND`
-2. Platform default
+1. **`--diffusion-attention-config`** — structured per-role config (highest priority).
+2. **`--diffusion-attention-backend` / `DIFFUSION_ATTENTION_BACKEND` env var** — global shorthand that sets the default backend.
+3. **Platform default** — used when nothing is configured.
 
-Example:
+`--diffusion-attention-backend` is shorthand for `--diffusion-attention-config.default.backend`. It may be combined with `--diffusion-attention-config.per_role.*` overrides, but is mutually exclusive with `--diffusion-attention-config.default.backend`.
+
+### Global default
+
+Set the default backend for every diffusion attention layer:
 
 ```bash
+# CLI flag
+vllm-omni serve <model> --diffusion-attention-backend SAGE_ATTN
+
+# Environment variable (also recognized for backwards compatibility)
 export DIFFUSION_ATTENTION_BACKEND=SAGE_ATTN
 ```
 
+### Per-role configuration
+
+Roles are free-form strings declared by each diffusion model. The two common categories are `"self"` and `"cross"`; model-specific roles (e.g. `"ltx2.audio_to_video"`) may also be declared. A role string is matched in this order:
+
+1. Exact `per_role[role]` match
+2. `per_role[role_category]` fallback (e.g. `"ltx2.audio_to_video"` → `"cross"`)
+3. `default`
+4. Platform default
+
+Use vLLM-style dotted flags or one JSON blob:
+
+```bash
+# Dotted flags
+vllm-omni serve <model> \
+    --diffusion-attention-config.default.backend FLASH_ATTN \
+    --diffusion-attention-config.per_role.cross.backend TORCH_SDPA
+
+# JSON
+vllm-omni serve <model> \
+    --diffusion-attention-config '{"default":{"backend":"FLASH_ATTN"},"per_role":{"cross":{"backend":"TORCH_SDPA"}}}'
+```
+
+Backends may also accept backend-specific parameters via `extra`:
+
+```bash
+--diffusion-attention-config.per_role.self.backend SPARSE_BLOCK \
+--diffusion-attention-config.per_role.self.extra.block_size 128
+```
+
+### Programmatic API
+
+When constructing `OmniDiffusionConfig` directly:
+
+```python
+from vllm_omni.diffusion.data import AttentionConfig, AttentionSpec, OmniDiffusionConfig
+
+config = OmniDiffusionConfig(
+    attention_config=AttentionConfig(
+        default=AttentionSpec(backend="FLASH_ATTN"),
+        per_role={
+            "cross": AttentionSpec(backend="TORCH_SDPA"),
+        },
+    ),
+    ...,
+)
+```
+
+A plain dict is also accepted and normalized to `AttentionConfig`.
+
 ## SageAttention Installation
 
 vLLM-Omni expects SageAttention to be installed into the same Python environment as vLLM-Omni.
@@ -85,7 +143,21 @@ DIFFUSION_ATTENTION_BACKEND=SAGE_ATTN python examples/offline_inference/text_to_
     --output outputs/wan22_sage.mp4
 ```
 
-### Compare Against FlashAttention
+### Mixed backends across roles
+
+Use `FLASH_ATTN` for self-attention and `TORCH_SDPA` for cross-attention:
+
+```bash
+python examples/offline_inference/text_to_video/text_to_video.py \
+    --model Wan-AI/Wan2.2-TI2V-5B-Diffusers \
+    --prompt "A dog running across a field of golden wheat." \
+    --diffusion-attention-config.per_role.self.backend FLASH_ATTN \
+    --diffusion-attention-config.per_role.cross.backend TORCH_SDPA \
+    --tensor-parallel-size 2 \
+    --output outputs/wan22_mixed.mp4
+```
+
+### Compare against FlashAttention
 
 Unset the backend override, or explicitly use `FLASH_ATTN`:
 

docs/user_guide/diffusion/cpu_offload_diffusion.md

@@ -17,7 +17,7 @@ Both strategies use pinned memory for faster CPU-GPU transfers. The strategies a
 Model-level offloading implements mutual exclusion between DiT transformer and encoder modules using pre forward hooks:
 
 - **When encoders run**: DiT transformer is offloaded to CPU
-- **When DiT runs**: Encoders are offloaded to CPU
+- **When DiT runs**: Encoders are offloaded to CPU, if more than one dit models, only one loaded on GPU, others get offloaded to CPU.
 - **VAE**: Stays resident on GPU
 
 Before each module's forward pass, the hook automatically moves it to GPU while offloading the other module group to CPU. Transfers use pinned memory for speed.
@@ -38,14 +38,15 @@ vllm-omni serve diffusion Wan-AI/Wan2.2-T2V-A14B-Diffusers --enable-cpu-offload
 
 ### To Support a Model
 
-Implement the `SupportsModuleOffload` protocol to declare which
-submodules participate in offloading:
+Implement the `SupportsComponentDiscovery` protocol to declare which
+submodules serve as pipeline components (used by offloading, HSDP
+sharding, and other framework features):
 
 ```python
 from typing import ClassVar
-from vllm_omni.diffusion.models.interface import SupportsModuleOffload
+from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery
 
-class MyPipeline(nn.Module, SupportsModuleOffload):
+class MyPipeline(nn.Module, SupportsComponentDiscovery):
     _dit_modules: ClassVar[list[str]] = ["transformer"]
     _encoder_modules: ClassVar[list[str]] = ["text_encoder", "vision_model"]
     _vae_modules: ClassVar[list[str]] = ["vae"]
@@ -158,7 +159,7 @@ class Flux2Transformer2DModel(nn.Module):
 The offloader discovers pipeline components in two ways:
 
 1. **Protocol-based** (preferred): If the pipeline implements
-    `SupportsModuleOffload`, its `_dit_modules`, `_encoder_modules`,
+    `SupportsComponentDiscovery`, its `_dit_modules`, `_encoder_modules`,
     `_vae_modules`, and `_resident_modules` class variables are used
     directly.  All attribute names support dotted paths (e.g.
     `"pipe.transformer"`, `"bagel.time_embedder"`) for nested submodules.

docs/user_guide/diffusion/quantization/autoround.md

@@ -1,93 +0,0 @@
-# AutoRound Quantization
-
-## Overview
-
-[AutoRound](https://github.com/intel/auto-round) is an advanced quantization toolkit designed for Large Language Models (LLMs), Vision-Language Models (VLMs), and diffusion models. It achieves high accuracy at ultra-low bit widths (2–4 bits) with minimal tuning by leveraging sign-gradient descent, while providing broad hardware compatibility with multi-datatype support.
-
-The quantization config is auto-detected from the checkpoint's `config.json` (`quantization_config.quant_method = "auto-round"`). No extra CLI flags are needed.
-
-### Supported Schemes
-
-| Scheme | Bits | Status |
-|--------|------|--------|
-| W4A16 | 4 | ✅ Supported |
-| W8A16 | 8 | Planned |
-
-W4A16 is the first supported scheme. Additional schemes will be added in future releases.
-
-## Configuration
-
-1. **Python API**: point `model` at a pre-quantized checkpoint. The quantization is detected automatically.
-
-```python
-from vllm_omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-
-omni = Omni(model="vllm-project-org/FLUX.1-dev-AutoRound-w4a16")
-
-outputs = omni.generate(
-    "A cat sitting on a windowsill",
-    OmniDiffusionSamplingParams(num_inference_steps=28),
-)
-outputs[0].save_images("output.png")
-```
-
-2. **CLI**: pass the quantized model path directly.
-
-```bash
-python examples/offline_inference/text_to_image/text_to_image.py \
-  --model vllm-project-org/FLUX.1-dev-AutoRound-w4a16 \
-  --prompt "A cat sitting on a windowsill" \
-  --num-inference-steps 28 \
-  --output outputs/flux_w4a16.png
-```
-
-No `--quantization` flag is needed — the quantization method is read from the checkpoint.
-
-## How It Works
-
-The checkpoint's `config.json` contains:
-
-```json
-{
-  "quantization_config": {
-    "quant_method": "auto-round",
-    "bits": 4,
-    "group_size": 128,
-    "sym": true,
-    "packing_format": "auto_round:auto_gptq",
-    "block_name_to_quantize": "transformer_blocks,single_transformer_blocks"
-  }
-}
-```
-
-At load time:
-
-1. `TransformerConfig.from_dict()` parses the `quantization_config` section and builds a vLLM `INCConfig` via `build_quant_config("auto-round", ...)`.
-2. `OmniDiffusionConfig.set_tf_model_config()` propagates the detected config to the engine.
-3. The appropriate compute kernel (e.g. GPTQ-Marlin for W4A16) is selected automatically based on the checkpoint's bit-width and packing format.
-
-## Supported Models
-
-| Model | HF Checkpoint | Scheme | Group Size | Backend |
-|-------|--------------|--------|------------|---------|
-| FLUX.1-dev | `vllm-project-org/FLUX.1-dev-AutoRound-w4a16` | W4A16 | 128 | GPTQ-Marlin |
-| Qwen2.5-Omni-7B | `Intel/Qwen2.5-Omni-7B-int4-AutoRound` | W4A16 | 128 | GPTQ-Marlin |
-| Qwen3-Omni-30B-A3B-Instruct | `Intel/Qwen3-Omni-30B-A3B-Instruct-int4-AutoRound` | W4A16 | 128 | GPTQ-Marlin |
-
-## Creating a Quantized Checkpoint
-
-Use [AutoRound](https://github.com/intel/auto-round) to quantize a BF16 model. The `--scheme` flag selects the quantization scheme:
-
-```bash
-# W4A16 (4-bit weight, 16-bit activation)
-auto-round \
-    --model black-forest-labs/FLUX.1-dev \
-    --scheme W4A16 \
-    --batch_size 1 \
-    --disable_opt_rtn \
-    --dataset coco2014 \
-    --iters 0
-```
-
-The output directory can be used directly as the `model` argument. See the [AutoRound documentation](https://github.com/intel/auto-round) for all available schemes and options.

docs/user_guide/diffusion/quantization/fp8.md

@@ -1,82 +0,0 @@
-# FP8 Quantization
-
-## Overview
-
-FP8 quantization converts BF16/FP16 weights to FP8 at model load time. No calibration or pre-quantized checkpoint needed.
-
-Depending on the model, either all layers can be quantized, or some sensitive layers should stay in BF16. See the [per-model table](#supported-models) for which case applies.
-
-Common sensitive layers in DiT-based diffusion models include **image-stream MLPs** (`img_mlp`). These are particularly vulnerable to FP8 precision loss because they process denoising latents whose dynamic range shifts significantly across timesteps, and unlike attention projections (which benefit from QK-Norm stabilization), MLPs have no built-in normalization to absorb quantization error. In deep architectures (e.g., 60+ residual blocks), small per-layer errors compound and degrade output quality. Other layers such as **attention projections** (`to_qkv`, `to_out`) and **text-stream MLPs** (`txt_mlp`) are generally more robust due to normalization or more stable input statistics.
-
-## Configuration
-
-1. **Python API**: set `quantization="fp8"`. To skip sensitive layers, use `quantization_config` with `ignored_layers`.
-
-```python
-from vllm_omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-
-# All layers quantized
-omni = Omni(model="<your-model>", quantization="fp8")
-
-# Skip sensitive layers
-omni = Omni(
-    model="<your-model>",
-    quantization_config={
-        "method": "fp8",
-        "ignored_layers": ["<layer-name>"],
-    },
-)
-
-outputs = omni.generate(
-    "A cat sitting on a windowsill",
-    OmniDiffusionSamplingParams(num_inference_steps=50),
-)
-```
-
-2. **CLI**: pass `--quantization fp8` and optionally `--ignored-layers`.
-
-```bash
-# All layers
-python text_to_image.py --model <your-model> --quantization fp8
-
-# Skip sensitive layers
-python text_to_image.py --model <your-model> --quantization fp8 --ignored-layers "img_mlp"
-
-# Online serving
-vllm serve <your-model> --omni --quantization fp8
-```
-
-| Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
-| `method` | str | — | Quantization method (`"fp8"`) |
-| `ignored_layers` | list[str] | `[]` | Layer name patterns to keep in BF16 |
-| `activation_scheme` | str | `"dynamic"` | `"dynamic"` (no calibration) or `"static"` |
-| `weight_block_size` | list[int] \| None | `None` | Block size for block-wise weight quantization |
-
-The available `ignored_layers` names depend on the model architecture (e.g., `to_qkv`, `to_out`, `img_mlp`, `txt_mlp`). Consult the transformer source for your target model.
-
-## Supported Models
-
-| Model | HF Models | Recommendation | `ignored_layers` |
-|-------|-----------|---------------|------------------|
-| Z-Image | `Tongyi-MAI/Z-Image-Turbo` | All layers | None |
-| Qwen-Image | `Qwen/Qwen-Image`, `Qwen/Qwen-Image-2512` | Skip sensitive layers | `img_mlp` |
-| Flux | `black-forest-labs/FLUX.1-dev` | All layers | None |
-| HunyuanImage-3 | `tencent/HunyuanImage3` | All layers | None |
-| HunyuanVideo-1.5 | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `720p_t2v`, `480p_i2v` | All layers | None |
-| GLM-Image | `zai-org/GLM-Image` | All layers | None |
-| Helios | `BestWishYsh/Helios-Base`, `BestWishYsh/Helios-Mid`, `BestWishYsh/Helios-Distilled` | All layers | None |
-
-## Combining with Other Features
-
-FP8 quantization can be combined with cache acceleration:
-
-```python
-omni = Omni(
-    model="<your-model>",
-    quantization="fp8",
-    cache_backend="tea_cache",
-    cache_config={"rel_l1_thresh": 0.2},
-)
-```

docs/user_guide/diffusion/quantization/gguf.md

@@ -1,227 +0,0 @@
-# GGUF Quantization
-
-## Goals
-1. Reuse vLLM quantization configs and weight loaders as much as possible.
-2. Add native GGUF support to diffusion transformers without changing model definitions.
-3. Keep user-facing knobs minimal and consistent across offline and online flows.
-
-## Scope
-1. Models: Z-Image, Flux2-klein, and Qwen-Image family.
-2. Components: diffusion transformer weights, loader paths, and quantization configs.
-3. Modes: native GGUF (transformer-only weights).
-
-## Architecture Overview
-1. `OmniDiffusionConfig` accepts `quantization` or `quantization_config`.
-2. Diffusion quantization wrapper (`DiffusionGGUFConfig`) produces vLLM `QuantizationConfig` objects for linear layers.
-3. `DiffusersPipelineLoader` branches on quantization method and loads either HF weights or GGUF weights for the transformer.
-4. GGUF transformer loading is routed through model-specific adapters (e.g., Flux2Klein).
-5. vLLM GGUF path uses `GGUFConfig` and `GGUFLinearMethod` for matmul.
-
-## Call Chain (Offline)
-```
-CLI (examples/offline_inference/text_to_image/text_to_image.py)
-  |
-  v
-Omni (vllm_omni/entrypoints/__init__.py)
-  |
-  v
-AsyncOmniEngine
-  |
-  v
-Orchestrator
-  |
-  v
-StageDiffusionClient
-  |
-  v
-DiffusionWorker
-  |
-  v
-DiffusionModelRunner
-  |
-  v
-DiffusersPipelineLoader
-  |
-  v
-Pipeline.forward (Flux2/Qwen/Z-Image)
-  |
-  v
-DiffusionEngine
-  |
-  v
-OmniRequestOutput
-  |
-  v
-Client (saved PNG)
-```
-
-## Call Chain (Online)
-```
-Client
-  |
-  | POST /v1/images/generations
-  v
-APIServer (vllm_omni/entrypoints/openai/api_server.py)
-  |
-  v
-_generate_with_async_omni
-  |
-  v
-AsyncOmni
-  |
-  v
-AsyncOmniEngine
-  |
-  v
-Orchestrator
-  |
-  v
-StageDiffusionClient
-  |
-  v
-OmniRequestOutput
-  |
-  v
-encode_image_base64
-  |
-  v
-ImageGenerationResponse
-  |
-  v
-Client
-```
-
-## Call Chain (GGUF Operator Path)
-```
-Pipeline.forward (Flux2/Qwen/Z-Image)
-  |
-  v
-Transformer blocks
-  |
-  v
-QKVParallelLinear / ColumnParallelLinear / RowParallelLinear
-  |
-  v
-LinearBase.forward
-  |
-  v
-QuantMethod.apply (GGUFLinearMethod.apply)
-  |
-  v
-fused_mul_mat_gguf
-  |
-  v
-_fused_mul_mat_gguf (custom op)
-  |
-  v
-ops.ggml_dequantize
-  |
-  v
-x @ weight.T
-```
-
-## GGUF Weight Loading Path (Transformer-Only)
-1. `DiffusersPipelineLoader.load_model` detects `quantization_config.method == "gguf"`.
-2. `gguf_model` is resolved as one of: local file, `repo/file.gguf`, or `repo:quant_type`.
-3. GGUF weights are routed through adapters in `vllm_omni/diffusion/model_loader/gguf_adapters/`.
-4. Name mapping is applied per-architecture (Z-Image, Flux2Klein, Qwen-Image).
-5. GGUF weights are loaded into transformer modules, remaining non-transformer weights come from the HF checkpoint.
-
-## GGUF Adapter Design
-1. `GGUFAdapter` is an abstract base class for model-specific adapters.
-2. `Flux2KleinGGUFAdapter` implements Flux2-Klein remapping + qkv split + adaLN swap.
-3. `ZImageGGUFAdapter` implements Z-Image qkv + ffn shard handling and linear qweight routing.
-4. `QwenImageGGUFAdapter` implements Qwen-Image fused attention projection remapping.
-5. `get_gguf_adapter(...)` strictly selects by model class/config; unsupported models raise an error (no fallback adapter).
-
-Adapter paths:
-- Base: `vllm_omni/diffusion/model_loader/gguf_adapters/base.py`
-- Z-Image: `vllm_omni/diffusion/model_loader/gguf_adapters/z_image.py`
-- Flux2-Klein: `vllm_omni/diffusion/model_loader/gguf_adapters/flux2_klein.py`
-- Qwen-Image: `vllm_omni/diffusion/model_loader/gguf_adapters/qwen_image.py`
-
-## User Usage (Offline)
-
-### Baseline BF16
-```bash
-python examples/offline_inference/text_to_image/text_to_image.py \
-  --model /workspace/models/black-forest-labs/FLUX.2-klein-4B \
-  --prompt "a photo of a forest with mist swirling around the tree trunks. The word 'FLUX.2' is painted over it in big, red brush strokes with visible texture" \
-  --height 768 \
-  --width 1360 \
-  --seed 42 \
-  --cfg-scale 4.0 \
-  --num-images-per-prompt 1 \
-  --num-inference-steps 4 \
-  --output outputs/flux2_klein_4b.png
-```
-
-### Native GGUF (Transformer Only)
-```bash
-python examples/offline_inference/text_to_image/text_to_image.py \
-  --model /workspace/models/black-forest-labs/FLUX.2-klein-4B \
-  --gguf-model "/workspace/models/unsloth/FLUX.2-klein-4B-GGUF/flux-2-klein-4b-Q8_0.gguf" \
-  --quantization gguf \
-  --prompt "a photo of a forest with mist swirling around the tree trunks. The word 'FLUX.2' is painted over it in big, red brush strokes with visible texture" \
-  --height 768 \
-  --width 1360 \
-  --seed 42 \
-  --cfg-scale 4.0 \
-  --num-images-per-prompt 1 \
-  --num-inference-steps 4 \
-  --output outputs/flux2_klein_4b_gguf.png
-```
-
-Notes for GGUF:
-1. Many GGUF repos do not ship `model_index.json` and configs. Use the base repo for `--model` and only pass the GGUF file via `--gguf-model`.
-2. `gguf_model` supports local path, `repo/file.gguf`, or `repo:quant_type`.
-
-## User Usage (Online)
-
-### Start Server (Native GGUF via CLI)
-```bash
-vllm serve /workspace/models/black-forest-labs/FLUX.2-klein-4B \
-  --omni \
-  --port 8000 \
-  --quantization-config '{"method":"gguf","gguf_model":"/workspace/models/unsloth/FLUX.2-klein-4B-GGUF/flux-2-klein-4b-Q8_0.gguf"}'
-```
-
-### Online Request (Images API)
-```bash
-curl -X POST http://localhost:8000/v1/images/generations \
-  -H "Content-Type: application/json" \
-  -d '{
-    "prompt": "a dragon laying over the spine of the Green Mountains of Vermont",
-    "size": "1024x1024",
-    "seed": 42,
-    "num_inference_steps": 4
-  }'
-```
-
-## Qwen-Image Example
-
-### Offline
-```bash
-python examples/offline_inference/text_to_image/text_to_image.py \
-  --model Qwen/Qwen-Image \
-  --gguf-model QuantStack/Qwen-Image-GGUF/Qwen_Image-Q4_K_M.gguf \
-  --quantization gguf \
-  --prompt "a red paper kite hanging from a pine tree in a winter courtyard" \
-  --height 1024 \
-  --width 1024 \
-  --seed 42 \
-  --num_inference_steps 20 \
-  --output outputs/qwen_image_q4km.png
-```
-
-### Online
-```bash
-vllm serve Qwen/Qwen-Image \
-  --omni \
-  --port 8000 \
-  --quantization-config '{"method":"gguf","gguf_model":"QuantStack/Qwen-Image-GGUF/Qwen_Image-Q4_K_M.gguf"}'
-```
-
-Notes:
-1. The GGUF repo provides transformer weights only; the base model repo still provides tokenizer, text encoder, scheduler, and VAE.
-2. The same adapter path covers `QwenImagePipeline`, `QwenImageEditPipeline`, `QwenImageEditPlusPipeline`, and `QwenImageLayeredPipeline`.

docs/user_guide/diffusion/quantization/int8.md

@@ -1,75 +0,0 @@
-# Int8 Quantization
-
-## Overview
-
-Int8 quantization converts BF16/FP16 weights to Int8 at model load time. No calibration or pre-quantized checkpoint needed.
-
-Depending on the model, either all layers can be quantized, or some sensitive layers should stay in BF16/FP16. See the [per-model table](#supported-models) for which case applies.
-
-## Configuration
-
-1. **Python API**: set `quantization="int8"`. To skip sensitive layers, use `quantization_config` with `ignored_layers`.
-
-```python
-from vllm_omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-
-# All layers quantized
-omni = Omni(model="<your-model>", quantization="int8")
-
-# Skip sensitive layers
-omni = Omni(
-    model="<your-model>",
-    quantization_config={
-        "method": "int8",
-        "ignored_layers": ["<layer-name>"],
-    },
-)
-
-outputs = omni.generate(
-    "A cat sitting on a windowsill",
-    OmniDiffusionSamplingParams(num_inference_steps=50),
-)
-```
-
-2. **CLI**: pass `--quantization int8` and optionally `--ignored-layers`.
-
-```bash
-# All layers
-python text_to_image.py --model <your-model> --quantization int8
-
-# Skip sensitive layers
-python text_to_image.py --model <your-model> --quantization int8 --ignored-layers "img_mlp"
-
-# Online serving
-vllm serve <your-model> --omni --quantization int8
-```
-
-| Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
-| `method` | str | — | Quantization method (`"int8"`) |
-| `ignored_layers` | list[str] | `[]` | Layer name patterns to keep in BF16/FP16 |
-| `activation_scheme` | str | `"dynamic"` | `"dynamic"` (no calibration) |
-
-
-The available `ignored_layers` names depend on the model architecture (e.g., `to_qkv`, `to_out`, `img_mlp`, `txt_mlp`). Consult the transformer source for your target model.
-
-## Supported Models
-
-| Model | HF Models | Recommendation | `ignored_layers` |
-|-------|-----------|---------------|------------------|
-| Z-Image | `Tongyi-MAI/Z-Image-Turbo` | All layers | None |
-| Qwen-Image | `Qwen/Qwen-Image`, `Qwen/Qwen-Image-2512` | All layers | None |
-
-## Combining with Other Features
-
-Int8 quantization can be combined with cache acceleration:
-
-```python
-omni = Omni(
-    model="<your-model>",
-    quantization="int8",
-    cache_backend="tea_cache",
-    cache_config={"rel_l1_thresh": 0.2},
-)
-```

docs/user_guide/diffusion/quantization/msmodelslim.md

@@ -1,56 +0,0 @@
-# msModelSlim Quantization
-
-## Overview
-
-[msModelSlim](https://github.com/Ascend/msmodelslim) is an Ascend-friendly compression tool focused on acceleration, using compression techniques, and built for Ascend hardware. It includes a series of inference optimization technologies such as quantization and compression, aiming to accelerate large language dense models, MoE models, multimodal understanding models, multimodal generation models, etc.
-
-Once you have a quantized model which is generated by **msModelSlim**, you can use vLLM Omni for inference by specifying the --quantization ascend parameter to enable quantization features.
-
-### Supported Schemes
-
-| Scheme | Bits | Status |
-|--------|------|--------|
-| W8A8 | 8 | ✅ Supported |
-| W4A4 | 4 | Planned |
-
-W8A8 is the first supported scheme. Additional schemes will be added in future releases.
-
-## Model Quantization
-
-The following example shows how to generate W8A8 quantized weights for the [Wan2_2 model](https://gitcode.com/Ascend/msmodelslim/blob/master/example/multimodal_sd/Wan2_2/README.md).
-
-**Quantization Script:**
-
-```bash
-msmodelslim quant \
-  --model_path /path/to/wan2_2_t2v_float_weights \
-  --save_path /path/to/wan2_2_t2v_quantized_weights \
-  --device npu \
-  --model_type Wan2_2 \
-  --config_path /lab_practice/wan2_2/wan2_2_w8a8f8_mxfp_t2v.yaml \
-  --trust_remote_code True
-```
-
-After quantization completes, the output directory will contain the quantized model files.
-
-For more examples, refer to the [official examples](https://gitcode.com/Ascend/msit/tree/master/msmodelslim/example).
-
-## Configuration
-
-1. **CLI**: pass `--quantization ascend`.
-
-```bash
-# Offline inference
-python text_to_image.py --model <your-model> --quantization ascend
-
-# Online serving
-vllm serve <your-model> --omni --quantization ascend
-```
-
-## Supported Models
-
-| Model | HF Models | Recommendation | `ignored_layers` |
-|-------|-----------|---------------|------------------|
-| HunyuanImage-3.0 | - | All layers | None |
-
-Currently, quantized HunyuanImage-3.0 weights have not been uploaded to public model platforms such as Hugging Face. You can use a [HunyuanImage-3.0-adapted msModelSlim version](https://gitcode.com/betta18/msmodelslim/tree/hyimage3_mxfp8) to generate the quantized weights manually. We will upload the quantized weights as soon as possible.

docs/user_guide/diffusion/quantization/overview.md

@@ -1,153 +0,0 @@
-# Quantization
-
-vLLM-Omni provides a unified quantization framework that supports both diffusion models (DiT, text encoders, VAEs) and multi-stage omni models (e.g., Qwen3-Omni thinker). Quantization reduces memory usage and accelerates inference.
-
-## Supported Methods
-
-### Diffusion Models
-
-| Method | Guide | Description | Tested Models | Min GPU |
-|--------|-------|-------------|---------------|---------|
-| FP8 | [FP8](fp8.md) | FP8 W8A8, dynamic or static | Z-Image, Qwen-Image, Flux, Bagel | SM 89 (Ada) |
-| Int8 | [Int8](int8.md) | Int8 W8A8 | Z-Image, Qwen-Image | SM 89 (Ada) / Ascend NPU |
-| GGUF | [GGUF](gguf.md) | GGUF format, dequant+GEMM for N-D tensors | Z-Image, Flux | SM 60 |
-| AutoRound | [AutoRound](autoround.md) | W4A16 (pre-quantized) | Flux | SM 80 (Ampere) |
-
-### Multi-stage Omni Models (Pre-quantized Checkpoints)
-
-| Method | Description | Tested Models | Min GPU |
-|--------|-------------|---------------|---------|
-| ModelOpt FP8 | Pre-quantized FP8 via NVIDIA ModelOpt | Qwen3-Omni (thinker) | SM 89 (Ada/Hopper) |
-| ModelOpt NVFP4 | Pre-quantized NVFP4 via NVIDIA ModelOpt | Qwen3-Omni (experimental, quality issues) | SM 100 (Blackwell) |
-
-!!! note
-    AWQ, GPTQ, and BitsAndBytes are supported by vLLM's upstream quantization registry but have **not been tested** with vLLM-Omni pipelines. They may work via `build_quant_config()` but are not validated.
-
-### Pre-quantized LLM Checkpoints (Multi-stage Models)
-
-For multi-stage models like Qwen3-Omni, the unified quantization framework auto-detects
-pre-quantized checkpoints via `quantization_config` in the HF config. Quantization is
-automatically scoped to the thinker's `language_model` — audio encoder, vision encoder,
-talker, and code2wav remain in BF16.
-
-| Format | `quant_algo` | Hardware | Example | Status |
-|--------|-------------|----------|---------|--------|
-| ModelOpt FP8 | `FP8` | Ada/Hopper (SM 89+) | `asdazd/Qwen3-Omni-30B-A3B-Instruct_modelopt_FP8` | Tested — 47% memory reduction, comparable throughput |
-| ModelOpt NVFP4 | `NVFP4` | Blackwell (SM 100+) | `shunyang90/Qwen3-Omni-30B-A3B-Instruct-NVFP4` | Experimental — loads and runs but output quality is unacceptable |
-
-**Tested FP8 results (Qwen3-Omni, 1×H200):**
-
-| Config | Model Memory (GiB) | Mem Reduction | Decode (tok/s) |
-|--------|-------------------|---------------|----------------|
-| BF16 baseline | 59.26 | — | 41.6 |
-| FP8 (ModelOpt) | 31.41 | 47% | 39.9 |
-
-FP8 enables the full 3-stage pipeline (thinker + talker + code2wav) on a single 64GB GPU,
-which is impossible with BF16 (thinker alone requires 59.26 GiB).
-
-## Quantization Scope
-
-### Diffusion Models
-
-When `--quantization fp8` is enabled for diffusion models:
-
-| Component | What Gets Quantized | Mechanism |
-|-----------|-------------------|-----------|
-| **DiT (transformer)** | `nn.Linear` layers | vLLM W8A8 FP8 compute (Ada/Hopper) or weight-only (older GPUs) |
-| **Text encoder** | `nn.Linear` layers | FP8 weight storage, BF16 compute |
-| **VAE** | `nn.Conv2d`, `nn.Conv3d` layers | FP8 weight storage, BF16 compute |
-
-### Multi-stage Omni Models
-
-For pre-quantized omni model checkpoints, quantization is scoped per component:
-
-| Component | Quantized? | Notes |
-|-----------|-----------|-------|
-| **Thinker (language_model)** | Yes | FP8 or NVFP4 via ModelOpt |
-| **Audio encoder** | No | Stays BF16 |
-| **Vision encoder** | No | Stays BF16 |
-| **Talker** | No | Stays BF16 |
-| **Code2Wav** | No | Stays BF16 |
-
-!!! note
-    Not all models support all components. See the [FP8 supported models table](fp8.md#supported-models) for per-model details.
-
-## Device Compatibility for FP8
-
-| Device | Example Hardware | FP8 | NVFP4 |
-|--------|-----------------|-----|-------|
-| Blackwell GPU (SM 100+) | RTX 5090, B100, B200 | Yes | Yes (native FP4 HW) |
-| Ada/Hopper GPU (SM 89+) | RTX 4090, H100, H200 | Yes (W8A8 native) | No |
-| Turing/Ampere GPU (SM 75-86) | RTX 3090, A100 | Yes (weight-only Marlin) | No |
-| Ascend NPU | Atlas 800T A2 (910B) | Not yet supported | No |
-
-Kernel selection is automatic on CUDA GPUs.
-
-## Device Compatibility for Int8
-
-| Device Type | Generation | Example | Int8 Mode |
-|-------------|---------------|-------------------|----------|
-| NVIDIA GPU | Ada/Hopper (SM 89+) | RTX 4090, H100, H200 | Full W8A8 with native hardware |
-| Ascend NPU | Atlas A2/Atlas A3 | Atlas 800T A2/Atlas 900 A3 | Full W8A8 with native hardware |
-
-## Python API
-
-The `build_quant_config()` factory accepts multiple input formats:
-
-```python
-from vllm_omni.quantization import build_quant_config
-
-# String
-config = build_quant_config("fp8")
-
-# Dict with parameters
-config = build_quant_config({"method": "fp8", "activation_scheme": "static"})
-
-# AutoRound / INC (auto-detected from checkpoint, or explicit)
-config = build_quant_config("auto-round", bits=4, group_size=128)
-
-# Per-component dict
-config = build_quant_config({
-    "transformer": {"method": "fp8"},
-    "vae": None,
-})
-
-# None / "none"
-config = build_quant_config(None)  # returns None
-```
-
-## Architecture
-
-```
-build_quant_config(spec)
-    ├── str  → _build_single(method) → vLLM registry / _OVERRIDES
-    ├── dict with "method" → _build_single(method, **kwargs)
-    ├── per-component dict → ComponentQuantizationConfig
-    │       routes get_quant_method() by longest-prefix match
-    ├── QuantizationConfig → passthrough
-    └── None / "none" → None
-```
-
-## Migration Guide
-
-### Before (v0.14.x)
-
-```python
-# Old diffusion-specific API (removed)
-from vllm_omni.diffusion.quantization import get_diffusion_quant_config
-config = get_diffusion_quant_config("fp8", activation_scheme="static")
-```
-
-### After (v0.16.0+)
-
-```python
-# Unified API — delegates to vLLM's registry
-from vllm_omni.quantization import build_quant_config
-config = build_quant_config({"method": "fp8", "activation_scheme": "static"})
-
-# Per-component (new)
-config = build_quant_config({
-    "transformer": {"method": "fp8"},
-    "vae": None,
-})
-```

docs/user_guide/diffusion/step_execution.md

@@ -8,6 +8,8 @@ implement the stepwise contract support it today.
 
 ## Quick Start
 
+### Python API
+
 ```python
 from vllm_omni import Omni
 from vllm_omni.inputs.data import OmniDiffusionSamplingParams
@@ -25,6 +27,18 @@ outputs = omni.generate(
 )
 ```
 
+### Serving
+
+```bash
+vllm serve Qwen/Qwen-Image --omni \
+  --port 8091 \
+  --step-execution \
+  --max-num-seqs 8
+```
+
+For serving, `--step-execution` enables the step-wise runtime. Continuous
+batching only becomes relevant when `--max-num-seqs > 1`.
+
 ## Supported Pipelines
 
 | Pipeline | Example models | Step execution |
@@ -32,9 +46,17 @@ outputs = omni.generate(
 | `QwenImagePipeline` | `Qwen/Qwen-Image`, `Qwen/Qwen-Image-2512` | Yes |
 | All other diffusion pipelines | `QwenImageEditPipeline`, `QwenImageEditPlusPipeline`, `QwenImageLayeredPipeline`, GLM-Image, Wan, Flux, etc. | No |
 
+!!! warning "Experimental continuous batching"
+    When `--step-execution` is enabled and `max_num_seqs > 1` is configured,
+    the step-wise path can batch
+    compatible requests together. This is experimental. Requests with
+    incompatible sampling parameters are intentionally kept in separate batches,
+    and `max_num_seqs=1` remains the conservative default.
+
 ## Current Limitations
 
-- `step_execution` currently supports `batch_size=1` only.
+- Continuous batching under `step_execution` is experimental and only batches
+  compatible requests.
 - `cache_backend` is not supported together with step execution.
 - Unsupported pipelines fail early during model loading.
 - Request-mode extras such as KV transfer are not wired into step mode yet.
@@ -45,6 +67,11 @@ Use step execution only when you specifically need the pipeline to run through
 its stepwise request state machine. For normal diffusion inference, leave it
 disabled unless your workflow depends on this mode.
 
+For Qwen-Image online serving, the usual progression is:
+
+- start with `--step-execution --max-num-seqs 1` if you only need the step-wise path
+- increase `--max-num-seqs` after that if you want the experimental compatible-request batching behavior
+
 If you are looking for general diffusion speedups, see
 [Diffusion Features Overview](../diffusion_features.md).
 
@@ -59,3 +86,7 @@ pipeline does not support step execution.
 If you want to add step execution support to a new diffusion pipeline, see the
 implementation guide:
 [Diffusion Step Execution Design](../../design/feature/diffusion_step_execution.md).
+
+If you also want that pipeline to participate in the experimental batched
+step-wise path, see:
+[Continuous Batching for Step-Wise Diffusion](../../design/feature/diffusion_continuous_batching.md).

docs/user_guide/diffusion_features.md

@@ -59,7 +59,7 @@ Memory optimization methods help reduce GPU memory usage, enabling inference on
 | Method | Description | Best For |
 |--------|-------------|----------|
 | **[CPU Offload](diffusion/cpu_offload_diffusion.md)** | Offloads model components to CPU memory | Limited VRAM, large models on consumer GPUs |
-| **[Quantization](diffusion/quantization/overview.md)** | Reduces DiT layers from BF16 to FP8/INT8/etc. | Limited VRAM, minimal accuracy loss    |
+| **[Quantization](quantization/overview.md)** | Reduces transformer stages from BF16 to FP8/INT8/etc. | Limited VRAM, minimal accuracy loss    |
 | **[VAE Patch Parallelism](diffusion/parallelism/vae_patch_parallel.md)** | Distributes VAE decode tiling across GPUs | High-resolution generation with reduced VAE memory peak |
 
 ### Extensions
@@ -86,9 +86,9 @@ Execution modes control how the diffusion pipeline processes denoise steps.
 
 | Method | Configuration | Description | Best For |
 |--------|--------------|-------------|----------|
-| **[FP8](diffusion/quantization/fp8.md)** | `quantization="fp8"` | FP8 W8A8 on Ada/Hopper, weight-only on older GPUs | Memory reduction, inference speedup |
-| **[INT8](diffusion/quantization/int8.md)** | `quantization="int8"` | INT8 weight-only, no calibration or pre-quantized checkpoint needed | Memory reduction, broad GPU compatibility |
-| **[GGUF](diffusion/quantization/gguf.md)** | `quantization="gguf"` | Native GGUF transformer-only weights (Q4, Q8, etc.) | Memory reduction on consumer GPUs |
+| **[FP8](quantization/fp8.md)** | `quantization="fp8"` | FP8 W8A8 on validated transformer stages | Memory reduction, inference speedup |
+| **[INT8](quantization/int8.md)** | `quantization="int8"` | INT8 W8A8 on validated transformer stages | Memory reduction, broad GPU compatibility |
+| **[GGUF](quantization/gguf.md)** | `quantization="gguf"` | Native GGUF transformer-only weights (Q4, Q8, etc.) | Memory reduction on consumer GPUs |
 
 ## Supported Models
 
@@ -101,13 +101,13 @@ The following tables show which models support each feature:
 > Notes:
 
 > 1. CPU Offload has two methods: Module-wise (default for models with DiT + text encoder) and Layerwise. The tables below show **Layerwise support** only.
-> 2. The **💾Quantization** column is collapsed for readability. See [Quantization Overview](diffusion/quantization/overview.md) for per-method (FP8, GGUF, …) and per-model support details.
+> 2. The **💾Quantization** column is collapsed for readability. See [Quantization Overview](quantization/overview.md) for per-method and per-model support details.
 
 ### ImageGen
 
 | Model | ⚡TeaCache | ⚡Cache-DiT | 🔀SP (Ulysses & Ring) | 🔀CFG-Parallel | 🔀Tensor-Parallel | 🔀HSDP | 💾CPU Offload (Layerwise) | 💾VAE-Patch-Parallel | 💾Quantization | 🔄Step Execution |
 |-------|:----------:|:-----------:|:---------------------:|:--------------:|:-----------------:|:------:|:------------------------:|:--------------------:|:--------------:|:----------------:|
-| **Bagel** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅  | ❌ | ❌ | ❌ |
+| **Bagel** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅  | ❌ | ❌ | ❌ |
 | **FLUX.1-dev** | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ |
 | **FLUX.1-schnell** | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ |
 | **FLUX.2-klein** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ |
@@ -127,8 +127,10 @@ The following tables show which models support each feature:
 | **Qwen-Image-Edit** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ |
 | **Qwen-Image-Edit-2509** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ✅ | ❌ | ❌ |
 | **Qwen-Image-Layered** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ |
+| **SenseNova-U1** | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ |
 | **Stable-Diffusion3.5** | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ (decode) | ❌ | ❌ |
 | **Z-Image** | ✅ | ✅ | ✅ | ❓ | ✅ (TP=2 only) | ✅ | ❌ | ✅ (decode) | ✅ | ❌ |
+| **ERNIE-Image** | ❌ | ✅ | ✅ | ❓ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
 
 > Notes:
 > 1. Nextstep_1(T2I) does not support cache acceleration methods such as TeaCache or Cache-DiT.
@@ -142,9 +144,9 @@ The following tables show which models support each feature:
 | **Wan2.1-VACE** | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ |
 | **LTX-2** | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
 | **LTX-2.3** | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
-| **Helios** | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
+| **Helios** | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
 | **HunyuanVideo-1.5 T2V I2V** | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ✅ | ❌ |
-| **DreamID-Omni** | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
+| **DreamID-Omni** | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
 
 **Frame Interpolation Support**
 
@@ -155,7 +157,7 @@ The following tables show which models support each feature:
 
 | Model | ⚡TeaCache | ⚡Cache-DiT | 🔀SP (Ulysses & Ring) | 🔀CFG-Parallel | 🔀Tensor-Parallel | 🔀HSDP | 💾CPU Offload (Layerwise) | 💾VAE-Patch-Parallel | 💾Quantization | 🔄Step Execution |
 |-------|:----------:|:-----------:|:---------------------:|:--------------:|:-----------------:|:------:|:------------------------:|:--------------------:|:--------------:|:----------------:|
-| **Stable-Audio-Open** | ✅ | ❌ | ❓ | ❓ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ |
+| **Stable-Audio-Open** | ✅ | ❌ | ❓ | ❓ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ |
 
 
 ## Feature Compatibility
@@ -261,7 +263,7 @@ Measured on NVIDIA H800:
 
 - **[CPU Offload Guide](diffusion/cpu_offload_diffusion.md)** - Offload model components to CPU, reduce GPU memory usage
 - **[VAE Patch Parallelism Guide](diffusion/parallelism/vae_patch_parallel.md)** - Distribute VAE decode tiling across GPUs for high-resolution images
-- **[Quantization Overview](diffusion/quantization/overview.md)** - Overview of quantization methods for diffusion models
+- **[Quantization Overview](quantization/overview.md)** - Overview of quantization methods for diffusion, multi-stage omni/TTS, and multi-stage diffusion models
 
 **Extensions:**
 

docs/user_guide/examples/offline_inference/cosyvoice3.md

@@ -1,81 +0,0 @@
-# CosyVoice3
-
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/cosyvoice3>.
-
-
-## Setup
-
-Install dependencies:
-```
-uv pip install -e .
-```
-
-> **Note:** This includes required libraries such as `soundfile`,
-> `onnxruntime`, `x-transformers`, and `einops` via
-> `requirements/common.txt` and platform-specific requirements files.
-
-Download the model snapshot:
-```
-from huggingface_hub import snapshot_download
-snapshot_download('FunAudioLLM/Fun-CosyVoice3-0.5B-2512', local_dir='pretrained_models/Fun-CosyVoice3-0.5B')
-```
-
-Add `config.json` in `pretrained_models/Fun-CosyVoice3-0.5B/`:
-```json
-{
-    "model_type": "cosyvoice3",
-    "architectures": [
-        "CosyVoice3Model"
-    ]
-}
-```
-
-> **Why `config.json` is required:**
-> `AutoConfig.register("cosyvoice3", CosyVoice3Config)` only registers a class mapping.
-> The loader still needs `model_type: "cosyvoice3"` from `config.json` to select that class.
-> If no `config.json` is present, model type cannot be inferred automatically.
-> If your downloaded checkpoint already includes a valid `config.json` with
-> `model_type: "cosyvoice3"`, this manual step can be skipped.
-
-Run the offline verification script:
-```
-python examples/offline_inference/cosyvoice3/verify_e2e_cosyvoice.py \
-  --model pretrained_models/Fun-CosyVoice3-0.5B \
-  --tokenizer pretrained_models/Fun-CosyVoice3-0.5B/CosyVoice-BlankEN
-```
-
-## Implementation Overview
-
-CosyVoice3 runs as a **2-stage Omni pipeline**.
-
-- **Stage 0** (`talker`) converts text + prompt audio to speech tokens.
-- **Stage 1** (`code2wav`) converts speech tokens + prompt features to waveform via flow matching and HiFiGAN.
-
-Key components live in `vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py`.
-
-- `CosyVoice3MultiModalProcessor` builds the multimodal inputs:
-    - Tokenizes `prompt` and `prompt_text`.
-    - Extracts speech tokens and mel features from the prompt audio.
-    - Extracts a speaker embedding.
-- `CosyVoice3Model` implements both stages:
-    - Stage 0 uses `CosyVoice3LM` and outputs speech tokens + conditioning features.
-    - Stage 1 runs the flow model (DiT-based CFM) and HiFiGAN to synthesize waveform.
-
-Pipeline topology lives in `vllm_omni/model_executor/models/cosyvoice3/pipeline.py`;
-runtime tunables (batch size, memory limits, sampling) live in
-`vllm_omni/deploy/cosyvoice3.yaml`. The deploy config auto-loads by
-HF `model_type` and defaults to `async_chunk: true` (shared-memory
-streaming). Pass `--no-async-chunk` on `vllm serve` to switch to the
-legacy sync path where stage 1 runs `text2flow` over the full
-speech-token sequence.
-
-- Stage 0 emits latent speech tokens.
-- Stage 1 consumes them via `sync_process_input_func` (sync mode) or the
-  shared-memory connector (async-chunk mode) and outputs audio.
-
-## Example materials
-
-??? abstract "verify_e2e_cosyvoice.py"
-    ``````py
-    --8<-- "examples/offline_inference/cosyvoice3/verify_e2e_cosyvoice.py"
-    ``````

docs/user_guide/examples/offline_inference/fish_speech.md

@@ -1,85 +0,0 @@
-# Fish Speech S2 Pro
-
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/fish_speech>.
-
-
-This directory contains an offline demo for running Fish Speech S2 Pro (`fishaudio/s2-pro`) with vLLM Omni. It supports text-only synthesis and voice cloning with reference audio.
-
-## Model Overview
-
-[Fish Speech S2 Pro](https://huggingface.co/fishaudio/s2-pro) is a 4B dual-AR text-to-speech model by FishAudio. It uses:
-
-- **Slow AR**: Generates semantic tokens autoregressively (Qwen3-based backbone)
-- **Fast AR**: Predicts residual codebook tokens from semantic tokens
-- **DAC Decoder**: Converts codec codes to 44.1 kHz audio waveform
-
-The model produces high-quality speech with voice cloning capabilities.
-
-## Prerequisites
-
-Install the `fish-speech` package for the DAC codec:
-
-```bash
-pip install fish-speech
-```
-
-## Setup
-
-Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup.
-
-## Quick Start
-
-### Text-Only Synthesis
-
-```bash
-python end2end.py --text "Hello, this is a test of the Fish Speech text to speech system."
-```
-
-Generated audio files are saved to `output_audio/` by default.
-
-### Voice Cloning
-
-Provide a reference audio file and its transcript to clone a voice:
-
-```bash
-python end2end.py \
-    --text "Hello, this is a cloned voice." \
-    --ref-audio /path/to/reference.wav \
-    --ref-text "Transcript of the reference audio."
-```
-
-## Streaming Mode
-
-Add `--streaming` to stream audio chunks progressively via `AsyncOmni` (requires `async_chunk: true` in the stage config):
-
-```bash
-python end2end.py --text "Hello, this is a test." --streaming
-```
-
-Each DAC decoder chunk is logged as it arrives. The final WAV file is written once generation completes.
-
-## Configuration
-
-| Argument | Default | Description |
-|----------|---------|-------------|
-| `--text` | `"Hello, this is a test..."` | Text to synthesize |
-| `--ref-audio` | None | Path to reference audio for voice cloning |
-| `--ref-text` | None | Transcript of the reference audio |
-| `--model` | `fishaudio/s2-pro` | HuggingFace model path |
-| `--stage-configs-path` | `fish_speech_s2_pro.yaml` | Path to stage configs YAML |
-| `--output-dir` | `output_audio` | Output directory for WAV files |
-| `--streaming` | False | Enable streaming via AsyncOmni |
-| `--stage-init-timeout` | 600 | Stage initialization timeout (seconds) |
-
-## Notes
-
-- Output audio is 44.1 kHz mono WAV.
-- The DAC codec weights (`codec.pth`) are loaded lazily from the model directory.
-- Voice cloning encodes the reference audio using the DAC codec to extract semantic codes, then prepends them as a system message.
-
-## Example materials
-
-??? abstract "end2end.py"
-    ``````py
-    --8<-- "examples/offline_inference/fish_speech/end2end.py"
-    ``````

docs/user_guide/examples/offline_inference/qwen3_tts.md

@@ -1,172 +0,0 @@
-# Qwen3-TTS
-
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/qwen3_tts>.
-
-
-This directory contains an offline demo for running Qwen3 TTS models with vLLM Omni. It builds task-specific inputs and generates WAV files locally.
-
-## Model Overview
-
-Qwen3 TTS provides multiple task variants for speech generation:
-
-- **CustomVoice**: Generate speech with a known speaker identity (speaker ID) and optional instruction.
-- **VoiceDesign**: Generate speech from text plus a descriptive instruction that designs a new voice.
-- **Base**: Voice cloning using a reference audio + reference transcript, with optional mode selection. The `ref_audio` field accepts a local file path, HTTP URL, or base64 data URL.
-
-## Setup
-Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup.
-
-### ROCm Dependencies
-
-You will need to install the dependency `onnxruntime-rocm`.
-
-```
-pip uninstall onnxruntime # should be removed before we can install onnxruntime-rocm
-pip install onnxruntime-rocm
-```
-
-## Quick Start
-
-Run a single sample for a task:
-
-```
-python end2end.py --query-type CustomVoice
-```
-
-Generated audio files are saved to `output_audio/` by default.
-
-## Task Usage
-
-### CustomVoice
-
-Single sample:
-
-```
-python end2end.py --query-type CustomVoice
-```
-
-Batch sample (multiple prompts in one run):
-
-```
-python end2end.py --query-type CustomVoice --use-batch-sample
-```
-
-### VoiceDesign
-
-Single sample:
-
-```
-python end2end.py --query-type VoiceDesign
-```
-
-Batch sample:
-
-```
-python end2end.py --query-type VoiceDesign --use-batch-sample
-```
-
-### Base (Voice Clone)
-
-Single sample:
-
-```
-python end2end.py --query-type Base
-```
-
-Batch sample:
-
-```
-python end2end.py --query-type Base --use-batch-sample
-```
-
-Mode selection for Base:
-
-- `--mode-tag icl` (default): standard mode
-- `--mode-tag xvec_only`: enable `x_vector_only_mode` in the request
-
-Examples:
-
-```
-python end2end.py --query-type Base --mode-tag icl
-```
-
-## Voice and Language Control
-
-### Supported Voices (CustomVoice)
-
-Predefined speaker voices are set via the `speaker` (or `voice_type`) field in `additional_information`. Available speakers depend on the loaded checkpoint; check `talker_config.spk_id` in the model config for the full list. Common voices include `vivian`, `ryan`, `aiden`, `ethan`, `serena` (case-insensitive).
-
-Pass the speaker name in your request:
-
-```python
-additional_information = {
-    "text": ["你好，我是通义千问"],
-    "task_type": ["CustomVoice"],
-    "speaker": ["Vivian"],  
-    "language": ["Chinese"],
-}
-```
-
-### Supported Languages
-
-The `language` field controls the codec-level language tag. Use `"Auto"` (default) for automatic detection.
-
-Supported values: `Auto`, `Chinese`, `English`, `Japanese`, `Korean`, `German`, `French`, `Russian`, `Portuguese`, `Spanish`, `Italian`.
-
-```python
-additional_information = {
-    "text": ["Hello, nice to meet you."],
-    "task_type": ["CustomVoice"],
-    "speaker": ["Aiden"],
-    "language": ["English"],
-}
-```
-
-### VoiceDesign and Base
-
-- **VoiceDesign**: Use `instruct` for natural-language voice description; no `speaker` needed.
-- **Base**: Use `ref_audio` and `ref_text` for voice cloning; `language` is optional.
-
-## Streaming Mode
-
-Add `--streaming` to stream audio chunks progressively via `AsyncOmni` (requires `async_chunk: true` in the stage config):
-
-```bash
-python end2end.py --query-type CustomVoice --streaming --output-dir /tmp/out_stream
-```
-
-Each Code2Wav chunk is logged as it arrives (default 25 frames; configurable via `codec_chunk_frames`
-in the stage config). The initial chunk size is dynamically selected based on server load for reduced
-TTFA, and can be overridden per-request via the `initial_codec_chunk_frames` API field. The final WAV file is written once generation
-completes. This demonstrates that audio data is available progressively rather than only at the end.
-
-> **Note:** Streaming uses `AsyncOmni` internally. The non-streaming path (`Omni`) is unchanged.
-
-## Batched Decoding
-
-The Code2Wav stage (stage 1) supports batched decoding, where multiple requests are decoded in a single forward pass through the SpeechTokenizer. To use it, set `max_num_seqs > 1` on both stages via `--stage-overrides` and pass multiple prompts via `--txt-prompts` with a matching `--batch-size`.
-
-```
-python end2end.py --query-type CustomVoice \
-    --txt-prompts benchmark_prompts.txt \
-    --batch-size 4 \
-    --stage-overrides '{"0":{"max_num_seqs":4,"gpu_memory_utilization":0.2},"1":{"max_num_seqs":4,"gpu_memory_utilization":0.2}}'
-```
-
-**Important:** `--batch-size` must match a CUDA graph capture size (1, 2, 4, 8, 16...) because the Talker's code predictor KV cache is sized to `max_num_seqs`, and CUDA graphs pad the batch to the next capture size. Both stages need `max_num_seqs >= batch_size` in the stage config for batching to take effect. If only stage 1 has a higher `max_num_seqs`, it won't help — stage 1 can only batch chunks from requests that are in-flight simultaneously, which requires stage 0 to also process multiple requests concurrently.
-
-## Notes
-
-- The script uses the model paths embedded in `end2end.py`. Update them if your local cache path differs.
-- Use `--output-dir` to change the output folder.
-
-## Example materials
-
-??? abstract "benchmark_prompts.txt"
-    ``````txt
-    --8<-- "examples/offline_inference/qwen3_tts/benchmark_prompts.txt"
-    ``````
-??? abstract "end2end.py"
-    ``````py
-    --8<-- "examples/offline_inference/qwen3_tts/end2end.py"
-    ``````

docs/user_guide/examples/offline_inference/text_to_audio.md

@@ -57,6 +57,8 @@ Key arguments:
 - `--hsdp-shard-size`: number of GPUs used for HSDP sharding.
 - `--hsdp-replicate-size`: number of HSDP replica groups.
 - `--output`: path to save the generated WAV file.
+- `--enable-cpu-offload`: enabling model-wise offloading to save gpu memory
+- `--enable-layerwise-offload`: enabling layerwise offloading to save gpu memory
 
 ## Example materials
 

docs/user_guide/examples/offline_inference/text_to_speech.md

@@ -0,0 +1,395 @@
+# Text-To-Speech
+
+Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_speech>.
+
+
+vLLM-Omni supports several autoregressive TTS models. They share a common
+CLI shape (`--text`, `--ref-audio`, `--ref-text`, `--output-dir`) and live
+together in this hub. Each model has its own subdirectory containing a
+single `end2end.py` script; this README is the single doc entry point.
+
+For online serving, see `examples/online_serving/<model>/`. For the full
+list of supported architectures across all modalities, see
+[Supported Models](https://github.com/vllm-project/vllm-omni/tree/main/docs/models/supported_models.md).
+
+## Supported Models
+
+| Model | HuggingFace repo | Stages | Voice cloning | Streaming | Special modes | Sample rate |
+|---|---|---|---|---|---|---|
+| VoxCPM2 | `openbmb/VoxCPM2` | single (native AR) | ✓ | — | continuation (`--ref-audio` + `--ref-text`) | 48 kHz |
+| VoxCPM | local model dir | split | ✓ | ✓ (`voxcpm_async_chunk.yaml`) | — | 24 kHz |
+| CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | 2 (talker + code2wav) | ✓ | ✓ (`async_chunk: true` default) | — | 22.05 kHz |
+| Fish Speech S2 Pro | `fishaudio/s2-pro` | dual-AR | ✓ | ✓ (`--streaming`) | — | 44.1 kHz |
+| OmniVoice | `k2-fsa/OmniVoice` | 2 (gen + dec) | ✓ | — | voice design (`--instruct`), language (`--lang`) | 24 kHz |
+| Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | 2 (talker + code2wav) | ✓ (Base) | ✓ | 3 task variants (`--query-type`) | 24 kHz |
+| Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | varies | ✓ | ✓ | voice presets (`--voice`) | 24 kHz |
+
+## Common Quick Start
+
+Most models share this invocation shape:
+
+```bash
+python examples/offline_inference/text_to_speech/<model>/end2end.py \
+    --text "Hello, this is a test." \
+    --ref-audio /path/to/reference.wav \
+    --ref-text  "Transcript of the reference audio."
+```
+
+`--ref-audio` and `--ref-text` are optional (text-only synthesis works without them) and must be provided together for voice cloning. The exotic scripts — Qwen3-TTS, Voxtral TTS, CosyVoice3 — accept additional model-specific flags documented in their per-model section below.
+
+---
+
+## VoxCPM2
+
+Single-stage native AR TTS at 48 kHz. Pipeline: `feat_encoder → MiniCPM4 → FSQ → residual_lm → LocDiT → AudioVAE`.
+
+### Prerequisites
+```bash
+pip install voxcpm
+# or, for a local source checkout:
+export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/voxcpm
+```
+
+### Quick start
+```bash
+python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \
+    --model openbmb/VoxCPM2 \
+    --text "Hello, this is a VoxCPM2 demo."
+```
+
+### Voice cloning
+Pass a reference audio for isolated cloning, or both `--ref-audio` + `--ref-text` for prompt continuation:
+```bash
+python examples/offline_inference/text_to_speech/voxcpm2/end2end.py \
+    --text "Hello, this is a voice clone demo." \
+    --ref-audio /path/to/reference.wav \
+    --ref-text  "Transcript of the reference audio."
+```
+
+### Notes
+- Output: 48 kHz mono WAV.
+- Stage config: `vllm_omni/model_executor/stage_configs/voxcpm2.yaml` (default).
+
+---
+
+## VoxCPM
+
+Split-stage TTS. The hub example covers single TTS, single voice cloning, and streaming. Use `benchmarks/voxcpm/` for warmup, batch JSONL prompts, profiler injection, and offline TTFP / RTF measurement.
+
+### Prerequisites
+```bash
+pip install voxcpm soundfile
+# or use a local source tree:
+export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/VoxCPM/src
+```
+
+If the native VoxCPM `config.json` does not contain HF metadata such as `model_type`, prepare a persistent HF-compatible config directory and point the stage configs to it via `VLLM_OMNI_VOXCPM_HF_CONFIG_PATH`:
+
+```bash
+export VOXCPM_MODEL=/path/to/voxcpm-model
+export VLLM_OMNI_VOXCPM_HF_CONFIG_PATH=/tmp/voxcpm_hf_config
+mkdir -p "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"
+cp "$VOXCPM_MODEL/config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/config.json"
+cp "$VOXCPM_MODEL/generation_config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/generation_config.json" 2>/dev/null || true
+python3 -c 'import json, os; p=os.path.join(os.environ["VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"], "config.json"); cfg=json.load(open(p, "r", encoding="utf-8")); cfg["model_type"]="voxcpm"; cfg.setdefault("architectures", ["VoxCPMForConditionalGeneration"]); json.dump(cfg, open(p, "w", encoding="utf-8"), indent=2, ensure_ascii=False)'
+```
+
+### Quick start
+```bash
+python examples/offline_inference/text_to_speech/voxcpm/end2end.py \
+    --model "$VOXCPM_MODEL" \
+    --text "This is a split-stage VoxCPM synthesis example running on vLLM Omni."
+```
+
+### Voice cloning
+```bash
+python examples/offline_inference/text_to_speech/voxcpm/end2end.py \
+    --model "$VOXCPM_MODEL" \
+    --text "This sentence is synthesized with a cloned voice." \
+    --ref-audio /path/to/reference.wav \
+    --ref-text  "The exact transcript spoken in reference.wav."
+```
+
+### Streaming
+Pass the async-chunk stage config:
+```bash
+python examples/offline_inference/text_to_speech/voxcpm/end2end.py \
+    --model "$VOXCPM_MODEL" \
+    --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \
+    --text "This is a split-stage VoxCPM streaming example running on vLLM Omni."
+```
+
+### Notes
+- `voxcpm.yaml` is the default non-streaming stage config; `voxcpm_async_chunk.yaml` enables streaming.
+- Streaming is currently single-request oriented.
+- `--ref-text` must be the real transcript of `--ref-audio`; mismatched text degrades quality.
+- For online serving, see [`examples/online_serving/voxcpm`](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/voxcpm/README.md). For benchmark reporting, see [`benchmarks/voxcpm`](https://github.com/vllm-project/vllm-omni/tree/main/benchmarks/voxcpm/README.md).
+
+---
+
+## CosyVoice3
+
+2-stage TTS pipeline (`talker` + `code2wav`) at 22.05 kHz.
+
+### Prerequisites
+```bash
+uv pip install -e .
+# Includes soundfile, onnxruntime, x-transformers, einops via requirements.
+```
+
+Download the model snapshot:
+```python
+from huggingface_hub import snapshot_download
+snapshot_download('FunAudioLLM/Fun-CosyVoice3-0.5B-2512',
+                  local_dir='pretrained_models/Fun-CosyVoice3-0.5B')
+```
+
+If your downloaded checkpoint lacks `config.json`, add it:
+```json
+{
+    "model_type": "cosyvoice3",
+    "architectures": ["CosyVoice3Model"]
+}
+```
+This is required because `AutoConfig.register("cosyvoice3", CosyVoice3Config)` only registers the class mapping; the loader still reads `model_type` from `config.json` to select the class.
+
+### Quick start
+```bash
+python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
+    --model pretrained_models/Fun-CosyVoice3-0.5B \
+    --tokenizer pretrained_models/Fun-CosyVoice3-0.5B/CosyVoice-BlankEN
+```
+
+### Voice cloning
+Pass a reference audio. Note that CosyVoice3's `--prompt-text` is a system-style prompt for the GPT stage, not a reference transcript:
+```bash
+python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
+    --model pretrained_models/Fun-CosyVoice3-0.5B \
+    --tokenizer pretrained_models/Fun-CosyVoice3-0.5B/CosyVoice-BlankEN \
+    --ref-audio prompt.wav \
+    --prompt-text "You are a helpful assistant.<|endofprompt|>Testing my voices. Why should I not?"
+```
+
+### Notes
+- Stage 0 (`talker`) emits speech tokens; stage 1 (`code2wav`) runs flow matching + HiFiGAN to synthesize waveform.
+- Deploy config auto-loads from `vllm_omni/deploy/cosyvoice3.yaml` based on HF `model_type`. Pass `--deploy-config <path>` to override.
+- `async_chunk: true` is the default; pass `--no-async-chunk` to switch to the legacy synchronous path.
+
+---
+
+## Fish Speech S2 Pro
+
+4B dual-AR text-to-speech model from FishAudio with the DAC codec at 44.1 kHz.
+
+### Prerequisites
+```bash
+pip install fish-speech
+```
+
+### Quick start
+```bash
+python examples/offline_inference/text_to_speech/fish_speech/end2end.py \
+    --text "Hello, this is a test of the Fish Speech text to speech system."
+```
+
+### Voice cloning
+```bash
+python examples/offline_inference/text_to_speech/fish_speech/end2end.py \
+    --text "Hello, this is a cloned voice." \
+    --ref-audio /path/to/reference.wav \
+    --ref-text  "Transcript of the reference audio."
+```
+
+### Streaming
+```bash
+python examples/offline_inference/text_to_speech/fish_speech/end2end.py \
+    --text "Hello, this is a streaming test." \
+    --streaming
+```
+Streaming requires `async_chunk: true` in the stage config.
+
+### Notes
+- Output: 44.1 kHz mono WAV.
+- DAC codec weights (`codec.pth`) are loaded lazily from the model directory.
+
+---
+
+## OmniVoice
+
+Zero-shot multilingual TTS supporting 600+ languages, with three modes (auto / clone / design).
+
+### Prerequisites
+```bash
+huggingface-cli download k2-fsa/OmniVoice
+```
+Voice cloning requires `transformers>=5.3.0`. Auto and design modes work with `transformers>=4.57.0`.
+
+### Quick start (auto voice)
+```bash
+python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
+    --model k2-fsa/OmniVoice \
+    --text "Hello, this is a test."
+```
+
+### Voice cloning
+```bash
+python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
+    --model k2-fsa/OmniVoice \
+    --text "Hello, this is a test." \
+    --ref-audio ref.wav \
+    --ref-text  "This is the reference transcription."
+```
+
+### Voice design
+```bash
+python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
+    --model k2-fsa/OmniVoice \
+    --text "Hello, this is a test." \
+    --instruct "female, low pitch, british accent"
+```
+
+### Language hint
+```bash
+python examples/offline_inference/text_to_speech/omnivoice/end2end.py \
+    --model k2-fsa/OmniVoice \
+    --text "你好，这是一个测试。" \
+    --lang zh
+```
+
+### Notes
+- Stage 0 (Generator): Qwen3-0.6B with 32-step iterative unmasking.
+- Stage 1 (Decoder): HiggsAudioV2 RVQ + DAC at 24 kHz.
+
+---
+
+## Qwen3-TTS
+
+3-task-variant TTS with 24 kHz output. Has its own argparse surface (this script does not follow the common `--text` / `--ref-audio` shape).
+
+### Prerequisites
+For ROCm builds, replace `onnxruntime` with `onnxruntime-rocm`:
+```bash
+pip uninstall onnxruntime
+pip install onnxruntime-rocm
+```
+
+### Task variants
+- `CustomVoice`: predefined speaker (speaker ID) with optional style instruction.
+- `VoiceDesign`: text + descriptive instruction designs a new voice.
+- `Base`: voice cloning from reference audio + transcript.
+
+```bash
+# Single sample
+python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py --query-type CustomVoice
+python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py --query-type VoiceDesign
+python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py --query-type Base
+
+# Base variant has an additional mode flag:
+python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py --query-type Base --mode-tag icl       # default
+python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py --query-type Base --mode-tag xvec_only # x_vector_only_mode
+
+# Batch (multiple prompts in one run)
+python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py --query-type CustomVoice --use-batch-sample
+```
+
+### Streaming
+```bash
+python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
+    --query-type CustomVoice \
+    --streaming \
+    --output-dir /tmp/out_stream
+```
+Streaming requires `async_chunk: true` in the stage config.
+
+### Batched decoding
+The Code2Wav stage supports batched decoding through the SpeechTokenizer. Configure both stages with `max_num_seqs > 1` via `--stage-overrides` and pass multiple prompts via `--txt-prompts`:
+```bash
+python examples/offline_inference/text_to_speech/qwen3_tts/end2end.py \
+    --query-type CustomVoice \
+    --txt-prompts examples/offline_inference/text_to_speech/qwen3_tts/benchmark_prompts.txt \
+    --batch-size 4 \
+    --stage-overrides '{"0":{"max_num_seqs":4,"gpu_memory_utilization":0.2},"1":{"max_num_seqs":4,"gpu_memory_utilization":0.2}}'
+```
+`--batch-size` must match a CUDA-graph capture size (1, 2, 4, 8, 16…).
+
+### Notes
+- Run `--help` for the full argument surface.
+- See `qwen3_tts/end2end.py` for the prompt-length-estimation logic the Talker uses.
+
+---
+
+## Voxtral TTS
+
+Voxtral-4B-TTS (Mistral). Has its own argparse surface; uses voice presets and the `mistral_common` `SpeechRequest` protocol.
+
+### Prerequisites
+Latest `mistral_common` with `SpeechRequest` support:
+```bash
+pip install -e /path/to/mistral-common  # or upgrade from PyPI when available
+```
+
+### Quick start (voice preset)
+```bash
+python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
+    --write-audio --voice cheerful_female \
+    --model mistralai/Voxtral-4B-TTS-2603 \
+    --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?"
+```
+
+### Voice cloning (capability gated upstream)
+```bash
+python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
+    --write-audio \
+    --model mistralai/Voxtral-4B-TTS-2603 \
+    --text "This is a test message." \
+    --ref-audio path/to/reference_audio.wav
+```
+
+### Streaming + concurrency
+```bash
+python examples/offline_inference/text_to_speech/voxtral_tts/end2end.py \
+    --num-prompts 32 --concurrency 8 --streaming --write-audio --voice neutral_female \
+    --model mistralai/Voxtral-4B-TTS-2603 \
+    --text "..."
+```
+Available voice presets are listed on the HF model card (`mistralai/Voxtral-4B-TTS-2603`).
+
+### Notes
+- `--num-prompts N` replicates the prompt for performance measurement.
+- `--concurrency M` requires `--streaming` and must evenly divide `--num-prompts`.
+- Run `--help` for the full argument surface.
+
+## Example materials
+
+??? abstract "cosyvoice3/end2end.py"
+    ``````py
+    --8<-- "examples/offline_inference/text_to_speech/cosyvoice3/end2end.py"
+    ``````
+??? abstract "fish_speech/end2end.py"
+    ``````py
+    --8<-- "examples/offline_inference/text_to_speech/fish_speech/end2end.py"
+    ``````
+??? abstract "omnivoice/end2end.py"
+    ``````py
+    --8<-- "examples/offline_inference/text_to_speech/omnivoice/end2end.py"
+    ``````
+??? abstract "qwen3_tts/benchmark_prompts.txt"
+    ``````txt
+    --8<-- "examples/offline_inference/text_to_speech/qwen3_tts/benchmark_prompts.txt"
+    ``````
+??? abstract "qwen3_tts/end2end.py"
+    ``````py
+    --8<-- "examples/offline_inference/text_to_speech/qwen3_tts/end2end.py"
+    ``````
+??? abstract "voxcpm/end2end.py"
+    ``````py
+    --8<-- "examples/offline_inference/text_to_speech/voxcpm/end2end.py"
+    ``````
+??? abstract "voxcpm2/end2end.py"
+    ``````py
+    --8<-- "examples/offline_inference/text_to_speech/voxcpm2/end2end.py"
+    ``````
+??? abstract "voxtral_tts/end2end.py"
+    ``````py
+    --8<-- "examples/offline_inference/text_to_speech/voxtral_tts/end2end.py"
+    ``````

docs/user_guide/examples/offline_inference/voxtral_tts.md

@@ -1,69 +0,0 @@
-# Voxtral TTS Offline Inference
-
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/voxtral_tts>.
-
-
-`end2end.py` runs Voxtral TTS end-to-end offline inference using vLLM. It supports both blocking (`Omni`) and streaming (`AsyncOmni`) generation, batched prompts with configurable concurrency, and voice selection via preset name or reference audio file.
-
-When `mistral_common` has `SpeechRequest` support, prompt token IDs are built via `encode_speech_request`. Otherwise, the script falls back to manual token construction.
-
-## Usage Examples
-
-
-```bash
-# Basic single-prompt with cheerful_female voice preset
-python3 examples/offline_inference/voxtral_tts/end2end.py \
-    --write-audio --voice cheerful_female \
-    --model mistralai/Voxtral-4B-TTS-2603 \
-    --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?"
-
-# 32 replicate prompts with cheerful_female voice preset
-python3 examples/offline_inference/voxtral_tts/end2end.py \
-    --num-prompts 32 --write-audio --voice cheerful_female \
-    --model mistralai/Voxtral-4B-TTS-2603 \
-    --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?"
-
-# Streaming with neutral_female voice preset
-python3 examples/offline_inference/voxtral_tts/end2end.py \
-    --streaming --write-audio --voice neutral_female \
-    --model mistralai/Voxtral-4B-TTS-2603 \
-    --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?"
-
-# 32 prompts, 8 concurrent requests per wave, streaming with neutral_female voice
-python3 examples/offline_inference/voxtral_tts/end2end.py \
-    --num-prompts 32 --concurrency 8 --streaming --write-audio --voice neutral_female \
-    --model mistralai/Voxtral-4B-TTS-2603 \
-    --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?"
-
-# Short debug prompt with reference audio
-# Note: Reference audio capability is not yet released.
-python3 examples/offline_inference/voxtral_tts/end2end.py \
-    --write-audio \
-    --model mistralai/Voxtral-4B-TTS-2603 \
-    --text "This is a test message." \
-    --audio-path path/to/reference_audio.wav
-```
-
-## Arguments
-
-| Argument | Description |
-|---|---|
-| `--model PATH` | HuggingFace repo ID or local directory path (default: `mistralai/Voxtral-4B-TTS-2603`) |
-| `--text TEXT` | Text to synthesize (default: `"This is a test message."`) |
-| `--audio-path PATH` | Path to reference audio file for voice cloning |
-| `--output-dir DIR` | Directory to write output WAV files (default: `output_audio`) |
-| `--deploy-config PATH` | Override the deploy config path. If unset, auto-loads `vllm_omni/deploy/voxtral_tts.yaml` from the HF `model_type`. |
-| `--num-prompts N` | Number of replicate prompts to run for measuring performance (default: 1) |
-| `--streaming` | Use streaming generation via `AsyncOmni` (default: blocking `Omni`) |
-| `--concurrency N` | Max concurrent requests per wave (must be used with `--streaming`, must evenly divide `--num-prompts`) |
-| `--voice NAME` | Voice preset to use instead of reference audio file (e.g., casual_female, casual_male, cheerful_female, neutral_female, neutral_male) |
-| `--write-audio` | Write generated audio to WAV files |
-| `--profiling-mode` | Enable profiling mode (reduces max tokens to 50) |
-| `--log-stats` | Enable detailed statistics logging |
-
-## Example materials
-
-??? abstract "end2end.py"
-    ``````py
-    --8<-- "examples/offline_inference/voxtral_tts/end2end.py"
-    ``````

docs/user_guide/examples/offline_inference/x_to_video_audio.md

@@ -47,6 +47,24 @@ python x_to_video_audio.py \
 ```
 In the current test scenario (2 images + 2 audio inputs), the VRAM requirement is 72GB, regardless of whether cfg-parallel is enabled or disabled.
 The VRAM usage can be reduced by enabling CPU offload via --enable-cpu-offload.
+For multi-GPU memory reduction on the fused DreamID-Omni transformer, you can also enable HSDP:
+
+```python
+python x_to_video_audio.py \
+  --model /path/to/dreamid_omni \
+  --prompt "Two people walking together and singing happily" \
+  --image-path ./example0.png ./example1.png \
+  --audio-path ./example0.wav ./example1.wav \
+  --video-negative-prompt "jitter, bad hands, blur, distortion" \
+  --audio-negative-prompt "robotic, muffled, echo, distorted" \
+  --cfg-parallel-size 2 \
+  --num-inference-steps 45 \
+  --height 704 \
+  --width 1280 \
+  --use-hsdp \
+  --hsdp-shard-size 2 \
+  --output out_dreamid_omni_twoip.mp4
+```
 
 
 You could take reference images/audios from the test cases in the official repo: https://github.com/Guoxu1233/DreamID-Omni
@@ -77,6 +95,9 @@ Key arguments:
 - `--image-path`: path to the input image list.
 - `--audio-path`: path to the input audio list, indicate the timbre of the output video.
 - `--cfg-parallel-size`: number of parallel cfg parallel (defaults 1).
+- `--use-hsdp`: enable HSDP weight sharding for DreamID-Omni fused blocks.
+- `--hsdp-shard-size`: number of GPUs used for HSDP sharding.
+- `--hsdp-replicate-size`: number of HSDP replica groups.
 - `--num-inference-steps`: number of denoising steps (defaults 45).
 - `--video-negative-prompt`: negative prompt for video generation.
 - `--audio-negative-prompt`: negative prompt for audio generation.

docs/user_guide/examples/online_serving/diffusers_pipeline_adapter.md

@@ -1,18 +1,14 @@
-# Diffusers Backend Adapter Example
+# Diffusers Backend Adapter
 
 Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/diffusers_pipeline_adapter>.
 
 
-This example demonstrates how to serve any 🤗 Diffusers pipeline through vLLM-Omni
-using the `diffusers` load format.
-
-## Supported Models
-
-Any model loadable via `DiffusionPipeline.from_pretrained()` should be supported, including text-to-image, image-to-image, text-to-video, image-to-video, and text-to-audio.
+vLLM-Omni supports running diffusion models with the diffusers backend, directly serving any 🤗 Diffusers pipeline online without implementing them natively.
 
 ## Limitations
 
-The diffusers backend is a black-box adapter. The following features are NOT yet supported.
+The diffusers backend is a black-box adapter. Its primary focus is to serve diffusion models online.
+Currently, the following features are NOT yet supported.
 It is not guaranteed whether they will be supported in the future.
 
 - CFG parallel execution
@@ -22,72 +18,87 @@ It is not guaranteed whether they will be supported in the future.
 
 For these features, it is recommended to use natively supported pipelines instead.
 
-## Usage
+## Model Support
 
-### Option 1: CLI arguments
+Any model loadable via `DiffusionPipeline.from_pretrained()` should run, including text-to-image, image-to-image, text-to-video, image-to-video, and text-to-audio.
+
+However, as we strive to ensure output similarity between vLLM-Omni's diffuser backend and plain diffusers library, the following models are particularly verified:
+
+- Qwen/Qwen-Image
+- Tongyi-MAI/Z-Image-Turbo
+- Wan2.2-I2V-A14B-Diffusers
+
+If you find that a model not listed above also produces different outputs from running diffusers model directly.
+Please consider file an issue.
+
+## Usage
 
 ```bash
 vllm serve "stable-diffusion-v1-5/stable-diffusion-v1-5" \
     --omni \
-    --diffusion-load-format diffusers \
-    --diffusers-load-kwargs '{"use_safetensors": true}' \
-    --diffusers-call-kwargs '{"num_inference_steps": 30, "guidance_scale": 7.5}'
+    --diffusion-load-format diffusers
 ```
 
-`--diffusers-load-kwargs` and `--diffusers-call-kwargs` are only valid together with `--diffusion-load-format diffusers`.
+Users turn on the diffusers backend primarily through `--diffusion-load-format diffusers` argument.
+There are two more optional arguments, `--diffusers-load-kwargs` and `--diffusers-call-kwargs`,
+which are only valid together with `--diffusion-load-format diffusers`.
 
-### Option 2: Stage config YAML
-
-```bash
-vllm serve stable-diffusion-v1-5/stable-diffusion-v1-5 --stage-configs-path examples/online_serving/diffusers_pipeline_adapter/stage_config.yaml --omni
-```
-
-The particular fields of interest are `model`, `diffusion_load_format`, `diffusers_load_kwargs`, and `diffusers_call_kwargs` under `engine_args`. They are the same as the CLI arguments.
-
-## Send a Request
-
-```bash
-curl http://localhost:8000/v1/images/generations \
-  -H "Content-Type: application/json" \
-  -d '{
-    "model": "stable-diffusion-v1-5/stable-diffusion-v1-5",
-    "prompt": "a photo of an astronaut riding a horse on mars",
-    "n": 1,
-    "size": "512x512"
-  }'
-```
-
-Or refer to other documentation pages on how to request a particular input/output modality, such as `examples/online_serving/text_to_image/openai_chat_client.py`.
+After launching the model, users send a request as usual. Refer to other documentation pages on how to request a particular input/output modality, such as `examples/online_serving/text_to_image/openai_chat_client.py`.
 
 ## Configuration Reference
 
-For the diffusers adapter, set options under **`engine_args`**:
+### `--diffusers-load-kwargs`
 
-### `diffusion_load_format: "diffusers"`
+Passed as-is to `DiffusionPipeline.from_pretrained()`.
 
-This field selects the Hugging Face diffusers adapter path (see `DiffusersPipelineLoader`).
-
-### `diffusers_load_kwargs`
-
-Passed to `DiffusionPipeline.from_pretrained()`.
-
-This is suitable for model-specific configurations not available through the vLLM-Omni interface (such as `Omni.__init__()`, `vllm serve` CLI arguments, and stage config YAML fields outside `diffusers_load_kwargs`).
+This is suitable for model-specific configurations not available through the vLLM-Omni interface.
+For example: `--diffusers-load-kwargs '{"use_safetensors": true}'`.
 
 When a parameter is available in the vLLM-Omni interface, it will be adapted here.
 But if that parameter is simultaneously set in both the vLLM-Omni interface and `diffusers_load_kwargs`, the **latter** will take precedence.
 
-### `diffusers_call_kwargs`
+### `--diffusers-call-kwargs`
 
 Passed to `pipeline.__call__()`.
 
-This is suitable for sampling parameters not available through the vLLM-Omni interface (such as `Omni.generate()` and online serving payloads).
+This is suitable for sampling parameters not available through the vLLM-Omni interface (such as online serving payloads).
 
 When a parameter is available in the vLLM-Omni interface, it will be adapted here.
 But if that parameter is simultaneously set in both the vLLM-Omni interface and `diffusers_call_kwargs`, the **former** will take precedence (because it is set at request time).
 
-## Example materials
+### Attention Backends
 
-??? abstract "stage_config.yaml"
-    ``````yaml
-    --8<-- "examples/online_serving/diffusers_pipeline_adapter/stage_config.yaml"
-    ``````
+The diffusers backend converts
+[vLLM-Omni standard of attention backend setting](../../../docs/user_guide/diffusion/attention_backends.md)
+to [diffusers standard](https://huggingface.co/docs/diffusers/optimization/attention_backends#available-backends).
+
+Specifically for `FLASH_ATTN`, it will first attempt to use FlashAttention-3 and then FlashAttention-2.
+
+For each attempted version of `FLASH_ATTN` and `SAGE_ATTN`, it will first try to load the attention backend from HuggingFace `kernels` library, then without.
+
+For unsuccessful attention selection or `TORCH_SDPA`, it will use the PyTorch's default attention backend.
+
+The loaded attention backend and the failed attempts (if any) are logged to console.
+
+### Model Specific Settings
+
+The model loading and inference strictly follows the diffusers library, and they may be different from vLLM-Omni's native interface for some specific models.
+Users are encouraged to double-check the model pipeline's interface in [diffusers' official documentation](https://huggingface.co/docs/diffusers/api/pipelines/overview).
+Some particular examples are below.
+
+#### Wan Series
+
+The Wan series video generation models takes `boundary_ratio` and `flow_shift` during model initialization ([ref](https://huggingface.co/docs/diffusers/api/pipelines/wan)), not during inference.
+
+Since our `OmniDiffusionConfig` contains these two values ([source](https://github.com/vllm-project/vllm-omni/blob/main/vllm_omni/diffusion/data.py)), we can directly pass `--boundary-ratio` and `--flow-shift` arguments to `vllm serve` command.
+
+```bash
+vllm serve "Wan2.2-T2V-A14B-Diffusers" \
+    --omni \
+    --boundary-ratio 0.875 \
+    --flow-shift 3 \
+    --diffusion-load-format diffusers
+```
+
+These extra CLI args will be attempted to pass as-is to the `OmniDiffusionConfig` dataclass and being accessible during model loading time.
+Special routines inside the pipeline adapter ensures that they are set properly.

docs/user_guide/examples/online_serving/fish_speech.md

@@ -1,199 +0,0 @@
-# Fish Speech S2 Pro
-
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/fish_speech>.
-
-
-## Model
-
-| Model | Description |
-|-------|-------------|
-| `fishaudio/s2-pro` | Fish Speech S2 Pro -- 4B dual-AR TTS model with DAC codec (44.1 kHz) |
-
-## Gradio Demo
-
-!!! note "Gradio is an optional dependency"
-    The Gradio demo requires the `[demo]` extras. Install them first:
-
-    ```bash
-    pip install 'vllm-omni[demo]'
-    ```
-
-    Or, if installing from source: `pip install -e '.[demo]'`
-
-An interactive Gradio demo is available with text-to-speech synthesis, voice cloning, and streaming support.
-
-```bash
-# Option 1: Launch server + Gradio together
-./run_gradio_demo.sh
-
-# Option 2: If server is already running
-python gradio_demo.py --api-base http://localhost:8091
-```
-
-Then open http://localhost:7860 in your browser.
-
-Features:
-
-- Text-to-speech synthesis
-- Voice cloning from uploaded audio or URL
-- Streaming mode (progressive PCM playback)
-
-## Launch the Server
-
-```bash
-vllm serve fishaudio/s2-pro --omni --port 8091
-```
-
-The deploy config is auto-loaded from `vllm_omni/deploy/fish_qwen3_omni.yaml`.
-
-Or use the convenience script:
-
-```bash
-./run_server.sh
-```
-
-## Send TTS Request
-
-### Using curl
-
-```bash
-# Basic TTS
-curl -X POST http://localhost:8091/v1/audio/speech \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "Hello, how are you?",
-        "voice": "default",
-        "response_format": "wav"
-    }' --output output.wav
-```
-
-### Voice Cloning
-
-Provide a reference audio (URL or base64 data URL) and its transcript:
-
-```bash
-curl -X POST http://localhost:8091/v1/audio/speech \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "Hello, this is a cloned voice.",
-        "voice": "default",
-        "ref_audio": "https://example.com/reference.wav",
-        "ref_text": "Transcript of the reference audio."
-    }' --output cloned.wav
-```
-
-### Using Python
-
-```python
-import httpx
-
-# Basic TTS
-response = httpx.post(
-    "http://localhost:8091/v1/audio/speech",
-    json={
-        "input": "Hello, how are you?",
-        "voice": "default",
-        "response_format": "wav",
-    },
-    timeout=300.0,
-)
-
-with open("output.wav", "wb") as f:
-    f.write(response.content)
-```
-
-### Using the CLI Client
-
-```bash
-cd examples/online_serving/fish_speech
-
-# Basic TTS
-python speech_client.py --text "Hello, how are you?"
-
-# Voice cloning
-python speech_client.py \
-    --text "Hello, this is a cloned voice." \
-    --ref-audio /path/to/reference.wav \
-    --ref-text "Transcript of the reference audio."
-
-# Streaming PCM output
-python speech_client.py --text "Hello world" --stream --output output.pcm
-```
-
-The CLI client supports:
-
-- `--api-base`: API base URL (default: `http://localhost:8091`)
-- `--model` (or `-m`): Model name (default: `fishaudio/s2-pro`)
-- `--text`: Text to synthesize (required)
-- `--ref-audio`: Reference audio for voice cloning (local path or URL)
-- `--ref-text`: Transcript of the reference audio
-- `--stream`: Enable streaming (PCM output)
-- `--response-format`: Audio format: wav, mp3, flac, pcm, aac, opus (default: wav)
-- `--output` (or `-o`): Output file path
-
-## Streaming
-
-Set `stream=true` with `response_format="pcm"` to receive raw PCM audio chunks as they are decoded:
-
-```bash
-curl -X POST http://localhost:8091/v1/audio/speech \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "Hello, how are you?",
-        "voice": "default",
-        "stream": true,
-        "response_format": "pcm"
-    }' --no-buffer | play -t raw -r 44100 -e signed -b 16 -c 1 -
-```
-
-**Note:** Fish Speech outputs at 44.1 kHz (unlike Qwen3-TTS which outputs at 24 kHz).
-
-## API Parameters
-
-Fish Speech uses the same `/v1/audio/speech` endpoint as Qwen3-TTS. See the [Speech API reference](https://docs.vllm.ai/projects/vllm-omni/en/latest/serving/speech_api/) for full parameter documentation.
-
-Key parameters for Fish Speech:
-
-| Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
-| `input` | string | **required** | Text to synthesize |
-| `voice` | string | "default" | Voice name (use "default" for Fish Speech) |
-| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, aac, opus |
-| `ref_audio` | string | null | Reference audio URL or base64 data URL for voice cloning |
-| `ref_text` | string | null | Transcript of reference audio (required for voice cloning) |
-| `max_new_tokens` | int | 4096 | Maximum tokens to generate |
-| `stream` | bool | false | Stream raw PCM chunks |
-
-## Prerequisites
-
-Install the `fish-speech` package for the DAC codec:
-
-```bash
-pip install fish-speech
-```
-
-## Troubleshooting
-
-1. **No audio output**: Make sure the `fish-speech` package is installed for the DAC decoder
-2. **Connection refused**: Ensure the server is running on the correct port
-3. **Flashinfer version mismatch**: Set `FLASHINFER_DISABLE_VERSION_CHECK=1` if you see version warnings
-4. **Out of memory**: Reduce `--gpu-memory-utilization` or use a GPU with more VRAM
-
-## Example materials
-
-??? abstract "gradio_demo.py"
-    ``````py
-    --8<-- "examples/online_serving/fish_speech/gradio_demo.py"
-    ``````
-??? abstract "run_gradio_demo.sh"
-    ``````sh
-    --8<-- "examples/online_serving/fish_speech/run_gradio_demo.sh"
-    ``````
-??? abstract "run_server.sh"
-    ``````sh
-    --8<-- "examples/online_serving/fish_speech/run_server.sh"
-    ``````
-??? abstract "speech_client.py"
-    ``````py
-    --8<-- "examples/online_serving/fish_speech/speech_client.py"
-    ``````

docs/user_guide/examples/online_serving/qwen3_tts.md

@@ -1,451 +0,0 @@
-# Qwen3-TTS
-
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/qwen3_tts>.
-
-
-## 🛠️ Installation
-
-Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/README.md)
-
-## Supported Models
-
-| Model                                  | Task Type   | Description                                           |
-| -------------------------------------- | ----------- | ----------------------------------------------------- |
-| `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | CustomVoice | Predefined speaker voices with optional style control |
-| `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | VoiceDesign | Natural language voice style description              |
-| `Qwen/Qwen3-TTS-12Hz-1.7B-Base`        | Base        | Voice cloning from reference audio                    |
-| `Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice` | CustomVoice | Smaller/faster variant                                |
-| `Qwen/Qwen3-TTS-12Hz-0.6B-Base`        | Base        | Smaller/faster variant for voice cloning              |
-
-## Gradio Demo
-
-!!! note "Gradio is an optional dependency"
-    The Gradio demo requires the `[demo]` extras. Install them first:
-
-    ```bash
-    pip install 'vllm-omni[demo]'
-    ```
-
-    Or, if installing from source: `pip install -e '.[demo]'`
-
-Two interactive Gradio demos are available, both supporting all 3 task types:
-
-| Demo     | File                     | Transport    | Streaming Quality                                  |
-| -------- | ------------------------ | ------------ | -------------------------------------------------- |
-| Standard | `gradio_demo.py`         | HTTP chunked | May have small gaps between chunks                 |
-| FastRTC  | `gradio_fastrtc_demo.py` | WebRTC       | Gapless streaming (requires `pip install fastrtc`) |
-
-```bash
-# Option 1: Launch server + Standard Gradio together
-./run_gradio_demo.sh                                # CustomVoice (default)
-./run_gradio_demo.sh --task-type VoiceDesign        # VoiceDesign
-./run_gradio_demo.sh --task-type Base               # Voice cloning
-
-# Option 2: If server is already running
-python gradio_demo.py --api-base http://localhost:8000
-
-# Option 3: FastRTC demo (gapless streaming)
-pip install fastrtc
-python gradio_fastrtc_demo.py --api-base http://localhost:8000
-```
-
-Then open http://localhost:7860 in your browser.
-
-## Run examples (Qwen3-TTS)
-
-### Launch the Server
-
-```bash
-# CustomVoice model (predefined speakers)
-vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
-    --deploy-config vllm_omni/deploy/qwen3_tts.yaml \
-    --omni \
-    --port 8091 \
-    --trust-remote-code \
-    --enforce-eager
-
-# VoiceDesign model
-vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign \
-    --deploy-config vllm_omni/deploy/qwen3_tts.yaml \
-    --omni \
-    --port 8091 \
-    --trust-remote-code \
-    --enforce-eager
-
-# Base model (voice cloning)
-vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-Base \
-    --deploy-config vllm_omni/deploy/qwen3_tts.yaml \
-    --omni \
-    --port 8091 \
-    --trust-remote-code \
-    --enforce-eager
-```
-
-If you have custom stage configs file, launch the server with command below
-```bash
-vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
-    --stage-configs-path /path/to/stage_configs_file \
-    --omni \
-    --port 8091 \
-    --trust-remote-code \
-    --enforce-eager
-```
-
-Alternatively, use the convenience script:
-```bash
-./run_server.sh                  # Default: CustomVoice model
-./run_server.sh CustomVoice      # CustomVoice model
-./run_server.sh VoiceDesign      # VoiceDesign model
-./run_server.sh Base             # Base (voice clone) model
-```
-
-### Send TTS Request
-
-Get into the example folder
-```bash
-cd examples/online_serving/qwen3_tts
-```
-
-####  Send request via python
-
-```bash
-# CustomVoice: Use predefined speaker
-python openai_speech_client.py \
-    --text "你好，我是通义千问" \
-    --speaker vivian \
-    --language Chinese
-
-# CustomVoice with style instruction
-python openai_speech_client.py \
-    --text "今天天气真好" \
-    --speaker ryan \
-    --instructions "用开心的语气说"
-
-# VoiceDesign: Describe the voice style
-python openai_speech_client.py \
-    --model Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign \
-    --task-type VoiceDesign \
-    --text "哥哥，你回来啦" \
-    --instructions "体现撒娇稚嫩的萝莉女声，音调偏高"
-
-# Base: Voice cloning
-python openai_speech_client.py \
-    --model Qwen/Qwen3-TTS-12Hz-1.7B-Base \
-    --task-type Base \
-    --text "Hello, this is a cloned voice" \
-    --ref-audio /path/to/reference.wav \
-    --ref-text "Original transcript of the reference audio"
-```
-
-The Python client supports the following command-line arguments:
-
-- `--api-base`: API base URL (default: `http://localhost:8091`)
-- `--model` (or `-m`): Model name/path (default: `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice`)
-- `--task-type` (or `-t`): TTS task type. Options: `CustomVoice`, `VoiceDesign`, `Base`
-- `--text`: Text to synthesize (required)
-- `--speaker`: Speaker name (default: `vivian`). Options: `vivian`, `ryan`, `aiden`, etc.
-- `--language`: Language. Options: `Auto`, `Chinese`, `English`, `Japanese`, `Korean`, `German`, `French`, `Russian`, `Portuguese`, `Spanish`, `Italian`
-- `--instructions`: Voice style/emotion instructions
-- `--ref-audio`: Reference audio file path or URL for voice cloning (Base task). Local paths are automatically base64-encoded by the client before sending to the server.
-- `--ref-text`: Reference audio transcript for voice cloning (Base task).
-- `--response-format`: Audio output format (default: `wav`). Options: `wav`, `mp3`, `flac`, `pcm`, `aac`, `opus`
-- `--output` (or `-o`): Output audio file path (default: `tts_output.wav`)
-
-####  Send request via curl
-
-```bash
-# Simple TTS request
-curl -X POST http://localhost:8091/v1/audio/speech \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "Hello, how are you?",
-        "voice": "vivian",
-        "language": "English"
-    }' --output output.wav
-
-# With style instruction
-curl -X POST http://localhost:8091/v1/audio/speech \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "I am so excited!",
-        "voice": "vivian",
-        "instructions": "Speak with great enthusiasm"
-    }' --output excited.wav
-
-# List available voices in CustomVoice models
-curl http://localhost:8091/v1/audio/voices
-```
-
-### Using OpenAI SDK
-
-```python
-from openai import OpenAI
-
-client = OpenAI(base_url="http://localhost:8091/v1", api_key="none")
-
-response = client.audio.speech.create(
-    model="Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
-    voice="vivian",
-    input="Hello, how are you?",
-)
-
-response.stream_to_file("output.wav")
-```
-
-### Using Python httpx
-
-```python
-import httpx
-
-response = httpx.post(
-    "http://localhost:8091/v1/audio/speech",
-    json={
-        "input": "Hello, how are you?",
-        "voice": "vivian",
-        "language": "English",
-    },
-    timeout=300.0,
-)
-
-with open("output.wav", "wb") as f:
-    f.write(response.content)
-```
-
-## API Reference
-
-### Voices Endpoint
-
-#### GET /v1/audio/voices
-
-List all available voices/speakers from the loaded model, including both built-in model voices and uploaded custom voices.
-
-**Response Example:**
-```json
-{
-  "voices": ["vivian", "ryan", "custom_voice_1"],
-  "uploaded_voices": [
-    {
-      "name": "custom_voice_1",
-      "consent": "user_consent_id",
-      "created_at": 1738660000,
-      "file_size": 1024000,
-      "mime_type": "audio/wav",
-      "ref_text": "The exact transcript of the audio sample.",
-      "speaker_description": "warm narrator"
-    }
-  ]
-}
-```
-
-Fields `ref_text` and `speaker_description` are omitted per-entry when not provided at upload time.
-
-#### POST /v1/audio/voices
-
-Upload a new voice sample for voice cloning in Base task TTS requests.
-
-**Form Parameters:**
-- `audio_sample` (required): Audio file (max 10MB, supported formats: wav, mp3, flac, ogg, aac, webm, mp4)
-- `consent` (required): Consent recording ID
-- `name` (required): Name for the new voice
-- `ref_text` (optional): Transcript of the audio. Enables in-context voice cloning (higher quality).
-- `speaker_description` (optional): Free-form description of the voice (e.g. "warm narrator", "energetic presenter"). Stored as metadata.
-
-**Response Example:**
-```json
-{
-  "success": true,
-  "voice": {
-    "name": "custom_voice_1",
-    "consent": "user_consent_id",
-    "created_at": 1738660000,
-    "mime_type": "audio/wav",
-    "file_size": 1024000,
-    "ref_text": "The exact transcript of the audio sample.",
-    "speaker_description": "warm narrator"
-  }
-}
-```
-
-Fields `ref_text` and `speaker_description` are omitted when not provided at upload time.
-
-**Usage Example:**
-```bash
-curl -X POST http://localhost:8091/v1/audio/voices \
-  -F "audio_sample=@/path/to/voice_sample.wav" \
-  -F "consent=user_consent_id" \
-  -F "name=custom_voice_1" \
-  -F "ref_text=The exact transcript of the audio sample." \
-  -F "speaker_description=warm narrator"
-```
-
-### Endpoint
-
-```
-POST /v1/audio/speech
-Content-Type: application/json
-```
-
-This endpoint follows the [OpenAI Audio Speech API](https://platform.openai.com/docs/api-reference/audio/createSpeech) format with additional Qwen3-TTS parameters.
-
-### Request Body
-
-```json
-{
-    "input": "Text to synthesize",
-    "voice": "vivian",
-    "response_format": "wav",
-    "task_type": "CustomVoice",
-    "language": "Auto",
-    "instructions": "Optional style instructions",
-    "ref_audio":  "HTTP URL, base64 data URL, or file:// URI for voice cloning",
-    "ref_text": "Reference audio transcript",
-    "x_vector_only_mode": false,
-    "max_new_tokens": 2048
-}
-```
-
-> **Note:** The `model` field is optional when serving a single model, as the server already knows which model is loaded.
-
-### Response
-
-Returns binary audio data with appropriate `Content-Type` header (e.g., `audio/wav`).
-
-### Voice and language (summary)
-
-- **Speaker**: Use the `voice` request field to select the speaker (e.g., `vivian`, `ryan`, `aiden`). List available speakers with `GET /v1/audio/voices`.
-- **Language**: Use the `language` field for the codec language tag (`Auto`, `Chinese`, `English`, etc.). Default is `Auto` for automatic detection.
-- **CustomVoice**: Requires a valid `voice` from the model’s speaker set. **VoiceDesign**: Use `instructions` to describe the voice. **Base**: Use `ref_audio` and `ref_text` for voice cloning.
-
-## Parameters
-
-### OpenAI Standard Parameters
-
-| Parameter         | Type   | Default        | Description                                                 |
-| ----------------- | ------ | -------------- | ----------------------------------------------------------- |
-| `input`           | string | **required**   | Text to synthesize                                          |
-| `model`           | string | server's model | Model to use (optional, should match server if specified)   |
-| `voice`           | string | "vivian"       | Speaker name (e.g., vivian, ryan, aiden)                    |
-| `response_format` | string | "wav"          | Audio format: wav, mp3, flac, pcm, aac, opus                |
-| `speed`           | float  | 1.0            | Playback speed (0.25-4.0, not supported with `stream=true`) |
-
-### vLLM-Omni Extension Parameters
-
-| Parameter                    | Type   | Default       | Description                                                                                                          |
-| ---------------------------- | ------ | ------------- | -------------------------------------------------------------------------------------------------------------------- |
-| `task_type`                  | string | "CustomVoice" | Task: CustomVoice, VoiceDesign, or Base                                                                              |
-| `language`                   | string | "Auto"        | Language (see supported languages below)                                                                             |
-| `instructions`               | string | ""            | Voice style/emotion instructions                                                                                     |
-| `max_new_tokens`             | int    | 2048          | Maximum tokens to generate                                                                                           |
-| `initial_codec_chunk_frames` | int    | null          | Per-request initial chunk size override for TTFA tuning. When null, IC is computed dynamically based on server load. |
-| `stream`                     | bool   | false         | Stream raw PCM chunks as they are decoded (requires `response_format="pcm"`)                                         |
-
-**Supported languages:** Auto, Chinese, English, Japanese, Korean, German, French, Russian, Portuguese, Spanish, Italian
-
-### Voice Clone Parameters (Base task)
-
-| Parameter            | Type   | Required | Description                                                                                     |
-| -------------------- | ------ | -------- | ----------------------------------------------------------------------------------------------- |
-| `ref_audio`          | string | **Yes**  | Reference audio (HTTP URL, base64 data URL, or `file://` URI with `--allowed-local-media-path`) |
-| `ref_text`           | string | No       | Transcript of reference audio (for ICL mode)                                                    |
-| `x_vector_only_mode` | bool   | No       | Use speaker embedding only (no ICL)                                                             |
-
-## Streaming
-
-Set `stream=true` with `response_format="pcm"` to receive raw PCM audio chunks as they are decoded
-(one chunk per Code2Wav window, default 25 frames; configurable in the stage config):
-
-```bash
-curl -X POST http://localhost:8091/v1/audio/speech \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "Hello, how are you?",
-        "voice": "vivian",
-        "language": "English",
-        "stream": true,
-        "response_format": "pcm"
-    }' --no-buffer | play -t raw -r 24000 -e signed -b 16 -c 1 -
-```
-
-**Constraints:**
-- `stream=true` requires `response_format="pcm"` (raw 16-bit signed PCM, 24 kHz mono).
-- `speed` adjustment is not supported when streaming.
-- Requires the server stage config to have `async_chunk: true` (default in `qwen3_tts.yaml`).
-
-## Streaming Text Input (WebSocket)
-
-The `/v1/audio/speech/stream` WebSocket endpoint accepts text incrementally, buffers and splits it at sentence boundaries, and generates audio per sentence.
-
-When `stream_audio=true`, each sentence is emitted as `audio.start`, one or more binary PCM frames, and `audio.done`.
-
-### Quick Start
-
-```bash
-python streaming_speech_client.py \
-    --text "Hello world. How are you? I am fine."
-
-python streaming_speech_client.py \
-    --text "Hello world. How are you? I am fine." \
-    --simulate-stt --stt-delay 0.1
-```
-
-### WebSocket Protocol
-
-Client -> Server:
-
-```jsonc
-{"type": "session.config", "voice": "Vivian", "task_type": "CustomVoice", "language": "Auto", "split_granularity": "sentence", "stream_audio": true, "response_format": "pcm"}
-{"type": "input.text", "text": "Hello, how are you? "}
-{"type": "input.done"}
-```
-
-Server -> Client:
-
-```jsonc
-{"type": "audio.start", "sentence_index": 0, "sentence_text": "Hello, how are you?", "format": "pcm", "sample_rate": 24000}
-// binary PCM frame(s)
-{"type": "audio.done", "sentence_index": 0, "total_bytes": 96000, "error": false}
-{"type": "session.done", "total_sentences": 1}
-```
-
-## Limitations
-
-- **Single request**: Batch processing is not yet optimized for online serving.
-
-## Troubleshooting
-
-1. **TTS model did not produce audio output**: Ensure you're using the correct model variant for your task type (CustomVoice task → CustomVoice model, etc.)
-2. **Connection refused**: Make sure the server is running on the correct port
-3. **Out of memory**: Use smaller model variant (`Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice`) or reduce `--gpu-memory-utilization`
-4. **Unsupported speaker**: Use `/v1/audio/voices` to list available voices for the loaded model
-5. **Voice clone fails**: Ensure you're using the Base model variant for voice cloning
-
-## Example materials
-
-??? abstract "gradio_demo.py"
-    ``````py
-    --8<-- "examples/online_serving/qwen3_tts/gradio_demo.py"
-    ``````
-??? abstract "gradio_fastrtc_demo.py"
-    ``````py
-    --8<-- "examples/online_serving/qwen3_tts/gradio_fastrtc_demo.py"
-    ``````
-??? abstract "openai_speech_client.py"
-    ``````py
-    --8<-- "examples/online_serving/qwen3_tts/openai_speech_client.py"
-    ``````
-??? abstract "run_gradio_demo.sh"
-    ``````sh
-    --8<-- "examples/online_serving/qwen3_tts/run_gradio_demo.sh"
-    ``````
-??? abstract "run_server.sh"
-    ``````sh
-    --8<-- "examples/online_serving/qwen3_tts/run_server.sh"
-    ``````
-??? abstract "streaming_speech_client.py"
-    ``````py
-    --8<-- "examples/online_serving/qwen3_tts/streaming_speech_client.py"
-    ``````
-??? abstract "tts_common.py"
-    ``````py
-    --8<-- "examples/online_serving/qwen3_tts/tts_common.py"
-    ``````

docs/user_guide/examples/online_serving/text_to_audio.md

@@ -0,0 +1,193 @@
+# Text-To-Audio
+
+Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/stable_audio>.
+
+This example demonstrates how to deploy Stable Audio models for online text-to-audio generation using vLLM-Omni.
+
+## Supported Models
+
+| Model | Description |
+|-------|-------------|
+| `stabilityai/stable-audio-open-1.0` | Open-source audio generation, up to ~47 seconds, 44.1 kHz stereo |
+
+## Start Server
+
+### Basic Start
+
+```bash
+vllm-omni serve stabilityai/stable-audio-open-1.0 \
+    --host 0.0.0.0 \
+    --port 8091 \
+    --gpu-memory-utilization 0.9 \
+    --trust-remote-code \
+    --enforce-eager \
+    --omni
+```
+
+## API Calls
+
+### Method 1: Using curl
+
+```bash
+# Run all curl examples
+bash curl_examples.sh
+
+# Or execute directly
+curl -X POST http://localhost:8091/v1/audio/generate \
+    -H "Content-Type: application/json" \
+    -d '{
+        "input": "The sound of a cat purring",
+        "audio_length": 10.0
+    }' --output cat.wav
+```
+
+### Method 2: Using Python Client
+
+```bash
+cd examples/online_serving/stable_audio
+
+# Simple generation
+python stable_audio_client.py \
+    --text "The sound of a cat purring"
+
+# With custom duration
+python stable_audio_client.py \
+    --text "A dog barking" \
+    --audio_length 5.0
+
+# With all parameters
+python stable_audio_client.py \
+    --text "Thunder and rain" \
+    --audio_length 15.0 \
+    --negative_prompt "Low quality" \
+    --guidance_scale 7.0 \
+    --num_inference_steps 100 \
+    --seed 42 \
+    --output thunder.wav
+```
+
+The Python client supports the following command-line arguments:
+
+- `--api_url`: API endpoint URL (default: `http://localhost:8091/v1/audio/generate`)
+- `--text`: Text prompt for audio generation (default: `"The sound of a cat purring"`)
+- `--audio_length`: Audio length in seconds (default: `10.0`, max ~47s for `stable-audio-open-1.0`)
+- `--audio_start`: Audio start time in seconds (default: `0.0`)
+- `--negative_prompt`: Negative prompt for classifier-free guidance (default: `"Low quality"`)
+- `--guidance_scale`: Guidance scale for diffusion (default: `7.0`)
+- `--num_inference_steps`: Number of inference steps (default: `100`)
+- `--seed`: Random seed for reproducibility (default: `None`)
+- `--response_format`: Audio output format (default: `wav`). Options: `wav`, `mp3`, `flac`, `pcm`
+- `--output`: Output file path (default: `stable_audio_output.wav`)
+
+### Method 3: Using Python httpx
+
+```python
+import httpx
+
+response = httpx.post(
+    "http://localhost:8091/v1/audio/generate",
+    json={
+        "input": "The sound of ocean waves crashing on a beach",
+        "audio_length": 10.0,
+        "negative_prompt": "Low quality, distorted",
+        "guidance_scale": 7.0,
+        "num_inference_steps": 100,
+    },
+    timeout=300.0,
+)
+
+with open("ocean.wav", "wb") as f:
+    f.write(response.content)
+```
+
+## Request Format
+
+### Simple Generation
+
+```json
+{
+    "input": "The sound of ocean waves"
+}
+```
+
+### Generation with Parameters
+
+```json
+{
+    "input": "A piano playing a gentle melody",
+    "audio_length": 10.0,
+    "negative_prompt": "Low quality, distorted, noisy",
+    "guidance_scale": 8.0,
+    "num_inference_steps": 150,
+    "seed": 42,
+    "response_format": "wav"
+}
+```
+
+## API Reference
+
+### Endpoint
+
+```
+POST /v1/audio/generate
+Content-Type: application/json
+```
+
+### Generation Parameters
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `input` | string | **required** | Text prompt describing the audio to generate |
+| `model` | string | server's model | Model to use (optional, should match server if specified) |
+| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, aac, opus |
+| `speed` | float | 1.0 | Playback speed (0.25 - 4.0) |
+| `audio_length` | float | null | Audio duration in seconds (max ~47s for `stable-audio-open-1.0`) |
+| `audio_start` | float | 0.0 | Audio start time in seconds |
+| `negative_prompt` | string | null | Text describing what to avoid in generation |
+| `guidance_scale` | float | model default | Classifier-free guidance scale (higher = more adherence to prompt) |
+| `num_inference_steps` | int | model default | Number of denoising steps (higher = better quality, slower) |
+| `seed` | int | null | Random seed for reproducible generation |
+
+### Response Format
+
+Returns binary audio data with appropriate `Content-Type` header (e.g., `audio/wav`).
+
+## Tuning Tips
+
+1. **Audio Length**: Keep under 47 seconds for `stable-audio-open-1.0`.
+2. **Quality vs Speed**:
+   - 50 steps: Fast, decent quality (quick previews)
+   - 100 steps: Good balance (general purpose)
+   - 150+ steps: High quality, slower (final / critical audio)
+3. **Guidance Scale**:
+   - Lower (3 - 5): More creative / varied output
+   - Default (7): Good balance
+   - Higher (10+): Strict adherence to the prompt
+4. **Negative Prompts**: Use to avoid unwanted characteristics such as `"Low quality"`, `"distorted"`, `"noisy"`.
+5. **Seeds**: Set a fixed seed to get deterministic, reproducible results.
+
+## File Description
+
+| File | Description |
+|------|-------------|
+| `curl_examples.sh` | Curl examples covering common use cases |
+| `stable_audio_client.py` | Python client with full CLI argument support |
+
+## Troubleshooting
+
+1. **Audio generation model did not produce audio output**: Verify the server started successfully and the model loaded without errors.
+2. **Connection refused**: Make sure the server is running on the correct port.
+3. **Generation timeout**: Reduce `num_inference_steps` or `audio_length`, and check GPU memory with `nvidia-smi`.
+4. **Out of memory**: Lower `--gpu-memory-utilization` or reduce `audio_length`.
+5. **Audio quality issues**: Increase `num_inference_steps`, add a negative prompt, or raise `guidance_scale`.
+
+## Example materials
+
+??? abstract "stable_audio_client.py"
+    ``````py
+    --8<-- "examples/online_serving/stable_audio/stable_audio_client.py"
+    ``````
+??? abstract "curl_examples.sh"
+    ``````sh
+    --8<-- "examples/online_serving/stable_audio/curl_examples.sh"
+    ``````

docs/user_guide/examples/online_serving/text_to_speech.md

@@ -0,0 +1,489 @@
+# Text-To-Speech (Online Serving)
+
+Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/text_to_speech>.
+
+
+vLLM-Omni exposes TTS models through the OpenAI-compatible
+[`POST /v1/audio/speech`](https://github.com/vllm-project/vllm-omni/tree/main/docs/serving/speech_api.md) endpoint,
+launched with `vllm serve <model> --omni`. Each TTS model has its own
+subdirectory containing client snippets, gradio demos, and helper
+scripts; this README is the single doc entry point for all of them.
+
+For offline inference, see [`examples/offline_inference/text_to_speech`](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_speech/README.md).
+For the full list of supported architectures across all modalities, see
+[Supported Models](https://github.com/vllm-project/vllm-omni/tree/main/docs/models/supported_models.md).
+
+## Supported Models
+
+| Model | HuggingFace repo | Voice cloning | Streaming | Voice presets / upload | Gradio demo |
+|---|---|---|---|---|---|
+| Fish Speech S2 Pro | `fishaudio/s2-pro` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | — | ✓ |
+| OmniVoice | `k2-fsa/OmniVoice` | (offline only) | — | — | — |
+| Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | ✓ (Base) | ✓ (PCM + WebSocket) | ✓ (presets + `/v1/audio/voices` upload) | ✓ (standard + FastRTC) |
+| VoxCPM | local model dir | ✓ | ✓ (PCM stream) | — | — |
+| VoxCPM2 | `openbmb/VoxCPM2` | ✓ | ✓ (AudioWorklet via gradio) | — | ✓ |
+| Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✓ (gated upstream) | ✓ | ✓ (presets) | ✓ |
+
+CosyVoice3 is intentionally absent: no online example exists for it yet. See its [offline section](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_speech/README.md#cosyvoice3) instead.
+
+## Common Quick Start
+
+Launch the server (defaults shown — adjust `--port`, `--gpu-memory-utilization`, etc. as needed):
+
+```bash
+vllm serve <hf-repo-or-local-path> --omni --port 8091
+```
+
+Send a TTS request via curl:
+
+```bash
+curl -X POST http://localhost:8091/v1/audio/speech \
+    -H "Content-Type: application/json" \
+    -d '{
+        "input": "Hello, how are you?",
+        "voice": "default",
+        "response_format": "wav"
+    }' --output output.wav
+```
+
+Or via Python httpx:
+
+```python
+import httpx
+
+response = httpx.post(
+    "http://localhost:8091/v1/audio/speech",
+    json={
+        "input": "Hello, how are you?",
+        "voice": "default",
+        "response_format": "wav",
+    },
+    timeout=300.0,
+)
+open("output.wav", "wb").write(response.content)
+```
+
+Or via the OpenAI SDK:
+
+```python
+from openai import OpenAI
+
+client = OpenAI(base_url="http://localhost:8091/v1", api_key="none")
+response = client.audio.speech.create(
+    model="<hf-repo>",
+    voice="default",
+    input="Hello, how are you?",
+)
+response.stream_to_file("output.wav")
+```
+
+Streaming PCM output (where supported) — set `stream=true` with `response_format="pcm"`:
+
+```bash
+curl -X POST http://localhost:8091/v1/audio/speech \
+    -H "Content-Type: application/json" \
+    -d '{
+        "input": "Hello, how are you?",
+        "voice": "default",
+        "stream": true,
+        "response_format": "pcm"
+    }' --no-buffer | play -t raw -r 24000 -e signed -b 16 -c 1 -
+```
+
+Adjust the player's sample rate to match the model (44.1 kHz for Fish Speech, 48 kHz for VoxCPM2, 24 kHz for the others).
+
+For full request-shape documentation (all parameters, response formats, error codes), see the [Speech API reference](https://github.com/vllm-project/vllm-omni/tree/main/docs/serving/speech_api.md).
+
+---
+
+## Fish Speech S2 Pro
+
+4B dual-AR TTS at 44.1 kHz. Server uses the DAC codec.
+
+### Prerequisites
+```bash
+pip install fish-speech
+```
+
+### Launch
+```bash
+vllm serve fishaudio/s2-pro --omni --port 8091
+# or:
+./fish_speech/run_server.sh
+```
+The deploy config auto-loads from `vllm_omni/deploy/fish_qwen3_omni.yaml` (the HF `model_type` on the fishaudio checkpoint is `fish_qwen3_omni`).
+
+### Voice cloning
+```bash
+curl -X POST http://localhost:8091/v1/audio/speech \
+    -H "Content-Type: application/json" \
+    -d '{
+        "input": "Hello, this is a cloned voice.",
+        "voice": "default",
+        "ref_audio": "https://example.com/reference.wav",
+        "ref_text": "Transcript of the reference audio."
+    }' --output cloned.wav
+```
+
+### CLI client
+```bash
+cd examples/online_serving/text_to_speech/fish_speech
+python speech_client.py --text "Hello, how are you?"
+python speech_client.py --text "Hello world" --stream --output output.pcm
+```
+
+### Gradio demo
+```bash
+./fish_speech/run_gradio_demo.sh             # launches server + Gradio
+python fish_speech/gradio_demo.py --api-base http://localhost:8091  # if server already running
+```
+
+### Notes
+- Output: 44.1 kHz mono.
+- Streaming PCM player command must use `-r 44100`.
+
+---
+
+## OmniVoice
+
+Zero-shot multilingual TTS (600+ languages). Online serving currently exposes **auto voice** only; voice cloning and voice design are available offline.
+
+### Prerequisites
+```bash
+huggingface-cli download k2-fsa/OmniVoice
+```
+Voice cloning (offline) needs `transformers>=5.3.0`; auto voice works with `transformers>=4.57.0`.
+
+### Launch
+```bash
+vllm serve k2-fsa/OmniVoice --omni --port 8091 --trust-remote-code
+# or:
+./omnivoice/run_server.sh
+```
+
+### CLI client
+```bash
+cd examples/online_serving/text_to_speech/omnivoice
+python speech_client.py --text "Hello, how are you?"
+python speech_client.py --text "Bonjour, comment allez-vous?" --language French
+```
+
+The client supports `--api-base`, `--model`, `--text`, `--response-format`, `--language`, `--output`.
+
+### Notes
+- Voice cloning and voice design require offline inference; see the [offline OmniVoice section](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_speech/README.md#omnivoice).
+
+---
+
+## Qwen3-TTS
+
+Three model variants exposed via separate checkpoints:
+
+| Variant | HF repo | Use |
+|---|---|---|
+| CustomVoice | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | Predefined speakers (`vivian`, `ryan`, …) with optional style instructions |
+| VoiceDesign | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | Natural-language voice style description |
+| Base | `Qwen/Qwen3-TTS-12Hz-1.7B-Base` | Voice cloning from a reference audio |
+
+Each variant ships smaller `0.6B` companions where available.
+
+### Launch
+```bash
+vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni --port 8091
+# or:
+./qwen3_tts/run_server.sh                # default: CustomVoice
+./qwen3_tts/run_server.sh VoiceDesign
+./qwen3_tts/run_server.sh Base
+```
+
+### Choosing an executor backend (uniproc vs mp)
+Stage configs ship with the chunked-streaming default. To use the uniproc executor (lower IPC overhead for the Base cloning task), pass `--stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_tts_uniproc.yaml`. See [#2603](https://github.com/vllm-project/vllm-omni/issues/2603) and [#2604](https://github.com/vllm-project/vllm-omni/pull/2604) for the full investigation.
+
+To opt out of chunked streaming, pass `--no-async-chunk` instead — the pipeline auto-dispatches to the end-to-end codec processor.
+
+### Sending requests
+```bash
+# CustomVoice with a predefined speaker
+python qwen3_tts/openai_speech_client.py \
+    --model Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
+    --text "今天天气真好" \
+    --voice ryan \
+    --instructions "用开心的语气说"
+
+# VoiceDesign with a style description
+python qwen3_tts/openai_speech_client.py \
+    --model Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign \
+    --task-type VoiceDesign \
+    --text "哥哥，你回来啦" \
+    --instructions "体现撒娇稚嫩的萝莉女声，音调偏高"
+
+# Base voice cloning
+python qwen3_tts/openai_speech_client.py \
+    --model Qwen/Qwen3-TTS-12Hz-1.7B-Base \
+    --task-type Base \
+    --text "Hello, this is a cloned voice" \
+    --ref-audio /path/to/reference.wav \
+    --ref-text "Original transcript of the reference audio"
+```
+
+### Voices endpoint
+List available voices, or upload a custom one for Base cloning:
+```bash
+# List
+curl http://localhost:8091/v1/audio/voices
+
+# Upload
+curl -X POST http://localhost:8091/v1/audio/voices \
+    -F "audio_sample=@/path/to/voice_sample.wav" \
+    -F "consent=user_consent_id" \
+    -F "name=custom_voice_1" \
+    -F "ref_text=The exact transcript of the audio sample." \
+    -F "speaker_description=warm narrator"
+```
+Uploaded voices are then usable as `voice="custom_voice_1"` on subsequent requests.
+
+### Streaming PCM
+```bash
+curl -X POST http://localhost:8091/v1/audio/speech \
+    -H "Content-Type: application/json" \
+    -d '{
+        "input": "Hello, how are you?",
+        "voice": "vivian",
+        "language": "English",
+        "stream": true,
+        "response_format": "pcm"
+    }' --no-buffer | play -t raw -r 24000 -e signed -b 16 -c 1 -
+```
+Streaming requires `response_format="pcm"` and `async_chunk: true` on the stage config (default in `qwen3_tts.yaml`). `speed` is not supported when streaming.
+
+### Streaming WebSocket
+The `/v1/audio/speech/stream` endpoint accepts text incrementally, splits it at sentence boundaries, and emits one PCM stream per sentence:
+```bash
+python qwen3_tts/streaming_speech_client.py --text "Hello world. How are you? I am fine."
+python qwen3_tts/streaming_speech_client.py --text "..." --simulate-stt --stt-delay 0.1
+```
+
+### Gradio demos
+```bash
+./qwen3_tts/run_gradio_demo.sh                              # CustomVoice (default)
+./qwen3_tts/run_gradio_demo.sh --task-type VoiceDesign
+./qwen3_tts/run_gradio_demo.sh --task-type Base
+
+# FastRTC variant (gapless WebRTC streaming):
+pip install fastrtc
+python qwen3_tts/gradio_fastrtc_demo.py --api-base http://localhost:8000
+```
+
+### Speaker embedding interpolation
+`qwen3_tts/speaker_embedding_interpolation.py` blends two predefined speakers' embeddings to produce intermediate voices. See the script for usage.
+
+### Batch client
+`qwen3_tts/batch_speech_client.py` issues many concurrent requests for throughput measurement.
+
+### Notes
+- Base voice cloning has uniproc-vs-mp tradeoffs depending on per-request reference audio cost; see the executor-backend section above.
+- `vllm_omni/deploy/qwen3_tts.yaml` is the default deploy config (loaded by HF `model_type`); per-stage runtime overrides are available via `--stage-N-<field> <value>`.
+
+---
+
+## VoxCPM
+
+Split-stage TTS at 24 kHz.
+
+### Prerequisites
+```bash
+pip install voxcpm
+# or use a local source tree:
+export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/VoxCPM/src
+```
+
+If the native VoxCPM `config.json` lacks HF `model_type`, set up an HF-compatible config dir:
+```bash
+export VOXCPM_MODEL=/path/to/voxcpm-model
+export VLLM_OMNI_VOXCPM_HF_CONFIG_PATH=/tmp/voxcpm_hf_config
+mkdir -p "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"
+cp "$VOXCPM_MODEL/config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/config.json"
+cp "$VOXCPM_MODEL/generation_config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/generation_config.json" 2>/dev/null || true
+python3 -c 'import json, os; p=os.path.join(os.environ["VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"], "config.json"); cfg=json.load(open(p, "r", encoding="utf-8")); cfg["model_type"]="voxcpm"; cfg.setdefault("architectures", ["VoxCPMForConditionalGeneration"]); json.dump(cfg, open(p, "w", encoding="utf-8"), indent=2, ensure_ascii=False)'
+```
+
+### Launch
+```bash
+export VOXCPM_MODEL=/path/to/voxcpm-model
+./voxcpm/run_server.sh                # async-chunk streaming (default)
+./voxcpm/run_server.sh sync           # non-streaming
+```
+Or directly:
+```bash
+vllm serve "$VOXCPM_MODEL" \
+    --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \
+    --trust-remote-code --enforce-eager --omni --port 8091
+```
+
+### Sending requests
+```bash
+# Basic TTS
+python voxcpm/openai_speech_client.py \
+    --model "$VOXCPM_MODEL" \
+    --text "This is a VoxCPM online text-to-speech example."
+
+# Voice cloning
+python voxcpm/openai_speech_client.py \
+    --model "$VOXCPM_MODEL" \
+    --text "This sentence is synthesized with a cloned voice." \
+    --ref-audio /path/to/reference.wav \
+    --ref-text "The exact transcript spoken in reference.wav."
+
+# Streaming PCM
+python voxcpm/openai_speech_client.py \
+    --model "$VOXCPM_MODEL" \
+    --text "This is a streaming VoxCPM request." \
+    --stream --output voxcpm_stream.pcm
+```
+
+### Notes
+- `voxcpm.yaml` for one-shot decode; `voxcpm_async_chunk.yaml` for single-request streaming. Do not use the async-chunk config for concurrent requests or `/v1/audio/speech/batch`.
+- Generic TTS fields not supported by VoxCPM: `voice`, `instructions`, `language`, `speaker_embedding`, `x_vector_only_mode`.
+- For benchmark measurement, see [`benchmarks/voxcpm`](https://github.com/vllm-project/vllm-omni/tree/main/benchmarks/voxcpm/README.md).
+
+---
+
+## VoxCPM2
+
+Single-stage native AR TTS at 48 kHz.
+
+### Launch
+```bash
+vllm serve openbmb/VoxCPM2 --omni --host 0.0.0.0 --port 8000
+```
+Deploy config auto-loads from `vllm_omni/deploy/voxcpm2.yaml`. Pass `--deploy-config <path>` to override or `--stage-N-<field> <value>` for per-stage runtime tweaks.
+
+### Sending requests
+```bash
+# Zero-shot synthesis
+python voxcpm2/openai_speech_client.py --text "Hello, this is VoxCPM2."
+
+# Voice cloning
+python voxcpm2/openai_speech_client.py \
+    --text "This should sound like the reference speaker." \
+    --ref-audio /path/to/reference.wav
+```
+The `ref_audio` field accepts local file paths (auto-base64), HTTP URLs, or `data:audio/wav;base64,...` data URIs.
+
+### Gradio demo (gapless streaming via AudioWorklet)
+```bash
+python voxcpm2/gradio_demo.py
+```
+Uses an AudioWorklet-based player adapted from the Qwen3-TTS demo for gap-free playback. Audio is streamed from the OpenAI Speech endpoint with `stream=true`.
+
+---
+
+## Voxtral TTS
+
+Voxtral-4B-TTS (Mistral). Uses the `mistral_common` `SpeechRequest` protocol; voice presets are model-specific.
+
+### Prerequisites
+Latest `mistral_common` with `SpeechRequest` support:
+```bash
+pip install -e /path/to/mistral-common  # or upgrade from PyPI when available
+```
+
+### Launch
+```bash
+vllm serve mistralai/Voxtral-4B-TTS-2603 --omni --port 8091
+```
+Deploy config auto-loads from `vllm_omni/deploy/voxtral_tts.yaml`.
+
+### Gradio demo
+```bash
+python voxtral_tts/gradio_demo.py
+```
+The demo handles voice-preset selection and reference-audio upload. `voxtral_tts/text_preprocess.py` provides the text-normalization helpers used by the demo (also available for other clients).
+
+### Notes
+- Voice presets are listed on the HF model card (`mistralai/Voxtral-4B-TTS-2603`).
+- Voice cloning is gated upstream and may require a recent `mistral_common`.
+- A standalone CLI client is not yet shipped; the gradio demo is the canonical reference for now.
+
+## Example materials
+
+??? abstract "fish_speech/gradio_demo.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/fish_speech/gradio_demo.py"
+    ``````
+??? abstract "fish_speech/run_gradio_demo.sh"
+    ``````sh
+    --8<-- "examples/online_serving/text_to_speech/fish_speech/run_gradio_demo.sh"
+    ``````
+??? abstract "fish_speech/run_server.sh"
+    ``````sh
+    --8<-- "examples/online_serving/text_to_speech/fish_speech/run_server.sh"
+    ``````
+??? abstract "fish_speech/speech_client.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/fish_speech/speech_client.py"
+    ``````
+??? abstract "omnivoice/run_server.sh"
+    ``````sh
+    --8<-- "examples/online_serving/text_to_speech/omnivoice/run_server.sh"
+    ``````
+??? abstract "omnivoice/speech_client.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/omnivoice/speech_client.py"
+    ``````
+??? abstract "qwen3_tts/batch_speech_client.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/qwen3_tts/batch_speech_client.py"
+    ``````
+??? abstract "qwen3_tts/gradio_demo.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/qwen3_tts/gradio_demo.py"
+    ``````
+??? abstract "qwen3_tts/openai_speech_client.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/qwen3_tts/openai_speech_client.py"
+    ``````
+??? abstract "qwen3_tts/run_gradio_demo.sh"
+    ``````sh
+    --8<-- "examples/online_serving/text_to_speech/qwen3_tts/run_gradio_demo.sh"
+    ``````
+??? abstract "qwen3_tts/run_server.sh"
+    ``````sh
+    --8<-- "examples/online_serving/text_to_speech/qwen3_tts/run_server.sh"
+    ``````
+??? abstract "qwen3_tts/speaker_embedding_interpolation.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/qwen3_tts/speaker_embedding_interpolation.py"
+    ``````
+??? abstract "qwen3_tts/streaming_speech_client.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/qwen3_tts/streaming_speech_client.py"
+    ``````
+??? abstract "qwen3_tts/tts_common.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/qwen3_tts/tts_common.py"
+    ``````
+??? abstract "voxcpm/openai_speech_client.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/voxcpm/openai_speech_client.py"
+    ``````
+??? abstract "voxcpm/run_server.sh"
+    ``````sh
+    --8<-- "examples/online_serving/text_to_speech/voxcpm/run_server.sh"
+    ``````
+??? abstract "voxcpm2/gradio_demo.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/voxcpm2/gradio_demo.py"
+    ``````
+??? abstract "voxcpm2/openai_speech_client.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/voxcpm2/openai_speech_client.py"
+    ``````
+??? abstract "voxtral_tts/gradio_demo.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/voxtral_tts/gradio_demo.py"
+    ``````
+??? abstract "voxtral_tts/text_preprocess.py"
+    ``````py
+    --8<-- "examples/online_serving/text_to_speech/voxtral_tts/text_preprocess.py"
+    ``````

docs/user_guide/quantization/autoround.md

@@ -0,0 +1,127 @@
+# AutoRound Quantization
+
+## Overview
+
+[AutoRound](https://github.com/intel/auto-round) produces pre-quantized
+checkpoints for LLMs, VLMs, and diffusion models. vLLM-Omni reads the
+checkpoint's `config.json` and auto-detects
+`quantization_config.quant_method = "auto-round"`.
+
+AutoRound is static quantization: no `--quantization` flag is needed at
+inference time when the checkpoint already contains the quantization config.
+
+## Hardware Support
+
+| Device | Support |
+|--------|---------|
+| NVIDIA Blackwell GPU (SM 100+) | ✅ |
+| NVIDIA Ada/Hopper GPU (SM 89+) | ✅ |
+| NVIDIA Ampere GPU (SM 80+) | ✅ |
+| AMD ROCm | ⭕ |
+| Intel XPU | ✅ |
+| Ascend NPU | ❌ |
+
+Legend: `✅` supported, `❌` unsupported, `⭕` not verified in this
+guide. AutoRound is Intel-supported.
+
+## Model Type Support
+
+### Diffusion Model (Qwen-Image, Wan2.2)
+
+| Model | Checkpoint | Scope | Scheme | Backend |
+|-------|------------|-------|--------|---------|
+| FLUX.1-dev | `vllm-project-org/FLUX.1-dev-AutoRound-w4a16` | Diffusion transformer | W4A16 | GPTQ-Marlin or Intel-supported AutoRound backend |
+| Qwen-Image | Not listed | Diffusion transformer | W4A16 | Not validated |
+| Wan2.2 | Not listed | Diffusion transformer | W4A16 | Not validated |
+
+### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS)
+
+| Model | Checkpoint | Scope | Scheme | Backend |
+|-------|------------|-------|--------|---------|
+| Qwen2.5-Omni-7B | `Intel/Qwen2.5-Omni-7B-int4-AutoRound` | Language-model stage | W4A16 | AutoRound |
+| Qwen3-Omni-30B-A3B-Instruct | `Intel/Qwen3-Omni-30B-A3B-Instruct-int4-AutoRound` | Thinker language-model stage | W4A16 | AutoRound |
+| Qwen3-TTS | Not listed | TTS language-model stage | W4A16 | Not validated |
+
+AutoRound support is checkpoint-driven. A model is supported when its
+checkpoint uses a compatible INC/AutoRound config and the target stage maps to
+vLLM-Omni's runtime module names.
+
+### Multi-Stage Diffusion Model (BAGEL, GLM-Image)
+
+| Model | Scope | Status | Notes |
+|-------|-------|--------|-------|
+| BAGEL | Checkpoint-defined diffusion or transformer stage | Not validated | Requires a compatible AutoRound checkpoint |
+| GLM-Image | Checkpoint-defined diffusion or transformer stage | Not validated | Requires a compatible AutoRound checkpoint |
+
+## Configuration
+
+Python API:
+
+```python
+from vllm_omni import Omni
+from vllm_omni.inputs.data import OmniDiffusionSamplingParams
+
+omni = Omni(model="vllm-project-org/FLUX.1-dev-AutoRound-w4a16")
+
+outputs = omni.generate(
+    "A cat sitting on a windowsill",
+    OmniDiffusionSamplingParams(num_inference_steps=28),
+)
+outputs[0].save_images("output.png")
+```
+
+CLI:
+
+```bash
+python examples/offline_inference/text_to_image/text_to_image.py \
+  --model vllm-project-org/FLUX.1-dev-AutoRound-w4a16 \
+  --prompt "A cat sitting on a windowsill" \
+  --num-inference-steps 28 \
+  --output outputs/flux_w4a16.png
+```
+
+## Parameters
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `quant_method` | str | Must be `"auto-round"` |
+| `bits` | int | Quantized weight bit width, usually `4` |
+| `group_size` | int | Quantization group size |
+| `packing_format` | str | AutoRound packing format, for example `auto_round:auto_gptq` |
+| `block_name_to_quantize` | str | Checkpoint block names that should map to runtime module names |
+
+The checkpoint should contain a config like:
+
+```json
+{
+  "quantization_config": {
+    "quant_method": "auto-round",
+    "bits": 4,
+    "group_size": 128,
+    "sym": true,
+    "packing_format": "auto_round:auto_gptq",
+    "block_name_to_quantize": "transformer_blocks,single_transformer_blocks"
+  }
+}
+```
+
+## Validation and Notes
+
+At load time, vLLM-Omni builds an `OmniINCConfig`, remaps checkpoint block names
+to runtime module names, and selects the matching vLLM compute backend.
+
+Example checkpoint creation:
+
+```bash
+auto-round \
+  --model black-forest-labs/FLUX.1-dev \
+  --scheme W4A16 \
+  --batch_size 1 \
+  --disable_opt_rtn \
+  --dataset coco2014 \
+  --iters 0
+```
+
+Use the generated output directory directly as the `model` argument. See the
+[AutoRound documentation](https://github.com/intel/auto-round) for all
+available schemes and options.

docs/user_guide/quantization/fp8.md

@@ -0,0 +1,121 @@
+# FP8 Quantization
+
+## Overview
+
+FP8 quantization converts BF16/FP16 weights to FP8 at model load time. Online
+activation scaling is the default and does not require calibration. Static
+activation scaling is supported when calibrated scale information is available.
+For ModelOpt-produced pre-quantized checkpoints, see
+[ModelOpt Quantization](modelopt.md).
+
+Some architectures can quantize all linear layers. Others have
+quality-sensitive layers that should stay in BF16 through `ignored_layers`.
+Image-stream MLPs (`img_mlp`) are a common sensitive target because denoising
+latent ranges shift across timesteps and small per-layer errors can compound
+in deep DiT blocks.
+
+## Hardware Support
+
+| Device | Support |
+|--------|---------|
+| NVIDIA Blackwell GPU (SM 100+) | ✅ |
+| NVIDIA Ada/Hopper GPU (SM 89+) | ✅ |
+| NVIDIA Ampere GPU (SM 80+) | ✅ |
+| AMD ROCm | ⭕ |
+| Intel XPU | ⭕ |
+| Ascend NPU | ❌ |
+
+Legend: `✅` supported, `❌` unsupported, `⭕` not verified in this
+guide. FP8 on Ampere may use a weight-only path where available.
+
+## Model Type Support
+
+### Diffusion Model (Qwen-Image, Wan2.2)
+
+| Model | HF models | Online | Pre-calibrated | Recommendation | `ignored_layers` | Text-Encoder quantization |
+|-------|-----------|:-------:|:------:|----------------|------------------|------------------|
+| Qwen-Image | `Qwen/Qwen-Image`, `Qwen/Qwen-Image-2512` | Yes | Yes | Skip sensitive image-stream MLPs when quality regresses | `img_mlp` | |
+| Wan2.2 | Wan2.2 diffusion pipelines | Not validated | Not validated | Validate against BF16 before documenting as supported | TBD | |
+| Z-Image | `Tongyi-MAI/Z-Image-Turbo` | Yes | Yes | All layers | None | ✅︎ |
+| FLUX.1 | `black-forest-labs/FLUX.1-dev`, `black-forest-labs/FLUX.1-schnell` | Yes | Yes | All layers | None | |
+| FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B` | Yes | Yes | All layers | None | |
+| HunyuanImage-3.0 | `tencent/HunyuanImage-3.0`, `tencent/HunyuanImage-3.0-Instruct` | Yes | Yes | All layers; use the Hunyuan stage config for multi-stage runs | None | |
+| HunyuanVideo-1.5 | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `720p_t2v`, `480p_i2v` | Yes | Yes | All layers | None | |
+
+### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS)
+
+| Model | Scope | Format | Status |
+|-------|-------|--------|--------|
+| Qwen3-Omni | Thinker language-model stage | [ModelOpt](modelopt.md) `quant_algo=FP8` | Tested for thinker memory reduction |
+| Qwen3-TTS | TTS language-model stage | Checkpoint config | Not validated |
+
+Audio encoder, vision encoder, talker, and code2wav stay in BF16 unless a
+model-specific guide says otherwise.
+
+### Multi-Stage Diffusion Model (BAGEL, GLM-Image)
+
+| Model | Scope | Status | Notes |
+|-------|-------|--------|-------|
+| BAGEL | Stage-specific transformer or DiT module | Not validated | Route FP8 to the intended stage before enabling |
+| GLM-Image | Stage-specific transformer or DiT module | Not validated | Validate quality against BF16 baseline |
+
+## Configuration
+
+Python API:
+
+```python
+from vllm_omni import Omni
+from vllm_omni.inputs.data import OmniDiffusionSamplingParams
+
+omni = Omni(model="<your-model>", quantization="fp8")
+
+omni_with_skips = Omni(
+    model="<your-model>",
+    quantization_config={
+        "method": "fp8",
+        "ignored_layers": ["img_mlp"],
+    },
+)
+
+outputs = omni.generate(
+    "A cat sitting on a windowsill",
+    OmniDiffusionSamplingParams(num_inference_steps=50),
+)
+```
+
+CLI:
+
+```bash
+python text_to_image.py --model <your-model> --quantization fp8
+python text_to_image.py --model <your-model> --quantization fp8 --ignored-layers "img_mlp"
+vllm serve <your-model> --omni --quantization fp8
+```
+
+## Parameters
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `method` | str | - | Quantization method (`"fp8"`) |
+| `ignored_layers` | list[str] | `[]` | Layer name patterns to keep in BF16 |
+| `activation_scheme` | str | `"dynamic"` | `"dynamic"` selects online activation scaling, or `"static"` when scales are available |
+| `weight_block_size` | list[int] \| None | `None` | Block size for block-wise weight quantization |
+
+The available `ignored_layers` names depend on the model architecture, for
+example `to_qkv`, `to_out`, `img_mlp`, or `txt_mlp`.
+
+## Validation and Notes
+
+FP8 quantization can be combined with cache acceleration:
+
+```python
+omni = Omni(
+    model="<your-model>",
+    quantization="fp8",
+    cache_backend="tea_cache",
+    cache_config={"rel_l1_thresh": 0.2},
+)
+```
+
+Compare generated outputs with a BF16 baseline before adding a new model to the
+supported table. GLM-Image and Helios are not listed as FP8-supported diffusion
+models until they have method-specific validation.

docs/user_guide/quantization/gguf.md

@@ -0,0 +1,107 @@
+# GGUF Quantization
+
+## Overview
+
+GGUF support loads pre-quantized diffusion transformer weights while keeping
+the rest of the pipeline on the base Hugging Face checkpoint. Use the base
+model for tokenizer, text encoder, scheduler, and VAE, then pass the GGUF file
+for the transformer.
+
+GGUF is static quantization: the quantized weights are produced before serving.
+
+## Hardware Support
+
+| Device | Support |
+|--------|---------|
+| NVIDIA Blackwell GPU (SM 100+) | ✅ |
+| NVIDIA Ada/Hopper GPU (SM 89+) | ✅ |
+| NVIDIA Ampere GPU (SM 80+) | ✅ |
+| AMD ROCm | ⭕ |
+| Intel XPU | ⭕ |
+| Ascend NPU | ❌ |
+
+Legend: `✅` supported, `❌` unsupported, `⭕` not verified in this
+guide.
+
+## Model Type Support
+
+### Diffusion Model (Qwen-Image, Wan2.2)
+
+| Model | HF base model | GGUF input | Scope | Adapter |
+|-------|---------------|------------|-------|---------|
+| Qwen-Image family | `Qwen/Qwen-Image`, `Qwen/Qwen-Image-2512`, edit and layered Qwen-Image pipelines | Local `.gguf`, `repo/file.gguf`, or `repo:quant_type` | Transformer only | `QwenImageGGUFAdapter` |
+| Wan2.2 | Wan2.2 diffusion pipelines | Not validated | Transformer only | No validated adapter listed |
+| Z-Image | `Tongyi-MAI/Z-Image-Turbo` | Local `.gguf`, `repo/file.gguf`, or `repo:quant_type` | Transformer only | `ZImageGGUFAdapter` |
+| FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B` | Local `.gguf`, `repo/file.gguf`, or `repo:quant_type` | Transformer only | `Flux2KleinGGUFAdapter` |
+
+Generic FLUX.1 GGUF checkpoints are not listed here; the implemented adapter is
+for the FLUX.2-klein path.
+
+### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS)
+
+| Model | Scope | Status | Notes |
+|-------|-------|--------|-------|
+| Qwen3-Omni | Thinker language-model stage | Not validated | GGUF is not documented for omni/TTS AR stages |
+| Qwen3-TTS | TTS language-model stage | Not validated | GGUF is not documented for TTS stages |
+
+### Multi-Stage Diffusion Model (BAGEL, GLM-Image)
+
+| Model | Scope | Status | Notes |
+|-------|-------|--------|-------|
+| BAGEL | Stage-specific transformer weights | Not validated | Requires a model-specific GGUF adapter |
+| GLM-Image | Stage-specific transformer weights | Not validated | Requires a model-specific GGUF adapter |
+
+## Configuration
+
+Offline:
+
+```bash
+python examples/offline_inference/text_to_image/text_to_image.py \
+  --model Qwen/Qwen-Image \
+  --gguf-model QuantStack/Qwen-Image-GGUF/Qwen_Image-Q4_K_M.gguf \
+  --quantization gguf \
+  --prompt "a red paper kite hanging from a pine tree in a winter courtyard" \
+  --height 1024 \
+  --width 1024 \
+  --seed 42 \
+  --num_inference_steps 20 \
+  --output outputs/qwen_image_q4km.png
+```
+
+Online:
+
+```bash
+vllm serve Qwen/Qwen-Image \
+  --omni \
+  --port 8000 \
+  --quantization-config '{"method":"gguf","gguf_model":"QuantStack/Qwen-Image-GGUF/Qwen_Image-Q4_K_M.gguf"}'
+```
+
+## Parameters
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `method` | str | - | Quantization method (`"gguf"`) |
+| `gguf_model` | str | - | Local GGUF file, explicit Hugging Face file, or `repo:quant_type` selector |
+
+`gguf_model` accepts:
+
+| Form | Example |
+|------|---------|
+| Local file | `/models/z-image-Q4_K_M.gguf` |
+| Explicit HF file | `QuantStack/Qwen-Image-GGUF/Qwen_Image-Q4_K_M.gguf` |
+| HF repo plus quant type | `owner/repo:Q4_K_M` |
+
+## Validation and Notes
+
+1. `OmniDiffusionConfig` receives `{"method": "gguf", "gguf_model": ...}`.
+2. `DiffusersPipelineLoader` resolves the GGUF file.
+3. A model-specific adapter remaps GGUF tensor names to vLLM-Omni transformer
+   names.
+4. Only transformer weights are loaded from GGUF. Missing non-transformer
+   weights are loaded from the base model repository.
+5. vLLM's GGUF linear method performs dequantization and GEMM at runtime.
+
+Unsupported models fail fast with a clear "No GGUF adapter matched" error
+instead of falling back to a generic mapper. Many GGUF repositories do not
+include `model_index.json`; always pass the normal base model through `--model`.

docs/user_guide/quantization/int8.md

@@ -0,0 +1,107 @@
+# Int8 Quantization
+
+## Overview
+
+Int8 quantization supports W8A8 diffusion transformer inference on CUDA and
+Ascend NPU. It can quantize BF16/FP16 weights at load time, or load serialized
+Int8 checkpoints that already contain quantized weights and scales.
+
+Only online activation scaling is currently supported.
+
+## Hardware Support
+
+| Device | Support |
+|--------|---------|
+| NVIDIA Blackwell GPU (SM 100+) | ✅ |
+| NVIDIA Ada/Hopper GPU (SM 89+) | ✅ |
+| NVIDIA Ampere GPU (SM 80+) | ✅ |
+| AMD ROCm | ⭕ |
+| Intel XPU | ⭕ |
+| Ascend NPU | ✅ |
+
+Legend: `✅` supported, `❌` unsupported, `⭕` not verified in this
+guide.
+
+## Model Type Support
+
+### Diffusion Model (Qwen-Image, Wan2.2)
+
+| Model | HF models | CUDA | Ascend NPU | Mode | Recommendation |
+|-------|-----------|:----:|:----------:|------|----------------|
+| Qwen-Image | `Qwen/Qwen-Image`, `Qwen/Qwen-Image-2512` | Yes | Yes | Online W8A8 | All layers |
+| Wan2.2 | Wan2.2 diffusion pipelines | Not validated | Not validated | Online W8A8 | Validate before enabling in docs |
+| Z-Image | `Tongyi-MAI/Z-Image-Turbo` | Yes | Yes | Online W8A8 | All layers |
+
+Other diffusion models may work if their transformer uses supported linear
+layers, but they are not validated in this guide.
+
+### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS)
+
+| Model | Scope | Status | Notes |
+|-------|-------|--------|-------|
+| Qwen3-Omni | Thinker language-model stage | Not validated | Prefer checkpoint-supported ModelOpt FP8 or AutoRound paths |
+| Qwen3-TTS | TTS language-model stage | Not validated | No Int8 TTS stage support is documented |
+
+### Multi-Stage Diffusion Model (BAGEL, GLM-Image)
+
+| Model | Scope | Status | Notes |
+|-------|-------|--------|-------|
+| BAGEL | Stage-specific transformer or DiT module | Not validated | Requires explicit stage routing |
+| GLM-Image | Stage-specific transformer or DiT module | Not validated | Requires quality comparison with BF16 |
+
+## Configuration
+
+Python API:
+
+```python
+from vllm_omni import Omni
+from vllm_omni.inputs.data import OmniDiffusionSamplingParams
+
+omni = Omni(model="<your-model>", quantization="int8")
+
+omni_with_skips = Omni(
+    model="<your-model>",
+    quantization_config={
+        "method": "int8",
+        "ignored_layers": ["<layer-name>"],
+    },
+)
+
+outputs = omni.generate(
+    "A cat sitting on a windowsill",
+    OmniDiffusionSamplingParams(num_inference_steps=50),
+)
+```
+
+CLI:
+
+```bash
+python text_to_image.py --model <your-model> --quantization int8
+python text_to_image.py --model <your-model> --quantization int8 --ignored-layers "img_mlp"
+vllm serve <your-model> --omni --quantization int8
+```
+
+## Parameters
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `method` | str | - | Quantization method (`"int8"`) |
+| `activation_scheme` | str | `"dynamic"` | `"dynamic"` selects online activation scaling; static is not supported |
+| `ignored_layers` | list[str] | `[]` | Layer name patterns to keep in BF16/FP16 |
+| `is_checkpoint_int8_serialized` | bool | `False` | Set by checkpoint config when loading serialized Int8 weights |
+
+## Validation and Notes
+
+Int8 quantization can be combined with cache acceleration:
+
+```python
+omni = Omni(
+    model="<your-model>",
+    quantization="int8",
+    cache_backend="tea_cache",
+    cache_config={"rel_l1_thresh": 0.2},
+)
+```
+
+Only add a new model to the supported table after comparing the Int8 output
+against a BF16 baseline and documenting any required `ignored_layers`.

docs/user_guide/quantization/modelopt.md

@@ -0,0 +1,138 @@
+# ModelOpt Quantization
+
+## Overview
+
+ModelOpt quantization loads checkpoints produced by NVIDIA ModelOpt. The
+quantized weights and scale tensors are generated before serving, so inference
+does not run online calibration or convert a BF16 checkpoint at startup.
+
+vLLM-Omni currently validates the ModelOpt FP8 checkpoint path for diffusion
+transformers. The loader auto-detects supported ModelOpt FP8 checkpoint configs
+and keeps non-transformer components, such as the tokenizer, scheduler, text
+encoder, and VAE, on the base checkpoint unless a model-specific guide says
+otherwise.
+
+!!! note
+    `--force-cutlass-fp8` is an explicit runtime override for diffusion
+    checkpoints that already carry a supported ModelOpt FP8 config. It does not
+    quantize BF16 checkpoints and it does not apply to online `--quantization
+    fp8`. The flag only takes effect for ModelOpt FP8 diffusion stages on CUDA
+    SM89+ devices; other platforms and non-ModelOpt FP8 paths fall back to the
+    normal vLLM kernel selection.
+
+## Supported ModelOpt Checkpoint Formats
+
+vLLM-Omni treats ModelOpt checkpoints as pre-quantized checkpoints. The
+checkpoint config must identify ModelOpt as the quantization method or producer,
+and the quantization algorithm must be one of the validated FP8 algorithms.
+
+| Checkpoint field | Supported value |
+|------------------|-----------------|
+| `method` / `quant_method` | `modelopt` |
+| `producer.name` | `modelopt` |
+| `quant_algo` | `FP8`, `FP8_PER_CHANNEL_PER_TOKEN` |
+
+Other ModelOpt algorithms, such as NVFP4, are not enabled by this diffusion
+FP8 path until they have separate model and quality validation.
+
+## Hardware Support
+
+| Device | Support |
+|--------|---------|
+| NVIDIA Blackwell GPU (SM 100+) | ✅ |
+| NVIDIA Ada/Hopper GPU (SM 89+) | ✅ |
+| NVIDIA Ampere GPU (SM 80+) | ⭕ |
+| AMD ROCm | ⭕ |
+| Intel XPU | ⭕ |
+| Ascend NPU | ❌ |
+
+Legend: `✅` supported, `❌` unsupported, `⭕` not verified in this guide.
+The optional CUTLASS FP8 runtime override requires CUDA SM89+.
+
+## Model Type Support
+
+### Diffusion Model
+
+| Model | HF checkpoint | Scope | Status |
+|-------|---------------|-------|--------|
+| Qwen-Image 2512 | `feizhai123/qwen-image-2512-modelopt-fp8-dynamic-all` | Diffusion transformer | Validated for ModelOpt FP8 checkpoints |
+| Z-Image | `feizhai123/z-image-modelopt-fp8-conservative` | Diffusion transformer | Validated for ModelOpt FP8 checkpoints |
+| FLUX.2-dev | `feizhai123/flux2-dev-modelopt-fp8` | Diffusion transformer | Validated for ModelOpt FP8 checkpoints |
+| FLUX.2-klein 4B | `feizhai123/flux2-klein-4b-modelopt-fp8` | Diffusion transformer | Validated for ModelOpt FP8 checkpoints |
+| HunyuanImage-3.0 | `feizhai123/hunyuan-image3-modelopt-fp8` | MoE diffusion transformer | Validated for ModelOpt FP8 checkpoints |
+| Wan2.2 | Not available | Diffusion transformer | Not validated |
+
+### Multi-Stage Omni/TTS Model
+
+| Model | Scope | Status |
+|-------|-------|--------|
+| Qwen3-Omni | Thinker language-model stage | ModelOpt FP8 checkpoint path |
+| Qwen3-TTS | TTS language-model stage | Not validated |
+
+Audio encoder, vision encoder, talker, and code2wav stages stay in BF16 unless
+a model-specific guide documents otherwise.
+
+### Multi-Stage Diffusion Model
+
+ModelOpt checkpoints must be routed to the stage whose checkpoint contains the
+ModelOpt `quantization_config`. BAGEL and GLM-Image are not listed as validated
+ModelOpt targets yet.
+
+## Configuration
+
+For pre-quantized ModelOpt FP8 checkpoints, no `--quantization fp8` flag is
+needed. The checkpoint config selects the ModelOpt path.
+
+Online serving:
+
+```bash
+vllm serve <modelopt-fp8-checkpoint> \
+  --omni \
+  --tensor-parallel-size <N> \
+  --force-cutlass-fp8
+```
+
+Offline inference:
+
+```bash
+python examples/offline_inference/text_to_image/text_to_image.py \
+  --model <modelopt-fp8-checkpoint> \
+  --tensor-parallel-size <N> \
+  --prompt "a red ceramic teapot on a wooden table" \
+  --height 1024 \
+  --width 1024 \
+  --num-inference-steps 20 \
+  --seed 42 \
+  --output outputs/modelopt_fp8.png
+```
+
+Python API:
+
+```python
+from vllm_omni import Omni
+
+omni = Omni(
+    model="<modelopt-fp8-checkpoint>",
+    tensor_parallel_size=2,
+    force_cutlass_fp8=True,
+)
+```
+
+## Parameters
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `force_cutlass_fp8` / `--force-cutlass-fp8` | bool | `False` | Force CUTLASS FP8 linear kernels for supported ModelOpt FP8 diffusion stages on CUDA SM89+ |
+
+## Validation and Notes
+
+1. Compare the ModelOpt FP8 checkpoint against the BF16 baseline with the same
+   prompt, resolution, seed, and inference steps.
+2. Use `tests/diffusion/quantization/test_quantization_quality.py` with
+   `VLLM_OMNI_QUALITY_CONFIGS` to validate local baseline and quantized model
+   paths.
+3. Report LPIPS, PSNR, MAE, throughput, latency, and peak memory when adding a
+   new validated ModelOpt diffusion checkpoint.
+4. Keep `--quantization fp8` for online FP8 from BF16 checkpoints; use this
+   ModelOpt path only when the checkpoint already contains ModelOpt FP8 weights
+   and scales.

docs/user_guide/quantization/msmodelslim.md

@@ -0,0 +1,95 @@
+# msModelSlim Quantization
+
+## Overview
+
+[msModelSlim](https://github.com/Ascend/msmodelslim) is an Ascend compression
+toolkit for producing pre-quantized model checkpoints. In vLLM-Omni, these
+checkpoints run through the Ascend/NPU path with `--quantization ascend`.
+
+msModelSlim is static quantization: quantized weights are generated offline
+before vLLM-Omni inference starts.
+
+## Hardware Support
+
+| Device | Support |
+|--------|---------|
+| NVIDIA Blackwell GPU (SM 100+) | ❌ |
+| NVIDIA Ada/Hopper GPU (SM 89+) | ❌ |
+| NVIDIA Ampere GPU (SM 80+) | ❌ |
+| AMD ROCm | ❌ |
+| Intel XPU | ❌ |
+| Ascend NPU | ✅ |
+
+Legend: `✅` supported, `❌` unsupported, `⭕` not verified in this
+guide.
+
+## Model Type Support
+
+### Diffusion Model (Qwen-Image, Wan2.2)
+
+| Model | Base model | Scope | Hardware | Notes |
+|-------|------------|-------|----------|-------|
+| Wan2.2 | Wan2.2 diffusion weights | DiT or diffusion stage | Ascend NPU | Upstream msModelSlim provides a Wan2.2 quantization recipe; vLLM-Omni inference validation is not listed |
+| Qwen-Image | `Qwen/Qwen-Image`, `Qwen/Qwen-Image-2512` | DiT or diffusion stage | Ascend NPU | Not validated in this guide |
+| HunyuanImage-3.0 | `tencent/HunyuanImage-3.0`, `tencent/HunyuanImage-3.0-Instruct` | DiT or diffusion stage | Ascend A2/A3 NPU | Generate quantized weights with the HunyuanImage-3.0 msModelSlim adaptation |
+
+Public Hugging Face quantized weights are not available yet. Use the
+[HunyuanImage-3.0 msModelSlim adaptation](https://gitcode.com/betta18/msmodelslim/tree/hyimage3_mxfp8)
+to generate the checkpoint manually.
+
+### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS)
+
+| Model | Scope | Status | Notes |
+|-------|-------|--------|-------|
+| Qwen3-Omni | Thinker or language-model stage | Not validated | No msModelSlim omni checkpoint path is documented |
+| Qwen3-TTS | TTS language-model stage | Not validated | No msModelSlim TTS checkpoint path is documented |
+
+### Multi-Stage Diffusion Model (BAGEL, GLM-Image)
+
+| Model | Scope | Status | Notes |
+|-------|-------|--------|-------|
+| BAGEL | Stage-specific diffusion or transformer weights | Not validated | Requires a model-specific Ascend adaptation |
+| GLM-Image | Stage-specific diffusion or transformer weights | Not validated | Requires a model-specific Ascend adaptation |
+
+## Configuration
+
+Offline inference:
+
+```bash
+python text_to_image.py --model <quantized-model-path> --quantization ascend
+```
+
+Online serving:
+
+```bash
+vllm serve <quantized-model-path> --omni --quantization ascend
+```
+
+## Parameters
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `--quantization` | str | - | Use `ascend` for msModelSlim-produced checkpoints |
+| `model` | str | - | Path to the quantized checkpoint generated by Ascend tooling |
+
+Example msModelSlim command for a Wan2.2 W8A8 checkpoint:
+
+```bash
+msmodelslim quant \
+  --model_path /path/to/wan2_2_t2v_float_weights \
+  --save_path /path/to/wan2_2_t2v_quantized_weights \
+  --device npu \
+  --model_type Wan2_2 \
+  --config_path /path/to/wan2_2_w8a8f8_mxfp_t2v.yaml \
+  --trust_remote_code True
+```
+
+For HunyuanImage-3.0, use the Hunyuan-specific adaptation linked above.
+
+## Validation and Notes
+
+1. Run with the Ascend/NPU installation and environment.
+2. The `ascend` quantization method expects weights produced by the Ascend
+   tooling; it is not a load-time CUDA quantizer.
+3. Keep the quantized checkpoint aligned with the same model architecture and
+   stage config used for BF16 inference.