[CI] Fix moss_tts_nano test import paths to unblock collection

The test files imported from `tests.utils` (non-existent) and
`tests.conftest.OmniServerParams` (not lazy-exported), causing pytest
collection to fail for the whole suite. This broke unrelated CI jobs
(Diffusion Cache Backend Test, Simple Unit Test) because collection
runs across all test files even when they are deselected by marker.

Align with the pattern used by sibling TTS tests
(e.g. test_cosyvoice3_tts.py, test_voxcpm.py):
- hardware_test -> tests.helpers.mark
- OmniServerParams -> tests.helpers.runtime

https://claude.ai/code/session_01JYESRH7Ua1mc1wqKCQQs5q

.buildkite/bootstrap-amd-omni.sh

@@ -56,88 +56,6 @@ check_run_all_label() {
     fi
 }
 
-is_docs_only_change() {
-    local file_path
-    local has_any=0
-
-    while IFS= read -r file_path; do
-        [[ -z "${file_path}" ]] && continue
-        has_any=1
-
-        if [[ "${file_path}" == docs/* ]]; then
-            continue
-        fi
-        if [[ "${file_path}" == *.md ]]; then
-            continue
-        fi
-        if [[ "${file_path}" == "mkdocs.yml" ]]; then
-            continue
-        fi
-        return 1
-    done
-
-    [[ "${has_any}" -eq 1 ]]
-}
-
-resolve_skip_ci() {
-    local is_pr_build=0
-    local files
-    local base_branch base_ref
-
-    if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" && -n "${BUILDKITE_PULL_REQUEST:-}" ]]; then
-        is_pr_build=1
-    fi
-
-    if [[ "${is_pr_build}" -eq 1 ]]; then
-        base_branch="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}"
-        if ! git rev-parse --verify "origin/${base_branch}" >/dev/null 2>&1; then
-            echo "resolve_skip_ci: origin/${base_branch} not found locally; trying fetch" >&2
-            git fetch --depth=200 origin "${base_branch}" >/dev/null 2>&1 || true
-        fi
-
-        base_ref=""
-        if git rev-parse --verify "origin/${base_branch}" >/dev/null 2>&1; then
-            base_ref="origin/${base_branch}"
-        elif git rev-parse --verify "${base_branch}" >/dev/null 2>&1; then
-            base_ref="${base_branch}"
-        else
-            echo "resolve_skip_ci: cannot resolve PR base ${base_branch}; skip-ci=0" >&2
-            echo -n 0
-            return 0
-        fi
-
-        if ! files="$(git diff --name-only "${base_ref}...${BUILDKITE_COMMIT}" 2>/dev/null)"; then
-            echo "resolve_skip_ci: failed to compute PR changed files; skip-ci=0" >&2
-            echo -n 0
-            return 0
-        fi
-    elif [[ "${BUILDKITE_BRANCH:-}" == "main" ]]; then
-        if ! git rev-parse --verify "${BUILDKITE_COMMIT}^" >/dev/null 2>&1; then
-            echo "resolve_skip_ci: commit has no parent on main; skip-ci=0" >&2
-            echo -n 0
-            return 0
-        fi
-        if ! files="$(git diff --name-only "${BUILDKITE_COMMIT}^..${BUILDKITE_COMMIT}" 2>/dev/null)"; then
-            echo "resolve_skip_ci: failed to compute main changed files; skip-ci=0" >&2
-            echo -n 0
-            return 0
-        fi
-    else
-        echo "resolve_skip_ci: not PR/main build; skip-ci=0" >&2
-        echo -n 0
-        return 0
-    fi
-
-    if is_docs_only_change <<< "${files}"; then
-        echo "resolve_skip_ci: docs-only change detected; skip-ci=1" >&2
-        echo -n 1
-        return 0
-    fi
-
-    echo "resolve_skip_ci: non-doc changes detected; skip-ci=0" >&2
-    echo -n 0
-}
-
 if [[ -z "${COV_ENABLED:-}" ]]; then
     COV_ENABLED=0
 fi
@@ -173,24 +91,8 @@ upload_pipeline() {
 
     cd .buildkite
 
-    # Select test definition file: merge suite for main, ready suite for PRs.
-    # For debugging, DEBUG_TEST_YAML can override the selection — accepts
-    # "merge" or "ready" (case-insensitive).
-    if [[ -n "${DEBUG_TEST_YAML:-}" ]]; then
-        case "${DEBUG_TEST_YAML,,}" in
-            merge)
-                TEST_YAML="test-amd-merge.yml"
-                ;;
-            ready)
-                TEST_YAML="test-amd-ready.yaml"
-                ;;
-            *)
-                echo "ERROR: DEBUG_TEST_YAML must be 'merge' or 'ready', got '$DEBUG_TEST_YAML'" >&2
-                exit 1
-                ;;
-        esac
-        echo "DEBUG_TEST_YAML override: using $TEST_YAML"
-    elif [[ $BUILDKITE_BRANCH == "main" ]]; then
+    # Select test definition file: merge suite for main, ready suite for PRs
+    if [[ $BUILDKITE_BRANCH == "main" ]]; then
         TEST_YAML="test-amd-merge.yml"
     else
         TEST_YAML="test-amd-ready.yaml"
@@ -238,13 +140,30 @@ fi
 # Early exit start: skip pipeline if conditions are met
 # ----------------------------------------------------------------------
 
-# Match CUDA Buildkite skip-ci behavior for docs/markdown-only changes.
+# skip pipeline if all changed files are under docs/
 if [[ "${DOCS_ONLY_DISABLE}" != "1" ]]; then
-    SKIP_CI="$(resolve_skip_ci)"
-    if [[ "${SKIP_CI}" == "1" ]]; then
-        echo "[docs-only] Docs/markdown-only changes detected. Exiting before pipeline upload."
-        exit 0
+  if [[ -n "${file_diff:-}" ]]; then
+    docs_only=1
+    # Robust iteration over newline-separated file_diff
+    while IFS= read -r f; do
+      [[ -z "$f" ]] && continue
+      # **Policy:** only skip if *every* path starts with docs/
+      if [[ "$f" != docs/* ]]; then
+        docs_only=0
+        break
+      fi
+    done < <(printf '%s\n' "$file_diff" | tr ' ' '\n' | tr -d '\r')
+
+    if [[ "$docs_only" -eq 1 ]]; then
+      buildkite-agent annotate ":memo: CI skipped — docs/** only changes detected
+
+\`\`\`
+${file_diff}
+\`\`\`" --style "info" || true
+      echo "[docs-only] All changes are under docs/. Exiting before pipeline upload."
+      exit 0
     fi
+  fi
 fi
 
 # ----------------------------------------------------------------------

.buildkite/nightly-release-pipeline.yaml

@@ -0,0 +1,20 @@
+steps:
+  - label: "Build and upload wheel"
+    key: "build-wheel"
+    agents:
+      queue: cpu_queue_release
+    commands:
+      - "curl -LsSf https://astral.sh/uv/install.sh | sh"
+      - 'export PATH="$HOME/.local/bin:$PATH"'
+      - "uv venv --python=3.12 && source .venv/bin/activate"
+      - "uv pip install --upgrade build"
+      - "python3 -m build"
+      - "bash .buildkite/scripts/upload-nightly-wheels.sh"
+
+  - label: "Generate and upload wheel indices"
+    depends_on: "build-wheel"
+    allow_dependency_failure: true
+    agents:
+      queue: small_cpu_queue_release
+    commands:
+      - "bash .buildkite/scripts/generate-and-upload-nightly-index.sh"

.buildkite/pipeline-intel.yaml

@@ -10,7 +10,7 @@ steps:
           DOCKER_BUILDKIT: "1"
           # Buildkite will automatically replace this with the actual commit hash
           VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
-          VLLM_VERSION: "v0.21.0"
+          VLLM_VERSION: "v0.19.0"
         priority: 100
         timeout_in_minutes: 60
-        soft_fail: false
+        soft_fail: true

.buildkite/pipeline-npu-a3.yaml

@@ -1,110 +0,0 @@
-# steps:
-#   - label: ":buildkit: Build and Push NPU Test Image - A3"
-#     key: image-build-a3
-#     agents:
-#       queue: "ascend-a3"
-#       resource_class: "npu-2"
-#     plugins:
-#       - kubernetes:
-#           metadata:
-#             annotations:
-#               vault.hashicorp.com/agent-init-first: "true"
-#               vault.hashicorp.com/agent-inject: "true"
-#               vault.hashicorp.com/agent-inject-perms-ca.pem: "0400"
-#               vault.hashicorp.com/agent-inject-perms-cert.pem: "0400"
-#               vault.hashicorp.com/agent-inject-perms-config.json: "0400"
-#               vault.hashicorp.com/agent-inject-perms-key.pem: "0400"
-#               vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd
-#               vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd
-#               vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd
-#               vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd
-#               vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}"
-#               vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}"
-#               vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}"
-#               vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}"
-#               vault.hashicorp.com/agent-pre-populate-only: "true"
-#               vault.hashicorp.com/agent-run-as-group: "1000"
-#               vault.hashicorp.com/agent-run-as-user: "1000"
-#               vault.hashicorp.com/agent-service-account-token-volume-name: token-vol
-#               vault.hashicorp.com/role: ascend-gha-runners
-#               vault.hashicorp.com/secret-volume-path: /home/user/.docker/
-#               vault.hashicorp.com/tls-skip-verify: "true"
-#           podSpecPatch:
-#             volumes:
-#             - name: token-vol
-#               projected:
-#                 defaultMode: 420
-#                 sources:
-#                 - serviceAccountToken:
-#                     audience: api
-#                     expirationSeconds: 600
-#                     path: token
-#     env:
-#       VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
-#       IMAGE_NAME: "vllm-omni-ci-npu"
-#       IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
-#       BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
-#     command: |
-#       set -ex
-
-#       echo "--- Building and pushing NPU Test Image"
-#       echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
-#       echo "buildkitd address: $${BUILDKITD_ADDR}"
-
-#       git config --global url."https://gh-proxy.test.osinfra.cn/https://github.com/".insteadOf "https://github.com/"
-
-#       if ! command -v buildctl &> /dev/null; then
-#         echo "Installing buildctl..."
-#         mkdir -p /tmp/buildkit
-#         BUILDKIT_VERSION="v0.29.0"
-#         wget -q "https://gh-proxy.test.osinfra.cn/https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz
-#         tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
-#         cp /tmp/buildkit/bin/buildctl /usr/local/bin/
-#       fi
-
-#       export DOCKER_CONFIG=/home/user/.docker
-#       buildctl \
-#         --addr="$${BUILDKITD_ADDR}" \
-#         --tlscacert=/home/user/.docker/ca.pem \
-#         --tlscert=/home/user/.docker/cert.pem \
-#         --tlskey=/home/user/.docker/key.pem \
-#         build \
-#         --frontend dockerfile.v0 \
-#         --local context=. \
-#         --local dockerfile=./docker \
-#         --opt filename=Dockerfile.npu.ci.a3 \
-#         --secret id=dockerconfig,src=/home/user/.docker/config.json \
-#         --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \
-#         --progress=plain
-
-#       echo "--- Image pushed successfully"
-#       echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
-
-#   # - label: "Upload Ready Pipeline - A3"
-#   #   depends_on: image-build-a3
-#   #   key: upload-ready-pipeline-a3
-#   #   if: build.branch != "main" && build.pull_request.labels includes "npu-test"
-#   #   agents:
-#   #     queue: "ascend-a3"
-#   #     resource_class: "npu-2"
-#   #   command: |
-#   #     buildkite-agent pipeline upload .buildkite/test-npu-ready.yaml
-
-  # # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
-  # - label: "Upload Nightly Pipeline"
-  #   depends_on: image-build
-  #   key: upload-nightly-pipeline
-  #   if: >-
-  #     (build.branch == "main" && build.env("NIGHTLY") == "1") ||
-  #     (build.branch != "main" && (
-  #       build.pull_request.labels includes "nightly-test" ||
-  #       build.pull_request.labels includes "omni-test" ||
-  #       build.pull_request.labels includes "tts-test" ||
-  #       build.pull_request.labels includes "diffusion-x2iat-test" ||
-  #       build.pull_request.labels includes "diffusion-x2v-test"
-  #     ))
-  #   commands:
-  #     - buildkite-agent pipeline upload .buildkite/test-npu-nightly.yaml
-  #   agents:
-  #     queue: "ascend-a3"
-  #     resource_class: "npu-2"

.buildkite/pipeline-npu.yaml

@@ -1,112 +0,0 @@
-steps:
-  - label: ":buildkit: Build and Push NPU Test Image - A2"
-    key: image-build
-    agents:
-      queue: "ascend-a2b3"
-      resource_class: "npu-1"
-    plugins:
-      - kubernetes:
-          metadata:
-            annotations:
-              vault.hashicorp.com/agent-init-first: "true"
-              vault.hashicorp.com/agent-inject: "true"
-              vault.hashicorp.com/agent-inject-perms-ca.pem: "0400"
-              vault.hashicorp.com/agent-inject-perms-cert.pem: "0400"
-              vault.hashicorp.com/agent-inject-perms-config.json: "0400"
-              vault.hashicorp.com/agent-inject-perms-key.pem: "0400"
-              vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd
-              vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}"
-              vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}"
-              vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}"
-              vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}"
-              vault.hashicorp.com/agent-pre-populate-only: "true"
-              vault.hashicorp.com/agent-run-as-group: "1000"
-              vault.hashicorp.com/agent-run-as-user: "1000"
-              vault.hashicorp.com/agent-service-account-token-volume-name: token-vol
-              vault.hashicorp.com/role: ascend-gha-runners
-              vault.hashicorp.com/secret-volume-path: /home/user/.docker/
-              vault.hashicorp.com/tls-skip-verify: "true"
-          podSpecPatch:
-            volumes:
-            - name: token-vol
-              projected:
-                defaultMode: 420
-                sources:
-                - serviceAccountToken:
-                    audience: api
-                    expirationSeconds: 600
-                    path: token
-    env:
-      VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
-      IMAGE_NAME: "vllm-omni-ci-npu"
-      IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
-      BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
-    command: |
-      set -ex
-
-      echo "--- Building and pushing NPU Test Image"
-      echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
-      echo "buildkitd address: $${BUILDKITD_ADDR}"
-
-      if ! command -v buildctl &> /dev/null; then
-        echo "Installing buildctl..."
-        mkdir -p /tmp/buildkit
-        BUILDKIT_VERSION="v0.29.0"
-        wget -q "https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz
-        tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
-        cp /tmp/buildkit/bin/buildctl /usr/local/bin/
-      fi
-
-      export DOCKER_CONFIG=/home/user/.docker
-      buildctl \
-        --addr="$${BUILDKITD_ADDR}" \
-        --tlscacert=/home/user/.docker/ca.pem \
-        --tlscert=/home/user/.docker/cert.pem \
-        --tlskey=/home/user/.docker/key.pem \
-        build \
-        --frontend dockerfile.v0 \
-        --local context=. \
-        --local dockerfile=./docker \
-        --opt filename=Dockerfile.npu.ci \
-        --secret id=dockerconfig,src=/home/user/.docker/config.json \
-        --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG},push=true \
-        --progress=plain
-
-      echo "--- Image pushed successfully"
-      echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
-
-  - label: "Upload Ready Pipeline"
-    depends_on: image-build
-    key: upload-ready-pipeline
-    if: >-
-      build.branch != "main" && (
-        build.pull_request.labels includes "npu-test" ||
-        build.pull_request.labels includes "ready"
-      )
-    agents:
-      queue: "ascend-a2b3"
-      resource_class: "npu-1"
-    command: |
-      buildkite-agent pipeline upload .buildkite/test-npu-ready.yaml
-
-  # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
-  - label: "Upload Nightly Pipeline"
-    depends_on: image-build
-    key: upload-nightly-pipeline
-    if: >-
-      (build.branch == "main" && build.env("NIGHTLY") == "1") ||
-      (build.branch != "main" && (
-        build.pull_request.labels includes "nightly-test" ||
-        build.pull_request.labels includes "omni-test" ||
-        build.pull_request.labels includes "tts-test" ||
-        build.pull_request.labels includes "diffusion-x2iat-test" ||
-        build.pull_request.labels includes "diffusion-x2v-test"
-      ))
-    commands:
-      - buildkite-agent pipeline upload .buildkite/test-npu-nightly.yaml
-    agents:
-      queue: "ascend-a2b3"
-      resource_class: "npu-1"

.buildkite/pipeline.yml

@@ -1,9 +1,8 @@
 # Document 1: Buildkite loads only this block on first parse. The next step resolves docs-only skip-ci
-# from git diff, then uploads document 2. When docs-only skip applies, image-build still runs for the nightly
-# exception in upload_pipeline_with_skip_ci.sh (nightly-test label or main NIGHTLY=1); otherwise upload-nightly
-# can be skipped together with test-ready/test-merge. Weekly uploads also depend on image-build.
-
-# Document 2: appended after `---`; same file, read by upload_pipeline_with_skip_ci.sh (not evaluated as a second pipeline by Buildkite). Child uploads include test-ready.yml, test-merge.yml, test-nightly.yml, and test-weekly.yml.
+# from git diff, then uploads document 2. When docs-only skip applies, image-build still runs if nightly-test
+# / main NIGHTLY so upload-nightly is not skipped together with test-ready/test-merge.
+#
+# Document 2: appended after `---`; same file, read by upload_pipeline_with_skip_ci.sh (not evaluated as a second pipeline by Buildkite).
 steps:
   - label: ":github: Resolve skip-ci & upload pipeline"
     key: upload-ci-pipeline
@@ -62,15 +61,3 @@ steps:
       - buildkite-agent pipeline upload .buildkite/test-nightly.yml
     agents:
       queue: "cpu_queue_premerge"
-
-  # L5 Test — main+WEEKLY=1 (scheduled), or PR with weekly-test label
-  - label: "Upload Weekly Pipeline"
-    depends_on: image-build
-    key: upload-weekly-pipeline
-    if: >-
-      (build.branch == "main" && build.env("WEEKLY") == "1") ||
-      (build.branch != "main" && build.pull_request.labels includes "weekly-test")
-    commands:
-      - buildkite-agent pipeline upload .buildkite/test-weekly.yml
-    agents:
-      queue: "cpu_queue_premerge"

.buildkite/rebase-pipeline.yaml

@@ -1,38 +0,0 @@
-steps:
-  - label: ":docker: Build image"
-    key: image-build
-    commands:
-      - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
-      - "docker build --progress=plain --file docker/Dockerfile.ci -t vllm-omni-ci ."
-      - "docker tag vllm-omni-ci public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
-      - "docker push public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
-    agents:
-      queue: "cpu_queue_premerge"
-
-  # L2 Test
-  - label: "Upload Ready Pipeline"
-    depends_on: image-build
-    key: upload-ready-pipeline
-    commands:
-      - buildkite-agent pipeline upload .buildkite/test-ready.yml
-    agents:
-      queue: "cpu_queue_premerge"
-
-  # L3 Test
-  - label: "Upload Merge Pipeline"
-    depends_on: image-build
-    key: upload-merge-pipeline
-    commands:
-      - buildkite-agent pipeline upload .buildkite/test-merge.yml
-    agents:
-      queue: "cpu_queue_premerge"
-
-  # L4 Test
-  - label: "Upload Nightly Pipeline"
-    depends_on: image-build
-    key: upload-nightly-pipeline
-    if: build.env("NIGHTLY") == "1"
-    commands:
-      - buildkite-agent pipeline upload .buildkite/test-nightly.yml
-    agents:
-      queue: "cpu_queue_premerge"

.buildkite/release-pipeline.yaml

@@ -1,110 +1,38 @@
 steps:
-  - input: "Provide Release version here"
-    id: input-release-version
-    fields:
-      - text: "What is the release version?"
-        key: release-version
-
-  - label: "Build and upload wheel"
-    depends_on: ~
-    key: "build-wheel"
-    agents:
-      queue: cpu_queue_release
+  - label: ":docker: Build image"
+    key: image-build
     commands:
-      - "curl -LsSf https://astral.sh/uv/install.sh | sh"
-      - 'export PATH="$HOME/.local/bin:$PATH"'
-      - "uv venv --python=3.12 && source .venv/bin/activate"
-      - "uv pip install --upgrade build"
-      - "python3 -m build"
-      - "bash .buildkite/scripts/upload-nightly-wheels.sh"
-
-  - label: "Generate and upload wheel indices"
-    depends_on: "build-wheel"
-    allow_dependency_failure: true
+      - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
+      - "docker build --progress=plain --file docker/Dockerfile.ci -t vllm-omni-ci ."
+      - "docker tag vllm-omni-ci public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
+      - "docker push public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
     agents:
-      queue: small_cpu_queue_release
+      queue: "cpu_queue_premerge"
+
+  # L2 Test
+  - label: "Upload Ready Pipeline"
+    depends_on: image-build
+    key: upload-ready-pipeline
     commands:
-      - "bash .buildkite/scripts/generate-and-upload-nightly-index.sh"
-
-  - block: "Upload release wheels to PyPI"
-    depends_on: "build-wheel"
-    key: block-upload-release-wheels
-
-  - label: "Upload release wheels to PyPI"
-    depends_on:
-      - block-upload-release-wheels
-      - input-release-version
-    key: upload-release-wheels
+      - buildkite-agent pipeline upload .buildkite/test-ready.yml
     agents:
-      queue: small_cpu_queue_release
+      queue: "cpu_queue_premerge"
+
+  # L3 Test
+  - label: "Upload Merge Pipeline"
+    depends_on: image-build
+    key: upload-merge-pipeline
     commands:
-      - "bash .buildkite/scripts/publish-release-wheels-pypi.sh"
-
-  - block: "Unblock to build release Docker images"
-    depends_on: ~
-    key: block-build-release-images
-
-  - group: "Build release Docker images"
-    key: "build-release-images"
-    depends_on: block-build-release-images
-    allow_dependency_failure: true
-    steps:
-      - label: "Build release image - x86_64"
-        key: build-release-image-x86
-        depends_on: ~
-        agents:
-          queue: cpu_queue_release
-        commands:
-          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
-          - |
-            DOCKER_BUILDKIT=1 docker build \
-              --build-arg max_jobs=16 \
-              --tag public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m) \
-              --progress plain \
-              -f docker/Dockerfile.cuda .
-          - "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m)"
-          # re-tag to default image tag and push, just in case arm64 build fails
-          - "docker tag public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m) public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT"
-          - "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT"
-        env:
-          DOCKER_BUILDKIT: "1"
-
-      - label: "Build release image - aarch64"
-        key: build-release-image-arm64
-        depends_on: ~
-        agents:
-          queue: arm64_cpu_queue_release
-        commands:
-          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
-          - |
-            DOCKER_BUILDKIT=1 docker build \
-              --build-arg max_jobs=16 \
-              --tag public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m) \
-              --progress plain \
-              -f docker/Dockerfile.cuda .
-          - "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m)"
-        env:
-          DOCKER_BUILDKIT: "1"
-
-  - block: "Publish release images to DockerHub"
-    key: block-publish-release-images
-    depends_on:
-      - build-release-image-x86
-      - build-release-image-arm64
-      - input-release-version
-
-  - label: "Publish release images to DockerHub"
-    depends_on:
-      - block-publish-release-images
-    key: publish-release-images-dockerhub
+      - buildkite-agent pipeline upload .buildkite/test-merge.yml
     agents:
-      queue: small_cpu_queue_release
+      queue: "cpu_queue_premerge"
+
+  # L4 Test
+  - label: "Upload Nightly Pipeline"
+    depends_on: image-build
+    key: upload-nightly-pipeline
+    if: build.env("NIGHTLY") == "1"
     commands:
-      - "bash .buildkite/scripts/publish-release-images.sh"
-    plugins:
-      - docker-login#v3.0.0:
-          username: vllmbot
-          password-env: DOCKERHUB_TOKEN
-    env:
-      DOCKER_BUILDKIT: "1"
-      DOCKERHUB_USERNAME: "vllmbot"
+      - buildkite-agent pipeline upload .buildkite/test-nightly.yml
+    agents:
+      queue: "cpu_queue_premerge"

.buildkite/scripts/hardware_ci/run-xpu-test.sh

@@ -52,6 +52,5 @@ time timeout -k 30 30m docker run \
     pip install tblib==3.1.0
     cd /workspace/vllm-omni
     pytest -v -s -m "core_model and xpu and B60"
-    export VLLM_XPU_USE_SAMPLER_KERNEL=0    # NOTE: Remove this after vLLM v0.21.1 is merged. Fixes Qwen2-5 omni-expansion tests.
     pytest -v -s -m "advanced_model and xpu and B60"
 '

.buildkite/scripts/hardware_ci/run_npu_test.sh

@@ -141,5 +141,5 @@ docker run \
     "${image_name}" \
     bash -c '
     set -e
-    VLLM_USE_MODELSCOPE=True pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py
+    VLLM_USE_MODELSCOPE=True pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py
 '

.buildkite/scripts/publish-release-images.sh

@@ -1,50 +0,0 @@
-#!/bin/bash
-# SPDX-License-Identifier: Apache-2.0
-#
-# Publish release Docker images from ECR to DockerHub.
-# Pulls per-arch images, tags with `latest` and versioned tags, pushes them,
-# then creates and pushes a multi-arch manifest.
-
-set -euo pipefail
-
-RELEASE_VERSION=$(buildkite-agent meta-data get release-version --default "" | sed 's/^v//')
-if [ -z "${RELEASE_VERSION}" ]; then
-  echo "ERROR: release-version metadata not set"
-  exit 1
-fi
-
-COMMIT="$BUILDKITE_COMMIT"
-DOCKERHUB_REPO="vllm/vllm-omni"
-ECR_REPO="public.ecr.aws/q9t5s3a7/vllm-omni-release-repo"
-
-echo "========================================"
-echo "Publishing release images v${RELEASE_VERSION}"
-echo "  Commit: ${COMMIT}"
-echo "========================================"
-
-# Login to ECR to pull staging images
-aws ecr-public get-login-password --region us-east-1 | \
-  docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
-
-docker pull "${ECR_REPO}:${COMMIT}-x86_64"
-docker pull "${ECR_REPO}:${COMMIT}-aarch64"
-
-docker tag "${ECR_REPO}:${COMMIT}-x86_64" "${DOCKERHUB_REPO}:latest-x86_64"
-docker tag "${ECR_REPO}:${COMMIT}-x86_64" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-x86_64"
-docker push "${DOCKERHUB_REPO}:latest-x86_64"
-docker push "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-x86_64"
-
-docker tag "${ECR_REPO}:${COMMIT}-aarch64" "${DOCKERHUB_REPO}:latest-aarch64"
-docker tag "${ECR_REPO}:${COMMIT}-aarch64" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-aarch64"
-docker push "${DOCKERHUB_REPO}:latest-aarch64"
-docker push "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-aarch64"
-
-docker manifest rm "${DOCKERHUB_REPO}:latest" || true
-docker manifest rm "${DOCKERHUB_REPO}:v${RELEASE_VERSION}" || true
-docker manifest create "${DOCKERHUB_REPO}:latest" "${DOCKERHUB_REPO}:latest-x86_64" "${DOCKERHUB_REPO}:latest-aarch64"
-docker manifest create "${DOCKERHUB_REPO}:v${RELEASE_VERSION}" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-x86_64" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-aarch64"
-docker manifest push "${DOCKERHUB_REPO}:latest"
-docker manifest push "${DOCKERHUB_REPO}:v${RELEASE_VERSION}"
-
-echo ""
-echo "Successfully published release images for v${RELEASE_VERSION}"

.buildkite/scripts/publish-release-wheels-pypi.sh

@@ -1,52 +0,0 @@
-#!/usr/bin/env bash
-# SPDX-License-Identifier: Apache-2.0
-
-set -e
-
-BUCKET="vllm-wheels"
-SUBPATH="omni/$BUILDKITE_COMMIT"
-S3_COMMIT_PREFIX="s3://$BUCKET/$SUBPATH/"
-
-RELEASE_VERSION=$(buildkite-agent meta-data get release-version | sed 's/^v//')
-if [[ -z "$RELEASE_VERSION" ]]; then
-  echo "[FATAL] release-version metadata not set."
-  exit 1
-fi
-echo "Release version: $RELEASE_VERSION"
-
-if [[ -z "$PYPI_TOKEN" ]]; then
-  echo "[FATAL] PYPI_TOKEN is not set."
-  exit 1
-else
-  export TWINE_USERNAME="__token__"
-  export TWINE_PASSWORD="$PYPI_TOKEN"
-fi
-
-set -x
-
-if ! command -v uv &> /dev/null; then
-  curl -LsSf https://astral.sh/uv/install.sh | sh
-  export PATH="$HOME/.local/bin:$PATH"
-fi
-
-uv venv --python 3.12 /tmp/vllm-omni-release-env
-source /tmp/vllm-omni-release-env/bin/activate
-uv pip install twine
-
-DIST_DIR=/tmp/vllm-omni-release-dist
-mkdir -p "$DIST_DIR"
-
-echo "Downloading wheels from S3:"
-aws s3 ls "$S3_COMMIT_PREFIX"
-aws s3 cp --recursive --exclude "*" --include "vllm_omni-${RELEASE_VERSION}*.whl" --exclude "*dev*" "$S3_COMMIT_PREFIX" "$DIST_DIR"
-ls -la "$DIST_DIR"
-
-PYPI_WHEEL_FILES=$(find "$DIST_DIR" -name "vllm_omni-${RELEASE_VERSION}*.whl" -not -name "*+*")
-if [[ -z "$PYPI_WHEEL_FILES" ]]; then
-  echo "[FATAL] No wheels found for version ${RELEASE_VERSION}"
-  exit 1
-fi
-
-python3 -m twine check $PYPI_WHEEL_FILES
-python3 -m twine upload --non-interactive --verbose $PYPI_WHEEL_FILES
-echo "Wheels uploaded to PyPI"

.buildkite/scripts/upload_pipeline_with_skip_ci.sh

@@ -1,18 +1,10 @@
 #!/usr/bin/env bash
-# Evaluate docs-only skip-ci and upload continuation steps from the given pipeline YAML
+# Evaluate docs-only skip-ci and upload continuation steps from the same `.buildkite/pipeline.yml`
 # (YAML document after the first `---`). Buildkite `if` is evaluated at upload time.
-#
-# Usage: upload_pipeline_with_skip_ci.sh [pipeline_yaml]
-#   pipeline_yaml: path relative to repo root or absolute (default: .buildkite/pipeline.yml)
 set -euo pipefail
 
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
-PIPELINE_ARG="${1:-.buildkite/pipeline.yml}"
-if [[ "${PIPELINE_ARG}" = /* ]]; then
-  PIPELINE_YML="${PIPELINE_ARG}"
-else
-  PIPELINE_YML="${ROOT}/${PIPELINE_ARG}"
-fi
+PIPELINE_YML="${ROOT}/.buildkite/pipeline.yml"
 
 # Prints a single digit to stdout: 1 = skip image CI, 0 = run. Logs go to stderr.
 is_docs_only_change() {
@@ -29,7 +21,7 @@ is_docs_only_change() {
     if [[ "${file_path}" == *.md ]]; then
       continue
     fi
-    if [[ "${file_path}" == "mkdocs.yml" ]]; then
+    if [[ "${file_path}" == "mkdocs.yaml" ]]; then
       continue
     fi
     return 1
@@ -112,13 +104,12 @@ import pathlib
 path = pathlib.Path(os.environ["PIPELINE_YML"])
 text = path.read_text(encoding="utf-8")
 sep = "\n---\n"
-# Two supported layouts:
-#   - multi-doc: doc 1 = bootstrap (loaded by Buildkite), doc 2 (after `---`) = uploaded steps.
-#   - single-doc: whole file is the uploaded steps; caller (UI init step) is the bootstrap.
-if sep in text:
-    _, continuation = text.split(sep, 1)
-else:
-    continuation = text
+if sep not in text:
+    raise SystemExit(
+        "upload_pipeline_with_skip_ci: .buildkite/pipeline.yml must contain a '\\n---\\n' separator "
+        "(document 1 = bootstrap, document 2 = uploaded steps)"
+    )
+_, continuation = text.split(sep, 1)
 
 skip = os.environ.get("SKIP_CI") == "1"
 # When docs-only skip-ci: skip default CI image, but still build for L4 nightly (PR label nightly-test or
@@ -135,7 +126,7 @@ if skip:
 else:
     rep = "'true'"
     ready_rep = "'build.branch != \"main\" && build.pull_request.labels includes \"ready\"'"
-    merge_rep = "'(build.branch == \"main\" && build.env(\"NIGHTLY\") != \"1\" && build.env(\"WEEKLY\") != \"1\") || (build.branch != \"main\" && build.pull_request.labels includes \"merge-test\")'"
+    merge_rep = "'(build.branch == \"main\" && build.env(\"NIGHTLY\") != \"1\") || (build.branch != \"main\" && build.pull_request.labels includes \"merge-test\")'"
 rendered = (
     continuation
     .replace("__IMAGE_BUILD_IF__", rep)

.buildkite/test-amd-merge.yml

@@ -7,9 +7,16 @@ steps:
   grade: Blocking
   commands:
     - export VLLM_ROCM_USE_AITER=0
-    # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
-    # that is not supported on CPU
-    - "timeout 60m pytest -v -s -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+    - "timeout 20m pytest -v -s -m 'core_model and cpu' --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml"
+
+- label: "Diffusion Model Test"
+  agent_pool: mi325_2
+  depends_on: amd-build
+  mirror_hardwares: [amdproduction]
+  grade: Blocking
+  commands:
+    - export GPU_ARCHS=gfx942
+    - timeout 20m pytest -s -v tests/e2e/offline_inference/test_t2i_model.py -m "advanced_model and diffusion" --run-level "advanced_model"
 
 - label: "Diffusion Images API LoRA E2E"
   agent_pool: mi325_1
@@ -17,7 +24,7 @@ steps:
   mirror_hardwares: [amdproduction]
   grade: Blocking
   commands:
-    - timeout 20m pytest -s -v tests/diffusion/lora/test_images_generations_lora.py
+    - timeout 20m pytest -s -v tests/e2e/online_serving/test_images_generations_lora.py
 
 - label: "Diffusion Model CPU offloading Test"
   agent_pool: mi325_1
@@ -30,9 +37,9 @@ steps:
     - |
       timeout 20m bash -c '
         set +e
-        pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
+        pytest -s -v tests/e2e/offline_inference/test_diffusion_cpu_offload.py
         EXIT1=\$?
-        pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
+        pytest -s -v tests/e2e/offline_inference/test_diffusion_layerwise_offload.py
         EXIT2=\$?
         exit \$((EXIT1 | EXIT2))
       '
@@ -67,7 +74,7 @@ steps:
   commands:
     - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py
+    - timeout 20m pytest -s -v tests/e2e/offline_inference/test_sequence_parallel.py
     - timeout 20m pytest -s -v tests/diffusion/distributed/test_ulysses_uaa_perf.py
 
 # merge-only tests
@@ -77,15 +84,7 @@ steps:
   mirror_hardwares: [amdproduction]
   grade: Blocking
   commands:
-    - pytest -s -v tests/diffusion/distributed/test_zimage_parallelism.py
-
-- label: "Diffusion Model Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - timeout 30m pytest -s -v tests/e2e/offline_inference/test_z_image.py
+    - pytest -s -v tests/e2e/offline_inference/test_zimage_parallelism.py
 
 - label: "Diffusion GPU Worker Test"
   agent_pool: mi325_2
@@ -104,17 +103,16 @@ steps:
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
     - timeout 20m pytest -s -v tests/engine/test_async_omni_engine_abort.py
 
-# This is from test-nightly.yml.
-# This model seems to be important, so I am trying to run it here.
-# Since this is from test-nightly.yml,
-# I keep this non-blocking.
 - label: "Omni Model Test Qwen2-5-Omni"
-  agent_pool: mi325_4
+  agent_pool: mi325_2
   depends_on: amd-build
   mirror_hardwares: [amdproduction]
+  grade: Blocking
   commands:
-    - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-    - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model"
+    - export VLLM_LOGGING_LEVEL=DEBUG
+    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+    - timeout 20m pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py
+    - timeout 20m pytest -s -v tests/e2e/online_serving/test_qwen2_5_omni.py -m "advanced_model" --run-level "advanced_model"
 
 - label: "Omni Model Test Qwen3-Omni"
   agent_pool: mi325_2
@@ -122,8 +120,10 @@ steps:
   mirror_hardwares: [amdproduction]
   grade: Blocking
   commands:
+    - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/online_serving/test_mimo_audio.py -m "advanced_model" --run-level "advanced_model"
+    - export VLLM_TEST_CLEAN_GPU_MEMORY=1
+    - timeout 30m pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/online_serving/test_mimo_audio.py -m "advanced_model" --run-level "advanced_model"
 
 - label: "Qwen3-TTS CustomVoice E2E Test"
   agent_pool: mi325_1
@@ -153,6 +153,17 @@ steps:
         pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m "advanced_model" --run-level "advanced_model"
       '
 
+- label: "Diffusion Image Edit Test"
+  agent_pool: mi325_1
+  depends_on: amd-build
+  mirror_hardwares: [amdproduction]
+  grade: Blocking
+  commands:
+    - export GPU_ARCHS=gfx942
+    - export VLLM_LOGGING_LEVEL=DEBUG
+    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+    - timeout 20m pytest -s -v tests/e2e/online_serving/test_image_gen_edit.py
+
 # TODO: Bagel test on ROCm is very unstable. @tjtanaa
 # Need to debug before reneable numerical changes across large PRs
 # # split Bagel Model Test with H100 (Real Weights) into three tests
@@ -163,10 +174,11 @@ steps:
 #   grade: Blocking
 #   commands:
 #     - export GPU_ARCHS=gfx942
+#     - export VLLM_TEST_CLEAN_GPU_MEMORY=1
 #     - export VLLM_LOGGING_LEVEL=DEBUG
 #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
 #     - export VLLM_ROCM_USE_AITER_RMSNORM=0
-#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "advanced_model" --run-level "advanced_model" -k "shared_memory" -k "rocm"
+#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel_text2img.py -m "advanced_model" --run-level "advanced_model" -k "shared_memory" -k "rocm"
 
 # - label: "Bagel Img2Img Model Test (2/3)"
 #   agent_pool: mi325_1
@@ -175,10 +187,11 @@ steps:
 #   grade: Blocking
 #   commands:
 #     - export GPU_ARCHS=gfx942
+#     - export VLLM_TEST_CLEAN_GPU_MEMORY=1
 #     - export VLLM_LOGGING_LEVEL=DEBUG
 #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
 #     - export VLLM_ROCM_USE_AITER_RMSNORM=0
-#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "advanced_model" --run-level "advanced_model" -k "rocm"
+#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel_img2img.py -m "advanced_model" --run-level "advanced_model" -k "rocm"
 
 # - label: "Bagel Online Serving Test (3/3)"
 #   agent_pool: mi325_1
@@ -187,23 +200,12 @@ steps:
 #   grade: Blocking
 #   commands:
 #     - export GPU_ARCHS=gfx942
+#     - export VLLM_TEST_CLEAN_GPU_MEMORY=1
 #     - export VLLM_IMAGE_FETCH_TIMEOUT=60
 #     - export VLLM_LOGGING_LEVEL=DEBUG
 #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
 #     - export VLLM_ROCM_USE_AITER_RMSNORM=0
-#     - timeout 40m pytest -s -v tests/e2e/online_serving/test_bagel.py -m "advanced_model" --run-level "advanced_model" -k "rocm"
-
-- label: "Diffusion · Wan22 Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - |
-      timeout 40m bash -c "
-        export VLLM_IMAGE_FETCH_TIMEOUT=60
-        pytest -s -v tests/e2e/offline_inference/test_wan22_t2v.py tests/e2e/online_serving/test_wan22_t2v.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
-      "
+#     - timeout 40m pytest -s -v tests/e2e/online_serving/test_bagel_online.py -m "advanced_model" --run-level "advanced_model" -k "rocm"
 
 - label: "Voxtral-TTS E2E Test"
   agent_pool: mi325_1

.buildkite/test-amd-ready.yaml

@@ -7,9 +7,7 @@ steps:
   grade: Blocking
   commands:
     - export VLLM_ROCM_USE_AITER=0
-    # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
-    # that is not supported on CPU
-    - "timeout 60m pytest -v -s -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
+    - "timeout 20m pytest -v -s -m 'core_model and cpu' --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml"
 
 - label: "Voxtral TTS CUDA Unit Test"
   agent_pool: mi325_1
@@ -19,13 +17,21 @@ steps:
   commands:
     - timeout 10m pytest -s -v tests/model_executor/models/voxtral_tts/test_cuda_graph_acoustic_transformer.py
 
+- label: "Diffusion Model Test"
+  agent_pool: mi325_1
+  depends_on: amd-build
+  mirror_hardwares: [amdproduction]
+  grade: Blocking
+  commands:
+    - timeout 30m pytest -s -v tests/e2e/offline_inference/test_t2i_model.py -m "core_model and diffusion" --run-level "core_model"
+
 - label: "Diffusion Batching Test"
   agent_pool: mi325_1
   depends_on: amd-build
   mirror_hardwares: [amdproduction]
   grade: Blocking
   commands:
-    - timeout 20m pytest -s -v tests/diffusion/batching/test_diffusion_batching.py -m "core_model and diffusion" --run-level "core_model"
+    - timeout 20m pytest -s -v tests/e2e/offline_inference/test_qwen_image_diffusion_batching.py -m "core_model and diffusion" --run-level "core_model"
 
 - label: "Custom Pipeline Test"
   agent_pool: mi325_1
@@ -46,9 +52,9 @@ steps:
     - |
       timeout 20m bash -c '
         set +e
-        pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
+        pytest -s -v tests/e2e/offline_inference/test_diffusion_cpu_offload.py
         EXIT1=\$?
-        pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
+        pytest -s -v tests/e2e/offline_inference/test_diffusion_layerwise_offload.py
         EXIT2=\$?
         exit \$((EXIT1 | EXIT2))
       '
@@ -84,7 +90,7 @@ steps:
   commands:
     - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py -m core_model
+    - timeout 20m pytest -s -v tests/e2e/offline_inference/test_sequence_parallel.py -m core_model
 
 - label: "Diffusion GPU Worker Test"
   agent_pool: mi325_2
@@ -94,27 +100,6 @@ steps:
   commands:
     - timeout 20m pytest -s -v tests/diffusion/test_diffusion_worker.py
 
-- label: "Diffusion · Model Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model"
-
-- label: "Diffusion · Wan22 Test"
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - |
-      timeout 40m bash -c "
-        export VLLM_TEST_CLEAN_GPU_MEMORY=1
-        export VLLM_IMAGE_FETCH_TIMEOUT=60
-        pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model' --run-level 'core_model'
-      "
-
 - label: "Engine Test"
   agent_pool: mi325_1
   depends_on: amd-build
@@ -127,6 +112,19 @@ steps:
               pytest -s -v tests/engine/test_async_omni_engine_abort.py
       '
 
+
+# NOTE: This test is not running any thing. It is skipped and deselected.
+# Currently it is = 1 skipped, 1 deselected, 17 warnings in 0.03s ======
+# - label: "Omni Model Test Qwen2-5-Omni"
+#   agent_pool: mi325_2
+#   depends_on: amd-build
+#   mirror_hardwares: [amdproduction]
+#   grade: Blocking
+#   commands:
+#     - export VLLM_LOGGING_LEVEL=DEBUG
+#     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+#     - timeout 20m pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py -m "core_model" --run-level "core_model"
+
 # - label: "Omni Model Test Qwen3-Omni"
 #   agent_pool: mi325_2
 #   depends_on: amd-build
@@ -135,6 +133,7 @@ steps:
 #   commands:
 #     - export VLLM_LOGGING_LEVEL=DEBUG
 #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+#     - export VLLM_TEST_CLEAN_GPU_MEMORY=1
 #     - timeout 10m pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py
 #     - timeout 20m pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m "core_model" --run-level "core_model"
 
@@ -176,6 +175,17 @@ steps:
         pytest -s -v tests/e2e/offline_inference/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model"
       '
 
+- label: "Diffusion Image Edit Test"
+  agent_pool: mi325_1
+  depends_on: amd-build
+  mirror_hardwares: [amdproduction]
+  grade: Blocking
+  commands:
+    - export GPU_ARCHS=gfx942
+    - export VLLM_LOGGING_LEVEL=DEBUG
+    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+    - timeout 20m pytest -s -v tests/e2e/online_serving/test_image_gen_edit.py
+
 # TODO: Bagel test on ROCm is very unstable. @tjtanaa
 # Need to debug before reneable numerical changes across large PRs
 # - label: "Bagel Text2Img Model Test"
@@ -185,10 +195,11 @@ steps:
 #   grade: Blocking
 #   commands:
 #     - export GPU_ARCHS=gfx942
+#     - export VLLM_TEST_CLEAN_GPU_MEMORY=1
 #     - export VLLM_LOGGING_LEVEL=DEBUG
 #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
 #     - export VLLM_ROCM_USE_AITER_RMSNORM=0
-#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"
+#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel_text2img.py -m "core_model" --run-level "core_model" -k "rocm"
 
 # - label: "Bagel Img2Img Model Test"
 #   agent_pool: mi325_1
@@ -197,10 +208,11 @@ steps:
 #   grade: Blocking
 #   commands:
 #     - export GPU_ARCHS=gfx942
+#     - export VLLM_TEST_CLEAN_GPU_MEMORY=1
 #     - export VLLM_LOGGING_LEVEL=DEBUG
 #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
 #     - export VLLM_ROCM_USE_AITER_RMSNORM=0
-#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"
+#     - timeout 30m pytest -s -v tests/e2e/offline_inference/test_bagel_img2img.py -m "core_model" --run-level "core_model" -k "rocm"
 
 # - label: "Bagel Online Serving Test"
 #   agent_pool: mi325_1
@@ -209,11 +221,12 @@ steps:
 #   grade: Blocking
 #   commands:
 #     - export GPU_ARCHS=gfx942
+#     - export VLLM_TEST_CLEAN_GPU_MEMORY=1
 #     - export VLLM_IMAGE_FETCH_TIMEOUT=60
 #     - export VLLM_LOGGING_LEVEL=DEBUG
 #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
 #     - export VLLM_ROCM_USE_AITER_RMSNORM=0
-#     - timeout 40m pytest -s -v tests/e2e/online_serving/test_bagel.py -m "core_model" --run-level "core_model" -k "rocm"
+#     - timeout 40m pytest -s -v tests/e2e/online_serving/test_bagel_online.py -m "core_model" --run-level "core_model" -k "rocm"
 
 - label: "CosyVoice3-TTS E2E Test"
   agent_pool: mi325_1
@@ -223,5 +236,5 @@ steps:
   commands:
     - |
       timeout 20m bash -c '
-        pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "full_model" --run-level "core_model"
+        pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts.py -m "core_model" --run-level "core_model"
       '

.buildkite/test-amd.yaml

@@ -8,7 +8,7 @@ steps:
   grade: Blocking
   commands:
     - export GPU_ARCHS=gfx942
-    - pytest -s -v tests/e2e/offline_inference/test_z_image.py
+    - pytest -s -v tests/e2e/offline_inference/test_t2i_model.py
 
 - label: "Diffusion Images API LoRA E2E"
   timeout_in_minutes: 30
@@ -20,7 +20,7 @@ steps:
     - export GPU_ARCHS=gfx942
     - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/diffusion/lora/test_images_generations_lora.py
+    - pytest -s -v tests/e2e/online_serving/test_images_generations_lora.py
 
 - label: "Diffusion Model CPU offloading Test"
   timeout_in_minutes: 20
@@ -32,8 +32,8 @@ steps:
     - export GPU_ARCHS=gfx942
     - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
-    - pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
+    - pytest -s -v tests/e2e/offline_inference/test_diffusion_cpu_offload.py
+    - pytest -s -v tests/e2e/offline_inference/test_diffusion_layerwise_offload.py
 
 - label: "Diffusion Cache Backend Test"
   timeout_in_minutes: 15
@@ -57,7 +57,7 @@ steps:
     - export GPU_ARCHS=gfx942
     - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py -m core_model
+    - pytest -s -v tests/e2e/offline_inference/test_sequence_parallel.py -m core_model
 
 - label: "Diffusion Tensor Parallelism Test"
   timeout_in_minutes: 20
@@ -67,7 +67,7 @@ steps:
     - export GPU_ARCHS=gfx942
     - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/diffusion/distributed/test_zimage_parallelism.py
+    - pytest -s -v tests/e2e/offline_inference/test_zimage_parallelism.py
 
 - label: "Diffusion GPU Worker Test"
   timeout_in_minutes: 20
@@ -88,7 +88,7 @@ steps:
     - export GPU_ARCHS=gfx942
     - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py
+    - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py
     - pytest -s -v tests/engine/test_async_omni_engine_abort.py
 
 - label: "Omni Model Test Qwen3-Omni"
@@ -100,11 +100,25 @@ steps:
   commands:
     - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+    - export VLLM_TEST_CLEAN_GPU_MEMORY="1"
     - export VLLM_ROCM_USE_AITER=0
     - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py
     - pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py
 
 
+- label: "Diffusion Image Edit Test"
+  timeout_in_minutes: 15
+  agent_pool: mi325_1
+  depends_on: amd-build
+  mirror_hardwares: [amdproduction]
+  grade: Blocking
+  commands:
+    - export GPU_ARCHS=gfx942
+    - export VLLM_LOGGING_LEVEL=DEBUG
+    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+    - pytest -s -v tests/e2e/online_serving/test_image_gen_edit.py
+
+
 - label: "Omni Sleep Mode Test"
   timeout_in_minutes: 40
   agent_pool: mi325_2
@@ -115,4 +129,5 @@ steps:
     - export GPU_ARCHS=gfx942
     - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/entrypoints/test_omni_sleep_mode.py -m "advanced_model and omni and MI325" --run-level "advanced_model"
+    - export VLLM_TEST_CLEAN_GPU_MEMORY="1"
+    - pytest -s -v tests/e2e/offline_inference/test_omni_sleep_mode.py -m "advanced_model and omni and MI325" --run-level "advanced_model"

.buildkite/test-nightly.yml

@@ -1,6 +1,5 @@
 env:
   VLLM_WORKER_MULTIPROC_METHOD: spawn
-  VLLM_USE_FLASHINFER_MOE_FP16: 0
   HF_HUB_DOWNLOAD_TIMEOUT: 300
   HF_HUB_ETAG_TIMEOUT: 60
 
@@ -11,10 +10,10 @@ steps:
     depends_on: upload-nightly-pipeline
     if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "omni-test"
     steps:
-      - label: ":full_moon: Omni · Function Test with H100"
+      - label: ":full_moon: Omni · Function Test"
         timeout_in_minutes: 90
         commands:
-          - pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py
+          - pytest -s -v tests/e2e/ -m "full_model and H100 and omni" --run-level "full_model" --ignore=tests/e2e/accuracy
         agents:
           queue: "mithril-h100-pool"
         plugins:
@@ -49,25 +48,6 @@ steps:
                       path: /mnt/hf-cache
                       type: DirectoryOrCreate
 
-      - label: ":full_moon: Omni · Function Test with L4"
-        timeout_in_minutes: 90
-        commands:
-          - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-          - pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model"
-        agents:
-          queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
       - label: ":full_moon: Omni · Doc Test with L4"
         timeout_in_minutes: 90
         commands:
@@ -170,14 +150,14 @@ steps:
                       path: /mnt/hf-cache
                       type: DirectoryOrCreate
 
-      - label: ":full_moon: Omni · Perf Test · No Async Chunk"
-        key: nightly-omni-performance-no-async-chunk
+      - label: ":full_moon: Omni · Perf Test"
+        key: nightly-omni-performance
         timeout_in_minutes: 180
         commands:
           - export BENCHMARK_DIR=tests/dfx/perf/results
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json
+            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_omni.json
             EXIT=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
             exit $$EXIT
@@ -215,134 +195,6 @@ steps:
                       path: /mnt/hf-cache
                       type: DirectoryOrCreate
 
-      - label: ":full_moon: Omni · Perf Test · Async Chunk"
-        key: nightly-omni-performance-async-chunk
-        timeout_in_minutes: 180
-        commands:
-          - export BENCHMARK_DIR=tests/dfx/perf/results
-          - |
-            set +e
-            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json
-            EXIT=$$?
-            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
-            exit $$EXIT
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 2
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: ":full_moon: Omni · Perf Test · vLLM Text"
-        key: nightly-omni-performance-vllm-text
-        timeout_in_minutes: 180
-        commands:
-          - export BENCHMARK_DIR=tests/dfx/perf/results
-          - |
-            set +e
-            pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json
-            EXIT=$$?
-            buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
-            exit $$EXIT
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 2
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-
-      - label: ":full_moon: Omni · Multi-Replica Startup Test with 4x H100"
-        timeout_in_minutes: 45
-        commands:
-          - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py -m "full_model" --run-level "core_model"
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 4
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
 
   - group: ":card_index_dividers: TTS Model Test"
     key: nightly-tts-test-group
@@ -353,7 +205,7 @@ steps:
         timeout_in_minutes: 90
         commands:
           - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-          - pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy
+          - pytest -s -v tests/e2e/ -m "full_model and L4 and omni" --run-level "full_model" --ignore=tests/e2e/accuracy
         agents:
           queue: "gpu_1_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
         plugins:
@@ -389,7 +241,7 @@ steps:
                   - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                     resources:
                       limits:
-                        nvidia.com/gpu: 2
+                        nvidia.com/gpu: 1
                     volumeMounts:
                       - name: devshm
                         mountPath: /dev/shm
@@ -423,65 +275,10 @@ steps:
       build.pull_request.labels includes "nightly-test" ||
       build.pull_request.labels includes "diffusion-x2iat-test"
     steps:
-      # Use explicit file shards so single-card cases do not reserve 2+ GPUs,
-      # and Bagel feature coverage is not duplicated via a broad tests/e2e sweep.
-      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Single-GPU"
+      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100"
         timeout_in_minutes: 120
         commands:
-          - >-
-            pytest -sv
-            tests/e2e/online_serving/test_qwen_image_expansion.py
-            tests/e2e/online_serving/test_qwen_image_edit_expansion.py
-            tests/e2e/online_serving/test_qwen_image_layered_expansion.py
-            tests/e2e/online_serving/test_longcat_image_expansion.py
-            tests/e2e/online_serving/test_longcat_image_edit_expansion.py
-            tests/e2e/online_serving/test_flux_2_dev_expansion.py
-            tests/e2e/online_serving/test_bagel_expansion.py
-            -m "full_model and diffusion and H100 and not distributed_cuda"
-            --run-level "full_model"
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Qwen-Image"
-        timeout_in_minutes: 120
-        commands:
-          - >-
-            pytest -sv
-            tests/e2e/online_serving/test_qwen_image_expansion.py
-            tests/e2e/online_serving/test_qwen_image_edit_expansion.py
-            -m "full_model and diffusion and H100 and distributed_cuda"
-            --run-level "full_model"
+          - pytest -sv tests/e2e/ -k "not test_wan and not hunyuan" -m "full_model and diffusion and H100" --run-level "full_model" --ignore=tests/e2e/accuracy
         agents:
           queue: "mithril-h100-pool"
         plugins:
@@ -516,98 +313,10 @@ steps:
                       path: /mnt/hf-cache
                       type: DirectoryOrCreate
 
-      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Layered/LongCat/FLUX"
-        timeout_in_minutes: 90
-        commands:
-          - >-
-            pytest -sv
-            tests/e2e/online_serving/test_qwen_image_layered_expansion.py
-            tests/e2e/online_serving/test_longcat_image_expansion.py
-            tests/e2e/online_serving/test_longcat_image_edit_expansion.py
-            tests/e2e/online_serving/test_flux_2_dev_expansion.py
-            -m "full_model and diffusion and H100 and distributed_cuda"
-            --run-level "full_model"
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 2
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Bagel"
-        timeout_in_minutes: 120
-        commands:
-          - >-
-            pytest -sv
-            tests/e2e/online_serving/test_bagel_expansion.py
-            -m "full_model and diffusion and H100 and distributed_cuda"
-            --run-level "full_model"
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 3
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-
       - label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4"
         timeout_in_minutes: 60
         commands:
-          - pytest -sv tests/e2e/ -k "not test_wan and not test_bagel_expansion and not hunyuan" -m "full_model and diffusion and L4" --run-level "full_model" --ignore=tests/e2e/accuracy
+          - pytest -sv tests/e2e/ -k "not test_wan and not hunyuan" -m "full_model and diffusion and L4" --run-level "full_model" --ignore=tests/e2e/accuracy
         agents:
           queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
         plugins:
@@ -625,6 +334,7 @@ steps:
       - label: ":full_moon: Diffusion X2I(&A&T) · Doc Test"
         timeout_in_minutes: 60
         commands:
+          - export VLLM_TEST_CLEAN_GPU_MEMORY="1"
           - pytest -s -v tests/examples/*/test_text_to_image.py -m "full_model and example and H100" --run-level "full_model"
         agents:
           queue: "mithril-h100-pool"
@@ -660,11 +370,51 @@ steps:
                       path: /mnt/hf-cache
                       type: DirectoryOrCreate
 
-      - label: ":full_moon: Diffusion X2I(&A&T) · Accuracy Test"
-        timeout_in_minutes: 180
+      - label: ":full_moon: Diffusion X2I(&A&T) · GEBench Accuracy Test"
+        timeout_in_minutes: 60
         commands:
-          - pytest -s -v tests/e2e/accuracy/test_qwen_image*.py --run-level full_model
-          - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2i_matches_diffusers' -m full_model --run-level full_model
+          - pytest -s -v tests/e2e/accuracy/test_gebench_h100_smoke.py --run-level full_model --gebench-model Qwen/Qwen-Image-2512 --accuracy-judge-model QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ --accuracy-gpu 0 --gebench-port 8093 --accuracy-workers 1
+          - buildkite-agent artifact upload "tests/e2e/accuracy/artifacts/gebench_qwen-image-2512/summary*.json"
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          - kubernetes:
+              podSpec:
+                containers:
+                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                    resources:
+                      limits:
+                        nvidia.com/gpu: 1
+                    volumeMounts:
+                      - name: devshm
+                        mountPath: /dev/shm
+                      - name: hf-cache
+                        mountPath: /root/.cache/huggingface
+                    env:
+                      - name: HF_HOME
+                        value: /root/.cache/huggingface
+                      - name: HF_TOKEN
+                        valueFrom:
+                          secretKeyRef:
+                            name: hf-token-secret
+                            key: token
+                nodeSelector:
+                  node.kubernetes.io/instance-type: gpu-h100-sxm
+                volumes:
+                  - name: devshm
+                    emptyDir:
+                      medium: Memory
+                  - name: hf-cache
+                    hostPath:
+                      path: /mnt/hf-cache
+                      type: DirectoryOrCreate
+
+      - label: ":full_moon: Diffusion X2I(&A&T) · GEdit-Bench Accuracy Test"
+        timeout_in_minutes: 60
+        commands:
+          - pytest -s -v tests/e2e/accuracy/test_gedit_bench_h100_smoke.py --run-level full_model --gedit-model Qwen/Qwen-Image-Edit --accuracy-judge-model QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ --accuracy-gpu 0 --gedit-port 8093 --gedit-samples-per-group 20 --accuracy-workers 1
+          - buildkite-agent artifact upload "tests/e2e/accuracy/artifacts/gedit_scores_qwen-image-edit/qwen-image-edit_all_all_vie_score_*.csv"
+          - buildkite-agent artifact upload "tests/e2e/accuracy/artifacts/gedit_scores_qwen-image-edit/qwen-image-edit_all_all_summary_*.json"
         agents:
           queue: "mithril-h100-pool"
         plugins:
@@ -701,164 +451,10 @@ steps:
                       path: /mnt/hf-cache
                       type: DirectoryOrCreate
 
-      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image"
-        key: nightly-diffusion-x2iat-performance-qwen-image
+      - label: ":full_moon: Diffusion X2I(&A&T) · Accuracy Test"
         timeout_in_minutes: 180
         commands:
-          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
-          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
-          - export CACHE_DIT_VERSION=1.3.0
-          - |
-            set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
-            EXIT=$$?
-            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
-            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
-            exit $$EXIT
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 4
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit"
-        key: nightly-diffusion-x2iat-performance-qwen-image-edit
-        timeout_in_minutes: 180
-        commands:
-          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
-          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
-          - export CACHE_DIT_VERSION=1.3.0
-          - |
-            set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json
-            EXIT=$$?
-            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
-            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
-            exit $$EXIT
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 4
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit-2509"
-        key: nightly-diffusion-x2iat-performance-qwen-image-edit-2509
-        timeout_in_minutes: 180
-        commands:
-          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
-          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
-          - export CACHE_DIT_VERSION=1.3.0
-          - |
-            set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2509_vllm_omni.json
-            EXIT=$$?
-            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
-            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
-            exit $$EXIT
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 4
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Layered"
-        key: nightly-diffusion-x2iat-performance-qwen-image-layered
-        timeout_in_minutes: 120
-        commands:
-          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
-          - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
-          - export CACHE_DIT_VERSION=1.3.0
-          - |
-            set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json
-            EXIT=$$?
-            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
-            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
-            exit $$EXIT
+          - pytest -s -v tests/e2e/accuracy/test_qwen_image*.py --run-level full_model
         agents:
           queue: "mithril-h100-pool"
         plugins:
@@ -877,6 +473,8 @@ steps:
                     env:
                       - name: HF_HOME
                         value: /root/.cache/huggingface
+                      - name: VLLM_HTTP_TIMEOUT_KEEP_ALIVE
+                        value: "120"
                       - name: HF_TOKEN
                         valueFrom:
                           secretKeyRef:
@@ -893,26 +491,28 @@ steps:
                       path: /mnt/hf-cache
                       type: DirectoryOrCreate
 
-      - label: ":full_moon: Diffusion X2I(&A&T) · HunyuanImage3 · DiT Perf Test"
-        key: nightly-hunyuan-image3-performance
-        soft_fail: true
-        timeout_in_minutes: 120
-        if: build.env("RUN_HUNYUAN_IMAGE3_PERF") == "1"
+      - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test"
+        key: nightly-diffusion-x2iat-performance
+        timeout_in_minutes: 180
         commands:
           - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
           - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
           - export CACHE_DIT_VERSION=1.3.0
+          # [HACK]: run upload in the same command block as pytest.
+          # Because `exit` aborts the entire commands list.
           - |
             set +e
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp4_fp8.json
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
             EXIT1=$$?
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp2_fp8_sp2.json
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json
             EXIT2=$$?
-            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp2_fp8_cfgp2.json
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2509_vllm_omni.json
             EXIT3=$$?
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json
+            EXIT4=$$?
             buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
             buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
-            exit $$((EXIT1 | EXIT2 | EXIT3))
+            exit $$((EXIT1 | EXIT2 | EXIT3 | EXIT4))
         agents:
           queue: "mithril-h100-pool"
         plugins:
@@ -959,7 +559,7 @@ steps:
       - label: ":full_moon: Diffusion X2V · Function Test"
         timeout_in_minutes: 90
         commands:
-          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_hunyuan_video_15_expansion.py  -m "full_model and cuda" --run-level "full_model"
+          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_hunyuan_video_15_expansion.py  --run-level "full_model"
         agents:
           queue: "mithril-h100-pool"
         plugins:
@@ -998,8 +598,7 @@ steps:
         timeout_in_minutes: 180
         commands:
           - pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py -m full_model --run-level full_model
-          - pytest -s -v tests/e2e/accuracy/test_ltx2_3_video_similarity.py -m full_model --run-level full_model
-          - pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m full_model --run-level full_model
+          - pytest -s -v tests/e2e/accuracy/test_ltx2_3_video_similarity.py -m advanced_model --run-level advanced_model
         agents:
           queue: "mithril-h100-pool"
         plugins:
@@ -1081,65 +680,6 @@ steps:
                       path: /mnt/hf-cache
                       type: DirectoryOrCreate
 
-  - group: ":card_index_dividers: Diffusion · Quantization Test"
-    depends_on: upload-nightly-pipeline
-    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "diffusion-x2iat-test"
-    steps:
-      - label: ":full_moon: Quantization Test with H100"
-        timeout_in_minutes: 90
-        commands:
-          - timeout 30m pytest -sv tests/diffusion/quantization -m 'full_model and cuda and H100' --run-level "full_model"
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: ":full_moon: Quantization Test with L4"
-        timeout_in_minutes: 60
-        commands:
-          - pytest -sv tests/diffusion/quantization -m 'full_model and cuda and L4' --run-level "full_model"
-        agents:
-          queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"
-
   - label: ":bar_chart: Testcase Statistics"
     key: nightly-testcase-statistics
     timeout_in_minutes: 120
@@ -1157,7 +697,7 @@ steps:
               - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                 resources:
                   limits:
-                    nvidia.com/gpu: 1
+                    nvidia.com/gpu: 2
                 volumeMounts:
                   - name: devshm
                     mountPath: /dev/shm
@@ -1186,14 +726,9 @@ steps:
   - label: ":email: Nightly Collection & Email"
     key: nightly-perf-distribution
     depends_on:
-      - nightly-omni-performance-no-async-chunk
-      - nightly-omni-performance-async-chunk
-      - nightly-omni-performance-vllm-text
+      - nightly-omni-performance
       - nightly-tts-performance
-      - nightly-diffusion-x2iat-performance-qwen-image
-      - nightly-diffusion-x2iat-performance-qwen-image-edit
-      - nightly-diffusion-x2iat-performance-qwen-image-edit-2509
-      - nightly-diffusion-x2iat-performance-qwen-image-layered
+      - nightly-diffusion-x2iat-performance
       - nightly-diffusion-x2v-performance
       - nightly-testcase-statistics
     if: build.env("NIGHTLY") == "1"
@@ -1202,13 +737,8 @@ steps:
       - export DEFAULT_INPUT_DIR=tests/dfx/perf/results
       - export DEFAULT_OUTPUT_DIR=tests/dfx/perf/results
       - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance
-      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-no-async-chunk
-      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-async-chunk
-      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-vllm-text
-      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image
-      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit
-      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit-2509
-      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-layered
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance
       - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance
       - buildkite-agent artifact download "tests/dfx/perf/results/*.html" . --step nightly-testcase-statistics
       - python tools/nightly/generate_nightly_perf_excel.py

.buildkite/test-npu-nightly.yaml

@@ -1,24 +0,0 @@
-steps:
-  - group: ":card_index_dividers: Diffusion X2V Model Test"
-    key: nightly-diffusion-x2v-group
-    depends_on: upload-nightly-pipeline
-    if: >-
-      build.env("NIGHTLY") == "1" ||
-      build.pull_request.labels includes "nightly-test" ||
-      build.pull_request.labels includes "diffusion-x2v-test"
-    steps:
-      - label: ":full_moon: Diffusion X2V · Function Test"
-        agents:
-          queue: "ascend-a2b3"
-          resource_class: "npu-4"
-        image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}"
-        plugins:
-          - kubernetes:
-              podSpecPatch:
-                imagePullSecrets:
-                  - name: swr-secret
-        env:
-          VLLM_WORKER_MULTIPROC_METHOD: spawn
-          HF_TOKEN: "${HF_TOKEN}"
-        commands:
-          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k npu_i2v --run-level "full_model"

.buildkite/test-npu-ready.yaml

@@ -1,21 +0,0 @@
-steps:
-  - group: ":card_index_dividers: TTS Model Test"
-    key: ready-tts-group
-    depends_on: upload-ready-pipeline
-    steps:
-      - label: ":sound: TTS · Qwen3-TTS CustomVoice Test"
-        agents:
-          queue: "ascend-a2b3"
-          resource_class: "npu-1"
-        image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}"
-        plugins:
-          - kubernetes:
-              podSpecPatch:
-                imagePullSecrets:
-                  - name: swr-secret
-        env:
-          VLLM_WORKER_MULTIPROC_METHOD: spawn
-          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
-          HF_TOKEN: "${HF_TOKEN}"
-        commands:
-          - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'

.buildkite/test-weekly.yml

@@ -1,148 +0,0 @@
-env:
-  VLLM_WORKER_MULTIPROC_METHOD: spawn
-  VLLM_USE_FLASHINFER_MOE_FP16: 0
-  HF_HUB_DOWNLOAD_TIMEOUT: 300
-  HF_HUB_ETAG_TIMEOUT: 60
-
-steps:
-  - label: "Reliability Test - qwen3-omni"
-    timeout_in_minutes: 180
-    depends_on: upload-weekly-pipeline
-    if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
-    commands:
-      - pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m "slow"
-    agents:
-      queue: "mithril-h100-pool"
-    plugins:
-      - kubernetes:
-          podSpec:
-            containers:
-              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                resources:
-                  limits:
-                    nvidia.com/gpu: 2
-                volumeMounts:
-                  - name: devshm
-                    mountPath: /dev/shm
-                  - name: hf-cache
-                    mountPath: /root/.cache/huggingface
-                env:
-                  - name: HF_HOME
-                    value: /root/.cache/huggingface
-                  - name: HF_TOKEN
-                    valueFrom:
-                      secretKeyRef:
-                        name: hf-token-secret
-                        key: token
-            nodeSelector:
-              node.kubernetes.io/instance-type: gpu-h100-sxm
-            volumes:
-              - name: devshm
-                emptyDir:
-                  medium: Memory
-              - name: hf-cache
-                hostPath:
-                  path: /mnt/hf-cache
-                  type: DirectoryOrCreate
-
-  - label: "Reliability Test - wan22"
-    timeout_in_minutes: 180
-    depends_on: upload-weekly-pipeline
-    if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
-    commands:
-      - pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m "slow"
-    agents:
-      queue: "mithril-h100-pool"
-    plugins:
-      - kubernetes:
-          podSpec:
-            containers:
-              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                resources:
-                  limits:
-                    nvidia.com/gpu: 2
-                volumeMounts:
-                  - name: devshm
-                    mountPath: /dev/shm
-                  - name: hf-cache
-                    mountPath: /root/.cache/huggingface
-                env:
-                  - name: HF_HOME
-                    value: /root/.cache/huggingface
-                  - name: HF_TOKEN
-                    valueFrom:
-                      secretKeyRef:
-                        name: hf-token-secret
-                        key: token
-            nodeSelector:
-              node.kubernetes.io/instance-type: gpu-h100-sxm
-            volumes:
-              - name: devshm
-                emptyDir:
-                  medium: Memory
-              - name: hf-cache
-                hostPath:
-                  path: /mnt/hf-cache
-                  type: DirectoryOrCreate
-
-  # Invalid HTTP / WS payloads vs live Omni servers (Issue #3649 tracks known skips).
-  - group: "Reliability Test - Invalid parameters Test"
-    depends_on: upload-weekly-pipeline
-    if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
-    steps:
-      - label: "Invalid parameters Test · H100"
-        timeout_in_minutes: 120
-        commands:
-          - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100"
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 2
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      # Stable Audio Open, Qwen3-TTS speech/batch/voices paths marked ``L4``.
-      - label: "Invalid parameters Test · L4"
-        timeout_in_minutes: 120
-        commands:
-          - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and L4"
-        agents:
-          queue: "gpu_1_queue"
-        plugins:
-          - docker#v5.2.0:
-              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-              always-pull: true
-              propagate-environment: true
-              shm-size: "8gb"
-              environment:
-                - "HF_HOME=/fsx/hf_cache"
-                - "HF_TOKEN"
-              volumes:
-                - "/fsx/hf_cache:/fsx/hf_cache"

.claude/skills/add-diffusion-model/SKILL.md

@@ -515,15 +515,15 @@ After adding parallelism support, update:
 
 ### Step 11: Add CPU Offload Support
 
-Implement `SupportsComponentDiscovery` on your pipeline class to enable
+Implement `SupportsModuleOffload` on your pipeline class to enable
 `--enable-cpu-offload` and `--enable-layerwise-offload`. The protocol
 declares which submodules the offloader should manage:
 
 ```python
 from typing import ClassVar
-from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery
+from vllm_omni.diffusion.models.interface import SupportsModuleOffload
 
-class YourPipeline(nn.Module, SupportsComponentDiscovery):
+class YourPipeline(nn.Module, SupportsModuleOffload):
     _dit_modules: ClassVar[list[str]] = ["transformer"]
     _encoder_modules: ClassVar[list[str]] = ["text_encoder"]
     _vae_modules: ClassVar[list[str]] = ["vae"]
@@ -540,19 +540,10 @@ class YourPipeline(nn.Module, SupportsComponentDiscovery):
 All attribute names support dotted paths for nested submodules
 (e.g. `"pipe.transformer"`, `"bagel.time_embedder"`).
 
-Pipelines without `SupportsComponentDiscovery` fall back to scanning
+Pipelines without `SupportsModuleOffload` fall back to scanning
 well-known attribute names (`transformer`, `text_encoder`, `vae`,
 etc.), which fails for non-standard names.
 
-### Step 12: Performance Profiling
-
-After verifying correctness and implementing parallelism/caching, profile the model's performance to identify bottlenecks and ensure optimal execution.
-
-See the [Profiling Single-Stage Diffusion](../../../docs/contributing/profiling.md#3-profiling-single-stage-diffusion) guide for detailed instructions on:
-1. Using the PyTorch profiler (`profiler: "torch"`) to capture detailed CPU/CUDA traces.
-2. Using Nsight Systems (`nsys`) with `profiler: "cuda"` for low-overhead CUDA traces.
-3. Controlling profiling via `omni.start_profile()` and `omni.stop_profile()`.
-
 ---
 
 ## Iterative Development Tips

.claude/skills/add-diffusion-model/references/parallelism-patterns.md

@@ -92,7 +92,6 @@ class TPSelfAttention(nn.Module):
             softmax_scale=1.0 / (self.head_dim ** 0.5),
             causal=False,
             num_kv_heads=self.to_qkv.num_kv_heads,  # Local KV heads per GPU
-            role="self",
         )
 
     def forward(self, x):

.claude/skills/add-diffusion-model/references/transformer-adaptation.md

@@ -61,7 +61,6 @@ class SelfAttentionBlock(nn.Module):
             softmax_scale=1.0 / (self.head_dim ** 0.5),
             causal=False,
             num_kv_heads=num_heads,
-            role="self",
         )
 
     def forward(self, x, attn_mask=None):
@@ -76,24 +75,6 @@ class SelfAttentionBlock(nn.Module):
         return self.to_out(out)
 ```
 
-**Declaring the attention role.** Always pass a `role` string to `Attention(...)`. It is the lookup key users target with `--diffusion-attention-config.per_role.<role>.backend` to swap kernels per site without touching model code.
-
-| Convention | When to use |
-|---|---|
-| `role="self"` | Q/K/V come from the same hidden state |
-| `role="cross"` | K/V come from a separate `encoder_hidden_states`; pair with `skip_sequence_parallel=True` if K/V is replicated across SP ranks |
-
-For multi-modal sites that don't fit `self` / `cross`, use a dot-namespaced role and provide `role_category` so user config can fall back to the generic category:
-
-```python
-self.audio_to_video_attn = Attention(
-    num_heads=num_heads, head_size=self.head_dim,
-    softmax_scale=1.0 / (self.head_dim ** 0.5), causal=False,
-    role="mymodel.audio_to_video",
-    role_category="cross",
-)
-```
-
 #### Fused QKV with TP (Advanced)
 
 For tensor parallelism, use vLLM's parallel linear layers:
@@ -123,7 +104,6 @@ class TPSelfAttention(nn.Module):
             softmax_scale=1.0 / (self.head_dim ** 0.5),
             causal=False,
             num_kv_heads=num_heads,
-            role="self",
         )
 ```
 

.claude/skills/add-diffusion-model/references/troubleshooting.md

@@ -68,7 +68,7 @@ out = out.reshape(batch, seq_len, -1)
 ### Different Output vs Diffusers Reference
 
 **Common causes**:
-1. **Attention backend difference**: FlashAttention vs SDPA may produce slightly different results. Force SDPA via `--diffusion-attention-backend TORCH_SDPA` (or `DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA`) to match diffusers; for finer-grained per-role overrides see `--diffusion-attention-config.per_role.<role>.backend`
+1. **Attention backend difference**: FlashAttention vs SDPA may produce slightly different results. Set `DIFFUSION_ATTENTION_BACKEND=TORCH_SDPA` to match diffusers
 2. **Float precision**: vLLM-Omni may use bfloat16 where diffusers uses float32 for some operations
 3. **Missing normalization**: Check all LayerNorm/RMSNorm are preserved
 4. **Scheduler rounding**: Some schedulers have numerical sensitivity

.claude/skills/add-tts-model/SKILL.md

@@ -211,8 +211,8 @@ See `plan/voxcpm2_native_ar_design.md`.
 
 - Model files in `vllm_omni/model_executor/models/<model_name>/`
 - Stage config YAML
-- Working `end2end.py` at `examples/offline_inference/text_to_speech/<model>/end2end.py`
-- New section in `examples/offline_inference/text_to_speech/README.md` (table row + per-model section). Do **not** create a top-level `examples/offline_inference/<model>/` dir or a per-model `README.md` inside `text_to_speech/<model>/` — the hub README is the documented surface and the mkdocs `generate_examples` hook only descends one level into `examples/<category>/`.
+- Working `end2end.py` with correct audio output
+- README.md in the example directory
 
 ## Phase 3: Online Serving
 
@@ -308,11 +308,11 @@ def build_voice_clone_prompt(ref_audio_path: str, text: str, codec) -> list:
 ### Deliverables
 
 - Updated `serving_speech.py` with all 5 integration points (single commit)
-- Client scripts and server launcher under `examples/online_serving/text_to_speech/<model>/`
-- Gradio demo with streaming and voice cloning UI in the same dir
+- Client scripts and server launcher
+- Gradio demo with streaming and voice cloning UI
 - E2E online serving test (`tests/e2e/online_serving/test_<model>.py`)
 - Buildkite CI entry in `.buildkite/test-merge.yml`
-- New section in `examples/online_serving/text_to_speech/README.md` (table row + per-model section). Do **not** create a top-level `examples/online_serving/<model>/` dir or a per-model `README.md` inside `text_to_speech/<model>/`.
+- Documentation (offline + online serving docs)
 
 ### E2E test pitfalls to avoid
 

.claude/skills/diffusion-perf-opt/SKILL.md

@@ -1,488 +0,0 @@
----
-name: diffusion-perf-opt
-description: Diagnose and optimize vLLM Omni diffusion workloads, especially Wan/Qwen/Flux-style image and video generation. Use when Codex is asked to analyze profiling traces, choose parallel strategies, inspect torch profiler trace.json or trace.json.gz timelines, estimate optimization ROI, investigate GPU idle/free bubbles, compare USP/CFG/HSDP/VAE parallelism, or design operator/host/quantization optimizations for vLLM Omni.
----
-
-# vLLM Omni Performance Optimization
-
-Use this skill to run a disciplined optimization loop for vLLM Omni diffusion workloads. Keep two ideas separate: real performance baselines are collected with low overhead, while torch profiler traces are diagnostic artifacts and may distort latency.
-
-## First Questions
-
-Before proposing changes, ask for the optimization scene if it is not already known:
-
-- GPU model, card count, topology, and whether NVLink is present.
-- Model and pipeline, for example Wan2.2 I2V A14B.
-- User workload: resolution, frames, steps, batch/concurrency, CFG scales, prompt/image inputs.
-- Three runnable commands for each model/workload, or scripts that generate them:
-  - **Server startup command**: exact `vllm serve` command, environment variables, model path, port, parallelism flags, profiler flags, and precision/compile settings.
-  - **User request command**: exact single-request client command (for example curl or Python client) with fixed prompt/media/seed/size/steps. Use this to validate correctness and collect per-request stage timings.
-  - **User benchmark command**: exact repeatable benchmark command or script with warmup count, measured iteration count, concurrency/batch policy, output directory, and summary format.
-- Current enabled strategies: USP/SP, CFG parallel, HSDP/FSDP, VAE patch parallel, torch.compile, profiler options.
-- Optimization target: latency, throughput, memory, cost, or quality-preserving speed.
-- Precision/quality tolerance: bf16/fp8/quantization/sparsity/approximate attention allowed or not.
-
-## Workflow
-
-1. **Freeze the measurement protocol and commands.**
-   - Before analyzing parallel strategies or traces, establish the three commands for the target model/workload: server startup, single user request, and user benchmark.
-   - Prefer checked-in scripts or one-off shell scripts over free-form commands in chat. The scripts should make workload variables explicit, including model path, port/base URL, prompt/media inputs, resolution, frames, steps, seed, warmup, measured iterations, concurrency, and output path.
-   - Measurement reports must preserve the concrete process, not just the final numbers. For each tested configuration, write down the server startup command, the user request command, the user benchmark command or polling loop, the final server response metrics, and the client-side timing/HTTP result.
-   - Present all duration and latency values in **milliseconds (ms)** in tables and summaries. If an API returns seconds, convert to ms before reporting. Keep raw units only inside quoted raw logs or source snippets.
-   - Keep profiler modes separate in the command set:
-     - Baseline/benchmark commands should avoid torch profiler and stack collection.
-     - Diagnostic commands may enable torch profiler after a baseline identifies a bottleneck.
-   - If any of the three commands is missing, ask for it or create a proposed script before proceeding.
-   - Make clear which command is authoritative for correctness validation and which command is authoritative for performance numbers.
-
-2. **Collect a real baseline.**
-   - Disable PyTorch profiler and stack collection.
-   - Disable or fix torch.compile state so A/B is fair.
-   - Avoid `--enforce-eager` for production-speed baselines unless eager is the target.
-   - Prefer `--log-stats` and `--enable-diffusion-pipeline-profiler` for low-overhead stage timing. PR 3069 has the relevant metrics/log-stats changes; if local code does not include them, fetch or cherry-pick the minimal metrics changes rather than merging unrelated PR drift.
-   - Run warmup requests and exclude first-request lazy init.
-
-3. **Model the parallel strategy before testing.**
-   - Estimate compute and communication for self-attention, cross-attention, FFN, CFG, VAE encode/decode, and HSDP.
-   - Select a small candidate matrix rather than exhaustively testing everything.
-   - Typical 2-card candidates: `USP=2`, `CFG=2`, `USP=1/HSDP on-off`, VAE parallel on-off if memory allows.
-   - Typical 4-card candidates: `USP=4`, `CFG=2 x USP=2`, `USP=2 x HSDP`, VAE parallel on-off.
-   - Typical 8-card candidates for official CFG-enabled video diffusion models:
-     - Primary candidate: `CFG=2 x USP=4` with VAE patch parallel across all 8 ranks.
-     - Compare against: `CFG=1 x USP=8` to test whether larger sequence parallel groups beat CFG branch parallelism.
-     - Isolate VAE parallelism: keep the DiT strategy fixed and compare VAE patch parallel on/off or different VAE patch sizes.
-     - If long-video `diffuse` remains dominant and Ulysses all-to-all is suspected, test a hybrid sequence strategy such as `CFG=2 x USP=2 x Ring=2`.
-     - Test HSDP off only if the model fits without it; treat HSDP primarily as a memory strategy until A/B proves otherwise.
-   - Prefer USP/SP for long video token sequences; prefer CFG parallel when CFG doubles transformer forwards and sequence length is modest.
-   - Convert workload shape to latent/patch token counts before choosing candidates. For video models, record frames, latent frames, latent HxW, patch size, approximate token count, and which stages should scale with the token count.
-
-4. **Search for the best parallel configuration.**
-   - Start with a small matrix that answers one question per comparison:
-     - `CFG parallel vs larger USP`: compare `CFG=2 x USP=world/2` against `CFG=1 x USP=world` for CFG-enabled workloads.
-     - `VAE patch parallel value`: compare the best DiT strategy with VAE patch parallel enabled and disabled.
-     - `HSDP cost`: compare HSDP on/off only when both configurations fit memory.
-     - `Ulysses vs Ring`: test a Ring/Ulysses hybrid only after long-sequence `diffuse` is confirmed dominant.
-   - For each configuration, create or record a stable config id, for example `A_cfg2_usp4_vaepp8_hsdp_tiling`.
-   - For each config id, capture:
-     - exact server command and environment variables,
-     - observed distributed setup from logs, such as SP groups, CFG groups, HSDP shard/replicate sizes, VAE patch size,
-     - exact request command for every scenario,
-     - final server response metrics and client-side elapsed time,
-     - output artifact paths and any failed/empty responses.
-   - Use one warmup request per scenario, then at least three measured repeats for the shortlisted config. For exploratory matrix pruning, one measured request is acceptable only if the margin is large; label it as one-shot.
-   - Compare configurations by stage, not only end-to-end latency. A configuration can improve `diffuse` while hurting `vae.decode`; record both effects.
-   - Select the best config only after checking the target metric, dominant stages, memory headroom, and output correctness.
-
-5. **Run targeted A/B tests.**
-   - Change one variable per test.
-   - Keep model, input, seed, request parameters, GPU placement, and warmup policy fixed.
-   - Record latency, stage timings, memory, output quality, and logs.
-   - Report comparison tables in ms. Include at least end-to-end client time, server `inference_time`, server stage generation time if available, `vae.encode`, `diffuse`, `vae.decode`, and peak memory.
-
-6. **Collect diagnostic trace only after narrowing hypotheses.**
-   - Use torch profiler for a small number of requests.
-   - Run two separate diagnostic traces instead of mixing concerns:
-     - **Operator/shape trace**: enable `torch_profiler_record_shapes=True` and keep stack collection disabled. Use this to rank CUDA kernels, NCCL collectives, attention/MLP/norm/RoPE work, and shape-specific hot operators.
-     - **Host-stack trace**: enable `torch_profiler_with_stack=True` and normally keep shape collection disabled. Use this to map CPU/Python host gaps, synchronization points, scheduler paths, and request handling overhead.
-   - Keep both trace commands and reports separate from baseline/benchmark commands. Torch profiler latency is diagnostic only and must not be used as the final latency claim.
-   - Prefer profiling only the narrowed dominant scenario, for example the highest-resolution/video-length case where `diffuse` dominates.
-   - If profiler endpoints are available, run one warmup request first, then call `/start_profile`, run one profiled request, and call `/stop_profile`. This keeps model initialization and warmup out of the diagnostic trace.
-   - Start by analyzing rank 0 only. Expand to more ranks only if rank 0 suggests imbalance, unclear GPU idle/free bubbles, high NCCL wait, server timing mismatch, CFG branch imbalance, or USP group stragglers.
-   - Diagnostic reports must be written to disk and preserve: server command, profiler config, warmup/request/polling commands, trace artifact paths, rank analyzed, analyzer output or summary, and the decision about whether additional ranks are necessary.
-   - Analyze both rank-level balance and device-level free bubbles when additional ranks are opened.
-
-7. **Analyze host, communication, and operators.**
-   - Find GPU idle/free intervals and map each large gap to the enclosing CPU/Python code.
-   - Separate real GPU idle from profiler overhead such as CUPTI `Command Buffer Full`.
-   - Compare NCCL kernel time to user annotations; annotations can overcount nested intervals.
-   - Rank operator work by total CUDA time and by repeated small-kernel launch count.
-
-8. **Produce an optimization plan.**
-   - Classify candidates as P0/P1/P2.
-   - For each candidate, state necessity, expected benefit, implementation path, validation plan, and quality risk.
-   - Do not implement high-risk operator rewrites before proving the operator is a bottleneck for the target shapes.
-   - End the plan with a user-facing candidate selection table. The assistant
-     should not automatically choose a risky optimization just because it is
-     technically possible. Present the options clearly and let the user decide
-     which item is worth implementing based on latency target, engineering
-     budget, memory headroom, and quality tolerance.
-   - Organize the plan by optimization layer, not by a flat list of ideas:
-     - host/runtime optimization,
-     - measurement/benchmark reliability,
-     - parallelism and communication,
-     - VAE encode/decode and media pre/post processing,
-     - operator fusion and layout cleanup,
-     - attention main-path optimization,
-     - algorithmic, precision, or approximation changes.
-   - For each layer, tie every candidate back to evidence from baseline
-     metrics, diagnostic trace, source code, or output quality requirements.
-     Do not list generic optimizations without a trace or workload reason.
-
-## Priority Rules
-
-- **P0:** low risk, likely useful, or required for trustworthy measurement. Examples: real baseline, warmup, targeted parallel A/B, disabling avoidable `empty_cache`, scheduler coefficient caching.
-- **P1:** meaningful code changes with contained risk. Examples: cross-attention KV caching, VAE gather/broadcast reduction, AdaLayerNorm/RMSNorm/RoPE fusion after trace evidence.
-- **P2:** high implementation or quality risk. Examples: FA to LA replacement, custom Triton/CUDA fused kernels, FP8/quantization, sparsity/Rainfusion-style acceleration.
-
-Every implemented optimization needs A/B validation and quality regression. A/B means same workload and hardware before/after. Quality regression means checking generated image/video stability, artifacts, temporal flicker, and seed behavior when precision or approximate kernels change.
-
-## Optimization Layers
-
-After baseline, parallel-search, and diagnostic traces, summarize optimization
-opportunities by layer. This is the core of the performance analysis: the goal
-is to connect evidence to a scoped implementation and a validation plan.
-
-### Host and Runtime Optimization
-
-Purpose: remove CPU/Python stalls, synchronization points, allocator overhead,
-and request-path overhead that leave GPU lanes empty.
-
-Evidence to look for:
-
-- High `idle_pct` or large `GAP` blocks in `trace_analyzer.py`.
-- Host-stack trace lines such as `torch.cuda.empty_cache`,
-  `cudaStreamSynchronize`, `cudaDeviceSynchronize`, Python locks, scheduler
-  waits, image/video preprocessing, or repeated small allocation paths.
-- Difference between client wall-clock time and server `inference_time_s`.
-
-Typical candidates:
-
-- Make avoidable `torch.cuda.empty_cache()` optional or guard it by memory
-  headroom.
-- Cache scheduler coefficients, timesteps, masks, or other tiny repeated CPU
-  computations when the request shape/steps are fixed.
-- Remove avoidable host-device synchronizations and blocking logging/stat calls.
-- Move expensive preprocessing out of the critical path or cache fixed prompt,
-  image, and transform work for benchmark scenarios.
-- Ensure benchmark scripts record client-side elapsed time, HTTP status, output
-  path, and server response metrics.
-
-Priority guidance:
-
-- Usually P0 when the change is measurement reliability or an obvious removable
-  synchronization.
-- Usually P1 when it changes scheduling, memory lifetime, or request execution
-  order.
-
-Validation:
-
-- Re-run non-profiler baseline with same workload and seed.
-- Confirm peak memory headroom if disabling cache cleanup.
-- Confirm generated output exists and quality/seed behavior is unchanged.
-
-### Parallelism and Communication Optimization
-
-Purpose: choose the right decomposition for CFG branches, sequence tokens, model
-weights, VAE tiles, and rank topology.
-
-Evidence to look for:
-
-- Baseline A/B across `CFG`, `USP/SP`, `Ring`, `HSDP/FSDP`, and VAE patch
-  parallelism.
-- Stage timing shifts: `diffuse`, `vae.encode`, `vae.decode`, and server
-  end-to-end.
-- NCCL kernel time from trace, not only `user_annotation` time.
-- Rank imbalance across SP group ranks or CFG branch ranks.
-- Memory headroom and OOM risk.
-
-Typical candidates:
-
-- `CFG=2 x USP=world/2` versus `CFG=1 x USP=world` for CFG-enabled models.
-- VAE patch parallel on/off or patch size tuning.
-- HSDP/FSDP on/off only if both configurations fit memory.
-- Ulysses versus Ulysses+Ring only after long-sequence `diffuse` is confirmed
-  dominant and all-to-all is suspected.
-- Rank mapping/topology changes if all-rank traces show stragglers or NCCL wait.
-- Buffer reuse or preallocation for FSDP/HSDP all-gather paths.
-
-Priority guidance:
-
-- P0/P1 for configuration-only changes with strong measured wins.
-- P1 for buffer reuse or rank mapping changes.
-- P2 for invasive distributed algorithm changes.
-
-Validation:
-
-- Measure by stage and memory, not only end-to-end.
-- Use one-variable A/B with identical prompt/media/seed/shape/steps.
-- When communication is suspected, compare rank0-3 in one USP group and rank0
-  versus rank4 across CFG branches for `CFG=2 x USP=4`.
-
-### VAE and Media Pipeline Optimization
-
-Purpose: reduce encode/decode, tiling, split/gather, and media conversion time.
-
-Evidence to look for:
-
-- Large `vae.encode` or `vae.decode` in low-overhead stage timings.
-- Host-stack gaps in VAE tile split, gather, merge, broadcast, or image/video
-  transforms.
-- VAE kernels or cuDNN convolution in operator trace.
-- Whether every rank needs the final decoded tensor.
-
-Typical candidates:
-
-- Keep VAE patch parallel enabled when it has clear measured benefit.
-- Reduce VAE gather/broadcast to only ranks that need the final media output.
-- Reuse tile metadata, split buffers, or gather buffers.
-- Evaluate bf16/autocast behavior for VAE only with visual quality checks.
-- Avoid redundant image conversion, resize, or tensor construction in repeated
-  benchmark runs.
-
-Priority guidance:
-
-- P0/P1 if VAE is a large share of the target workload or if a host gap is
-  obvious and low risk.
-- Lower priority when `diffuse` dominates and VAE is already patch-parallelized.
-
-Validation:
-
-- Compare `vae.encode`, `vae.decode`, server end-to-end, and peak memory.
-- Check output video integrity, artifacts, flicker, and seed stability.
-
-### Operator Fusion and Layout Cleanup
-
-Purpose: reduce high-frequency small kernels, memory bandwidth pressure, layout
-conversions, and launch overhead in transformer and VAE blocks.
-
-Evidence to look for:
-
-- Top operator tables showing many `aten::copy_`, `aten::cat`,
-  `split_with_sizes_copy`, `aten::add`, `aten::mul`, `aten::div`, norm,
-  activation, RoPE, or reshape/layout kernels.
-- `ops_rankN.xlsx` `by_shape` sheet showing repeated small shapes inside the
-  same block path.
-- Trace lanes showing many short kernels between larger GEMM/attention kernels.
-- Source code patterns with repeated elementwise chains or layout conversions.
-
-Typical fusion targets:
-
-- AdaLayerNorm / RMSNorm / LayerNorm plus scale/shift fusion.
-- RoPE fusion with Q/K layout preparation when shapes are stable.
-- Residual add, scale, gate, and elementwise chains.
-- MLP gate/up/down cleanup, such as fusing activation and multiply around
-  GEMM outputs when feasible.
-- QKV projection and reshape/split/cat path cleanup.
-- Attention pre/post layout cleanup to avoid unnecessary copies, cats, and
-  splits around sequence parallel all-to-all.
-
-Priority guidance:
-
-- P1 when implemented with existing PyTorch/Triton/local helper patterns and
-  validated against exact outputs or tolerances.
-- P2 when it requires custom CUDA/Triton kernels, changes numerics, or touches
-  attention math directly.
-
-Validation:
-
-- First prove the operator family is material for the target shape.
-- Use non-profiler A/B for latency and stage timing.
-- Use quality regression checks for generated video stability.
-- Check compile behavior and graph breaks if using `torch.compile`.
-
-### Attention Main-Path Optimization
-
-Purpose: address the dominant self-attention cost when FlashAttention or other
-attention kernels dominate CUDA time.
-
-Evidence to look for:
-
-- Operator trace where FlashAttention/SDPA kernels dominate total CUDA time.
-- Attention shape from `ops_rankN.xlsx` `by_shape`, model code, or trace
-  metadata.
-- Whether attention cost scales with latent frames, latent H/W, patch size, or
-  CFG duplication.
-- Layout/copy/all-to-all work around attention.
-
-Typical candidates:
-
-- Verify the attention backend and shape are on the intended fast kernel path.
-- Compare supported attention backends only with identical workload and quality
-  settings.
-- Reduce attention input size by safe model/config choices when allowed:
-  latent resolution, frame count, patching, boundary ratio, or windowing.
-- Remove avoidable layout conversions before/after attention.
-- Reuse condition-side KV or other static inputs if the model structure allows.
-- Consider custom kernels, sparse/window/linear attention, or approximation only
-  after quality risk is accepted.
-
-Priority guidance:
-
-- P1 for backend/config/layout changes with preserved math.
-- P2 for approximate attention, sparsity, custom kernels, or any change that can
-  alter quality/temporal consistency.
-
-Validation:
-
-- Always include output quality and seed behavior checks.
-- Compare `diffuse`, server end-to-end, peak memory, and attention kernel time
-  in diagnostic traces if needed.
-
-### Algorithmic, Precision, and Approximation Optimization
-
-Purpose: reduce mathematical work or precision cost beyond local code cleanup.
-
-Evidence to look for:
-
-- A single operator family dominates even after low-risk runtime, parallel, and
-  fusion work.
-- Memory bandwidth or compute utilization suggests precision or quantization
-  could matter.
-- The user explicitly allows quality-preserving or approximate methods.
-
-Typical candidates:
-
-- FP8/quantization for transformer or selected projections.
-- Sparsity or Rainfusion-style acceleration.
-- Reduced steps, scheduler changes, distillation, or caching across frames.
-- Approximate attention or linear attention.
-
-Priority guidance:
-
-- Usually P2 because quality, numerics, and implementation risk are high.
-
-Validation:
-
-- Requires strict A/B, visual quality review, temporal flicker checks, seed
-  stability, and possibly human evaluation.
-
-### Interpolation, Super-Resolution, and E2E Pipeline Optimization
-
-Purpose: optimize the whole user-visible video product, not only the base
-diffusion invocation. Some deployments trade base-model latency against
-post-processing, interpolation, or super-resolution stages.
-
-Evidence to look for:
-
-- E2E latency breakdown across base generation, interpolation, super-resolution,
-  encoding, storage, and response streaming.
-- Fast/slow GPU or fast/slow stage analysis across multiple cards and pipeline
-  stages.
-- User quality target: resolution, FPS, temporal smoothness, and acceptable
-  post-processing artifacts.
-
-Typical candidates:
-
-- Add or optimize a frame interpolation stage when it reduces required base
-  model frames for the same perceived FPS.
-- Add or optimize a super-resolution model when generating lower base
-  resolution plus SR is faster for the target quality.
-- Analyze E2E2 pipeline behavior: client request, service scheduling, diffusion,
-  VAE/media, post-process, file write, and response.
-- Identify fast/slow cards or stages and rebalance pipeline placement.
-
-Priority guidance:
-
-- P1 when using proven interpolation/SR components without changing diffusion
-  math.
-- P2 when quality risk is high or the pipeline adds significant operational
-  complexity.
-
-Validation:
-
-- Measure E2E wall-clock, per-stage server timings, output FPS/resolution,
-  artifacts, flicker, and user-visible quality.
-
-### Optimization Candidate Library
-
-Use this table as a compact menu, not as automatic recommendations. Pick items
-only when baseline metrics, trace evidence, source inspection, or quality
-tolerance supports them.
-
-| Layer | Candidate | Evidence | Priority | Validation focus |
-|---|---|---|---|---|
-| Measurement | Freeze server/request/benchmark commands | Missing or drifting commands | P0 | Repeatable non-profiler A/B |
-| Measurement | Separate baseline and diagnostic profiler runs | Profiler used for latency claims | P0 | Low-overhead stage timings |
-| Host/runtime | Guard or remove avoidable `empty_cache` | Host-stack gaps or sync stalls | P0/P1 | Latency, peak memory, OOM safety |
-| Host/runtime | Cache scheduler coefficients/timesteps | Repeated tiny CPU/GPU work | P0/P1 | Same seed/output, stage timing |
-| Host/runtime | Reduce framework scheduling overhead | Client time exceeds server time | P1 | E2E latency, throughput |
-| Parallel | `CFG=2 x USP=world/2` vs `USP=world` | CFG doubles forward work | P0/P1 | `diffuse`, NCCL, memory |
-| Parallel | Tune VAE patch parallelism | VAE encode/decode is material | P0/P1 | VAE time, output correctness |
-| Parallel | HSDP on/off or buffer reuse | HSDP affects memory/all-gather | P1 | Memory, latency, OOM risk |
-| Parallel | Ulysses vs Ulysses+Ring | Long sequence all-to-all suspected | P1/P2 | Rank balance, NCCL kernels |
-| Cross-attn | Disable SP for short condition tokens | Cross-attn comm exceeds compute | P1 | `diffuse`, correctness |
-| VAE/media | Reduce VAE gather/broadcast | Rank traces show VAE wait | P1 | Rank balance, output file |
-| VAE/media | Reuse tile metadata/buffers | Tile split/merge host gaps | P1 | `vae.encode/decode`, memory |
-| VAE/media | VAE bf16/autocast | VAE float kernels are slow | P1/P2 | Artifacts, flicker, seed stability |
-| Operator fusion | AdaLayerNorm/LayerNorm fusion | Norm plus scale/shift kernels | P1 | Numeric tolerance, latency |
-| Operator fusion | RMSNorm fusion | Many small RMSNorm kernels | P1 | Numeric tolerance, latency |
-| Operator fusion | RoPE cache/fuse/layout cleanup | RoPE copy/reshape kernels | P1/P2 | Kernel count, correctness |
-| Operator layout | QKV or attention layout cleanup | Copy/cat/split around attention | P1 | Copy kernels, compile behavior |
-| Attention | Verify backend fast path | FA/SDPA dominates trace | P1 | `diffuse`, attention kernels |
-| Attention | FA to LA or selected-head LA | Attention remains dominant | P2 | Quality, temporal stability |
-| Precision | Transformer FP8/quantization | Compute/bandwidth bound and allowed | P2 | Quality, speed, stability |
-| Sparsity | Rainfusion-style acceleration | DiT compute remains dominant | P2 | Prompt diversity, quality |
-| Pipeline | Frame interpolation | Fewer base frames can meet FPS | P1/P2 | E2E latency, motion artifacts |
-| Pipeline | Super-resolution | Lower base res plus SR may win | P1/P2 | Detail quality, artifacts |
-| E2E | Fast/slow-card analysis | Multi-card stragglers | P0/P1 | Per-rank/stage wall-clock |
-
-### Optimization Plan Template
-
-Use this table shape when reporting the next work items:
-
-| Priority | Layer | Candidate | Evidence | Expected benefit | Implementation path | Validation | Quality risk |
-|---|---|---|---|---|---|---|---|
-| P0 | Host/runtime | Guard `empty_cache` | Host-stack gap points to `torch.cuda.empty_cache` | Small latency reduction, less idle | Add config/env guard | Non-profiler A/B, memory check | Low |
-| P1 | Operator fusion | RMSNorm/AdaLayerNorm fusion | High-frequency norm/elementwise kernels | Lower launch/bandwidth overhead | Use existing fusion helper or targeted Triton | A/B + output check | Medium |
-| P1/P2 | Attention | Attention layout/backend investigation | FA kernel dominates CUDA time | Potentially large | Inspect shapes/backend and remove layout copies | A/B + trace + quality | Medium/high |
-
-Then present a short selection prompt using the same rows:
-
-```text
-Which candidate should we implement next?
-
-1. P0 Host/runtime: guard empty_cache
-   - Expected benefit: small but low-risk latency reduction.
-   - Risk: possible memory increase/OOM if memory headroom is insufficient.
-
-2. P1 Operator fusion: inspect by_shape and implement first norm/RoPE/layout fusion
-   - Expected benefit: medium if high-frequency small kernels are confirmed.
-   - Risk: numerical/compile/quality validation needed.
-
-3. P1/P2 Attention: FA/LA/backend/layout investigation
-   - Expected benefit: potentially large.
-   - Risk: high quality and implementation risk.
-```
-
-If the user has not chosen an item, default to explaining tradeoffs and asking
-which candidate to execute. Only proceed autonomously on low-risk P0 measurement
-or instrumentation fixes.
-
-## Analysis Helpers
-
-PyTorch profiler traces are Chrome/Perfetto-compatible JSON files, usually
-`trace_rankN.json` or `trace_rankN.json.gz`. They normally contain a top-level
-`traceEvents` list, though some exporters emit the raw event list directly.
-
-Use the checked-in analyzer from the repository root:
-
-```bash
-python3 .claude/skills/diffusion-perf-opt/scripts/trace_analyzer.py \
-  vllm_profile/.../trace_rank0.json.gz \
-  --min-gap-ms 5 \
-  --topn 20
-```
-
-For rank imbalance or communication questions, pass all relevant rank traces in
-one command. For host gaps, lower `--min-gap-ms` to `1` and use a host-stack
-trace. Read `gpu_span_s`, `busy_union_s`, `idle_union_s`, `idle_pct`, `GAP`
-blocks, `Top GPU/operator events by total duration`, and `Top NCCL-like events
-by category`. Treat `cat=user_annotation` NCCL ranges as enclosing annotations;
-prefer `cat=kernel` or `cat=gpu_user_annotation` for real device work.
-
-The analyzer summarizes timing only. It does not parse tensor shapes, attribute
-overlap to individual streams, prove quality, or provide final latency claims.
-Use `ops_rankN.xlsx` or PyTorch key averages for shape analysis, and re-test any
-optimization with non-profiler baseline commands.
-
-Read `references/optimization-playbook.md` when drafting the optimization table or comparing candidate techniques.
-
-## vLLM Omni Heuristics
-
-- Cross-attention usually should not use USP/SP when text/image condition token count is much smaller than latent video tokens. Confirm via trace; in Wan2.2 I2V, self-attention dominates cross-attention.
-- VAE bf16/autocast is often worthwhile but requires visual quality checks.
-- VAE patch parallel can help decode/encode but may add gather/merge/broadcast overhead. Check whether all ranks need the final decoded tensor.
-- HSDP/FSDP is primarily a memory strategy. If the model fits without it, run an on/off latency comparison.
-- Scheduler work can create small host/device gaps; cache tiny solve coefficients when timesteps/order are known.
-- `torch.cuda.empty_cache()` can prevent OOM but creates synchronization/idle. Make it optional if memory headroom is sufficient.
-- `Command Buffer Full` in profiler output is profiler overhead, not a model optimization target.

.claude/skills/diffusion-perf-opt/agents/openai.yaml

@@ -1,4 +0,0 @@
-interface:
-  display_name: "vLLM Omni Perf Opt"
-  short_description: "Profile and optimize vLLM Omni diffusion performance."
-  default_prompt: "Analyze and optimize a vLLM Omni diffusion workload using baselines, traces, parallel strategy modeling, and prioritized optimization plans."

.claude/skills/diffusion-perf-opt/references/optimization-playbook.md

@@ -1,82 +0,0 @@
-# Optimization Playbook
-
-Use this reference when turning trace evidence into prioritized vLLM Omni optimizations.
-
-## Parallel Strategy
-
-Start from workload shape:
-
-- Long video tokens, high resolution, many frames: self-attention and FFN dominate. USP/SP is usually valuable.
-- CFG enabled with positive and negative prompts: transformer forward count doubles. CFG parallel may beat USP on small/moderate sequence lengths.
-- Cross-attention: skip SP/USP when condition tokens are much smaller than latent tokens; extra all-to-all can exceed compute saved.
-- HSDP/FSDP: use to fit memory first. If memory allows, compare without HSDP because all-gathers and lazy init add overhead.
-- VAE parallel: useful when encode/decode is material. Check gather/merge/broadcast and whether every rank needs final decoded output.
-
-Candidate matrices:
-
-- 2 cards: `USP=2`, `CFG=2`, HSDP on/off, VAE PP on/off.
-- 4 cards: `USP=4`, `CFG=2 x USP=2`, `USP=2 x HSDP`, VAE PP on/off.
-- 8 cards: add pipeline/stage parallel only after measuring orchestration and inter-stage transfer overhead.
-
-## Host And Free Bubble Analysis
-
-Look for device intervals where no kernel, memcpy, or memset is active.
-
-Common causes:
-
-- First request FSDP/HSDP lazy init.
-- Python preprocessing: image load, resize, `to_tensor`, tokenizer, request marshaling.
-- `torch.cuda.empty_cache()` or explicit synchronizations.
-- Scheduler small linear algebra such as tiny `torch.linalg.solve`.
-- VAE tile split/merge Python loops.
-- Profiler overhead such as CUPTI command buffer flush/full.
-
-Also inspect trace-external time:
-
-- API request parsing and multipart upload.
-- Engine/orchestrator dispatch.
-- Worker RPC and queue wait.
-- Output serialization, video write, base64, HTTP response.
-
-## Operator Optimization Candidates
-
-Evaluate necessity before implementation:
-
-- **FA to LA / LA for head:** high potential only if attention is the main CUDA hotspot and target shapes fit LA well.
-- **Cross-attention BSND/no transpose:** do only if transpose/copy around cross-attn is measurable.
-- **AdaLayerNorm fusion:** fuse norm, scale/shift, gate, residual, and dtype conversion if repeated elementwise kernels are significant.
-- **RMSNorm fusion:** useful for Q/K norms and VAE norms when small-kernel launch count is high.
-- **FP32 LayerNorm to bf16:** quality-sensitive; run visual regression.
-- **LA preprocess:** integrate only if preprocess overhead is cached or amortized.
-- **RoPE move/cache/fuse:** useful when RoPE small kernels are visible around every self-attn block.
-- **VAE RMSNorm replacement:** only after VAE internal trace shows norm as a real component of decode/encode.
-- **Scheduler fusion/cache:** cache coefficient solves and fuse CFG combine with scheduler arithmetic when possible.
-
-Before writing a custom kernel, check:
-
-- Local vLLM Omni implementations.
-- Upstream vLLM kernels and attention backends.
-- SGLang, FlashInfer, FlashAttention, and Triton examples.
-- Shape constraints and layout expectations.
-
-## Quantization And Sparsity
-
-Treat these as high-risk optimizations:
-
-- Rainfusion/sparse diffusion can be high reward but needs model-specific quality validation.
-- VAE decode quantization can reduce decode time but usually has smaller end-to-end upside than transformer attention/FFN work.
-- FP8 or approximate kernels require seed-stable visual regression, temporal flicker checks, and prompt diversity.
-
-## Output Format For Recommendations
-
-Return a table with:
-
-- Priority: P0/P1/P2.
-- Optimization.
-- Evidence from trace or code.
-- Expected benefit.
-- Risk/quality impact.
-- Implementation path.
-- Validation command or A/B test.
-
-State clearly when a recommendation is a hypothesis requiring new trace with shapes.

.claude/skills/diffusion-perf-opt/scripts/trace_analyzer.py

@@ -1,144 +0,0 @@
-#!/usr/bin/env python3
-"""Summarize torch profiler Chrome traces for vLLM Omni diffusion optimization."""
-
-from __future__ import annotations
-
-import argparse
-import collections
-import gzip
-import json
-from pathlib import Path
-from typing import Any
-
-GPU_CATS = {"kernel", "gpu_memcpy", "gpu_memset"}
-CPU_CATS = {"python_function", "user_annotation", "cpu_op", "cuda_runtime", "cuda_driver"}
-
-
-def open_trace(path: Path):
-    if path.suffix == ".gz":
-        return gzip.open(path, "rt")
-    return path.open("rt")
-
-
-def event_name(event: dict[str, Any]) -> str:
-    return str(event.get("name", ""))
-
-
-def summarize_trace(path: Path, min_gap_us: float, topn: int) -> None:
-    with open_trace(path) as f:
-        data = json.load(f)
-
-    events = data if isinstance(data, list) else data.get("traceEvents", [])
-    gpu: list[tuple[float, float, float, str, str, Any, Any]] = []
-    cpu: list[tuple[float, float, float, str, str, Any, Any]] = []
-    by_gpu_name: dict[str, list[float]] = collections.defaultdict(lambda: [0, 0.0, 0.0])
-    by_name: dict[str, list[float]] = collections.defaultdict(lambda: [0, 0.0, 0.0])
-    nccl: dict[tuple[str, str], list[float]] = collections.defaultdict(lambda: [0, 0.0, 0.0])
-
-    for event in events:
-        dur = event.get("dur")
-        ts = event.get("ts")
-        if dur is None or ts is None or dur <= 0:
-            continue
-        cat = str(event.get("cat", ""))
-        name = event_name(event)
-        row = (float(ts), float(ts + dur), float(dur), name, cat, event.get("pid"), event.get("tid"))
-        stat = by_name[name]
-        stat[0] += 1
-        stat[1] += dur
-        stat[2] = max(stat[2], dur)
-        if cat in GPU_CATS:
-            gpu.append(row)
-            gstat = by_gpu_name[name]
-            gstat[0] += 1
-            gstat[1] += dur
-            gstat[2] = max(gstat[2], dur)
-        elif cat in CPU_CATS:
-            cpu.append(row)
-        if "nccl" in name.lower():
-            nstat = nccl[(cat, name)]
-            nstat[0] += 1
-            nstat[1] += dur
-            nstat[2] = max(nstat[2], dur)
-
-    print(f"\n== {path}")
-    print(f"events={len(events)} gpu_events={len(gpu)} cpu_events={len(cpu)}")
-    if not gpu:
-        print("No GPU events found.")
-        return
-
-    gpu.sort()
-    merged: list[list[Any]] = []
-    for start, end, dur, name, cat, pid, tid in gpu:
-        if not merged or start > merged[-1][1]:
-            merged.append([start, end, [(start, end, dur, name, cat, pid, tid)]])
-        else:
-            merged[-1][1] = max(merged[-1][1], end)
-            merged[-1][2].append((start, end, dur, name, cat, pid, tid))
-
-    span = max(end for _, end, *_ in gpu) - min(start for start, *_ in gpu)
-    busy = sum(end - start for start, end, _ in merged)
-    idle = span - busy
-    print(
-        f"gpu_span_s={span / 1e6:.3f} "
-        f"busy_union_s={busy / 1e6:.3f} "
-        f"idle_union_s={idle / 1e6:.3f} "
-        f"idle_pct={idle / span * 100:.2f}"
-    )
-
-    interesting_cpu = [
-        row
-        for row in cpu
-        if row[2] >= 1000
-        and (
-            row[4] in {"python_function", "user_annotation"}
-            or "cudaStreamSynchronize" in row[3]
-            or "cudaDeviceSynchronize" in row[3]
-            or "cudaLaunch" in row[3]
-            or "cudaMemcpy" in row[3]
-        )
-    ]
-
-    gaps = []
-    for idx in range(1, len(merged)):
-        gap_start = merged[idx - 1][1]
-        gap_end = merged[idx][0]
-        gap_dur = gap_end - gap_start
-        if gap_dur >= min_gap_us:
-            prev_event = max(merged[idx - 1][2], key=lambda x: x[1])
-            next_event = min(merged[idx][2], key=lambda x: x[0])
-            mid = (gap_start + gap_end) / 2
-            containers = [row for row in interesting_cpu if row[0] <= mid <= row[1]]
-            containers = sorted(containers, key=lambda x: x[2])[:8]
-            gaps.append((gap_dur, gap_start, gap_end, prev_event, next_event, containers))
-
-    print(f"gaps_ge_{min_gap_us / 1000:.3f}ms count={len(gaps)} sum_s={sum(g[0] for g in gaps) / 1e6:.3f}")
-    for gap_dur, gap_start, gap_end, prev_event, next_event, containers in sorted(gaps, reverse=True)[:topn]:
-        print(f"\nGAP {gap_dur / 1000:.3f} ms ts={gap_start:.0f}->{gap_end:.0f}")
-        print(f"  prev {prev_event[4]} {prev_event[2] / 1000:.3f} ms {prev_event[3][:160]}")
-        print(f"  next {next_event[4]} {next_event[2] / 1000:.3f} ms {next_event[3][:160]}")
-        for row in containers:
-            print(f"  in   {row[4]} {row[2] / 1000:.3f} ms {row[3][:180]}")
-
-    print("\nTop GPU/operator events by total duration:")
-    for name, (count, total, max_dur) in sorted(by_gpu_name.items(), key=lambda kv: kv[1][1], reverse=True)[:topn]:
-        print(f"  {int(count):8d} total={total / 1e6:9.3f}s max={max_dur / 1000:9.3f}ms {name[:180]}")
-
-    print("\nTop NCCL-like events by category:")
-    for (cat, name), (count, total, max_dur) in sorted(nccl.items(), key=lambda kv: kv[1][1], reverse=True)[:topn]:
-        print(f"  {int(count):8d} total={total / 1e6:9.3f}s max={max_dur / 1000:9.3f}ms cat={cat} {name[:160]}")
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("traces", nargs="+", type=Path, help="trace.json or trace.json.gz files")
-    parser.add_argument("--min-gap-ms", type=float, default=5.0, help="minimum GPU idle gap to print")
-    parser.add_argument("--topn", type=int, default=20, help="number of gaps/hotspots to print")
-    args = parser.parse_args()
-
-    for trace in args.traces:
-        summarize_trace(trace, args.min_gap_ms * 1000.0, args.topn)
-
-
-if __name__ == "__main__":
-    main()

.claude/skills/readme.md

@@ -18,9 +18,6 @@ include:
 
 - `add-diffusion-model`: guides integration of a new diffusion model into
   `vllm-omni`
-- `diffusion-perf-opt`: guides diffusion model performance optimization,
-  including profiling traces, parallel strategies, stage timing analysis, and
-  benchmark-driven tuning
 - `add-omni-model`: covers addition of new omni-modality model support
 - `add-tts-model`: covers integration of new TTS models and related serving
   workflows

.gitattributes

@@ -27,7 +27,6 @@
 *.mov binary
 *.mp4 binary
 *.mp3 binary
-*.wav binary
 *.flv binary
 *.fla binary
 *.swf binary

.github/CODEOWNERS

@@ -3,64 +3,15 @@
 # More info: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners
 
 # Default owners (fallback reviewers for all files)
-* @Gaohan123 @hsliuustc0106 @ywang96
+# later, we can add more reviewers here
+* @hsliuustc0106
 
-# Repository operations
-/.github/ @hsliuustc0106
+# Assign reviewers by directory
+/.github/ @owner1
+/tests/ @owner1
+/docs/ @owner1
 
-# Documentation and Recipes
-/docs/ @hsliuustc0106 @Gaohan123 @david6666666
-/recipes @hsliuustc0106 @Gaohan123 @david6666666
-
-# Skills
-/.claude/ @hsliuustc0106
-
-# Tests
-/tests/ @yenuo26
-
-# CI
-/.buildkite/ @yenuo26 @congw729
-
-# Engine and model_executor
-/vllm_omni/core/ @tzhouam @yinpeiqi @fake0fan
-/vllm_omni/engine/ @tzhouam @yinpeiqi @fake0fan
-/vllm_omni/entrypoints/ @tzhouam @yinpeiqi @fake0fan
-/vllm_omni/model_executor/ @tzhouam @gcanlin
-/vllm_omni/worker/ @tzhouam @gcanlin
-
-# Diffusion
-/vllm_omni/diffusion/ @Isotr0py @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa @david6666666
-/vllm_omni/lora/ @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa @yuanheng-zhao
-
-# Diffusion CPU offloading
-/vllm_omni/diffusion/offloader/ @Isotr0py @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa @yuanheng-zhao
-
-# Omni and TTS support
-/vllm_omni/transformers_utils/ @tzhouam @yuanheng-zhao
-/vllm_omni/model_executor/models/ @linyueqian @ZeldaHuang @princepride @yuanheng-zhao @amy-why-3459 @Bounty-hunter
-
-# Benchmarks
-/benchmarks/accuracy/ @david6666666
-/benchmarks/build_dataset/ @linyueqian @yuanheng-zhao
-/benchmarks/diffusion/ @Isotr0py @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa
-/benchmarks/distributed/ @princepride
-/benchmarks/glm_image/ @princepride @JaredforReal
-/benchmarks/tts/ @linyueqian @ZeldaHuang @princepride
-
-# Usability and Observability
-/vllm_omni/deploy/ @tzhouam @lishunyang12
-/vllm_omni/metrics/ @lishunyang12
-/vllm_omni/profiler/ @gcanlin
-
-# ComfyUI
-/apps/ @fhfuih
-
-# Large Scale Serving
-/vllm_omni/distributed/ @princepride @yuanheng-zhao @xuechendi @natureofnature @fake0fan
-
-# Quantization and configuration
-/vllm_omni/config/ @lishunyang12
-/vllm_omni/quantization/ @david6666666 @Isotr0py @lishunyang12
-
-# Hardware plugins and accelerators
-/vllm_omni/platforms/ @gcanlin @tjtanaa @xuechendi
+# Example: reviewers for specific directories
+# /examples/offline_inference/ @offline-inference-team
+# /vllm_omni/core/ @core-team
+# /vllm_omni/model_executor/ @model-team

.github/mergify.yml

@@ -1,223 +0,0 @@
-pull_request_rules:
-- name: label-documentation
-  description: Automatically apply documentation label
-  conditions:
-    - label != stale
-    - or:
-      - files~=^[^/]+\.md$
-      - files~=^docs/
-      - files~=^examples/
-  actions:
-    label:
-      add:
-        - documentation
-    comment:
-      message: "Documentation preview: https://vllm--{{number}}.org.readthedocs.build/en/{{number}}/"
-
-- name: comment-pre-commit-failure
-  description: Comment on PR when pre-commit check fails
-  conditions:
-    - status-failure=pre-commit
-    - -closed
-    - -draft
-  actions:
-    comment:
-      message: |
-        Hi @{{author}}, the pre-commit checks have failed. Please run:
-
-        ```bash
-        uv pip install pre-commit
-        pre-commit install
-        pre-commit run --all-files
-        ```
-
-        Then, commit the changes and push to your branch.
-
-        For future commits, `pre-commit` will run automatically on changed files before each commit.
-
-        > [!TIP]
-        > <details>
-        > <summary>Is <code>mypy</code> or <code>markdownlint</code> failing?</summary>
-        > <br/>
-        > <code>mypy</code> and <code>markdownlint</code> are run differently in CI. If the failure is related to either of these checks, please use the following commands to run them locally:
-        >
-        > ```bash
-        > # For mypy (substitute "3.10" with the failing version if needed)
-        > pre-commit run --hook-stage manual mypy-3.10
-        > # For markdownlint
-        > pre-commit run --hook-stage manual markdownlint
-        > ```
-        > </details>
-
-- name: comment-dco-failure
-  description: Comment on PR when DCO check fails
-  conditions:
-    - status-failure=dco
-    - -closed
-    - -draft
-  actions:
-    comment:
-      message: |
-        Hi @{{author}}, the DCO check has failed. Please click on DCO in the Checks section for instructions on how to resolve this.
-
-- name: label-ci-build
-  description: Automatically apply ci/build label
-  conditions:
-    - label != stale
-    - or:
-      - files~=^\.github/
-      - files~=\.buildkite/
-      - files~=^cmake/
-      - files=CMakeLists.txt
-      - files~=^docker/Dockerfile
-      - files~=^requirements.*\.txt
-      - files=setup.py
-  actions:
-    label:
-      add:
-        - ci/build
-
-- name: label-frontend
-  description: Automatically apply frontend label
-  conditions:
-    - label != stale
-    - files~=^vllm_omni/entrypoints/
-  actions:
-    label:
-      add:
-        - frontend
-
-- name: label-diffusion
-  description: Automatically apply diffusion label
-  conditions:
-    - label != stale
-    - files~=^vllm_omni/diffusion/
-  actions:
-    label:
-      add:
-        - diffusion
-
-- name: label-new-model
-  description: Automatically apply new-model label
-  conditions:
-    - label != stale
-    - or:
-        - files=vllm_omni/model_executor/models/registry.py
-        - files=vllm_omni/diffusion/registry.py
-  actions:
-    label:
-      add:
-        - new-model
-
-- name: label-npu
-  description: Automatically apply npu label
-  conditions:
-    - label != stale
-    - or:
-      - files~=^vllm_omni/platforms/npu/
-      - title~=(?i)NPU
-      - title~=(?i)Ascend
-  actions:
-    label:
-      add:
-        - npu
-    assign:
-      users:
-        - "gcanlin"
-
-- name: label-rocm
-  description: Automatically apply rocm label
-  conditions:
-    - label != stale
-    - or:
-      - files~=^vllm_omni/platforms/rocm/
-      - title~=(?i)AMD
-      - title~=(?i)ROCm
-  actions:
-    label:
-      add:
-        - rocm
-    assign:
-      users:
-        - "tjtanaa"
-
-- name: label-xpu
-  description: Automatically apply xpu label
-  conditions:
-    - label != stale
-    - or:
-      - files~=^vllm_omni/platforms/xpu/
-      - title~=(?i)Intel
-      - title~=(?i)XPU
-  actions:
-    label:
-      add:
-        - xpu
-    assign:
-      users:
-        - "yma11"
-
-- name: label-modelrunner
-  description: Automatically apply modelrunner label
-  conditions:
-    - label != stale
-    - or:
-      - files~=^vllm_omni/worker/
-      - title~=(?i)ModelRunner
-      - title~=(?i)worker
-  actions:
-    label:
-      add:
-        - ModelRunner
-    assign:
-      users:
-        - "gcanlin" # To check whether need to sync to NPU
-
-- name: auto-rebase if approved, ready, and 30 commits behind main
-  conditions:
-    - base = main
-    - label=ready
-    - "#approved-reviews-by >= 1"
-    - "#commits-behind >= 30"
-    - -closed
-    - -draft
-    - -conflict
-  actions:
-    rebase: {}
-
-- name: ping author on conflicts and add 'needs-rebase' label
-  conditions:
-    - label != stale
-    - conflict
-    - -closed
-  actions:
-    label:
-      add:
-        - needs-rebase
-    comment:
-      message: |
-       This pull request has merge conflicts that must be resolved before it can be
-       merged. Please rebase the PR, @{{author}}.
-
-       https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/syncing-a-fork
-
-- name: remove 'needs-rebase' label when conflict is resolved
-  conditions:
-    - -conflict
-    - -closed
-  actions:
-    label:
-      remove:
-        - needs-rebase
-
-- name: label-bug
-  description: Automatically apply bug label
-  conditions:
-    - label != stale
-    - or:
-      - title~=(?i)\bbug\b
-      - title~=(?i)\bbugfix\b
-  actions:
-    label:
-      add:
-        - bug

.gitignore

@@ -239,7 +239,6 @@ data/
 datasets/
 *.csv
 *.json
-!tests/dfx/perf/tests/*.json
 !apps/ComfyUI-vLLM-Omni/example_workflows/*.json
 *.jsonl
 *.parquet
@@ -265,8 +264,6 @@ tmp_test
 vllm_omni/_version.py
 # output files
 *.wav
-# Vendored test reference audio (small, intentionally checked in)
-!tests/assets/**/*.wav
 .worktrees/
 # CI overlay yamls materialized from tests/utils.py:_CI_OVERLAYS at test time
 tests/.ci_generated/

README.md

@@ -9,19 +9,19 @@ Easy, fast, and cheap omni-modality model serving for everyone
 </h3>
 
 <p align="center">
-| <a href="https://vllm-omni.readthedocs.io/en/latest/"><b>Documentation</b></a> | <a href="https://deepwiki.com/vllm-project/vllm-omni"><b>DeepWiki</b></a> | <a href="https://discuss.vllm.ai"><b>User Forum</b></a> | <a href="https://slack.vllm.ai"><b>Developer Slack</b></a> | <a href="docs/assets/WeChat.jpg"><b>WeChat</b></a> | <a href="https://arxiv.org/abs/2602.02204"><b>Paper</b></a> | <a href="https://docs.google.com/presentation/d/111-L8zF7A1j_YI_cR8JsblofdScdRr2f/edit?usp=sharing&ouid=110473603432222024453&rtpof=true&sd=true"><b>Slides</b></a> |
+| <a href="https://vllm-omni.readthedocs.io/en/latest/"><b>Documentation</b></a> | <a href="https://discuss.vllm.ai"><b>User Forum</b></a> | <a href="https://slack.vllm.ai"><b>Developer Slack</b></a> | <a href="docs/assets/WeChat.jpg"><b>WeChat</b></a> | <a href="https://arxiv.org/abs/2602.02204"><b>Paper</b></a> | <a href="https://docs.google.com/presentation/d/1XJWgv79lORl8rbaVvp2d5Sqs6ZEBgAgj/edit?slide=id.p1#slide=id.p1"><b>Slides</b></a> |
 </p>
 
 
 ---
 
 *Latest News* 🔥
-- [2026/05] We released [0.20.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.20.0) - refreshes the serving/runtime stack for large-scale omni workloads, and improves diffusion model performance, quantization, and hardware readiness across CUDA, ROCm, MUSA, NPU, and XPU backends.
 - [2026/03] We released [0.18.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.18.0) - strengthens the core runtime through a large entrypoint refactor and scheduler/runtime cleanups, expands unified quantization and diffusion execution, broadens multimodal model coverage, and improves production readiness across audio, omni, image, video, RL, and multi-platform deployments.
 - [2026/03] Check out our first public [project deepdive](https://youtu.be/sgwNfsNnR9I) at the vLLM Hong Kong Meetup!
 - [2026/03] **[vllm-omni-skills](https://github.com/hsliuustc0106/vllm-omni-skills)** is a community-driven collection of AI assistant skills that help developers work with vLLM-Omni more effectively. These skills can be used with popular agentic AI coding assistants like **Cursor IDE**, **Claude**, **Codex**, and more.
 - [2026/02] We released [0.16.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.16.0) - A major alignment + capability release that rebases onto **upstream vLLM v0.16.0** and significantly expands performance, distributed execution, and production readiness across **Qwen3-Omni / Qwen3-TTS**, **Bagel**, **MiMo-Audio**, **GLM-Image** and the **Diffusion (DiT) image/video stack**—while also improving platform coverage (CUDA / ROCm / NPU / XPU), CI quality, and documentation.
 - [2026/02] We released [0.14.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.14.0) - This is the first **stable release** of vLLM-Omni that expands Omni’s diffusion / image-video generation and audio / TTS stack, improves distributed execution and memory efficiency, and broadens platform/backend coverage (GPU/ROCm/NPU/XPU). It also brings meaningful upgrades to serving APIs, profiling & benchmarking, and overall stability. Please check our latest [paper](https://arxiv.org/abs/2602.02204) for architecture design and performance results.
+- [2026/01] We released [0.12.0rc1](https://github.com/vllm-project/vllm-omni/releases/tag/v0.12.0rc1) - a major RC milestone focused on maturing the diffusion stack, strengthening OpenAI-compatible serving, expanding omni-model coverage, and improving stability across platforms (GPU/NPU/ROCm).
 - [2025/11] vLLM community officially released [vllm-project/vllm-omni](https://github.com/vllm-project/vllm-omni) in order to support omni-modality models serving.
 
 ---

benchmarks/build_dataset/seed_tts_design/en/meta.lst

@@ -1,30 +0,0 @@
-vd001|||The quick brown fox jumps over the lazy dog.|A warm, friendly female voice with a slight American Midwest accent, speaking at a moderate pace with natural inflection.
-vd002|||Welcome to the future of text-to-speech synthesis.|A deep, authoritative male news anchor voice, clear and professional with a measured cadence.
-vd003|||The sunset painted the sky in brilliant shades of orange and pink.|A gentle elderly female voice, soft and wise, with a slight Southern American accent.
-vd004|||Scientists have discovered a new species of deep-sea creature.|A young male voice with an Australian accent, curious and enthusiastic.
-vd005|||Breaking news: a major climate summit opens today in Geneva.|A crisp female newsreader voice, neutral accent, confident and precise.
-vd006|||In the beginning, there was darkness and silence across the universe.|A rich, dramatic bass male narrator voice, slow and deeply resonant.
-vd007|||Come closer, I have something important to tell you.|A soft, intimate female voice, slightly whispery, warm and gentle.
-vd008|||And they're off! The horses race toward the first turn at incredible speed.|An energetic male sports commentator, fast-paced and excited.
-vd009|||Once upon a time, in a land far away, lived a very clever fox.|A light, playful voice with childlike enthusiasm, bright and clear.
-vd010|||The ancient manuscript reveals secrets hidden for a thousand years.|A wise, measured elderly male voice, slow and deliberate, British English accent.
-vd011|||Good evening, ladies and gentlemen, and welcome to our show.|A sophisticated female voice with a slight French accent speaking English, elegant and refined.
-vd012|||System initialized. Running diagnostics. All systems nominal.|A clear, precise robotic-sounding voice, neutral and monotone with slight synthetic quality.
-vd013|||I hear what you are saying, and it is completely understandable to feel that way.|A warm, empathetic female therapist voice, calm and reassuring, unhurried pace.
-vd014|||Attention all units: proceed to grid reference seven-seven-alpha.|A firm, authoritative military male voice, clipped and commanding.
-vd015|||Oh my goodness, you have to try this amazing new recipe I just found!|An enthusiastic, bubbly female voice, high energy and friendly.
-vd016|||Dude, the waves were totally amazing out there today. Super happy about it!|A relaxed male voice with a California accent, casual and laid-back.
-vd017|||The quarterly results exceed expectations across all major metrics.|A sharp, businesslike female voice, confident and efficient, fast-paced delivery.
-vd018|||Chapter one. The morning sun filtered gently through the forest canopy.|A smooth, rich male audiobook narrator voice, expressive and engaging.
-vd019|||To be or not to be, that is the question.|A theatrical female voice, dramatic and expressive, stage projection quality.
-vd020|||And that is all for tonight. Stay well out there, everyone.|A warm, velvety male late-night radio DJ voice, smooth and intimate.
-vd021|||Please fasten your seatbelts; we will be experiencing some turbulence shortly.|A calm, professional female flight attendant voice, reassuring and clear.
-vd022|||The fossil record suggests these creatures lived more than sixty million years ago.|A measured male documentary narrator voice, scientific and curious.
-vd023|||Hey kiddo, did you finish your homework before dinner today?|A gentle, caring father voice, warm and slightly tired after a long day.
-vd024|||Live from courtside, the home team has just taken a commanding lead.|A high-energy male sports broadcaster, urgent and engaged.
-vd025|||Please remain on the line while I transfer you to the next available agent.|A polite, neutral female call-center voice, professional and patient.
-vd026|||I have a dream that one day this nation will rise up and live out the true meaning of its creed.|A powerful, resonant male orator voice, slow and impassioned.
-vd027|||Mama always said life was like a box of chocolates; you never know what you're gonna get.|A folksy male voice with a Southern American accent, warm and reflective.
-vd028|||Ahoy there, matey! Hoist the colors and prepare to set sail.|A theatrical pirate male voice, gruff and boisterous.
-vd029|||Ladies and gentlemen, please welcome to the stage our keynote speaker.|A polished female event MC voice, gracious and articulate.
-vd030|||Once you've completed the form, click submit and we'll review it within two business days.|A friendly female product-onboarding voice, encouraging and clear.

benchmarks/build_dataset/seed_tts_smoke/en/meta.lst

@@ -1,30 +0,0 @@
-smoke001|||The quick brown fox jumps over the lazy dog near the riverbank at sunset.
-smoke002|||Welcome to the future of text-to-speech synthesis in production systems.
-smoke003|||Yesterday the team finished rolling out the new authentication flow.
-smoke004|||She walked carefully across the wet cobblestones, careful not to slip.
-smoke005|||The conference call is scheduled for nine in the morning, Pacific time.
-smoke006|||Please remember to save your work before closing the editor.
-smoke007|||Two plus two equals four, but five hundred and forty three digits is long.
-smoke008|||I would like a coffee with oat milk and a chocolate croissant please.
-smoke009|||The library closes at eight on weekdays and six on Saturdays.
-smoke010|||During the Renaissance, art and science flourished in European cities.
-smoke011|||He whispered the secret word so quietly that no one else could hear.
-smoke012|||Our flight departs from gate twenty three at eleven fifteen.
-smoke013|||The storm knocked out power for six hours, but the backup generator kicked in.
-smoke014|||Reading a good book on a rainy afternoon is one of life's great pleasures.
-smoke015|||When the kettle whistled, she poured the hot water over the fresh tea leaves.
-smoke016|||The algorithm runs in linear time, which is a big improvement over the previous approach.
-smoke017|||In the distance, the mountains were shrouded in thick morning fog.
-smoke018|||Our company reported record revenue for the fourth quarter of the fiscal year.
-smoke019|||She explained the new policy in detail during the staff meeting this morning.
-smoke020|||The children laughed and played in the garden until the sun began to set.
-smoke021|||A long line of cars stretched along the coastal highway after the lighthouse opened.
-smoke022|||The chef recommended the seared scallops with a lemon butter sauce as the special.
-smoke023|||Our quarterly engineering review begins promptly at two in the afternoon on Thursday.
-smoke024|||Despite the heavy rain, the marathon runners kept a steady pace through downtown.
-smoke025|||She finished the crossword puzzle in under fifteen minutes while sipping her tea.
-smoke026|||The new release adds streaming output, faster startup, and better error reporting.
-smoke027|||His grandfather taught him how to whittle small wooden birds from cedar offcuts.
-smoke028|||Travelers were advised to check the timetable since trains were running on a holiday schedule.
-smoke029|||The startup launched a public beta after eighteen months of closed development.
-smoke030|||At night the desert sky filled with stars so bright the Milky Way was clearly visible.

benchmarks/diffusion/README.md

@@ -30,7 +30,7 @@ python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
 
 **Notes**
 
-- By default, image tasks talk to `http://<host>:<port>/v1/chat/completions`; video tasks talk to `/v1/videos`.
+- The benchmark talks to `http://<host>:<port>/v1/chat/completions`.
 - If you run the server on another host or port, pass `--base-url` accordingly.
 
 ## 2. Supported Datasets
@@ -95,9 +95,8 @@ You can point to your own trace using `--dataset-path`.
 
 ### Basic flags
 
-- `--base-url`: Server address; `--endpoint` selects the path appended to this base URL.
+- `--base-url`: Server address (the script calls `.../v1/chat/completions`).
 - `--model`: The OpenAI-compatible `model` field.
-- `--endpoint`: API endpoint path. Leading `/` is optional, e.g. `/v1/videos` or `v1/videos`.
 - `--task`: Task type (e.g., `t2i`, `t2v`, `i2i`, `i2v`).
 - `--dataset`: Dataset mode (`vbench` / `trace` / `random`).
 - `--num-prompts`: Number of requests to send.
@@ -131,21 +130,9 @@ Warmup flags:
 
 - `--warmup-requests`: Number of warmup requests.
 - `--warmup-num-inference-steps`: Steps used during warmup.
-- `--warmup-concurrency`: Maximum concurrent warmup requests. Use this to warm
-  the same batch shape as the measured run instead of warming only batch=`1`.
 - For `--task t2v`: warmup requests are forced to use `num_frames=1` to make warmup faster and less noisy.
 
 Traffic / concurrency flags:
 
 - `--request-rate`: Target request rate (requests/second). If set to `inf`, the script sends all requests immediately.
 - `--max-concurrency`: Max number of in-flight requests (default: `1`). This can hard-cap the achieved QPS: if it is too small, requests will queue behind the semaphore, and both achieved throughput and observed SLO attainment can be skewed.
-
-### Batched warmup note
-
-For batched serving runs, warm the same in-flight shape you plan to measure.
-For example, a run with `--max-concurrency 8` should usually also use
-`--warmup-requests 8 --warmup-concurrency 8`; otherwise the first measured
-batch may still pay compile or CUDA-graph capture cost.
-
-For a Qwen-Image continuous-batching replay example, see
-[`performance_dashboard/qwen_image_serving_performance.md`](./performance_dashboard/qwen_image_serving_performance.md).

benchmarks/diffusion/backends.py

@@ -151,7 +151,7 @@ async def async_request_chat_completions(
     return output
 
 
-async def async_request_openai_image_generations(
+async def async_request_openai_images(
     input: RequestFuncInput,
     session: aiohttp.ClientSession,
     pbar: tqdm | None = None,
@@ -278,7 +278,7 @@ async def async_request_v1_videos(
                 return output
 
         # invoke a poll request (GET /v1/videos/{video_id})
-        poll_interval = 0.2  # Unit(s)
+        poll_interval = 2.0  # Unit(s)
         timeout_seconds = 600.0
         deadline = time.perf_counter() + timeout_seconds
         job_url = f"{input.api_url}/{job_id}"
@@ -348,38 +348,12 @@ async def async_request_v1_videos(
     return output
 
 
-LEGACY_BACKEND_ENDPOINT_ALIASES = {
-    "vllm-omni": "/v1/chat/completions",
-    "openai": "/v1/images/generations",
-}
-
-
-def normalize_endpoint(value: str) -> str:
-    endpoint = str(value).strip()
-    if not endpoint:
-        raise ValueError("endpoint must not be empty.")
-    endpoint = LEGACY_BACKEND_ENDPOINT_ALIASES.get(
-        endpoint,
-        LEGACY_BACKEND_ENDPOINT_ALIASES.get(endpoint.lstrip("/"), endpoint),
-    )
-    if not endpoint.startswith("/"):
-        endpoint = f"/{endpoint}"
-    return endpoint
-
-
-def endpoint_filename_token(value: str) -> str:
-    token = normalize_endpoint(value).lstrip("/")
-    for bad in ("/", "\\", ":", "*", "?", '"', "<", ">", "|"):
-        token = token.replace(bad, "_")
-    return token or "endpoint"
-
-
 backends_function_mapping = {
     "2i": {
-        "/v1/chat/completions": (async_request_chat_completions, "/v1/chat/completions"),
-        "/v1/images/generations": (async_request_openai_image_generations, "/v1/images/generations"),
+        "vllm-omni": (async_request_chat_completions, "/v1/chat/completions"),
+        "openai": (async_request_openai_images, "/v1/images/generations"),
     },
     "2v": {
-        "/v1/videos": (async_request_v1_videos, "/v1/videos"),
+        "v1/videos": (async_request_v1_videos, "/v1/videos"),
     },
 }

benchmarks/diffusion/diffusion_benchmark_serving.py

@@ -6,31 +6,31 @@
 Benchmark online serving for diffusion models (Image/Video Generation).
 If you want to use i2v, i2i dataset, you should `uv pip install gdown` first
 
-Supports multiple endpoints:
-    - /v1/chat/completions: OpenAI chat-compatible image requests
-    - /v1/images/generations: OpenAI image generation requests
-    - /v1/videos: Async video jobs
+Supports multiple backends:
+    - vllm-omni: Uses /v1/chat/completions endpoint (default)
+    - openai: Uses /v1/images/generations endpoint
+    - v1/videos: Use /v1/videos endpoint
 
 Usage:
-    # Video (/v1/videos endpoint)
+    # Video (v1/videos backend)
     t2v:
     python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --endpoint /v1/videos --dataset vbench --task t2v --num-prompts 10 \
+        --backend v1/videos --dataset vbench --task t2v --num-prompts 10 \
         --height 480 --width 640 --fps 16 --num-frames 80
 
     i2v:
     python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --endpoint /v1/videos --dataset vbench --task i2v --num-prompts 10
+        --backend v1/videos --dataset vbench --task i2v --num-prompts 10
 
 
-    # Image (/v1/chat/completions endpoint)
+    # Image (vllm-omni backend)
     t2i:
     python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --endpoint /v1/chat/completions --dataset vbench --task t2i --num-prompts 10 \
+        --backend vllm-omni --dataset vbench --task t2i --num-prompts 10 \
         --height 1024 --width 1024
 
     python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --endpoint /v1/chat/completions --dataset random --task t2i --num-prompts 1 \
+        --backend vllm-omni --dataset random --task t2i --num-prompts 1 \
         --max-concurrency 1 --enable-negative-prompt \
         --random-request-config '[
             {"width":512,"height":512,"num_inference_steps":20,"weight":0.15},
@@ -41,18 +41,18 @@ Usage:
 
     i2i:
     python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --endpoint /v1/chat/completions --dataset vbench --task i2i --num-prompts 10
+        --backend vllm-omni --dataset vbench --task i2i --num-prompts 10
 
-    # Image (/v1/images/generations endpoint)
+    # Image (openai backend)
     t2i:
     python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --endpoint /v1/images/generations --dataset vbench --task t2i --num-prompts 10 \
+        --backend openai --dataset vbench --task t2i --num-prompts 10 \
         --height 1024 --width 1024 --port 3000
 
     # Video (v1/videos)
     t2v:
     python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --endpoint /v1/videos --dataset random --task t2v --num-prompts 1 \
+        --backend v1/videos --dataset random --task t2v --num-prompts 1 \
         --max-concurrency 1 --enable-negative-prompt \
         --random-request-config '[
             {"width":854,"height":480,"num_inference_steps":18,"num_frames":120,"fps":24,"weight":1}
@@ -68,7 +68,6 @@ import glob
 import json
 import logging
 import os
-import pathlib
 import random
 import tempfile
 import time
@@ -81,12 +80,7 @@ from typing import Any
 import aiohttp
 import numpy as np
 import requests
-from backends import (
-    RequestFuncInput,
-    RequestFuncOutput,
-    backends_function_mapping,
-    normalize_endpoint,
-)
+from backends import RequestFuncInput, RequestFuncOutput, backends_function_mapping
 from PIL import Image
 from tqdm.asyncio import tqdm
 
@@ -599,13 +593,6 @@ class RandomDataset(BaseDataset):
         if self.enable_negative_prompt:
             extra_body["negative_prompt"] = f"Negative prompt {idx} for benchmarking diffusion models"
 
-        request_fields = {
-            "width",
-            "height",
-            "num_frames",
-            "num_inference_steps",
-            "fps",
-        }
         params = {
             "width": self.args.width,
             "height": self.args.height,
@@ -615,11 +602,9 @@ class RandomDataset(BaseDataset):
         }
         if self._sampled_requests:
             profile = self._sampled_requests[idx]
-            params.update({key: value for key, value in profile.items() if key in request_fields})
-            extra_body.update({key: value for key, value in profile.items() if key not in request_fields})
-        prompt = self.args.random_prompt or f"Random prompt {idx} for benchmarking diffusion models"
+            params.update(profile)
         return RequestFuncInput(
-            prompt=prompt,
+            prompt=f"Random prompt {idx} for benchmarking diffusion models",
             api_url=self.api_url,
             model=self.model,
             seed=self.args.seed,
@@ -766,52 +751,6 @@ async def iter_requests(
         yield req
 
 
-def _make_warmup_request(
-    requests_list: list[RequestFuncInput],
-    index: int,
-    args,
-) -> RequestFuncInput:
-    warm_req = requests_list[index % len(requests_list)]
-    if args.warmup_num_inference_steps is not None:
-        warm_req = replace(
-            warm_req,
-            num_inference_steps=args.warmup_num_inference_steps,
-        )
-    if args.task == "t2v":
-        warm_req = replace(warm_req, num_frames=1)
-    return warm_req
-
-
-async def _run_warmups(
-    requests_list: list[RequestFuncInput],
-    args,
-    session: aiohttp.ClientSession,
-    request_func,
-) -> list[tuple[RequestFuncInput, RequestFuncOutput]]:
-    if not args.warmup_requests or not requests_list:
-        return []
-
-    warmup_requests = [_make_warmup_request(requests_list, i, args) for i in range(args.warmup_requests)]
-    warmup_concurrency = min(int(args.warmup_concurrency), len(warmup_requests))
-    warmup_semaphore = asyncio.Semaphore(warmup_concurrency)
-
-    print(
-        f"Running {len(warmup_requests)} warmup request(s) "
-        f"with num_inference_steps={args.warmup_num_inference_steps} "
-        f"and warmup_concurrency={warmup_concurrency}..."
-    )
-
-    async def limited_warmup_request_func(
-        req: RequestFuncInput,
-    ) -> RequestFuncOutput:
-        async with warmup_semaphore:
-            return await request_func(req, session, None)
-
-    warmup_tasks = [asyncio.create_task(limited_warmup_request_func(req)) for req in warmup_requests]
-    warmup_outputs = await asyncio.gather(*warmup_tasks)
-    return list(zip(warmup_requests, warmup_outputs))
-
-
 def calculate_metrics(
     outputs: list[RequestFuncOutput],
     total_duration: float,
@@ -879,35 +818,6 @@ def calculate_metrics(
     return metrics
 
 
-def _response_suffix(endpoint: str) -> str:
-    if endpoint == "/v1/videos":
-        return ".mp4"
-    return ".json"
-
-
-def _save_success_responses(
-    outputs: list[RequestFuncOutput],
-    requests_list: list[RequestFuncInput],
-    output_dir: str,
-    endpoint: str,
-) -> list[str]:
-    path = pathlib.Path(output_dir).expanduser().resolve()
-    path.mkdir(parents=True, exist_ok=True)
-    saved_paths: list[str] = []
-    suffix = _response_suffix(endpoint)
-
-    for index, (request, output) in enumerate(zip(requests_list, outputs, strict=False)):
-        if not output.success:
-            continue
-        item_path = path / f"{index:05d}_{request.request_id}{suffix}"
-        if isinstance(output.response_body, bytes):
-            item_path.write_bytes(output.response_body)
-        else:
-            item_path.write_text(json.dumps(output.response_body, indent=2, sort_keys=True), encoding="utf-8")
-        saved_paths.append(str(item_path))
-    return saved_paths
-
-
 def wait_for_service(base_url: str, timeout: int = 120) -> None:
     print(f"Waiting for service at {base_url}...")
     start_time = time.time()
@@ -927,14 +837,6 @@ def wait_for_service(base_url: str, timeout: int = 120) -> None:
         time.sleep(1)
 
 
-def _default_endpoint_for_task(task: str) -> str:
-    if task in {"t2v", "i2v", "ti2v"}:
-        return "/v1/videos"
-    if task in {"t2i", "i2i", "ti2i"}:
-        return "/v1/chat/completions"
-    raise ValueError(f"Unsupported task for endpoint resolution: {task}")
-
-
 async def benchmark(args):
     # Construct base_url if not provided
     if args.base_url is None:
@@ -954,23 +856,18 @@ async def benchmark(args):
             f"Valid image tasks: {sorted(IMAGE_TASKS)}"
         )
 
-    raw_endpoint = args.endpoint if args.endpoint is not None else args.backend
-    if raw_endpoint is None:
-        raw_endpoint = _default_endpoint_for_task(args.task)
-    args.endpoint = normalize_endpoint(raw_endpoint)
+    valid_backends = sorted(backends_function_mapping[task_type].keys())
 
-    valid_endpoints = sorted(backends_function_mapping[task_type].keys())
-
-    if args.endpoint not in valid_endpoints:
+    if args.backend not in valid_backends:
         logger.error(
-            f"Invalid endpoint '{args.endpoint}' for task '{args.task}' (task type: '{task_type}').\n"
-            f"Valid endpoints for this task type: {valid_endpoints}\n"
-            f"Example usage: --task {args.task} --endpoint {valid_endpoints[0]}"
+            f"Invalid backend '{args.backend}' for task '{args.task}' (task type: '{task_type}').\n"
+            f"Valid backends for this task type: {valid_backends}\n"
+            f"Example usage: --task {args.task} --backend {valid_backends[0]}"
         )
-        raise ValueError("Endpoint validation failed. See log above for valid options.")
+        raise ValueError("Backend validation failed. See log above for valid options.")
 
-    # Setup API URL and request function based on endpoint.
-    request_func, api_url = backends_function_mapping[task_type][args.endpoint]
+    # Setup API URL and request function based on backend
+    request_func, api_url = backends_function_mapping[task_type][args.backend]
     api_url = f"{args.base_url}{api_url}"
 
     if args.dataset == "vbench":
@@ -1003,12 +900,23 @@ async def benchmark(args):
     pbar = tqdm(total=len(requests_list), disable=args.disable_tqdm)
 
     async with aiohttp.ClientSession() as session:
-        warmup_pairs = await _run_warmups(
-            requests_list=requests_list,
-            args=args,
-            session=session,
-            request_func=request_func,
-        )
+        warmup_pairs: list[tuple[RequestFuncInput, RequestFuncOutput]] = []
+        if args.warmup_requests and requests_list:
+            print(
+                f"Running {args.warmup_requests} warmup request(s) \
+                with num_inference_steps={args.warmup_num_inference_steps}..."
+            )
+            for i in range(args.warmup_requests):
+                warm_req = requests_list[i % len(requests_list)]
+                if args.warmup_num_inference_steps is not None:
+                    warm_req = replace(
+                        warm_req,
+                        num_inference_steps=args.warmup_num_inference_steps,
+                    )
+                if args.task == "t2v":
+                    warm_req = replace(warm_req, num_frames=1)
+                warm_out = await limited_request_func(warm_req, session, None)
+                warmup_pairs.append((warm_req, warm_out))
 
         if args.slo:
             # Prefer trace-provided per-request slo_ms. Only populate when missing.
@@ -1033,22 +941,15 @@ async def benchmark(args):
     metrics = calculate_metrics(outputs, total_duration, requests_list, args, args.slo)
 
     # Add configuration info to metrics for JSON output
-    metrics["endpoint"] = args.endpoint
+    metrics["backend"] = args.backend
     metrics["model"] = args.model
     metrics["dataset"] = args.dataset
     metrics["task"] = args.task
-    if args.save_response_dir:
-        metrics["saved_responses"] = _save_success_responses(
-            outputs,
-            requests_list,
-            args.save_response_dir,
-            args.endpoint,
-        )
 
-    print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="="))
+    print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=60, c="="))
 
     # Section 1: Configuration
-    print("{:<40} {:<15}".format("Endpoint:", args.endpoint))
+    print("{:<40} {:<15}".format("Backend:", args.backend))
     print("{:<40} {:<15}".format("Model:", args.model))
     print("{:<40} {:<15}".format("Dataset:", args.dataset))
     print("{:<40} {:<15}".format("Task:", args.task))
@@ -1111,17 +1012,12 @@ if __name__ == "__main__":
     parser.add_argument("--host", type=str, default="localhost", help="Server host.")
     parser.add_argument("--port", type=int, default=8091, help="Server port.")
     parser.add_argument("--model", type=str, default="default", help="Model name.")
-    parser.add_argument(
-        "--endpoint",
-        type=str,
-        default=None,
-        help=("Endpoint path to target. Leading '/' is optional, e.g. /v1/videos or v1/videos."),
-    )
     parser.add_argument(
         "--backend",
         type=str,
-        default=None,
-        help=argparse.SUPPRESS,
+        default="vllm-omni",
+        choices=["vllm-omni", "openai", "v1/videos"],
+        help="Backend to target the benchmark to.",
     )
     parser.add_argument(
         "--dataset",
@@ -1176,13 +1072,6 @@ if __name__ == "__main__":
         default=1,
         help="num_inference_steps used for warmup requests.",
     )
-    parser.add_argument(
-        "--warmup-concurrency",
-        type=int,
-        default=1,
-        help="Maximum number of warmup requests to run concurrently. "
-        "Set this to match the real batch shape when warming up torch.compile or CUDA graphs.",
-    )
     parser.add_argument("--width", type=int, default=None, help="Image/Video width.")
     parser.add_argument("--height", type=int, default=None, help="Image/Video height.")
     parser.add_argument("--num-frames", type=int, default=None, help="Number of frames (for video).")
@@ -1200,12 +1089,6 @@ if __name__ == "__main__":
     )
     parser.add_argument("--fps", type=int, default=None, help="FPS (for video).")
     parser.add_argument("--output-file", type=str, default=None, help="Output JSON file for metrics.")
-    parser.add_argument(
-        "--save-response-dir",
-        type=str,
-        default=None,
-        help="Directory to save successful raw responses. Video responses are saved as MP4 files.",
-    )
     parser.add_argument(
         "--slo",
         action="store_true",
@@ -1241,12 +1124,6 @@ if __name__ == "__main__":
             '{"width":768,"height":768,"num_inference_steps":20,"weight":0.85}]'
         ),
     )
-    parser.add_argument(
-        "--random-prompt",
-        type=str,
-        default=None,
-        help="Prompt to use for every synthetic random request. Defaults to generated placeholder prompts.",
-    )
     parser.add_argument(
         "--num-input-images",
         type=int,
@@ -1258,4 +1135,5 @@ if __name__ == "__main__":
     )
 
     args = parser.parse_args()
+
     asyncio.run(benchmark(args))

benchmarks/diffusion/performance_dashboard/helios_distilled_serving_performance.md

@@ -1,222 +0,0 @@
-# Helios-Distilled Serving Performance Dashboard
-
-This document describes how to deploy and benchmark **BestWishYsh/Helios-Distilled** using vLLM-Omni. It follows the same serving benchmark structure as the Qwen-Image and Wan2.2 dashboards.
-
----
-
-# 1. Overview
-
-Helios-Distilled is a text-to-video diffusion model. Helios uses an autoregressive chunking pattern that generates 33 frames per chunk. For stable comparisons, choose `num_frames` as a multiple of 33, such as 99.
-
-This document covers:
-
-* Service launch configuration
-* Benchmark scripts and usage
-* Dataset and workload settings
-* Performance and accuracy comparison methodology
-* Reproducibility guidelines
-
----
-
-# 2. Test Environment
-
-| Component | Specification |
-|------------|----------------|
-| GPU | NVIDIA H100 for CI accuracy/perf gate; local bring-up also used L20X |
-| Diffusion Attention Backend | FlashAttention |
-| Model | `BestWishYsh/Helios-Distilled` |
-
----
-
-# 3. Service Launch Configuration
-
-## 3.1 Basic Serving Command
-
-```bash
-vllm serve BestWishYsh/Helios-Distilled --omni \
-    --port 8091
-```
-
-## 3.2 Key Parameters
-
-| Parameter | Description |
-| --- | --- |
-| `--port` | Serving port |
-| `--max-num-seqs` | Maximum number of active sequences |
-| `--enable-diffusion-pipeline-profiler` | Optional stage-level profiling |
-| `--profiler-config` | Optional torch profiler configuration |
-
-Record these parameters when reporting performance results.
-
----
-
-# 4. Benchmark Scripts
-
-## 4.1 Online Serving Benchmark
-
-Use the shared diffusion serving benchmark, as with Qwen-Image and Wan2.2:
-
-```bash
-python benchmarks/diffusion/diffusion_benchmark_serving.py \
-    --base-url http://localhost:8091 \
-    --endpoint /v1/videos \
-    --model BestWishYsh/Helios-Distilled \
-    --dataset random \
-    --task t2v \
-    --num-prompts 1 \
-    --max-concurrency 1 \
-    --disable-tqdm \
-    --random-request-config '[
-        {
-            "width": 640,
-            "height": 384,
-            "num_inference_steps": 50,
-            "num_frames": 99,
-            "fps": 16,
-            "guidance_scale": 1.0,
-            "is_enable_stage2": true,
-            "pyramid_num_stages": 3,
-            "pyramid_num_inference_steps_list": [1, 1, 1],
-            "is_amplify_first_chunk": false,
-            "weight": 1
-        }
-    ]'
-```
-
-`--random-request-config` forwards standard fields such as `width`, `height`, `num_frames`, `fps`, and `num_inference_steps` as first-class request fields. Model-specific fields are forwarded in the request body, which is required for Helios stage-2 settings.
-
-## 4.2 Warmed Serving Benchmark
-
-For local bring-up, use the same shared serving benchmark with one warmup request and one measured request:
-
-```bash
-python benchmarks/diffusion/diffusion_benchmark_serving.py \
-    --base-url http://localhost:8091 \
-    --endpoint /v1/videos \
-    --model BestWishYsh/Helios-Distilled \
-    --dataset random \
-    --task t2v \
-    --num-prompts 1 \
-    --max-concurrency 1 \
-    --warmup-requests 1 \
-    --warmup-num-inference-steps 1 \
-    --random-prompt "A cat wearing sunglasses dances on a beach at sunset, cinematic lighting." \
-    --seed 42 \
-    --disable-tqdm \
-    --output-file metrics.json \
-    --save-response-dir responses \
-    --random-request-config '[
-        {
-            "width": 640,
-            "height": 384,
-            "num_inference_steps": 50,
-            "num_frames": 99,
-            "fps": 16,
-            "guidance_scale": 1.0,
-            "is_enable_stage2": true,
-            "pyramid_num_stages": 3,
-            "pyramid_num_inference_steps_list": [1, 1, 1],
-            "is_amplify_first_chunk": false,
-            "weight": 1
-        }
-    ]'
-```
-
-## 4.3 PR-vs-Main Accuracy and Performance Comparison
-
-The PR-vs-main accuracy/performance gate lives in the Helios accuracy pytest. It checks out `origin/main`, starts a server for each checkout, and drives both with `diffusion_benchmark_serving.py`:
-
-```bash
-pytest -s -v tests/e2e/accuracy/test_helios_distilled.py::test_helios_distilled_pr_matches_main_acc_perf \
-    -m 'full_model and benchmark' \
-    --run-level 'full_model'
-```
-
----
-
-# 5. Dataset & Workload Settings
-
-## 5.1 Recommended Evaluation Configurations
-
-### Dataset A (99 frames, 384x640)
-
-* Dataset: `random`
-* Task: t2v
-* Concurrency: 1
-* Frame count: 99 (`33 x 3`)
-* FPS: 16
-* Stage-2 steps: `[1, 1, 1]`
-
-```json
-[
-  {
-    "width": 640,
-    "height": 384,
-    "num_inference_steps": 50,
-    "num_frames": 99,
-    "fps": 16,
-    "guidance_scale": 1.0,
-    "is_enable_stage2": true,
-    "pyramid_num_stages": 3,
-    "pyramid_num_inference_steps_list": [1, 1, 1],
-    "is_amplify_first_chunk": false,
-    "weight": 1
-  }
-]
-```
-
----
-
-# 6. Performance Metrics
-
-| Metric | Description | Unit |
-| --- | --- | --- |
-| Mean Latency | Mean request latency | seconds |
-| P99 Latency | P99 request latency | seconds |
-| Generated FPS | `num_frames / measured_time_s` | frames/second |
-| Candidate/reference latency ratio | PR latency divided by reference latency | ratio |
-
----
-
-# 7. Accuracy Metrics
-
-The PR-vs-main gate compares encoded video frames extracted from MP4 outputs:
-
-| Metric | Threshold | Direction |
-| --- | --- | --- |
-| PSNR | `>= 35 dB` | Higher is better |
-| MAE | `<= 3.0 / 255` | Lower is better |
-| Candidate/reference latency ratio | `<= 1.03` | Lower is better |
-
-Local L20X bring-up for this PR produced:
-
-| Variant | Measured Latency (ms) | Generated FPS |
-| --- | --- | --- |
-| `origin/main` | 5317.95 | 18.62 |
-| Candidate PR | 5283.27 | 18.74 |
-
-Encoded-video similarity:
-
-| Metric | Value |
-| --- | --- |
-| PSNR | 38.66 dB |
-| MAE | 1.89 / 255 |
-| Cosine similarity | 0.99978 |
-
----
-
-# 8. Reproducibility Checklist
-
-To ensure consistent and comparable benchmark results:
-
-* Record GPU type
-* Record vLLM and vLLM-Omni versions
-* Record benchmark parameters: resolution, frame count, FPS, seed, concurrency, and stage-2 steps
-* Use frame counts that are multiples of 33
-* Keep prompt, seed, resolution, and sampling parameters identical when comparing PR vs main
-* Ensure no background workload on GPUs during testing
-* Use the same MP4 export and frame extraction path for both variants
-
----
-
-This document serves as the Helios-Distilled serving performance reference under vLLM-Omni.

benchmarks/diffusion/performance_dashboard/qwen_image_serving_performance.md

@@ -33,12 +33,6 @@ vllm serve Qwen/Qwen-Image --omni \
     --port 8091
 ```
 
-To replay step-wise continuous batching, compare
-`vllm serve Qwen/Qwen-Image --omni --port 8089 --step-execution --max-num-seqs 1`
-against the same command with `--max-num-seqs 8`. `--step-execution` is the
-feature gate, and increasing `--max-num-seqs` above `1` lets the scheduler
-keep more compatible requests active at once.
-
 ## 3.2 Key Parameters
 
 | Parameter             | Description              |
@@ -58,7 +52,7 @@ Record these parameters when reporting performance results.
 
 ```bash
 python benchmarks/diffusion/diffusion_benchmark_serving.py \
-    --endpoint /v1/chat/completions \
+    --backend vllm-omni \
     --dataset <DATASET_NAME> \
     --task t2i \
     --num-prompts <N> \
@@ -71,12 +65,11 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \
 
 | Parameter              | Description                       |
 | ---------------------- | --------------------------------- |
-| `--endpoint`           | API endpoint (use `/v1/chat/completions`; leading `/` is optional) |
+| `--backend`            | Serving backend (use `vllm-omni`) |
 | `--dataset`            | Dataset name (`random` or custom) |
 | `--task`               | Task type (e.g., `t2i`)           |
 | `--num-prompts`        | Total number of requests          |
 | `--max-concurrency`    | Client-side concurrency           |
-| `--warmup-concurrency` | Warmup concurrency for batch-shape prewarming |
 | `--random-request-config`| JSON string defining random request |
 
 ---
@@ -129,7 +122,7 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \
 
 ```bash
 python benchmarks/diffusion/diffusion_benchmark_serving.py \
-    --endpoint /v1/chat/completions \
+    --backend vllm-omni \
     --dataset random \
     --task t2i \
     --num-prompts 1 \
@@ -140,12 +133,6 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \
     ]'
 ```
 
-For a continuous-batching A/B replay, run the same benchmark command twice
-against the two serving configs above and keep traffic plus warmup settings
-identical across both runs. If the measured run uses `--max-concurrency 8`,
-warm with `--warmup-requests 8 --warmup-concurrency 8` so the first measured
-batch does not include compile or CUDA-graph capture overhead.
-
 ---
 
 # 6. Performance Metrics
@@ -176,7 +163,6 @@ To ensure consistent and comparable benchmark results:
 * Record parallel configuration
 * Record benchmark parameters (resolution, concurrency, number of prompts)
 * Ensure no background workload on GPUs during testing
-* When comparing continuous batching, keep warmup settings the same across runs
 
 ---
 

benchmarks/diffusion/performance_dashboard/wan_2_2_serving_performance.md

@@ -53,7 +53,7 @@ Record these parameters when reporting performance results.
 
 ```bash
 python benchmarks/diffusion/diffusion_benchmark_serving.py \
-    --endpoint /v1/videos \
+    --backend v1/videos \
     --dataset <DATASET_NAME> \
     --task t2v \
     --num-prompts <N> \
@@ -66,7 +66,7 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \
 
 | Parameter              | Description                       |
 | ---------------------- | --------------------------------- |
-| `--endpoint`           | API endpoint (use `/v1/videos`; leading `/` is optional) |
+| `--backend`            | Serving backend (use `v1/videos`) |
 | `--dataset`            | Dataset name (`random` or custom) |
 | `--task`               | Task type (e.g., `t2v`)           |
 | `--num-prompts`        | Total number of requests          |
@@ -120,7 +120,7 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \
 
 ```bash
 python benchmarks/diffusion/diffusion_benchmark_serving.py \
-    --endpoint /v1/videos \
+    --backend v1/videos \
     --dataset random \
     --task t2v \
     --num-prompts 1 \

benchmarks/diffusion/quantization_quality.py

@@ -148,11 +148,10 @@ def _build_omni_kwargs(args, quantization=None):
 def _generate_image(omni, args, prompt, seed):
     """Generate a single image and return (PIL.Image, time_seconds, memory_gib)."""
     from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-    from vllm_omni.outputs import OmniRequestOutput
     from vllm_omni.platforms import current_omni_platform
 
     generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(seed)
-    torch.accelerator.reset_peak_memory_stats()
+    torch.cuda.reset_peak_memory_stats()
     start = time.perf_counter()
     outputs = omni.generate(
         {"prompt": prompt},
@@ -164,11 +163,10 @@ def _generate_image(omni, args, prompt, seed):
         ),
     )
     elapsed = time.perf_counter() - start
-    peak_mem = torch.accelerator.max_memory_allocated() / (1024**3)
+    peak_mem = torch.cuda.max_memory_allocated() / (1024**3)
 
-    req_out = OmniRequestOutput.unwrap_result(outputs)
-    if not req_out.images:
-        raise ValueError("Could not extract image output from result.")
+    first = outputs[0]
+    req_out = first.request_output[0] if hasattr(first, "request_output") else first
     img = req_out.images[0]
     return img, elapsed, peak_mem
 
@@ -180,7 +178,7 @@ def _generate_video(omni, args, prompt, seed):
     from vllm_omni.platforms import current_omni_platform
 
     generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(seed)
-    torch.accelerator.reset_peak_memory_stats()
+    torch.cuda.reset_peak_memory_stats()
     start = time.perf_counter()
     outputs = omni.generate(
         {"prompt": prompt, "negative_prompt": ""},
@@ -194,7 +192,7 @@ def _generate_video(omni, args, prompt, seed):
         ),
     )
     elapsed = time.perf_counter() - start
-    peak_mem = torch.accelerator.max_memory_allocated() / (1024**3)
+    peak_mem = torch.cuda.max_memory_allocated() / (1024**3)
 
     first = outputs[0]
     if hasattr(first, "request_output") and isinstance(first.request_output, list):
@@ -230,8 +228,8 @@ def _unload_omni(omni):
     del omni
     gc.collect()
     if torch.cuda.is_available():
-        torch.accelerator.empty_cache()
-        torch.accelerator.synchronize()
+        torch.cuda.empty_cache()
+        torch.cuda.synchronize()
 
 
 def run_benchmark(args):

benchmarks/distributed/omni_connectors/cross_node_mooncake_transfer_engine.py

@@ -614,8 +614,8 @@ Examples:
         if not torch.cuda.is_available():
             print("[ERROR] CUDA is not available but GPU mode was requested.")
             sys.exit(1)
-        if args.gpu_id >= torch.accelerator.device_count():
-            print(f"[ERROR] GPU {args.gpu_id} not available. Found {torch.accelerator.device_count()} GPUs.")
+        if args.gpu_id >= torch.cuda.device_count():
+            print(f"[ERROR] GPU {args.gpu_id} not available. Found {torch.cuda.device_count()} GPUs.")
             sys.exit(1)
         print(f"[INFO] Using GPU {args.gpu_id}: {torch.cuda.get_device_name(args.gpu_id)}")
 

benchmarks/fish-speech/bench_voice_cache.py

@@ -3,11 +3,11 @@
 Measures TTFP improvement from DAC-code caching when using uploaded voices.
 
 Setup:
-  1. Start vllm-omni with Fish Speech S2 Pro
+  1. Start vllm-omni with Fish Speech S2 Pro (use our feat branch)
   2. Provide a reference audio file for voice cloning
 
 Usage:
-    python bench_speaker_cache.py \
+    python bench_voice_cache.py \
         --ref-audio /path/to/reference.wav \
         --ref-text "Transcript of the reference audio." \
         --num-prompts 20 \

benchmarks/glm_image/README.md

@@ -1,157 +0,0 @@
-# GLM-Image Benchmarks
-
-Benchmark GLM-Image T2I (text-to-image) and I2I (image-to-image) performance across three backends: HuggingFace baseline, vLLM-Omni offline, and vLLM-Omni online serving.
-
-## Benchmarks
-
-| Benchmark | Script | Description |
-|-----------|--------|-------------|
-| HuggingFace Baseline | `huggingface/inference.py` | Single-GPU transformers + diffusers pipeline |
-| vLLM-Omni Offline | `vllm-omni/inference.py` | Offline inference with continuous batching |
-| vLLM-Omni Online | `benchmark_glm_image.py` | Online serving via `/v1/chat/completions` |
-
-## HuggingFace Baseline
-
-Single-request sequential inference using the reference HuggingFace pipeline.
-
-```bash
-# T2I
-CUDA_VISIBLE_DEVICES=0 python benchmarks/glm_image/huggingface/inference.py \
-    --model-path /path/to/GLM-Image --mode t2i --num-prompts 10
-
-# I2I
-CUDA_VISIBLE_DEVICES=0 python benchmarks/glm_image/huggingface/inference.py \
-    --model-path /path/to/GLM-Image --mode i2i --num-prompts 10
-```
-
-### Options
-
-| Flag | Default | Description |
-|------|---------|-------------|
-| `--model-path` | `zai-org/GLM-Image` | Model path |
-| `--mode` | `t2i` | `t2i` or `i2i` |
-| `--dataset-path` | `prompt/prompt.json` | Path to prompt.json |
-| `--num-prompts` | `10` | Number of images to generate |
-| `--width` / `--height` | `1024` | Output image size |
-| `--num-inference-steps` | `50` | Diffusion denoising steps |
-| `--output-dir` | `benchmarks/glm_image/huggingface/outputs` | Output directory |
-| `--output-file` | - | JSON file for metrics |
-
-## vLLM-Omni Offline
-
-Multi-GPU offline inference with pipeline parallelism and continuous batching.
-
-```bash
-# T2I
-CUDA_VISIBLE_DEVICES=0,1 python benchmarks/glm_image/vllm-omni/inference.py \
-    --model-path /path/to/GLM-Image --mode t2i --num-prompts 10
-
-# I2I
-CUDA_VISIBLE_DEVICES=0,1 python benchmarks/glm_image/vllm-omni/inference.py \
-    --model-path /path/to/GLM-Image --mode i2i --num-prompts 10
-```
-
-### Options
-
-| Flag | Default | Description |
-|------|---------|-------------|
-| `--model-path` | `zai-org/GLM-Image` | Model path |
-| `--deploy-config` | - | Deploy config YAML |
-| `--mode` | `t2i` | `t2i` or `i2i` |
-| `--dataset-path` | `prompt/prompt.json` | Path to prompt.json |
-| `--num-prompts` | `10` | Number of images to generate |
-| `--width` / `--height` | `1024` | Output image size |
-| `--num-inference-steps` | `50` | Diffusion denoising steps |
-| `--output-dir` | `benchmarks/glm_image/vllm-omni/outputs` | Output directory |
-| `--output-file` | - | JSON file for metrics |
-| `--stage-init-timeout` | `600` | Stage initialization timeout (s) |
-
-### Latency Computation
-
-In offline mode all requests are submitted simultaneously and processed with continuous batching. The per-request latency is computed by summing the actual per-stage times (with `stage_0_gen_ms` diffed against the previous request to remove accumulated queue/scheduling wait).
-
-## vLLM-Omni Online Serving
-
-### Start the server
-
-```bash
-CUDA_VISIBLE_DEVICES=0,1 vllm serve /path/to/GLM-Image \
-    --omni --port 8091 --host 0.0.0.0 \
-    --served-model-name glm-image
-```
-
-### Run the benchmark
-
-```bash
-# T2I
-python benchmarks/glm_image/benchmark_glm_image.py \
-    --mode t2i --num-prompts 10 --model glm-image
-
-# I2I
-python benchmarks/glm_image/benchmark_glm_image.py \
-    --mode i2i --num-prompts 10 --model glm-image
-
-# Custom dataset
-python benchmarks/glm_image/benchmark_glm_image.py \
-    --mode i2i --dataset custom \
-    --dataset-path prompts.json --num-prompts 5
-```
-
-### Options
-
-| Flag | Default | Description |
-|------|---------|-------------|
-| `--mode` | `t2i` | `t2i` or `i2i` |
-| `--dataset` | `prompt` | `prompt`, `random`, or `custom` |
-| `--dataset-path` | - | JSON file path (required for `custom`) |
-| `--num-prompts` | `10` | Number of benchmark requests |
-| `--max-concurrency` | `1` | Max concurrent requests |
-| `--request-rate` | `inf` | Requests per second (Poisson arrival) |
-| `--warmup-requests` | `1` | Warmup requests before measurement |
-| `--width` / `--height` | `1024` | Output image size |
-| `--num-inference-steps` | `50` | Diffusion denoising steps |
-| `--seed` | - | Random seed |
-| `--model` | `default` | Model name (must match `--served-model-name`) |
-| `--host` | `localhost` | Server host |
-| `--port` | `8091` | Server port |
-| `--output-file` | - | JSON output file for metrics |
-| `--num-input-images` | `1` | Number of input images for random I2I |
-
-## Dataset
-
-The default dataset is hosted on [HuggingFace](https://huggingface.co/datasets/JaredforReal/glm-image-bench) (`prompt.json`). It is automatically downloaded and cached to `prompt/prompt.json` on first run. No manual setup needed.
-
-Each entry contains:
-
-- `t2i_prompt`: Text prompt for text-to-image generation
-- `i2i_prompt`: Text prompt for image-to-image editing
-- `image_url`: Source image URL for I2I (downloaded and cached on first use)
-
-Custom datasets use the same JSON format and can be provided via `--dataset-path`.
-
-## Pipeline Timings
-
-All three benchmarks report per-stage pipeline timings (in milliseconds):
-
-| Key | Description |
-|-----|-------------|
-| `preprocess_ms` | Input preprocessing (tokenization, multimodal encoding) |
-| `stage_0_gen_ms` | AR (autoregressive) model generation time |
-| `ar2diffusion_ms` | AR output to diffusion input conversion |
-| `stage_1_gen_ms` | Diffusion model denoising time |
-| `queue_wait_ms` | Queue wait time before processing |
-
-The stages are ordered by execution: `preprocess → stage_0 (AR) → ar2diffusion → stage_1 (Diffusion)`.
-
-## Sample Results
-
-Tested on 2x GPU with 10 prompts, 1024x1024, 50 denoising steps:
-
-| Backend | Mode | Latency Mean (s) | Throughput (img/s) |
-|---------|------|-------------------|--------------------|
-| HuggingFace | T2I | 72.6 | 0.014 |
-| HuggingFace | I2I | 70.9 | 0.014 |
-| vLLM-Omni Offline | T2I | 35.0 | 0.044 |
-| vLLM-Omni Offline | I2I | 31.0 | 0.053 |
-| vLLM-Omni Online | T2I | 38.8 | 0.026 |
-| vLLM-Omni Online | I2I | 34.7 | 0.029 |

benchmarks/glm_image/benchmark_glm_image.py

@@ -1,464 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""
-Online serving benchmark for GLM-Image (T2I and I2I modes).
-
-Sends requests to the /v1/chat/completions endpoint and reports end-to-end
-latency, throughput, and per-stage durations (when the server is started with
---enable-diffusion-pipeline-profiler and/or --enable-ar-profiler).
-
-Supports three dataset types:
-  - prompt:   Use prompt.json (default). T2I uses t2i_prompt, I2I uses i2i_prompt
-              and sends source images from image_url.
-  - random:   Generate synthetic prompts (and random images for I2I).
-  - custom:   Load from a user-specified JSON file.
-
-Usage:
-    # T2I with prompt.json (default)
-    python benchmarks/glm_image/benchmark_glm_image.py \
-        --mode t2i --num-prompts 10
-
-    # I2I with prompt.json (downloads source images automatically)
-    python benchmarks/glm_image/benchmark_glm_image.py \
-        --mode i2i --num-prompts 10
-
-    # Random dataset
-    python benchmarks/glm_image/benchmark_glm_image.py \
-        --mode t2i --dataset random --num-prompts 20
-
-    # Custom dataset
-    python benchmarks/glm_image/benchmark_glm_image.py \
-        --mode i2i --dataset custom \
-        --dataset-path my_prompts.json --num-prompts 5
-"""
-
-import argparse
-import asyncio
-import base64
-import json
-import os
-import sys
-import tempfile
-import time
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Any
-
-import aiohttp
-import numpy as np
-import requests as sync_requests
-from PIL import Image
-from tqdm.asyncio import tqdm
-
-# Import backends from the diffusion benchmark (add parent dirs to path)
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "diffusion"))
-from backends import RequestFuncOutput
-
-BENCHMARK_DIR = Path(__file__).resolve().parent
-DEFAULT_PROMPT_JSON = BENCHMARK_DIR / "prompt" / "prompt.json"
-IMAGE_CACHE_DIR = BENCHMARK_DIR / "prompt" / "images"
-
-DATASET_REPO = "JaredforReal/glm-image-bench"
-DATASET_FILE = "prompt.json"
-
-
-def _ensure_prompt_json(dataset_path: str | None) -> str:
-    """Return path to prompt.json, downloading from HuggingFace if needed."""
-    if dataset_path:
-        return dataset_path
-    local = DEFAULT_PROMPT_JSON
-    if local.exists():
-        return str(local)
-    print(f"Downloading {DATASET_FILE} from {DATASET_REPO} ...")
-    try:
-        from huggingface_hub import hf_hub_download
-
-        downloaded = hf_hub_download(
-            repo_id=DATASET_REPO,
-            filename=DATASET_FILE,
-            repo_type="dataset",
-        )
-        local.parent.mkdir(parents=True, exist_ok=True)
-        import shutil
-
-        shutil.copy2(downloaded, local)
-        print(f"Saved to {local}")
-    except ImportError:
-        url = f"https://huggingface.co/datasets/{DATASET_REPO}/resolve/main/{DATASET_FILE}"
-        import urllib.request
-
-        local.parent.mkdir(parents=True, exist_ok=True)
-        urllib.request.urlretrieve(url, local)
-        print(f"Saved to {local}")
-    return str(local)
-
-
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
-
-@dataclass
-class GLMImageRequest:
-    prompt: str
-    image_path: str | None = None  # Only for I2I mode
-
-
-def download_image(url: str) -> str:
-    """Download an image to cache and return the local path."""
-    IMAGE_CACHE_DIR.mkdir(parents=True, exist_ok=True)
-    fname = url.rsplit("/", 1)[-1]
-    local_path = IMAGE_CACHE_DIR / fname
-    if local_path.exists():
-        return str(local_path)
-    resp = sync_requests.get(url, timeout=30)
-    resp.raise_for_status()
-    local_path.write_bytes(resp.content)
-    return str(local_path)
-
-
-def encode_image_as_data_url(path: str) -> str:
-    """Encode a local image file as a base64 data URL."""
-    with open(path, "rb") as f:
-        encoded = base64.b64encode(f.read()).decode("utf-8")
-    ext = Path(path).suffix.lower()
-    mime = {"png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg"}.get(ext, "image/png")
-    return f"data:{mime};base64,{encoded}"
-
-
-# ---------------------------------------------------------------------------
-# Datasets
-# ---------------------------------------------------------------------------
-
-
-class PromptDataset:
-    """Load from prompt.json. T2I uses t2i_prompt, I2I uses i2i_prompt + image_url."""
-
-    def __init__(self, args: argparse.Namespace):
-        path = _ensure_prompt_json(args.dataset_path)
-        with open(path, encoding="utf-8") as f:
-            raw = json.load(f)
-
-        prompt_key = "t2i_prompt" if args.mode == "t2i" else "i2i_prompt"
-        self.items: list[GLMImageRequest] = []
-
-        for entry in raw:
-            prompt = entry.get(prompt_key, "").strip()
-            if not prompt:
-                continue
-            image_path = None
-            if args.mode == "i2i":
-                url = entry.get("image_url", "")
-                if url:
-                    image_path = download_image(url)
-            self.items.append(GLMImageRequest(prompt=prompt, image_path=image_path))
-
-        if args.num_prompts and len(self.items) > args.num_prompts:
-            self.items = self.items[: args.num_prompts]
-
-    def __len__(self) -> int:
-        return len(self.items)
-
-    def __getitem__(self, idx: int) -> GLMImageRequest:
-        return self.items[idx]
-
-    def get_requests(self) -> list[GLMImageRequest]:
-        return list(self.items)
-
-
-class RandomDataset:
-    """Generate synthetic prompts (and optional random images for I2I)."""
-
-    def __init__(self, args: argparse.Namespace):
-        self.args = args
-        self.num_prompts = args.num_prompts
-        self._random_image_paths: list[str] | None = None
-        if args.mode == "i2i":
-            self._random_image_paths = self._generate_random_images()
-
-    def _generate_random_images(self) -> list[str]:
-        paths: list[str] = []
-        for i in range(self.args.num_input_images):
-            img = Image.new("RGB", (512, 512), (128 + i * 30 % 128, 64, 192))
-            path = os.path.join(tempfile.gettempdir(), f"glm_image_bench_input_{i}.png")
-            img.save(path)
-            paths.append(path)
-        return paths
-
-    def __len__(self) -> int:
-        return self.num_prompts
-
-    def __getitem__(self, idx: int) -> GLMImageRequest:
-        image_path = None
-        if self._random_image_paths is not None:
-            image_path = self._random_image_paths[idx % len(self._random_image_paths)]
-        return GLMImageRequest(
-            prompt=f"A beautiful scene with vivid colors and intricate details, prompt {idx}",
-            image_path=image_path,
-        )
-
-    def get_requests(self) -> list[GLMImageRequest]:
-        return [self[i] for i in range(len(self))]
-
-
-class CustomDataset:
-    """Load from a user-specified JSON file.
-
-    Expected format:
-    [
-        {"prompt": "A cat sitting on a windowsill"},
-        {"prompt": "Make it look like winter", "image_path": "/path/to/img.png"}
-    ]
-    """
-
-    def __init__(self, args: argparse.Namespace):
-        if not args.dataset_path:
-            raise ValueError("--dataset-path is required for custom dataset")
-        with open(args.dataset_path, encoding="utf-8") as f:
-            raw = json.load(f)
-        self.items: list[GLMImageRequest] = []
-        for item in raw:
-            self.items.append(
-                GLMImageRequest(
-                    prompt=item.get("prompt", ""),
-                    image_path=item.get("image_path"),
-                )
-            )
-        if args.num_prompts and len(self.items) > args.num_prompts:
-            self.items = self.items[: args.num_prompts]
-
-    def __len__(self) -> int:
-        return len(self.items)
-
-    def __getitem__(self, idx: int) -> GLMImageRequest:
-        return self.items[idx]
-
-    def get_requests(self) -> list[GLMImageRequest]:
-        return list(self.items)
-
-
-# ---------------------------------------------------------------------------
-# Async request for GLM-Image (chat completions with image support)
-# ---------------------------------------------------------------------------
-
-
-async def async_glm_image_request(
-    req: GLMImageRequest,
-    api_url: str,
-    model: str,
-    session: aiohttp.ClientSession,
-    pbar: Any,
-    args: argparse.Namespace,
-) -> RequestFuncOutput:
-    """Send a single T2I or I2I request via chat completions endpoint."""
-    output = RequestFuncOutput()
-    output.start_time = time.perf_counter()
-
-    # Build messages
-    if req.image_path and args.mode == "i2i":
-        data_url = encode_image_as_data_url(req.image_path)
-        content = [
-            {"type": "text", "text": req.prompt},
-            {"type": "image_url", "image_url": {"url": data_url}},
-        ]
-    else:
-        content = req.prompt
-
-    messages = [{"role": "user", "content": content}]
-
-    extra_body: dict[str, Any] = {}
-    if args.height:
-        extra_body["height"] = args.height
-    if args.width:
-        extra_body["width"] = args.width
-    if args.num_inference_steps:
-        extra_body["num_inference_steps"] = args.num_inference_steps
-    if args.seed is not None:
-        extra_body["seed"] = args.seed
-
-    payload: dict[str, Any] = {
-        "model": model,
-        "messages": messages,
-    }
-    if extra_body:
-        payload["extra_body"] = extra_body
-
-    try:
-        async with session.post(api_url, json=payload) as response:
-            if response.status == 200:
-                resp_json = await response.json()
-                output.response_body = resp_json
-                output.success = True
-                try:
-                    choices = resp_json.get("choices", [])
-                    if choices and isinstance(choices, list):
-                        msg = choices[0].get("message", {})
-                        if isinstance(msg, dict):
-                            resp_content = msg.get("content", [])
-                            if resp_content and isinstance(resp_content, list) and len(resp_content) > 0:
-                                first_item = resp_content[0]
-                                if isinstance(first_item, dict):
-                                    output.stage_durations = first_item.get("stage_durations") or {}
-                                    output.peak_memory_mb = first_item.get("peak_memory_mb", 0.0)
-                except (IndexError, TypeError, AttributeError):
-                    pass
-            else:
-                output.error = f"HTTP {response.status}: {await response.text()}"
-                output.success = False
-    except Exception as e:
-        output.error = str(e)
-        output.success = False
-
-    output.latency = time.perf_counter() - output.start_time
-    if pbar:
-        pbar.update(1)
-    return output
-
-
-# ---------------------------------------------------------------------------
-# Benchmark
-# ---------------------------------------------------------------------------
-
-
-async def iter_requests(n: int, request_rate: float) -> Any:
-    import random as _random
-
-    for i in range(n):
-        if request_rate != float("inf") and i > 0:
-            await asyncio.sleep(_random.expovariate(request_rate))
-        yield i
-
-
-def calculate_metrics(outputs: list[RequestFuncOutput], total_duration: float) -> dict[str, Any]:
-    success = [o for o in outputs if o.success]
-    errors = [o for o in outputs if not o.success]
-    latencies = [o.latency for o in success]
-    peak_mems = [o.peak_memory_mb for o in success if o.peak_memory_mb > 0]
-
-    stage_duration_lists: dict[str, list[float]] = {}
-    for o in success:
-        for stage, dur in (o.stage_durations or {}).items():
-            stage_duration_lists.setdefault(stage, []).append(dur)
-
-    return {
-        "duration": total_duration,
-        "completed_requests": len(success),
-        "failed_requests": len(errors),
-        "throughput_qps": len(success) / total_duration if total_duration > 0 else 0,
-        "latency_mean": float(np.mean(latencies)) if latencies else 0,
-        "latency_median": float(np.median(latencies)) if latencies else 0,
-        "latency_p99": float(np.percentile(latencies, 99)) if latencies else 0,
-        "latency_p95": float(np.percentile(latencies, 95)) if latencies else 0,
-        "peak_memory_mb_max": max(peak_mems) if peak_mems else 0,
-        "stage_durations_mean": {s: float(np.mean(v)) for s, v in stage_duration_lists.items()},
-        "stage_durations_p50": {s: float(np.percentile(v, 50)) for s, v in stage_duration_lists.items()},
-    }
-
-
-async def benchmark(args: argparse.Namespace) -> None:
-    api_url = f"http://{args.host}:{args.port}/v1/chat/completions"
-
-    # Load dataset
-    if args.dataset == "prompt":
-        dataset = PromptDataset(args)
-    elif args.dataset == "random":
-        dataset = RandomDataset(args)
-    elif args.dataset == "custom":
-        dataset = CustomDataset(args)
-    else:
-        raise ValueError(f"Unknown dataset: {args.dataset}")
-
-    glm_requests = dataset.get_requests()
-    print(f"Prepared {len(glm_requests)} requests (mode={args.mode}, dataset={args.dataset})")
-
-    semaphore = asyncio.Semaphore(args.max_concurrency) if args.max_concurrency else None
-
-    async def limited_request(idx: int, req: GLMImageRequest, session: aiohttp.ClientSession, pbar: Any):
-        if semaphore:
-            async with semaphore:
-                return await async_glm_image_request(req, api_url, args.model, session, pbar, args)
-        return await async_glm_image_request(req, api_url, args.model, session, pbar, args)
-
-    async with aiohttp.ClientSession() as session:
-        # Warmup
-        if args.warmup_requests and glm_requests:
-            print(f"Running {args.warmup_requests} warmup request(s)...")
-            for i in range(args.warmup_requests):
-                await limited_request(i, glm_requests[i % len(glm_requests)], session, None)
-
-        # Main benchmark
-        pbar = tqdm(total=len(glm_requests), disable=args.disable_tqdm)
-        start_time = time.perf_counter()
-        tasks = []
-        async for idx in iter_requests(len(glm_requests), args.request_rate):
-            tasks.append(asyncio.create_task(limited_request(idx, glm_requests[idx], session, pbar)))
-        outputs = await asyncio.gather(*tasks)
-        total_duration = time.perf_counter() - start_time
-        pbar.close()
-
-    # Metrics
-    metrics = calculate_metrics(outputs, total_duration)
-    metrics["mode"] = args.mode
-    metrics["model"] = args.model
-    metrics["dataset"] = args.dataset
-
-    print(f"\n{' GLM-Image Online Benchmark Result ':=^60}")
-    print(f"{'Mode:':<40} {args.mode}")
-    print(f"{'Model:':<40} {args.model}")
-    print(f"{'Dataset:':<40} {args.dataset}")
-    print("-" * 50)
-    print(f"{'Benchmark duration (s):':<40} {metrics['duration']:.2f}")
-    print(f"{'Request rate:':<40} {args.request_rate}")
-    print(f"{'Max concurrency:':<40} {args.max_concurrency}")
-    print(f"{'Successful requests:':<40} {metrics['completed_requests']}/{len(glm_requests)}")
-    print("-" * 50)
-    print(f"{'Throughput (req/s):':<40} {metrics['throughput_qps']:.2f}")
-    print(f"{'Latency Mean (s):':<40} {metrics['latency_mean']:.4f}")
-    print(f"{'Latency Median (s):':<40} {metrics['latency_median']:.4f}")
-    print(f"{'Latency P95 (s):':<40} {metrics['latency_p95']:.4f}")
-    print(f"{'Latency P99 (s):':<40} {metrics['latency_p99']:.4f}")
-
-    if metrics["peak_memory_mb_max"] > 0:
-        print("-" * 50)
-        print(f"{'Peak Memory Max (MB):':<40} {metrics['peak_memory_mb_max']:.2f}")
-
-    if metrics["stage_durations_mean"]:
-        print("-" * 50)
-        print("Stage Durations Mean:")
-        for stage, val in sorted(metrics["stage_durations_mean"].items()):
-            unit = "ms" if stage.endswith("_ms") else "s"
-            print(f"  {stage + ':':<38} {val:.4f} ({unit})")
-
-    print("=" * 60)
-
-    if args.output_file:
-        with open(args.output_file, "w") as f:
-            json.dump(metrics, f, indent=2)
-        print(f"Metrics saved to {args.output_file}")
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(description="Benchmark GLM-Image T2I/I2I online serving.")
-    parser.add_argument("--mode", type=str, default="t2i", choices=["t2i", "i2i"])
-    parser.add_argument("--dataset", type=str, default="prompt", choices=["prompt", "random", "custom"])
-    parser.add_argument("--dataset-path", type=str, default=None)
-    parser.add_argument("--num-prompts", type=int, default=10)
-    parser.add_argument("--max-concurrency", type=int, default=1)
-    parser.add_argument("--request-rate", type=float, default=float("inf"))
-    parser.add_argument("--warmup-requests", type=int, default=1)
-    parser.add_argument("--width", type=int, default=1024)
-    parser.add_argument("--height", type=int, default=1024)
-    parser.add_argument("--num-inference-steps", type=int, default=50)
-    parser.add_argument("--seed", type=int, default=None)
-    parser.add_argument("--model", type=str, default="default")
-    parser.add_argument("--host", type=str, default="localhost")
-    parser.add_argument("--port", type=int, default=8091)
-    parser.add_argument("--output-file", type=str, default=None)
-    parser.add_argument("--disable-tqdm", action="store_true")
-    parser.add_argument("--num-input-images", type=int, default=1, help="For random I2I dataset.")
-    args = parser.parse_args()
-    asyncio.run(benchmark(args))
-
-
-if __name__ == "__main__":
-    main()

benchmarks/glm_image/huggingface/inference.py

@@ -1,291 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""
-HuggingFace (transformers + diffusers) baseline benchmark for GLM-Image.
-
-Supports T2I and I2I modes with the prompt.json dataset.
-Downloads source images for I2I from image_url on first run and caches locally.
-
-Usage:
-    # T2I mode (text-to-image, no source images needed)
-    python benchmarks/glm_image/huggingface/inference.py \
-        --model-path zai-org/GLM-Image \
-        --mode t2i --num-prompts 10
-
-    # I2I mode (image-to-image, downloads source images)
-    python benchmarks/glm_image/huggingface/inference.py \
-        --model-path zai-org/GLM-Image \
-        --mode i2i --num-prompts 10
-
-    # With custom prompt.json
-    python benchmarks/glm_image/huggingface/inference.py \
-        --model-path zai-org/GLM-Image \
-        --mode i2i --dataset-path prompts.json --num-prompts 5
-"""
-
-import argparse
-import json
-import os
-import time
-from pathlib import Path
-
-import numpy as np
-import requests
-import torch
-from PIL import Image
-
-BENCHMARK_DIR = Path(__file__).resolve().parent.parent
-DEFAULT_PROMPT_JSON = BENCHMARK_DIR / "prompt" / "prompt.json"
-IMAGE_CACHE_DIR = BENCHMARK_DIR / "prompt" / "images"
-
-DATASET_REPO = "JaredforReal/glm-image-bench"
-DATASET_FILE = "prompt.json"
-
-
-def _ensure_prompt_json(dataset_path: str | None) -> str:
-    """Return path to prompt.json, downloading from HuggingFace if needed."""
-    if dataset_path:
-        return dataset_path
-    local = DEFAULT_PROMPT_JSON
-    if local.exists():
-        return str(local)
-    print(f"Downloading {DATASET_FILE} from {DATASET_REPO} ...")
-    try:
-        from huggingface_hub import hf_hub_download
-
-        downloaded = hf_hub_download(
-            repo_id=DATASET_REPO,
-            filename=DATASET_FILE,
-            repo_type="dataset",
-        )
-        local.parent.mkdir(parents=True, exist_ok=True)
-        import shutil
-
-        shutil.copy2(downloaded, local)
-        print(f"Saved to {local}")
-    except ImportError:
-        url = f"https://huggingface.co/datasets/{DATASET_REPO}/resolve/main/{DATASET_FILE}"
-        import urllib.request
-
-        local.parent.mkdir(parents=True, exist_ok=True)
-        urllib.request.urlretrieve(url, local)
-        print(f"Saved to {local}")
-    return str(local)
-
-
-HEIGHT = 1024
-WIDTH = 1024
-SEED = 42
-NUM_INFERENCE_STEPS = 50
-GUIDANCE_SCALE = 1.5
-
-
-# ---------------------------------------------------------------------------
-# Dataset
-# ---------------------------------------------------------------------------
-
-
-def load_dataset(
-    dataset_path: str | None,
-    mode: str,
-    num_prompts: int,
-) -> list[dict]:
-    """Load prompts from prompt.json and prepare per-request data."""
-    path = _ensure_prompt_json(dataset_path)
-    with open(path, encoding="utf-8") as f:
-        raw = json.load(f)
-
-    items = []
-    for entry in raw:
-        if mode == "t2i":
-            prompt_key = "t2i_prompt"
-        else:
-            prompt_key = "i2i_prompt"
-
-        prompt_text = entry.get(prompt_key, "").strip()
-        if not prompt_text:
-            continue
-
-        item = {"prompt": prompt_text}
-        if mode == "i2i":
-            item["image_url"] = entry.get("image_url", "")
-        items.append(item)
-
-    if num_prompts and len(items) > num_prompts:
-        items = items[:num_prompts]
-    return items
-
-
-def download_image(url: str, cache_dir: Path) -> str:
-    """Download an image to cache_dir and return the local path."""
-    cache_dir.mkdir(parents=True, exist_ok=True)
-    fname = url.rsplit("/", 1)[-1]
-    local_path = cache_dir / fname
-    if local_path.exists():
-        return str(local_path)
-    print(f"  Downloading {url} ...")
-    resp = requests.get(url, timeout=30)
-    resp.raise_for_status()
-    local_path.write_bytes(resp.content)
-    return str(local_path)
-
-
-# ---------------------------------------------------------------------------
-# Benchmark
-# ---------------------------------------------------------------------------
-
-
-def benchmark(args: argparse.Namespace) -> None:
-    from diffusers.pipelines.glm_image import GlmImagePipeline
-
-    print("=" * 60)
-    print("GLM-Image HuggingFace Baseline Benchmark")
-    print(f"Mode: {args.mode}  |  Model: {args.model_path}")
-    print(f"Size: {args.height}x{args.width}  |  Steps: {args.num_inference_steps}")
-    print("=" * 60)
-
-    # Load dataset
-    items = load_dataset(args.dataset_path, args.mode, args.num_prompts)
-    if not items:
-        print("No prompts loaded. Exiting.")
-        return
-    print(f"Loaded {len(items)} prompts for {args.mode} mode")
-
-    # Download I2I source images
-    if args.mode == "i2i":
-        print("Preparing source images...")
-        for item in items:
-            url = item.get("image_url", "")
-            if url:
-                item["image_path"] = download_image(url, IMAGE_CACHE_DIR)
-            else:
-                item["image_path"] = None
-
-    # Load pipeline
-    print(f"\nLoading pipeline from {args.model_path} ...")
-    t0 = time.perf_counter()
-    pipe = GlmImagePipeline.from_pretrained(
-        args.model_path,
-        torch_dtype=torch.bfloat16,
-        device_map="cuda",
-    )
-    init_time = time.perf_counter() - t0
-    print(f"Pipeline loaded in {init_time:.2f}s")
-
-    # Create output dir
-    os.makedirs(args.output_dir, exist_ok=True)
-
-    # Run benchmark
-    generator = torch.Generator(device="cuda").manual_seed(args.seed)
-    latencies = []
-    success = 0
-    failed = 0
-
-    print(f"\nRunning {len(items)} requests sequentially...")
-    print("-" * 60)
-
-    for i, item in enumerate(items):
-        prompt = item["prompt"]
-        gen_kwargs: dict = {
-            "prompt": prompt,
-            "height": args.height,
-            "width": args.width,
-            "num_inference_steps": args.num_inference_steps,
-            "guidance_scale": args.guidance_scale,
-            "generator": generator,
-        }
-
-        if args.mode == "i2i":
-            img_path = item.get("image_path")
-            if img_path and os.path.exists(img_path):
-                gen_kwargs["image"] = [Image.open(img_path).convert("RGB")]
-            else:
-                print(f"  [{i + 1}] SKIP: no source image")
-                failed += 1
-                continue
-
-        t_start = time.perf_counter()
-        try:
-            result = pipe(**gen_kwargs)
-            image = result.images[0]
-            elapsed = time.perf_counter() - t_start
-            latencies.append(elapsed)
-            success += 1
-
-            out_path = os.path.join(args.output_dir, f"{i:04d}.png")
-            image.save(out_path)
-            print(f"  [{i + 1}/{len(items)}] {elapsed:.3f}s -> {out_path}")
-        except Exception as e:
-            elapsed = time.perf_counter() - t_start
-            failed += 1
-            print(f"  [{i + 1}/{len(items)}] FAILED ({elapsed:.3f}s): {e}")
-
-    # Report
-    total_gen_time = sum(latencies) if latencies else 0
-    print("\n" + "=" * 60)
-    print("HuggingFace Baseline Results")
-    print("=" * 60)
-    print(f"{'Mode:':<40} {args.mode}")
-    print(f"{'Model:':<40} {args.model_path}")
-    print(f"{'Image size:':<40} {args.height}x{args.width}")
-    print(f"{'Num inference steps:':<40} {args.num_inference_steps}")
-    print("-" * 50)
-    print(f"{'Pipeline init time (s):':<40} {init_time:.2f}")
-    print(f"{'Successful:':<40} {success}/{len(items)}")
-    print(f"{'Failed:':<40} {failed}")
-    print("-" * 50)
-    if latencies:
-        arr = np.array(latencies)
-        print(f"{'Total generation time (s):':<40} {total_gen_time:.2f}")
-        print(f"{'Throughput (img/s):':<40} {success / total_gen_time:.4f}")
-        print(f"{'Latency Mean (s):':<40} {arr.mean():.4f}")
-        print(f"{'Latency Median (s):':<40} {np.median(arr):.4f}")
-        print(f"{'Latency P95 (s):':<40} {np.percentile(arr, 95):.4f}")
-        print(f"{'Latency P99 (s):':<40} {np.percentile(arr, 99):.4f}")
-
-    print(f"\n{'Output dir:':<40} {args.output_dir}")
-    print("=" * 60)
-
-    # Save metrics JSON
-    metrics = {
-        "backend": "huggingface",
-        "mode": args.mode,
-        "model": args.model_path,
-        "height": args.height,
-        "width": args.width,
-        "num_inference_steps": args.num_inference_steps,
-        "init_time_s": init_time,
-        "completed_requests": success,
-        "failed_requests": failed,
-        "total_gen_time_s": total_gen_time,
-        "throughput_qps": success / total_gen_time if total_gen_time > 0 else 0,
-        "latency_mean": float(np.mean(latencies)) if latencies else 0,
-        "latency_median": float(np.median(latencies)) if latencies else 0,
-        "latency_p95": float(np.percentile(latencies, 95)) if latencies else 0,
-        "latency_p99": float(np.percentile(latencies, 99)) if latencies else 0,
-    }
-    if args.output_file:
-        with open(args.output_file, "w") as f:
-            json.dump(metrics, f, indent=2)
-        print(f"Metrics saved to {args.output_file}")
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(description="GLM-Image HuggingFace baseline benchmark")
-    parser.add_argument("--model-path", type=str, default="zai-org/GLM-Image")
-    parser.add_argument("--mode", type=str, default="t2i", choices=["t2i", "i2i"])
-    parser.add_argument("--dataset-path", type=str, default=None, help="Path to prompt.json")
-    parser.add_argument("--num-prompts", type=int, default=10)
-    parser.add_argument("--height", type=int, default=HEIGHT)
-    parser.add_argument("--width", type=int, default=WIDTH)
-    parser.add_argument("--num-inference-steps", type=int, default=NUM_INFERENCE_STEPS)
-    parser.add_argument("--guidance-scale", type=float, default=GUIDANCE_SCALE)
-    parser.add_argument("--seed", type=int, default=SEED)
-    parser.add_argument("--output-dir", type=str, default="benchmarks/glm_image/huggingface/outputs")
-    parser.add_argument("--output-file", type=str, default=None, help="JSON file for metrics")
-    args = parser.parse_args()
-    benchmark(args)
-
-
-if __name__ == "__main__":
-    main()

benchmarks/glm_image/vllm-omni/inference.py

@@ -1,505 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""
-vLLM-Omni offline benchmark for GLM-Image.
-
-Supports T2I and I2I modes with the prompt.json dataset.
-Downloads source images for I2I from image_url on first run and caches locally.
-
-Usage:
-    # T2I mode
-    python benchmarks/glm_image/vllm-omni/inference.py \
-        --model-path zai-org/GLM-Image \
-        --mode t2i --num-prompts 10
-
-    # I2I mode (downloads source images)
-    python benchmarks/glm_image/vllm-omni/inference.py \
-        --model-path zai-org/GLM-Image \
-        --mode i2i --num-prompts 10
-"""
-
-import argparse
-import json
-import math
-import os
-import time
-from pathlib import Path
-
-import numpy as np
-import requests
-from PIL import Image
-from vllm import SamplingParams
-
-from vllm_omni.entrypoints.omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-
-BENCHMARK_DIR = Path(__file__).resolve().parent.parent
-DEFAULT_PROMPT_JSON = BENCHMARK_DIR / "prompt" / "prompt.json"
-IMAGE_CACHE_DIR = BENCHMARK_DIR / "prompt" / "images"
-DEFAULT_DEPLOY_CONFIG = "vllm_omni/deploy/glm_image.yaml"
-
-DATASET_REPO = "JaredforReal/glm-image-bench"
-DATASET_FILE = "prompt.json"
-
-
-def _ensure_prompt_json(dataset_path: str | None) -> str:
-    """Return path to prompt.json, downloading from HuggingFace if needed."""
-    if dataset_path:
-        return dataset_path
-    local = DEFAULT_PROMPT_JSON
-    if local.exists():
-        return str(local)
-    print(f"Downloading {DATASET_FILE} from {DATASET_REPO} ...")
-    try:
-        from huggingface_hub import hf_hub_download
-
-        downloaded = hf_hub_download(
-            repo_id=DATASET_REPO,
-            filename=DATASET_FILE,
-            repo_type="dataset",
-        )
-        local.parent.mkdir(parents=True, exist_ok=True)
-        import shutil
-
-        shutil.copy2(downloaded, local)
-        print(f"Saved to {local}")
-    except ImportError:
-        url = f"https://huggingface.co/datasets/{DATASET_REPO}/resolve/main/{DATASET_FILE}"
-        import urllib.request
-
-        local.parent.mkdir(parents=True, exist_ok=True)
-        urllib.request.urlretrieve(url, local)
-        print(f"Saved to {local}")
-    return str(local)
-
-
-SEED = 42
-HEIGHT = 1024
-WIDTH = 1024
-NUM_INFERENCE_STEPS = 50
-GUIDANCE_SCALE = 1.5
-
-GLM_IMAGE_EOS_TOKEN_ID = 16385
-GLM_IMAGE_VISION_VOCAB_SIZE = 16512
-
-
-# ---------------------------------------------------------------------------
-# Dataset
-# ---------------------------------------------------------------------------
-
-
-def load_dataset(
-    dataset_path: str | None,
-    mode: str,
-    num_prompts: int,
-) -> list[dict]:
-    path = _ensure_prompt_json(dataset_path)
-    with open(path, encoding="utf-8") as f:
-        raw = json.load(f)
-
-    items = []
-    for entry in raw:
-        prompt_key = "t2i_prompt" if mode == "t2i" else "i2i_prompt"
-        prompt_text = entry.get(prompt_key, "").strip()
-        if not prompt_text:
-            continue
-
-        item = {"prompt": prompt_text}
-        if mode == "i2i":
-            item["image_url"] = entry.get("image_url", "")
-        items.append(item)
-
-    if num_prompts and len(items) > num_prompts:
-        items = items[:num_prompts]
-    return items
-
-
-def download_image(url: str, cache_dir: Path) -> str:
-    cache_dir.mkdir(parents=True, exist_ok=True)
-    fname = url.rsplit("/", 1)[-1]
-    local_path = cache_dir / fname
-    if local_path.exists():
-        return str(local_path)
-    print(f"  Downloading {url} ...")
-    resp = requests.get(url, timeout=30)
-    resp.raise_for_status()
-    local_path.write_bytes(resp.content)
-    return str(local_path)
-
-
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
-
-def compute_max_tokens(height: int, width: int, is_i2i: bool = False) -> int:
-    factor = 32
-    token_h = height // factor
-    token_w = width // factor
-    large_tokens = token_h * token_w
-
-    # Small preview tokens (half resolution in each dimension)
-
-    ratio = token_h / token_w if token_w > 0 else 1.0
-    small_token_h = max(1, int(math.sqrt(ratio) * (factor // 2)))
-    small_token_w = max(1, int(math.sqrt(1 / ratio) * (factor // 2)))
-    small_tokens = small_token_h * small_token_w
-
-    # Mode-dependent totals:
-    # - t2i: small + large + EOS
-    # - i2i: large + EOS
-    if is_i2i:
-        return large_tokens + 1
-    return small_tokens + large_tokens + 1
-
-
-def build_prompt_t2i(prompt: str, height: int, width: int, **gen_kw) -> dict:
-    return {
-        "prompt": prompt,
-        "height": height,
-        "width": width,
-        "mm_processor_kwargs": {"target_h": height, "target_w": width},
-        **gen_kw,
-    }
-
-
-def build_prompt_i2i(prompt: str, image_path: str, height: int, width: int, **gen_kw) -> dict:
-    return {
-        "prompt": prompt,
-        "height": height,
-        "width": width,
-        "mm_processor_kwargs": {"target_h": height, "target_w": width},
-        "multi_modal_data": {"image": Image.open(image_path).convert("RGB")},
-        **gen_kw,
-    }
-
-
-def resolve_deploy_config(args: argparse.Namespace) -> str:
-    if args.deploy_config:
-        return args.deploy_config
-    if os.path.exists(DEFAULT_DEPLOY_CONFIG):
-        return DEFAULT_DEPLOY_CONFIG
-    fallback = Path(__file__).resolve().parents[3] / DEFAULT_DEPLOY_CONFIG
-    if fallback.exists():
-        return str(fallback)
-    raise FileNotFoundError("Deploy config not found. Specify --deploy-config.")
-
-
-# ---------------------------------------------------------------------------
-# Benchmark
-# ---------------------------------------------------------------------------
-
-
-def benchmark(args: argparse.Namespace) -> None:
-    is_i2i = args.mode == "i2i"
-
-    print("=" * 60)
-    print("GLM-Image vLLM-Omni Benchmark")
-    print(f"Mode: {args.mode}  |  Model: {args.model_path}")
-    print(f"Size: {args.height}x{args.width}  |  Steps: {args.num_inference_steps}")
-    print("=" * 60)
-
-    # Load dataset
-    items = load_dataset(args.dataset_path, args.mode, args.num_prompts)
-    if not items:
-        print("No prompts loaded. Exiting.")
-        return
-    print(f"Loaded {len(items)} prompts for {args.mode} mode")
-
-    # Download I2I source images
-    if is_i2i:
-        print("Preparing source images...")
-        for item in items:
-            url = item.get("image_url", "")
-            if url:
-                item["image_path"] = download_image(url, IMAGE_CACHE_DIR)
-            else:
-                item["image_path"] = None
-
-    # Init Omni
-    deploy_config = resolve_deploy_config(args)
-    print(f"\nInitializing vLLM-Omni (deploy config: {deploy_config}) ...")
-    t0 = time.perf_counter()
-
-    omni = Omni(
-        model=args.model_path,
-        deploy_config=deploy_config,
-        log_stats=args.log_stats,
-        stage_init_timeout=args.stage_init_timeout,
-        enable_diffusion_pipeline_profiler=args.enable_diffusion_pipeline_profiler,
-        enable_ar_profiler=args.enable_ar_profiler,
-    )
-
-    init_time = time.perf_counter() - t0
-    print(f"Initialized in {init_time:.2f}s")
-
-    # Sampling params
-    max_tokens = compute_max_tokens(args.height, args.width, is_i2i=is_i2i)
-    ar_params = SamplingParams(
-        temperature=0.9,
-        top_p=0.75,
-        top_k=GLM_IMAGE_VISION_VOCAB_SIZE,
-        max_tokens=max_tokens,
-        stop_token_ids=[GLM_IMAGE_EOS_TOKEN_ID],
-        seed=args.seed,
-        detokenize=False,
-        extra_args={"target_h": args.height, "target_w": args.width},
-    )
-    diff_params = OmniDiffusionSamplingParams(
-        num_inference_steps=args.num_inference_steps,
-        guidance_scale=args.guidance_scale,
-        height=args.height,
-        width=args.width,
-        seed=args.seed,
-    )
-    sampling_params_list = [ar_params, diff_params]
-
-    # Build all prompts
-    gen_kw = {
-        "seed": args.seed,
-        "num_inference_steps": args.num_inference_steps,
-        "guidance_scale": args.guidance_scale,
-    }
-    all_prompts = []
-    for item in items:
-        if is_i2i:
-            img_path = item.get("image_path")
-            if not img_path or not os.path.exists(img_path):
-                continue
-            all_prompts.append(build_prompt_i2i(item["prompt"], img_path, args.height, args.width, **gen_kw))
-        else:
-            all_prompts.append(build_prompt_t2i(item["prompt"], args.height, args.width, **gen_kw))
-
-    valid = len(all_prompts)
-    print(f"Valid prompts: {valid}")
-
-    # Create output dir
-    os.makedirs(args.output_dir, exist_ok=True)
-
-    # Warmup: run 1 request to prime caches, CUDA graphs, etc.
-    if all_prompts:
-        print("Running warmup request...")
-        try:
-            warmup_prompt = [all_prompts[0]]
-            omni.generate(warmup_prompt, sampling_params_list, py_generator=False)
-            print("Warmup done.\n")
-        except Exception as e:
-            print(f"Warmup failed (continuing): {e}")
-
-    # Run
-    print(f"\nRunning {valid} requests...")
-    print("-" * 60)
-
-    latencies = []
-    all_stage_durations: list[dict[str, float]] = []
-    success = 0
-    failed = 0
-    wall_start = time.perf_counter()
-
-    try:
-        output_idx = 0
-        for stage_outputs in omni.generate(all_prompts, sampling_params_list, py_generator=True):
-            if stage_outputs.final_output_type == "image":
-                request_output = stage_outputs.request_output
-                request_id = getattr(request_output, "request_id", "")
-
-                images = getattr(request_output, "images", [])
-                if not images and hasattr(request_output, "multimodal_output"):
-                    mm = request_output.multimodal_output
-                    if isinstance(mm, dict):
-                        images = mm.get("images", [])
-
-                elapsed = time.perf_counter() - wall_start
-                if images:
-                    for img in images:
-                        if isinstance(img, Image.Image):
-                            out_path = os.path.join(args.output_dir, f"{output_idx:04d}.png")
-                            img.save(out_path)
-                    success += 1
-                    latencies.append(elapsed)
-                    stage_durations = getattr(stage_outputs, "stage_durations", {})
-                    if stage_durations:
-                        all_stage_durations.append(stage_durations)
-                    # Show wall-clock elapsed and pipeline breakdown if available
-                    preprocess_str = ""
-                    if "preprocess_ms" in stage_durations:
-                        preprocess_str = f" preprocess={stage_durations['preprocess_ms'] / 1000.0:.2f}s"
-                    print(f"  [{success}/{valid}] id={request_id[:8]} {elapsed:.2f}s{preprocess_str}")
-                    output_idx += 1
-                else:
-                    failed += 1
-    except Exception as e:
-        print(f"Error: {e}")
-        failed = valid - success
-
-    total_gen_time = time.perf_counter() - wall_start
-
-    # Diff stage_0_gen_ms with previous request to remove accumulated wait time.
-    # stage_0_gen_ms is measured from submit_ts (same for all requests submitted
-    # at once), so it accumulates queue/scheduling overhead across requests.
-    # Other stages and pipeline timings are per-request already.
-    _TIMING_ORDER = [
-        "preprocess_ms",
-        "stage_0_gen_ms",
-        "ar2diffusion_ms",
-        "stage_1_gen_ms",
-        "queue_wait_ms",
-    ]
-
-    per_request_actual: list[dict[str, float]] = []
-    prev_stage_0_ms = 0.0
-    for sd in all_stage_durations:
-        actual = dict(sd)
-        s0 = sd.get("stage_0_gen_ms", 0.0)
-        actual["stage_0_gen_ms"] = s0 - prev_stage_0_ms
-        prev_stage_0_ms = s0
-        per_request_actual.append(actual)
-
-    per_request_e2e_ms: list[float] = []
-    for actual in per_request_actual:
-        e2e_ms = sum(v for k, v in actual.items() if k in _TIMING_ORDER)
-        if e2e_ms > 0:
-            per_request_e2e_ms.append(e2e_ms)
-
-    # Report
-    print("\n" + "=" * 60)
-    print("vLLM-Omni Benchmark Results")
-    print("=" * 60)
-    print(f"{'Mode:':<40} {args.mode}")
-    print(f"{'Model:':<40} {args.model_path}")
-    print(f"{'Image size:':<40} {args.height}x{args.width}")
-    print(f"{'Num inference steps:':<40} {args.num_inference_steps}")
-    print("-" * 50)
-    print(f"{'Init time (s):':<40} {init_time:.2f}")
-    print(f"{'Successful:':<40} {success}/{valid}")
-    print(f"{'Failed:':<40} {failed}")
-    print("-" * 50)
-
-    if per_request_e2e_ms:
-        per_request_s = np.array(per_request_e2e_ms) / 1000.0
-        print(f"{'Total generation time (s):':<40} {total_gen_time:.2f}")
-        print(f"{'Throughput (img/s):':<40} {success / total_gen_time:.4f}")
-        print(f"{'Latency Mean (s):':<40} {per_request_s.mean():.4f}")
-        print(f"{'Latency Median (s):':<40} {np.median(per_request_s):.4f}")
-        print(f"{'Latency P95 (s):':<40} {np.percentile(per_request_s, 95):.4f}")
-        print(f"{'Latency P99 (s):':<40} {np.percentile(per_request_s, 99):.4f}")
-        print(f"{'Latency Min (s):':<40} {per_request_s.min():.4f}")
-        print(f"{'Latency Max (s):':<40} {per_request_s.max():.4f}")
-    elif latencies:
-        per_request = np.diff([0.0] + list(latencies))
-        print(f"{'Total generation time (s):':<40} {total_gen_time:.2f}")
-        print(f"{'Throughput (img/s):':<40} {success / total_gen_time:.4f}")
-        print(f"{'Latency Mean (s) [wall-clock]:':<40} {per_request.mean():.4f}")
-        print(f"{'Latency Median (s) [wall-clock]:':<40} {np.median(per_request):.4f}")
-        print(f"{'Latency P95 (s) [wall-clock]:':<40} {np.percentile(per_request, 95):.4f}")
-        print(f"{'Latency P99 (s) [wall-clock]:':<40} {np.percentile(per_request, 99):.4f}")
-        print(f"{'Latency Min (s) [wall-clock]:':<40} {per_request.min():.4f}")
-        print(f"{'Latency Max (s) [wall-clock]:':<40} {per_request.max():.4f}")
-
-    if per_request_actual:
-        print("-" * 50)
-        print("Pipeline Timings Mean:")
-        for key in _TIMING_ORDER:
-            vals = [d.get(key, 0.0) for d in per_request_actual]
-            if any(v != 0 for v in vals):
-                unit = "ms" if key.endswith("_ms") else "s"
-                print(f"  {key + ':':<38} {np.mean(vals):.4f} ({unit})")
-        # Show any extra keys not in the ordered list
-        ordered_set = set(_TIMING_ORDER)
-        extra_keys = sorted(k for k in per_request_actual[0].keys() if k not in ordered_set)
-        for key in extra_keys:
-            vals = [d.get(key, 0.0) for d in per_request_actual]
-            if any(v != 0 for v in vals):
-                unit = "ms" if key.endswith("_ms") else "s"
-                print(f"  {key + ':':<38} {np.mean(vals):.4f} ({unit})")
-
-    print(f"\n{'Output dir:':<40} {args.output_dir}")
-    print("=" * 60)
-
-    # Metrics JSON
-    metrics = {
-        "backend": "vllm-omni",
-        "mode": args.mode,
-        "model": args.model_path,
-        "height": args.height,
-        "width": args.width,
-        "num_inference_steps": args.num_inference_steps,
-        "init_time_s": init_time,
-        "completed_requests": success,
-        "failed_requests": failed,
-        "total_gen_time_s": total_gen_time,
-        "throughput_qps": success / total_gen_time if total_gen_time > 0 else 0,
-    }
-    if per_request_e2e_ms:
-        per_request_s = np.array(per_request_e2e_ms) / 1000.0
-        metrics["latency_mean"] = float(per_request_s.mean())
-        metrics["latency_median"] = float(np.median(per_request_s))
-        metrics["latency_p95"] = float(np.percentile(per_request_s, 95))
-        metrics["latency_p99"] = float(np.percentile(per_request_s, 99))
-    elif latencies:
-        per_request = np.diff([0.0] + list(latencies))
-        metrics["latency_mean"] = float(per_request.mean())
-        metrics["latency_median"] = float(np.median(per_request))
-        metrics["latency_p95"] = float(np.percentile(per_request, 95))
-        metrics["latency_p99"] = float(np.percentile(per_request, 99))
-    else:
-        metrics["latency_mean"] = 0
-        metrics["latency_median"] = 0
-        metrics["latency_p95"] = 0
-        metrics["latency_p99"] = 0
-    if per_request_actual:
-        all_keys = list(_TIMING_ORDER) + sorted(k for k in per_request_actual[0].keys() if k not in set(_TIMING_ORDER))
-        stage_metrics = {}
-        for key in all_keys:
-            vals = [d.get(key, 0.0) for d in per_request_actual]
-            stage_metrics[key] = {
-                "mean": float(np.mean(vals)),
-                "median": float(np.median(vals)),
-                "p95": float(np.percentile(vals, 95)),
-            }
-        metrics["stage_durations"] = stage_metrics
-    if args.output_file:
-        with open(args.output_file, "w") as f:
-            json.dump(metrics, f, indent=2)
-        print(f"Metrics saved to {args.output_file}")
-
-    omni.close()
-    print("Done!")
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(description="GLM-Image vLLM-Omni offline benchmark")
-    parser.add_argument("--model-path", type=str, default="zai-org/GLM-Image")
-    parser.add_argument("--deploy-config", type=str, default=None, help="Deploy config YAML")
-    parser.add_argument("--mode", type=str, default="t2i", choices=["t2i", "i2i"])
-    parser.add_argument("--dataset-path", type=str, default=None, help="Path to prompt.json")
-    parser.add_argument("--num-prompts", type=int, default=10)
-    parser.add_argument("--height", type=int, default=HEIGHT)
-    parser.add_argument("--width", type=int, default=WIDTH)
-    parser.add_argument("--num-inference-steps", type=int, default=NUM_INFERENCE_STEPS)
-    parser.add_argument("--guidance-scale", type=float, default=GUIDANCE_SCALE)
-    parser.add_argument("--seed", type=int, default=SEED)
-    parser.add_argument("--output-dir", type=str, default="benchmarks/glm_image/vllm-omni/outputs")
-    parser.add_argument("--output-file", type=str, default=None, help="JSON file for metrics")
-    parser.add_argument("--stage-init-timeout", type=int, default=600)
-    parser.add_argument(
-        "--enable-diffusion-pipeline-profiler",
-        action="store_true",
-        help="Enable diffusion pipeline profiler for stage-level timing",
-    )
-    parser.add_argument(
-        "--enable-ar-profiler",
-        action="store_true",
-        help="Enable AR stage profiler to include AR timing in stage_durations",
-    )
-    parser.add_argument(
-        "--log-stats",
-        action="store_true",
-        help="Enable detailed per-request pipeline stats logging",
-    )
-    args = parser.parse_args()
-    benchmark(args)
-
-
-if __name__ == "__main__":
-    main()

benchmarks/qwen3-tts/README.md

@@ -0,0 +1,107 @@
+# Qwen3-TTS Benchmark
+
+Benchmarks for Qwen3-TTS text-to-speech models, comparing vLLM-Omni streaming serving against HuggingFace Transformers offline inference.
+
+## Prerequisites
+
+```bash
+pip install matplotlib aiohttp soundfile numpy tqdm
+pip install qwen_tts  # for HF baseline
+```
+
+## Quick Start
+
+Run the full benchmark (vllm-omni + HF baseline) with a single command:
+
+```bash
+cd benchmarks/qwen3-tts
+bash run_benchmark.sh
+```
+
+Results (JSON + PNG plots) are saved to `results/`.
+
+### Common options
+
+```bash
+# Only vllm-omni (skip HF baseline)
+bash run_benchmark.sh --async-only
+
+# Only HF baseline
+bash run_benchmark.sh --hf-only
+
+# Use a different model (e.g. 1.7B)
+MODEL=Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice bash run_benchmark.sh --async-only
+
+# Use a Voice Clone model
+MODEL=Qwen/Qwen3-TTS-12Hz-1.7B-Base TASK_TYPE=Base bash run_benchmark.sh --async-only
+
+# Use batch size 16 for higher throughput
+BATCH_SIZE=16 bash run_benchmark.sh --async-only
+
+# Custom GPU, prompt count, concurrency levels
+GPU_DEVICE=1 NUM_PROMPTS=20 CONCURRENCY="1 4" bash run_benchmark.sh
+```
+
+## Manual Steps
+
+### 1) Start the vLLM-Omni server
+
+```bash
+CUDA_VISIBLE_DEVICES=0 python -m vllm_omni.entrypoints.cli.main serve \
+    "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice" \
+    --omni --host 127.0.0.1 --port 8000 \
+    --deploy-config vllm_omni/deploy/qwen3_tts.yaml \
+    --stage-overrides '{"0":{"max_num_seqs":1,"gpu_memory_utilization":0.3,"max_num_batched_tokens":512},"1":{"max_num_seqs":1,"gpu_memory_utilization":0.3,"max_num_batched_tokens":8192}}' \
+    --trust-remote-code
+```
+
+### 2) Run online serving benchmark
+
+```bash
+python benchmarks/qwen3-tts/vllm_omni/bench_tts_serve.py \
+    --port 8000 \
+    --num-prompts 50 \
+    --max-concurrency 1 4 10 \
+    --config-name "async_chunk" \
+    --result-dir results/
+```
+
+### 3) Run HuggingFace baseline
+
+```bash
+python benchmarks/qwen3-tts/transformers/bench_tts_hf.py \
+    --model "Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice" \
+    --num-prompts 50 \
+    --gpu-device 0 \
+    --result-dir results/
+```
+
+### 4) Generate comparison plots
+
+```bash
+python benchmarks/qwen3-tts/plot_results.py \
+    --results results/bench_async_chunk_*.json results/bench_hf_transformers_*.json \
+    --labels "vllm-omni" "hf_transformers" \
+    --output results/comparison.png
+```
+
+## Batch-size presets
+
+The bench script loads the bundled production deploy (`vllm_omni/deploy/qwen3_tts.yaml`) and layers per-stage budgets on top via `--stage-overrides`, driven by the `BATCH_SIZE` env var. Each batch size picks compatible per-stage `max_num_seqs`, `max_num_batched_tokens`, and `gpu_memory_utilization` defaults:
+
+| `BATCH_SIZE` | Description |
+|:--:|-------------|
+| `1` (default) | Single-request processing (lowest latency) |
+| `4`  | Moderate-throughput concurrent processing |
+| `16` | High-throughput concurrent processing |
+
+The 2-stage pipeline (Talker -> Code2Wav) runs with `async_chunk` streaming enabled via the prod deploy; the `SharedMemoryConnector` streams codec frames (25-frame chunks with 25-frame context overlap) between stages.
+
+The model is specified via the CLI `--model` flag (or `MODEL` env var), so the same bench script works for both the 0.6B and 1.7B model variants.
+
+## Metrics
+
+- **TTFP (Time to First Audio Packet)**: Time from request to first audio chunk (streaming latency)
+- **E2E (End-to-End Latency)**: Total time from request to complete audio response
+- **RTF (Real-Time Factor)**: E2E latency / audio duration. RTF < 1.0 means faster-than-real-time synthesis
+- **Throughput**: Total audio seconds generated per wall-clock second

benchmarks/qwen3-tts/plot_results.py

@@ -0,0 +1,254 @@
+"""Plot Qwen3-TTS benchmark results.
+
+Generates comparison bar charts similar to the async_chunk design doc:
+- TTFP (Time-to-First-Packet) across concurrency levels
+- E2E latency across concurrency levels
+- RTF (Real-Time Factor) across concurrency levels
+
+Usage:
+    # Compare two configs (async_chunk vs no_async_chunk):
+    python plot_results.py \
+        --results results/bench_async_chunk_*.json results/bench_no_async_chunk_*.json \
+        --labels "async_chunk" "no_async_chunk" \
+        --output results/qwen3_tts_benchmark.png
+
+    # Single config:
+    python plot_results.py \
+        --results results/bench_async_chunk_*.json \
+        --labels "async_chunk" \
+        --output results/qwen3_tts_benchmark.png
+"""
+
+import argparse
+import json
+from pathlib import Path
+
+import matplotlib.pyplot as plt
+import numpy as np
+
+
+def load_results(result_files: list[str]) -> list[list[dict]]:
+    """Load benchmark results from JSON files."""
+    all_results = []
+    for f in result_files:
+        with open(f) as fh:
+            data = json.load(fh)
+        all_results.append(data)
+    return all_results
+
+
+def plot_comparison(
+    all_results: list[list[dict]],
+    labels: list[str],
+    output_path: str,
+    title_prefix: str = "Qwen3-TTS",
+):
+    """Generate comparison bar charts."""
+    n_configs = len(all_results)
+
+    # Collect concurrency levels present in ALL configs (skip missing data)
+    all_concurrencies = [set(r["concurrency"] for r in results) for results in all_results]
+    concurrencies = sorted(set.union(*all_concurrencies))
+
+    # Build data arrays, using None for missing concurrency levels
+    ttfp_data = {label: [] for label in labels}
+    e2e_data = {label: [] for label in labels}
+    rtf_data = {label: [] for label in labels}
+    throughput_data = {label: [] for label in labels}
+
+    for results, label in zip(all_results, labels):
+        conc_map = {r["concurrency"]: r for r in results}
+        for c in concurrencies:
+            r = conc_map.get(c)
+            ttfp_data[label].append(r["mean_ttfp_ms"] if r else None)
+            e2e_data[label].append(r["mean_e2e_ms"] if r else None)
+            rtf_data[label].append(r["mean_rtf"] if r else None)
+            throughput_data[label].append(r["audio_throughput"] if r else None)
+
+    fig, axes = plt.subplots(2, 2, figsize=(14, 10))
+    fig.suptitle(f"{title_prefix} Performance Benchmark", fontsize=16, fontweight="bold")
+
+    x = np.arange(len(concurrencies))
+    width = 0.35 if n_configs == 2 else 0.5
+    if n_configs > 1:
+        offsets = np.linspace(-width / 2 * (n_configs - 1), width / 2 * (n_configs - 1), n_configs)
+    else:
+        offsets = [0]
+
+    colors = ["#2196F3", "#FF5722", "#4CAF50", "#FFC107"]
+
+    def plot_metric(ax, data_dict, ylabel, title, fmt=".1f"):
+        bars = []
+        for i, (label, values) in enumerate(data_dict.items()):
+            # Replace None with 0 for plotting, but track which are missing
+            plot_values = [v if v is not None else 0 for v in values]
+            color = colors[i % len(colors)]
+            bar = ax.bar(x + offsets[i], plot_values, width, label=label, color=color, alpha=0.85)
+            bars.append(bar)
+            # Add value labels on bars (skip None/missing data)
+            max_val = max((v for v in values if v is not None), default=1)
+            for rect, val in zip(bar, values):
+                if val is not None and val > 0:
+                    ax.text(
+                        rect.get_x() + rect.get_width() / 2,
+                        rect.get_height() + max_val * 0.02,
+                        f"{val:{fmt}}",
+                        ha="center",
+                        va="bottom",
+                        fontsize=9,
+                        fontweight="bold",
+                    )
+        ax.set_xlabel("Concurrency", fontsize=12)
+        ax.set_ylabel(ylabel, fontsize=12)
+        ax.set_title(title, fontsize=13, fontweight="bold")
+        ax.set_xticks(x)
+        ax.set_xticklabels([str(c) for c in concurrencies])
+        ax.legend(fontsize=10)
+        ax.grid(axis="y", alpha=0.3)
+        ax.set_axisbelow(True)
+
+    plot_metric(axes[0, 0], ttfp_data, "TTFP (ms)", "Time to First Audio Packet (TTFP)")
+    plot_metric(axes[0, 1], e2e_data, "E2E Latency (ms)", "End-to-End Latency (E2E)")
+    plot_metric(axes[1, 0], rtf_data, "RTF", "Real-Time Factor (RTF)", fmt=".3f")
+    plot_metric(axes[1, 1], throughput_data, "Audio-sec / Wall-sec", "Audio Throughput", fmt=".2f")
+
+    plt.tight_layout()
+    plt.savefig(output_path, dpi=150, bbox_inches="tight")
+    print(f"Plot saved to {output_path}")
+    plt.close()
+
+
+def plot_single_summary(results: list[dict], label: str, output_path: str):
+    """Generate a single-config summary with percentile breakdown."""
+    concurrencies = [r["concurrency"] for r in results]
+
+    fig, axes = plt.subplots(1, 3, figsize=(16, 5))
+    fig.suptitle(f"Qwen3-TTS Benchmark - {label}", fontsize=15, fontweight="bold")
+
+    # TTFP breakdown
+    ax = axes[0]
+    means = [r["mean_ttfp_ms"] for r in results]
+    medians = [r["median_ttfp_ms"] for r in results]
+    p90s = [r["p90_ttfp_ms"] for r in results]
+    p99s = [r["p99_ttfp_ms"] for r in results]
+    x = np.arange(len(concurrencies))
+    w = 0.2
+    ax.bar(x - 1.5 * w, means, w, label="mean", color="#2196F3")
+    ax.bar(x - 0.5 * w, medians, w, label="median", color="#4CAF50")
+    ax.bar(x + 0.5 * w, p90s, w, label="p90", color="#FF9800")
+    ax.bar(x + 1.5 * w, p99s, w, label="p99", color="#F44336")
+    ax.set_xticks(x)
+    ax.set_xticklabels([str(c) for c in concurrencies])
+    ax.set_xlabel("Concurrency")
+    ax.set_ylabel("TTFP (ms)")
+    ax.set_title("Time to First Audio Packet")
+    ax.legend(fontsize=9)
+    ax.grid(axis="y", alpha=0.3)
+
+    # E2E breakdown
+    ax = axes[1]
+    means = [r["mean_e2e_ms"] for r in results]
+    medians = [r["median_e2e_ms"] for r in results]
+    p90s = [r["p90_e2e_ms"] for r in results]
+    p99s = [r["p99_e2e_ms"] for r in results]
+    ax.bar(x - 1.5 * w, means, w, label="mean", color="#2196F3")
+    ax.bar(x - 0.5 * w, medians, w, label="median", color="#4CAF50")
+    ax.bar(x + 0.5 * w, p90s, w, label="p90", color="#FF9800")
+    ax.bar(x + 1.5 * w, p99s, w, label="p99", color="#F44336")
+    ax.set_xticks(x)
+    ax.set_xticklabels([str(c) for c in concurrencies])
+    ax.set_xlabel("Concurrency")
+    ax.set_ylabel("E2E Latency (ms)")
+    ax.set_title("End-to-End Latency")
+    ax.legend(fontsize=9)
+    ax.grid(axis="y", alpha=0.3)
+
+    # RTF
+    ax = axes[2]
+    means = [r["mean_rtf"] for r in results]
+    medians = [r["median_rtf"] for r in results]
+    ax.bar(x - 0.15, means, 0.3, label="mean", color="#2196F3")
+    ax.bar(x + 0.15, medians, 0.3, label="median", color="#4CAF50")
+    ax.set_xticks(x)
+    ax.set_xticklabels([str(c) for c in concurrencies])
+    ax.set_xlabel("Concurrency")
+    ax.set_ylabel("RTF")
+    ax.set_title("Real-Time Factor")
+    ax.legend(fontsize=9)
+    ax.grid(axis="y", alpha=0.3)
+
+    plt.tight_layout()
+    plt.savefig(output_path, dpi=150, bbox_inches="tight")
+    print(f"Plot saved to {output_path}")
+    plt.close()
+
+
+def print_comparison_table(all_results: list[list[dict]], labels: list[str]):
+    """Print a markdown-formatted comparison table."""
+    concurrencies = sorted(set(r["concurrency"] for r in all_results[0]))
+
+    print("\n## Benchmark Results\n")
+    header = "| Metric | Concurrency |"
+    sep = "| --- | --- |"
+    for label in labels:
+        header += f" {label} |"
+        sep += " --- |"
+    print(header)
+    print(sep)
+
+    for metric, key, fmt in [
+        ("TTFP (ms)", "mean_ttfp_ms", ".1f"),
+        ("E2E (ms)", "mean_e2e_ms", ".1f"),
+        ("RTF", "mean_rtf", ".3f"),
+        ("Throughput (audio-s/s)", "audio_throughput", ".2f"),
+    ]:
+        for c in concurrencies:
+            row = f"| {metric} | {c} |"
+            for results in all_results:
+                conc_map = {r["concurrency"]: r for r in results}
+                val = conc_map.get(c, {}).get(key, 0)
+                row += f" {val:{fmt}} |"
+            print(row)
+
+    # Improvement calculation (only if 2 configs)
+    if len(all_results) == 2:
+        print(f"\n## Improvement ({labels[0]} vs {labels[1]})\n")
+        print("| Metric | Concurrency | Improvement |")
+        print("| --- | --- | --- |")
+        for metric, key in [("TTFP", "mean_ttfp_ms"), ("E2E", "mean_e2e_ms"), ("RTF", "mean_rtf")]:
+            for c in concurrencies:
+                m0 = {r["concurrency"]: r for r in all_results[0]}
+                m1 = {r["concurrency"]: r for r in all_results[1]}
+                v0 = m0.get(c, {}).get(key, 0)
+                v1 = m1.get(c, {}).get(key, 0)
+                if v1 > 0:
+                    pct = (v1 - v0) / v1 * 100
+                    print(f"| {metric} | {c} | {pct:+.1f}% |")
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Plot Qwen3-TTS benchmark results")
+    parser.add_argument(
+        "--results", type=str, nargs="+", required=True, help="Path(s) to result JSON files (one per config)"
+    )
+    parser.add_argument(
+        "--labels", type=str, nargs="+", required=True, help="Labels for each config (must match --results count)"
+    )
+    parser.add_argument("--output", type=str, default="results/qwen3_tts_benchmark.png", help="Output image path")
+    parser.add_argument("--title", type=str, default="Qwen3-TTS", help="Title prefix for the plot")
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    assert len(args.results) == len(args.labels), "--results and --labels must have the same count"
+
+    all_results = load_results(args.results)
+    print_comparison_table(all_results, args.labels)
+
+    Path(args.output).parent.mkdir(parents=True, exist_ok=True)
+
+    if len(all_results) == 1:
+        plot_single_summary(all_results[0], args.labels[0], args.output)
+    else:
+        plot_comparison(all_results, args.labels, args.output, title_prefix=args.title)

benchmarks/qwen3-tts/results/.gitignore

@@ -0,0 +1,3 @@
+# Benchmark results are machine-specific - do not commit
+*
+!.gitignore

benchmarks/qwen3-tts/run_benchmark.sh

@@ -0,0 +1,285 @@
+#!/bin/bash
+# Qwen3-TTS Benchmark Runner
+#
+# Compares vllm-omni streaming serving vs HuggingFace transformers offline inference.
+# Produces JSON results and comparison plots.
+#
+# Usage:
+#   # Full comparison (vllm-omni + HF):
+#   bash run_benchmark.sh
+#
+#   # Only vllm-omni async_chunk config:
+#   bash run_benchmark.sh --async-only
+#
+#   # Only HuggingFace baseline:
+#   bash run_benchmark.sh --hf-only
+#
+#   # vllm-omni only (skip HF):
+#   bash run_benchmark.sh --skip-hf
+#
+#   # Custom settings:
+#   GPU_DEVICE=1 NUM_PROMPTS=20 CONCURRENCY="1 4" bash run_benchmark.sh
+#
+#   # Use 1.7B model:
+#   MODEL=Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice bash run_benchmark.sh --async-only
+#
+#   # Use Voice Clone model
+#   MODEL=Qwen/Qwen3-TTS-12Hz-1.7B-Base TASK_TYPE=Base bash run_benchmark.sh --async-only
+#
+#   # Use batch_size=4:
+#   BATCH_SIZE=4 bash run_benchmark.sh --async-only
+#
+# Environment variables:
+#   GPU_DEVICE       - GPU index to use (default: 0)
+#   NUM_PROMPTS      - Number of prompts per concurrency level (default: 50)
+#   CONCURRENCY      - Space-separated concurrency levels (default: "1 4 10")
+#   MODEL            - Model name (default: Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice)
+#   PORT             - Server port (default: 8000)
+#   BATCH_SIZE       - Per-stage ``max_num_seqs`` for both talker and code2wav (default: 1)
+#   GPU_MEM_TALKER   - gpu_memory_utilization for talker stage (default: 0.3 at bs=1, else 0.2)
+#   GPU_MEM_CODE2WAV - gpu_memory_utilization for code2wav stage (default: 0.3 at bs=1, else 0.2)
+#   TASK_TYPE        - Task type: CustomVoice, VoiceDesign, Base (default: CustomVoice)
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+
+# Defaults
+GPU_DEVICE="${GPU_DEVICE:-0}"
+NUM_PROMPTS="${NUM_PROMPTS:-50}"
+CONCURRENCY="${CONCURRENCY:-1 4 10}"
+MODEL="${MODEL:-Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice}"
+PORT="${PORT:-8000}"
+BATCH_SIZE="${BATCH_SIZE:-1}"
+DEFAULT_MEM=$([ "${BATCH_SIZE}" = "1" ] && echo "0.3" || echo "0.2")
+GPU_MEM_TALKER="${GPU_MEM_TALKER:-${DEFAULT_MEM}}"
+GPU_MEM_CODE2WAV="${GPU_MEM_CODE2WAV:-${DEFAULT_MEM}}"
+NUM_WARMUPS="${NUM_WARMUPS:-3}"
+DEPLOY_CONFIG="vllm_omni/deploy/qwen3_tts.yaml"
+RESULT_DIR="${SCRIPT_DIR}/results"
+TIMESTAMP="$(date +%Y%m%d_%H%M%S)"
+TASK_TYPE="${TASK_TYPE:-CustomVoice}"
+
+# Build --stage-overrides JSON from BATCH_SIZE + GPU_MEM_*.
+STAGE_OVERRIDES=$(
+    BATCH_SIZE="${BATCH_SIZE}" \
+    GPU_MEM_TALKER="${GPU_MEM_TALKER}" \
+    GPU_MEM_CODE2WAV="${GPU_MEM_CODE2WAV}" \
+    python - <<'PYEOF'
+import json, os
+bs = int(os.environ["BATCH_SIZE"])
+mem_t = float(os.environ["GPU_MEM_TALKER"])
+mem_c = float(os.environ["GPU_MEM_CODE2WAV"])
+# Prefill budget grows with batch size on both stages.
+talker_batched = 512 if bs <= 4 else 4096
+code2wav_batched = 8192 if bs <= 4 else 32768
+print(json.dumps({
+    "0": {"max_num_seqs": bs, "gpu_memory_utilization": mem_t, "max_num_batched_tokens": talker_batched},
+    "1": {"max_num_seqs": bs, "gpu_memory_utilization": mem_c, "max_num_batched_tokens": code2wav_batched},
+}))
+PYEOF
+)
+
+# Parse args
+RUN_ASYNC=true
+RUN_HF=true
+for arg in "$@"; do
+    case "$arg" in
+        --async-only) RUN_HF=false ;;
+        --hf-only) RUN_ASYNC=false ;;
+        --skip-hf) RUN_HF=false ;;
+    esac
+done
+
+mkdir -p "${RESULT_DIR}"
+
+echo "============================================================"
+echo " Qwen3-TTS Benchmark"
+echo "============================================================"
+echo " GPU:            ${GPU_DEVICE}"
+echo " Model:          ${MODEL}"
+echo " Prompts:        ${NUM_PROMPTS}"
+echo " Concurrency:    ${CONCURRENCY}"
+echo " Port:           ${PORT}"
+echo " Deploy config:  ${DEPLOY_CONFIG}"
+echo " Batch size:     ${BATCH_SIZE}"
+echo " GPU mem T/C:    ${GPU_MEM_TALKER} / ${GPU_MEM_CODE2WAV}"
+echo " Results:        ${RESULT_DIR}"
+echo " Task type:      ${TASK_TYPE}"
+echo "============================================================"
+
+# Start server and wait for it to be ready
+start_server() {
+    local config_name="$1"
+    local log_file="${RESULT_DIR}/server_${config_name}_${TIMESTAMP}.log"
+
+    echo ""
+    echo "Starting server with config: ${config_name}"
+    echo "  Deploy config: ${DEPLOY_CONFIG}"
+    echo "  Stage overrides: ${STAGE_OVERRIDES}"
+    echo "  Log file: ${log_file}"
+
+    VLLM_WORKER_MULTIPROC_METHOD=spawn \
+    CUDA_VISIBLE_DEVICES="${GPU_DEVICE}" \
+    python -m vllm_omni.entrypoints.cli.main serve "${MODEL}" \
+        --omni \
+        --host 127.0.0.1 \
+        --port "${PORT}" \
+        --deploy-config "${DEPLOY_CONFIG}" \
+        --stage-overrides "${STAGE_OVERRIDES}" \
+        --stage-init-timeout 120 \
+        --trust-remote-code \
+        --disable-log-stats \
+        > "${log_file}" 2>&1 &
+
+    SERVER_PID=$!
+    echo "  Server PID: ${SERVER_PID}"
+
+    # Wait for server to be ready
+    echo "  Waiting for server to be ready..."
+    local max_wait=300
+    local waited=0
+    while [ ${waited} -lt ${max_wait} ]; do
+        if curl -sf "http://127.0.0.1:${PORT}/v1/models" > /dev/null 2>&1; then
+            echo "  Server is ready! (waited ${waited}s)"
+            return 0
+        fi
+        # Check if process is still alive
+        if ! kill -0 ${SERVER_PID} 2>/dev/null; then
+            echo "  ERROR: Server process died. Check log: ${log_file}"
+            tail -20 "${log_file}"
+            return 1
+        fi
+        sleep 2
+        waited=$((waited + 2))
+    done
+
+    echo "  ERROR: Server did not start within ${max_wait}s. Check log: ${log_file}"
+    kill ${SERVER_PID} 2>/dev/null || true
+    return 1
+}
+
+# Stop the server
+stop_server() {
+    if [ -n "${SERVER_PID:-}" ]; then
+        echo "  Stopping server (PID: ${SERVER_PID})..."
+        kill ${SERVER_PID} 2>/dev/null || true
+        wait ${SERVER_PID} 2>/dev/null || true
+        # Kill any remaining child processes on the port
+        local pids
+        pids=$(lsof -ti:${PORT} 2>/dev/null || true)
+        if [ -n "${pids}" ]; then
+            echo "  Cleaning up remaining processes on port ${PORT}..."
+            echo "${pids}" | xargs kill -9 2>/dev/null || true
+        fi
+        echo "  Server stopped."
+        SERVER_PID=""
+    fi
+}
+
+# Cleanup on exit
+trap 'stop_server' EXIT
+
+# Run benchmark for a given config
+run_bench() {
+    local config_name="$1"
+
+    echo ""
+    echo "============================================================"
+    echo " Benchmarking: ${config_name}"
+    echo "============================================================"
+
+    start_server "${config_name}"
+
+    # Convert concurrency string to args
+    local conc_args=""
+    for c in ${CONCURRENCY}; do
+        conc_args="${conc_args} ${c}"
+    done
+
+    cd "${PROJECT_ROOT}"
+    python "${SCRIPT_DIR}/vllm_omni/bench_tts_serve.py" \
+        --host 127.0.0.1 \
+        --port "${PORT}" \
+        --num-prompts "${NUM_PROMPTS}" \
+        --max-concurrency ${conc_args} \
+        --num-warmups "${NUM_WARMUPS}" \
+        --config-name "${config_name}" \
+        --result-dir "${RESULT_DIR}" \
+        --task-type "${TASK_TYPE}"
+
+    stop_server
+
+    # Allow GPU memory to settle
+    sleep 5
+}
+
+# Run vllm-omni benchmark
+if [ "${RUN_ASYNC}" = true ]; then
+    run_bench "async_chunk"
+fi
+
+# Run HuggingFace baseline benchmark
+if [ "${RUN_HF}" = true ]; then
+    echo ""
+    echo "============================================================"
+    echo " Benchmarking: HuggingFace transformers (offline)"
+    echo "============================================================"
+
+    cd "${PROJECT_ROOT}"
+    python "${SCRIPT_DIR}/transformers/bench_tts_hf.py" \
+        --model "${MODEL}" \
+        --num-prompts "${NUM_PROMPTS}" \
+        --num-warmups "${NUM_WARMUPS}" \
+        --gpu-device "${GPU_DEVICE}" \
+        --config-name "hf_transformers" \
+        --result-dir "${RESULT_DIR}" \
+        --task-type "${TASK_TYPE}"
+
+    # Allow GPU memory to settle
+    sleep 5
+fi
+
+# Plot results
+echo ""
+echo "============================================================"
+echo " Generating plots..."
+echo "============================================================"
+
+RESULT_FILES=""
+LABELS=""
+
+if [ "${RUN_ASYNC}" = true ]; then
+    ASYNC_FILE=$(ls -t "${RESULT_DIR}"/bench_async_chunk_*.json 2>/dev/null | head -1)
+    if [ -n "${ASYNC_FILE}" ]; then
+        RESULT_FILES="${ASYNC_FILE}"
+        LABELS="async_chunk"
+    fi
+fi
+
+if [ "${RUN_HF}" = true ]; then
+    HF_FILE=$(ls -t "${RESULT_DIR}"/bench_hf_transformers_*.json 2>/dev/null | head -1)
+    if [ -n "${HF_FILE}" ]; then
+        if [ -n "${RESULT_FILES}" ]; then
+            RESULT_FILES="${RESULT_FILES} ${HF_FILE}"
+            LABELS="${LABELS} hf_transformers"
+        else
+            RESULT_FILES="${HF_FILE}"
+            LABELS="hf_transformers"
+        fi
+    fi
+fi
+
+if [ -n "${RESULT_FILES}" ]; then
+    python "${SCRIPT_DIR}/plot_results.py" \
+        --results ${RESULT_FILES} \
+        --labels ${LABELS} \
+        --output "${RESULT_DIR}/qwen3_tts_benchmark_${TIMESTAMP}.png"
+fi
+
+echo ""
+echo "============================================================"
+echo " Benchmark complete!"
+echo " Results: ${RESULT_DIR}"
+echo "============================================================"

benchmarks/qwen3-tts/transformers/bench_tts_hf.py

@@ -0,0 +1,301 @@
+"""Benchmark Qwen3-TTS using HuggingFace transformers (qwen_tts library).
+
+Measures E2E latency, RTF, and audio duration for offline (non-serving) inference.
+Results are saved in the same JSON format as bench_tts_serve.py for unified plotting.
+
+Usage:
+    python bench_tts_hf.py \
+        --model Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice \
+        --num-prompts 50 \
+        --num-warmups 3 \
+        --gpu-device 0 \
+        --result-dir results/
+"""
+
+import argparse
+import json
+import time
+from dataclasses import asdict, dataclass, field
+from datetime import datetime
+from pathlib import Path
+
+import numpy as np
+import soundfile as sf
+import torch
+
+PROMPTS = [
+    "Hello, welcome to the voice synthesis benchmark test.",
+    "She said she would be here by noon, but nobody showed up.",
+    "The quick brown fox jumps over the lazy dog near the riverbank.",
+    "I can't believe how beautiful the sunset looks from up here on the mountain.",
+    "Please remember to bring your identification documents to the appointment tomorrow morning.",
+    "Have you ever wondered what it would be like to travel through time and visit ancient civilizations?",
+    "The restaurant on the corner serves the best pasta I have ever tasted in my entire life.",
+    "After the meeting, we should discuss the quarterly results and plan for the next phase.",
+    "Learning a new language takes patience, practice, and a genuine curiosity about other cultures.",
+    "The train leaves at half past seven, so we need to arrive at the station before then.",
+    "Could you please turn down the music a little bit, I'm trying to concentrate on my work.",
+    "It was a dark and stormy night when the old lighthouse keeper heard a knock at the door.",
+]
+
+REF_AUDIO = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-TTS-Repo/clone_2.wav"
+REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."
+INSTRUCT = "Speak in an incredulous tone, but with a hint of panic beginning to creep into your voice."
+
+
+@dataclass
+class BenchmarkResult:
+    config_name: str = ""
+    concurrency: int = 1  # always 1 for offline
+    num_prompts: int = 0
+    completed: int = 0
+    failed: int = 0
+    duration_s: float = 0.0
+    # TTFP stats - not applicable for HF offline, set to E2E for compatibility
+    mean_ttfp_ms: float = 0.0
+    median_ttfp_ms: float = 0.0
+    std_ttfp_ms: float = 0.0
+    p90_ttfp_ms: float = 0.0
+    p95_ttfp_ms: float = 0.0
+    p99_ttfp_ms: float = 0.0
+    # E2E stats (ms)
+    mean_e2e_ms: float = 0.0
+    median_e2e_ms: float = 0.0
+    std_e2e_ms: float = 0.0
+    p90_e2e_ms: float = 0.0
+    p95_e2e_ms: float = 0.0
+    p99_e2e_ms: float = 0.0
+    # RTF stats
+    mean_rtf: float = 0.0
+    median_rtf: float = 0.0
+    std_rtf: float = 0.0
+    p99_rtf: float = 0.0
+    # Audio stats
+    mean_audio_duration_s: float = 0.0
+    total_audio_duration_s: float = 0.0
+    audio_throughput: float = 0.0
+    request_throughput: float = 0.0
+    # Per-request details
+    per_request: list = field(default_factory=list)
+
+
+def generate_audio(model, prompt: str, args):
+    if args.task_type == "Base":
+        return model.generate_voice_clone(
+            text=prompt,
+            language=args.language,
+            ref_audio=REF_AUDIO,
+            ref_text=REF_TEXT,
+        )
+
+    if args.task_type == "VoiceDesign":
+        return model.generate_voice_design(
+            text=prompt,
+            language=args.language,
+            instruct=INSTRUCT,
+        )
+
+    return model.generate_custom_voice(
+        text=prompt,
+        language=args.language,
+        speaker=args.voice,
+    )
+
+
+def run_benchmark(args):
+    from qwen_tts import Qwen3TTSModel
+
+    device = f"cuda:{args.gpu_device}"
+    print(f"Loading model: {args.model} on {device}")
+    model = Qwen3TTSModel.from_pretrained(
+        args.model,
+        device_map=device,
+        dtype=torch.bfloat16,
+    )
+    print("Model loaded.")
+
+    # Build prompt list
+    prompts = [PROMPTS[i % len(PROMPTS)] for i in range(args.num_prompts)]
+
+    # Warmup
+    if args.num_warmups > 0:
+        print(f"Warming up with {args.num_warmups} requests...")
+        for i in range(args.num_warmups):
+            p = PROMPTS[i % len(PROMPTS)]
+            wavs, sr = generate_audio(model, p, args)
+        # Sync GPU
+        torch.cuda.synchronize(device)
+        print("Warmup done.")
+
+    # Benchmark
+    print(f"Running {args.num_prompts} requests sequentially...")
+    e2e_times = []
+    rtfs = []
+    audio_durations = []
+    per_request = []
+    failed = 0
+
+    audio_dir = None
+    if args.save_audio:
+        audio_dir = Path(args.result_dir) / "audio_hf"
+        audio_dir.mkdir(parents=True, exist_ok=True)
+
+    total_start = time.perf_counter()
+
+    for i, prompt in enumerate(prompts):
+        try:
+            torch.cuda.synchronize(device)
+            st = time.perf_counter()
+
+            wavs, sr = generate_audio(model, prompt, args)
+
+            torch.cuda.synchronize(device)
+            elapsed = time.perf_counter() - st
+
+            # Compute audio duration
+            audio_samples = wavs[0]
+            if isinstance(audio_samples, torch.Tensor):
+                audio_samples = audio_samples.cpu().numpy()
+            audio_dur = len(audio_samples) / sr
+
+            rtf = elapsed / audio_dur if audio_dur > 0 else 0.0
+
+            e2e_times.append(elapsed)
+            rtfs.append(rtf)
+            audio_durations.append(audio_dur)
+            per_request.append(
+                {
+                    "e2e_ms": elapsed * 1000,
+                    "ttfp_ms": elapsed * 1000,  # no streaming, TTFP = E2E
+                    "rtf": rtf,
+                    "audio_duration_s": audio_dur,
+                    "prompt": prompt,
+                }
+            )
+
+            if audio_dir:
+                sf.write(str(audio_dir / f"output_{i:04d}.wav"), audio_samples, sr)
+
+            if (i + 1) % 10 == 0 or i == 0:
+                print(
+                    f"  [{i + 1}/{args.num_prompts}] e2e={elapsed * 1000:.0f}ms  rtf={rtf:.3f}  audio={audio_dur:.2f}s"
+                )
+
+        except Exception as e:
+            print(f"  [{i + 1}/{args.num_prompts}] FAILED: {e}")
+            failed += 1
+
+    total_duration = time.perf_counter() - total_start
+    completed = len(e2e_times)
+
+    # Compute stats
+    result = BenchmarkResult(
+        config_name=args.config_name,
+        concurrency=1,
+        num_prompts=args.num_prompts,
+        completed=completed,
+        failed=failed,
+        duration_s=total_duration,
+    )
+
+    if e2e_times:
+        e2e_ms = [t * 1000 for t in e2e_times]
+
+        result.mean_e2e_ms = float(np.mean(e2e_ms))
+        result.median_e2e_ms = float(np.median(e2e_ms))
+        result.std_e2e_ms = float(np.std(e2e_ms))
+        result.p90_e2e_ms = float(np.percentile(e2e_ms, 90))
+        result.p95_e2e_ms = float(np.percentile(e2e_ms, 95))
+        result.p99_e2e_ms = float(np.percentile(e2e_ms, 99))
+
+        # For HF offline, TTFP = E2E (no streaming)
+        result.mean_ttfp_ms = result.mean_e2e_ms
+        result.median_ttfp_ms = result.median_e2e_ms
+        result.std_ttfp_ms = result.std_e2e_ms
+        result.p90_ttfp_ms = result.p90_e2e_ms
+        result.p95_ttfp_ms = result.p95_e2e_ms
+        result.p99_ttfp_ms = result.p99_e2e_ms
+
+        result.mean_rtf = float(np.mean(rtfs))
+        result.median_rtf = float(np.median(rtfs))
+        result.std_rtf = float(np.std(rtfs))
+        result.p99_rtf = float(np.percentile(rtfs, 99))
+
+        result.mean_audio_duration_s = float(np.mean(audio_durations))
+        result.total_audio_duration_s = float(np.sum(audio_durations))
+        result.audio_throughput = result.total_audio_duration_s / total_duration
+        result.request_throughput = completed / total_duration
+        result.per_request = per_request
+
+    # Print summary in standardized performance template
+    W = 50
+    print("")
+    print(f"{'=' * W}")
+    print(f"{'Serving Benchmark Result':^{W}}")
+    print(f"{'=' * W}")
+    print(f"{'Successful requests:':<40}{completed:<10}")
+    print(f"{'Failed requests:':<40}{failed:<10}")
+    print(f"{'Maximum request concurrency:':<40}{1:<10}")
+    print(f"{'Benchmark duration (s):':<40}{total_duration:<10.2f}")
+    print(f"{'Request throughput (req/s):':<40}{result.request_throughput:<10.2f}")
+    print(f"{'-' * W}")
+    print(f"{'End-to-end Latency':^{W}}")
+    print(f"{'-' * W}")
+    print(f"{'Mean E2EL (ms):':<40}{result.mean_e2e_ms:<10.2f}")
+    print(f"{'Median E2EL (ms):':<40}{result.median_e2e_ms:<10.2f}")
+    print(f"{'P99 E2EL (ms):':<40}{result.p99_e2e_ms:<10.2f}")
+    print(f"{'=' * W}")
+    print(f"{'Audio Result':^{W}}")
+    print(f"{'=' * W}")
+    print(f"{'Total audio duration generated (s):':<40}{result.total_audio_duration_s:<10.2f}")
+    print(f"{'Audio throughput (audio duration/s):':<40}{result.audio_throughput:<10.2f}")
+    print(f"{'-' * W}")
+    print(f"{'Time to First Packet':^{W}}")
+    print(f"{'-' * W}")
+    print(f"{'Mean AUDIO_TTFP (ms):':<40}{result.mean_ttfp_ms:<10.2f}")
+    print(f"{'Median AUDIO_TTFP (ms):':<40}{result.median_ttfp_ms:<10.2f}")
+    print(f"{'P99 AUDIO_TTFP (ms):':<40}{result.p99_ttfp_ms:<10.2f}")
+    print(f"{'-' * W}")
+    print(f"{'Real Time Factor':^{W}}")
+    print(f"{'-' * W}")
+    print(f"{'Mean AUDIO_RTF:':<40}{result.mean_rtf:<10.3f}")
+    print(f"{'Median AUDIO_RTF:':<40}{result.median_rtf:<10.3f}")
+    print(f"{'P99 AUDIO_RTF:':<40}{result.p99_rtf:<10.3f}")
+    print(f"{'=' * W}")
+    print("")
+
+    # Save results (as a list with single concurrency=1 entry, matching serve format)
+    result_dir = Path(args.result_dir)
+    result_dir.mkdir(parents=True, exist_ok=True)
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    result_file = result_dir / f"bench_{args.config_name}_{timestamp}.json"
+
+    with open(result_file, "w") as f:
+        json.dump([asdict(result)], f, indent=2)
+    print(f"Results saved to {result_file}")
+
+    return result
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Qwen3-TTS HuggingFace Benchmark")
+    parser.add_argument(
+        "--model", type=str, default="Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice", help="HuggingFace model name or path"
+    )
+    parser.add_argument("--num-prompts", type=int, default=50)
+    parser.add_argument("--num-warmups", type=int, default=3)
+    parser.add_argument("--gpu-device", type=int, default=0)
+    parser.add_argument("--voice", type=str, default="Vivian")
+    parser.add_argument("--language", type=str, default="English")
+    parser.add_argument("--task-type", type=str, default="CustomVoice", choices=["CustomVoice", "VoiceDesign", "Base"])
+    parser.add_argument(
+        "--config-name", type=str, default="hf_transformers", help="Label for this config (used in filenames)"
+    )
+    parser.add_argument("--result-dir", type=str, default="results")
+    parser.add_argument("--save-audio", action="store_true", help="Save generated audio files")
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    run_benchmark(args)

benchmarks/qwen3-tts/vllm_omni/bench_async_chunk.py

@@ -0,0 +1,301 @@
+"""Benchmark comparing async_chunk on vs off for Qwen3-TTS.
+
+Measures TTFP (Time-to-First-Packet), E2E latency, and RTF across
+concurrency levels for both async_chunk modes. Saves results as JSON.
+
+Usage:
+    # Run against a server already serving with a given config:
+    python bench_async_chunk.py \
+        --host 127.0.0.1 --port 8000 \
+        --config-name async_chunk_on \
+        --num-prompts 50 \
+        --max-concurrency 1 10 \
+        --result-dir results/
+"""
+
+import argparse
+import asyncio
+import json
+import time
+from dataclasses import asdict, dataclass, field
+from datetime import datetime
+from pathlib import Path
+
+import aiohttp
+import numpy as np
+from tqdm.asyncio import tqdm
+
+PROMPTS = [
+    "Hello, welcome to the voice synthesis benchmark test.",
+    "She said she would be here by noon, but nobody showed up.",
+    "The quick brown fox jumps over the lazy dog near the riverbank.",
+    "I can't believe how beautiful the sunset looks from up here on the mountain.",
+    "Please remember to bring your identification documents to the appointment tomorrow morning.",
+    "Have you ever wondered what it would be like to travel through time and visit ancient civilizations?",
+    "The restaurant on the corner serves the best pasta I have ever tasted in my entire life.",
+    "After the meeting, we should discuss the quarterly results and plan for the next phase.",
+    "Learning a new language takes patience, practice, and a genuine curiosity about other cultures.",
+    "The train leaves at half past seven, so we need to arrive at the station before then.",
+    "Could you please turn down the music a little bit, I'm trying to concentrate on my work.",
+    "It was a dark and stormy night when the old lighthouse keeper heard a knock at the door.",
+]
+
+
+@dataclass
+class RequestResult:
+    success: bool = False
+    ttfp: float = 0.0
+    e2e: float = 0.0
+    audio_bytes: int = 0
+    audio_duration: float = 0.0
+    rtf: float = 0.0
+    prompt: str = ""
+    error: str = ""
+
+
+@dataclass
+class BenchmarkResult:
+    config_name: str = ""
+    concurrency: int = 0
+    num_prompts: int = 0
+    completed: int = 0
+    failed: int = 0
+    duration_s: float = 0.0
+    mean_ttfp_ms: float = 0.0
+    median_ttfp_ms: float = 0.0
+    std_ttfp_ms: float = 0.0
+    p90_ttfp_ms: float = 0.0
+    p95_ttfp_ms: float = 0.0
+    p99_ttfp_ms: float = 0.0
+    mean_e2e_ms: float = 0.0
+    median_e2e_ms: float = 0.0
+    std_e2e_ms: float = 0.0
+    p90_e2e_ms: float = 0.0
+    p95_e2e_ms: float = 0.0
+    p99_e2e_ms: float = 0.0
+    mean_rtf: float = 0.0
+    median_rtf: float = 0.0
+    std_rtf: float = 0.0
+    mean_audio_duration_s: float = 0.0
+    total_audio_duration_s: float = 0.0
+    audio_throughput: float = 0.0
+    request_throughput: float = 0.0
+    per_request: list = field(default_factory=list)
+
+
+def pcm_bytes_to_duration(num_bytes: int, sample_rate: int = 24000, sample_width: int = 2) -> float:
+    return num_bytes / sample_width / sample_rate
+
+
+async def send_tts_request(
+    session: aiohttp.ClientSession,
+    api_url: str,
+    prompt: str,
+    voice: str = "vivian",
+    language: str = "English",
+    stream: bool = True,
+    pbar: tqdm | None = None,
+) -> RequestResult:
+    payload = {
+        "input": prompt,
+        "voice": voice,
+        "language": language,
+        "stream": stream,
+        "response_format": "pcm",
+    }
+
+    result = RequestResult(prompt=prompt)
+    st = time.perf_counter()
+
+    try:
+        async with session.post(api_url, json=payload) as response:
+            if response.status != 200:
+                result.error = f"HTTP {response.status}: {await response.text()}"
+                return result
+
+            first_chunk = True
+            total_bytes = 0
+
+            async for chunk in response.content.iter_any():
+                if first_chunk and len(chunk) > 0:
+                    result.ttfp = time.perf_counter() - st
+                    first_chunk = False
+                total_bytes += len(chunk)
+
+            result.e2e = time.perf_counter() - st
+            result.audio_bytes = total_bytes
+            result.audio_duration = pcm_bytes_to_duration(total_bytes)
+            if result.audio_duration > 0:
+                result.rtf = result.e2e / result.audio_duration
+            result.success = True
+
+    except Exception as e:
+        result.error = str(e)
+        result.e2e = time.perf_counter() - st
+
+    if pbar:
+        pbar.update(1)
+    return result
+
+
+async def run_benchmark(
+    host: str,
+    port: int,
+    num_prompts: int,
+    max_concurrency: int,
+    num_warmups: int = 3,
+    voice: str = "vivian",
+    language: str = "English",
+    stream: bool = True,
+) -> BenchmarkResult:
+    api_url = f"http://{host}:{port}/v1/audio/speech"
+
+    connector = aiohttp.TCPConnector(limit=max_concurrency, limit_per_host=max_concurrency, keepalive_timeout=60)
+    session = aiohttp.ClientSession(connector=connector, timeout=aiohttp.ClientTimeout(total=600))
+
+    if num_warmups > 0:
+        print(f"  Warming up with {num_warmups} requests...")
+        warmup_tasks = [
+            send_tts_request(session, api_url, PROMPTS[i % len(PROMPTS)], voice, language, stream)
+            for i in range(num_warmups)
+        ]
+        await asyncio.gather(*warmup_tasks)
+        print("  Warmup done.")
+
+    request_prompts = [PROMPTS[i % len(PROMPTS)] for i in range(num_prompts)]
+
+    print(f"  Running {num_prompts} requests with concurrency={max_concurrency}...")
+    semaphore = asyncio.Semaphore(max_concurrency)
+    pbar = tqdm(total=num_prompts, desc=f"  concurrency={max_concurrency}")
+
+    async def limited_request(prompt):
+        async with semaphore:
+            return await send_tts_request(session, api_url, prompt, voice, language, stream, pbar)
+
+    start_time = time.perf_counter()
+    tasks = [asyncio.create_task(limited_request(p)) for p in request_prompts]
+    results: list[RequestResult] = await asyncio.gather(*tasks)
+    duration = time.perf_counter() - start_time
+    pbar.close()
+
+    await session.close()
+
+    successful = [r for r in results if r.success]
+    failed = [r for r in results if not r.success]
+
+    bench = BenchmarkResult(
+        concurrency=max_concurrency,
+        num_prompts=num_prompts,
+        completed=len(successful),
+        failed=len(failed),
+        duration_s=duration,
+    )
+
+    if successful:
+        ttfps = [r.ttfp * 1000 for r in successful]
+        e2es = [r.e2e * 1000 for r in successful]
+        rtfs = [r.rtf for r in successful]
+        audio_durs = [r.audio_duration for r in successful]
+
+        bench.mean_ttfp_ms = float(np.mean(ttfps))
+        bench.median_ttfp_ms = float(np.median(ttfps))
+        bench.std_ttfp_ms = float(np.std(ttfps))
+        bench.p90_ttfp_ms = float(np.percentile(ttfps, 90))
+        bench.p95_ttfp_ms = float(np.percentile(ttfps, 95))
+        bench.p99_ttfp_ms = float(np.percentile(ttfps, 99))
+
+        bench.mean_e2e_ms = float(np.mean(e2es))
+        bench.median_e2e_ms = float(np.median(e2es))
+        bench.std_e2e_ms = float(np.std(e2es))
+        bench.p90_e2e_ms = float(np.percentile(e2es, 90))
+        bench.p95_e2e_ms = float(np.percentile(e2es, 95))
+        bench.p99_e2e_ms = float(np.percentile(e2es, 99))
+
+        bench.mean_rtf = float(np.mean(rtfs))
+        bench.median_rtf = float(np.median(rtfs))
+        bench.std_rtf = float(np.std(rtfs))
+
+        bench.mean_audio_duration_s = float(np.mean(audio_durs))
+        bench.total_audio_duration_s = float(np.sum(audio_durs))
+        bench.audio_throughput = bench.total_audio_duration_s / duration
+        bench.request_throughput = len(successful) / duration
+
+        bench.per_request = [
+            {
+                "ttfp_ms": r.ttfp * 1000,
+                "e2e_ms": r.e2e * 1000,
+                "rtf": r.rtf,
+                "audio_duration_s": r.audio_duration,
+                "prompt": r.prompt,
+            }
+            for r in successful
+        ]
+
+    print(f"\n{'=' * 60}")
+    print(f"  Concurrency: {max_concurrency}  |  Completed: {bench.completed}  |  Failed: {bench.failed}")
+    print(f"  Duration: {duration:.2f}s  |  Throughput: {bench.request_throughput:.2f} req/s")
+    print(
+        f"  TTFP (ms):  mean={bench.mean_ttfp_ms:.1f}  median={bench.median_ttfp_ms:.1f}"
+        f"  p90={bench.p90_ttfp_ms:.1f}  p99={bench.p99_ttfp_ms:.1f}"
+    )
+    print(
+        f"  E2E (ms):   mean={bench.mean_e2e_ms:.1f}  median={bench.median_e2e_ms:.1f}"
+        f"  p90={bench.p90_e2e_ms:.1f}  p99={bench.p99_e2e_ms:.1f}"
+    )
+    print(f"  RTF:        mean={bench.mean_rtf:.3f}  median={bench.median_rtf:.3f}")
+    print(f"  Throughput: {bench.audio_throughput:.2f} audio-sec/wall-sec")
+    print(f"{'=' * 60}\n")
+
+    if failed:
+        for r in failed[:3]:
+            print(f"  [ERROR] {r.error[:200]}")
+
+    return bench
+
+
+async def main(args):
+    all_results = []
+
+    for concurrency in args.max_concurrency:
+        result = await run_benchmark(
+            host=args.host,
+            port=args.port,
+            num_prompts=args.num_prompts,
+            max_concurrency=concurrency,
+            num_warmups=args.num_warmups,
+            voice=args.voice,
+            language=args.language,
+            stream=args.stream,
+        )
+        result.config_name = args.config_name
+        all_results.append(asdict(result))
+
+    result_dir = Path(args.result_dir)
+    result_dir.mkdir(parents=True, exist_ok=True)
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    result_file = result_dir / f"bench_{args.config_name}_{timestamp}.json"
+
+    with open(result_file, "w") as f:
+        json.dump(all_results, f, indent=2)
+    print(f"Results saved to {result_file}")
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Qwen3-TTS async_chunk benchmark client")
+    parser.add_argument("--host", type=str, default="127.0.0.1")
+    parser.add_argument("--port", type=int, default=8000)
+    parser.add_argument("--num-prompts", type=int, default=50)
+    parser.add_argument("--max-concurrency", type=int, nargs="+", default=[1, 10])
+    parser.add_argument("--num-warmups", type=int, default=3)
+    parser.add_argument("--voice", type=str, default="vivian")
+    parser.add_argument("--language", type=str, default="English")
+    parser.add_argument("--stream", action="store_true", default=True)
+    parser.add_argument("--no-stream", dest="stream", action="store_false")
+    parser.add_argument("--config-name", type=str, default="async_chunk_on")
+    parser.add_argument("--result-dir", type=str, default="results")
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    asyncio.run(main(args))

benchmarks/qwen3-tts/vllm_omni/bench_tts_serve.py

@@ -0,0 +1,371 @@
+"""Benchmark client for Qwen3-TTS via /v1/audio/speech endpoint.
+
+Measures TTFP (Time-to-First-Packet), E2E latency, and RTF (Real-Time Factor)
+across configurable concurrency levels. Saves results as JSON for plotting.
+
+Usage:
+    python bench_tts_serve.py \
+        --host 127.0.0.1 --port 8000 \
+        --num-prompts 50 \
+        --max-concurrency 1 4 10 \
+        --result-dir results/
+"""
+
+import argparse
+import asyncio
+import json
+import time
+from dataclasses import asdict, dataclass, field
+from datetime import datetime
+from pathlib import Path
+
+import aiohttp
+import numpy as np
+from tqdm.asyncio import tqdm
+
+PROMPTS = [
+    "Hello, welcome to the voice synthesis benchmark test.",
+    "She said she would be here by noon, but nobody showed up.",
+    "The quick brown fox jumps over the lazy dog near the riverbank.",
+    "I can't believe how beautiful the sunset looks from up here on the mountain.",
+    "Please remember to bring your identification documents to the appointment tomorrow morning.",
+    "Have you ever wondered what it would be like to travel through time and visit ancient civilizations?",
+    "The restaurant on the corner serves the best pasta I have ever tasted in my entire life.",
+    "After the meeting, we should discuss the quarterly results and plan for the next phase.",
+    "Learning a new language takes patience, practice, and a genuine curiosity about other cultures.",
+    "The train leaves at half past seven, so we need to arrive at the station before then.",
+    "Could you please turn down the music a little bit, I'm trying to concentrate on my work.",
+    "It was a dark and stormy night when the old lighthouse keeper heard a knock at the door.",
+]
+REF_AUDIO = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-TTS-Repo/clone_2.wav"
+REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."
+INSTRUCT = "Speak in an incredulous tone, but with a hint of panic beginning to creep into your voice."
+
+
+@dataclass
+class RequestResult:
+    success: bool = False
+    ttfp: float = 0.0  # Time to first audio packet (seconds)
+    e2e: float = 0.0  # End-to-end latency (seconds)
+    audio_bytes: int = 0  # Total audio bytes received
+    audio_duration: float = 0.0  # Audio duration in seconds (estimated from PCM)
+    rtf: float = 0.0  # Real-time factor = e2e / audio_duration
+    prompt: str = ""
+    error: str = ""
+
+
+@dataclass
+class BenchmarkResult:
+    config_name: str = ""
+    concurrency: int = 0
+    num_prompts: int = 0
+    completed: int = 0
+    failed: int = 0
+    duration_s: float = 0.0
+    # TTFP stats (ms)
+    mean_ttfp_ms: float = 0.0
+    median_ttfp_ms: float = 0.0
+    std_ttfp_ms: float = 0.0
+    p90_ttfp_ms: float = 0.0
+    p95_ttfp_ms: float = 0.0
+    p99_ttfp_ms: float = 0.0
+    # E2E stats (ms)
+    mean_e2e_ms: float = 0.0
+    median_e2e_ms: float = 0.0
+    std_e2e_ms: float = 0.0
+    p90_e2e_ms: float = 0.0
+    p95_e2e_ms: float = 0.0
+    p99_e2e_ms: float = 0.0
+    # RTF stats
+    mean_rtf: float = 0.0
+    median_rtf: float = 0.0
+    std_rtf: float = 0.0
+    p99_rtf: float = 0.0
+    # Audio stats
+    mean_audio_duration_s: float = 0.0
+    total_audio_duration_s: float = 0.0
+    audio_throughput: float = 0.0  # audio_duration / wall_time
+    request_throughput: float = 0.0  # requests / second
+    # Per-request details
+    per_request: list = field(default_factory=list)
+
+
+def pcm_bytes_to_duration(num_bytes: int, sample_rate: int = 24000, sample_width: int = 2) -> float:
+    """Convert raw PCM byte count to duration in seconds."""
+    num_samples = num_bytes / sample_width
+    return num_samples / sample_rate
+
+
+def create_payload(
+    prompt: str, task_type: str = "CustomVoice", voice: str = "vivian", language: str = "English"
+) -> dict:
+    payload = {
+        "input": prompt,
+        "language": language,
+        "stream": True,
+        "response_format": "pcm",
+        "task_type": task_type,
+    }
+
+    if task_type == "Base":
+        payload["ref_audio"] = REF_AUDIO
+        payload["ref_text"] = REF_TEXT
+    elif task_type == "CustomVoice":
+        payload["voice"] = voice
+    elif task_type == "VoiceDesign":
+        payload["instructions"] = INSTRUCT
+
+    return payload
+
+
+async def send_tts_request(
+    session: aiohttp.ClientSession,
+    api_url: str,
+    prompt: str,
+    task_type: str = "CustomVoice",
+    voice: str = "vivian",
+    language: str = "English",
+    pbar: tqdm | None = None,
+) -> RequestResult:
+    """Send a streaming TTS request and measure latency metrics."""
+    payload = create_payload(prompt, task_type, voice, language)
+
+    result = RequestResult(prompt=prompt)
+    st = time.perf_counter()
+
+    try:
+        async with session.post(api_url, json=payload) as response:
+            if response.status != 200:
+                result.error = f"HTTP {response.status}: {await response.text()}"
+                result.success = False
+                return result
+
+            first_chunk = True
+            total_bytes = 0
+
+            async for chunk in response.content.iter_any():
+                if first_chunk and len(chunk) > 0:
+                    result.ttfp = time.perf_counter() - st
+                    first_chunk = False
+                total_bytes += len(chunk)
+
+            result.e2e = time.perf_counter() - st
+            result.audio_bytes = total_bytes
+            result.audio_duration = pcm_bytes_to_duration(total_bytes)
+
+            if result.audio_duration > 0:
+                result.rtf = result.e2e / result.audio_duration
+            result.success = True
+
+    except Exception as e:
+        result.error = str(e)
+        result.success = False
+        result.e2e = time.perf_counter() - st
+
+    if pbar:
+        pbar.update(1)
+    return result
+
+
+async def run_benchmark(
+    host: str,
+    port: int,
+    num_prompts: int,
+    max_concurrency: int,
+    num_warmups: int = 3,
+    task_type: str = "CustomVoice",
+    voice: str = "vivian",
+    language: str = "English",
+) -> BenchmarkResult:
+    """Run benchmark at a given concurrency level."""
+    api_url = f"http://{host}:{port}/v1/audio/speech"
+
+    connector = aiohttp.TCPConnector(
+        limit=max_concurrency,
+        limit_per_host=max_concurrency,
+        keepalive_timeout=60,
+    )
+    session = aiohttp.ClientSession(
+        connector=connector,
+        timeout=aiohttp.ClientTimeout(total=600),
+    )
+
+    # Warmup
+    if num_warmups > 0:
+        print(f"  Warming up with {num_warmups} requests...")
+        warmup_tasks = []
+        for i in range(num_warmups):
+            prompt = PROMPTS[i % len(PROMPTS)]
+            warmup_tasks.append(send_tts_request(session, api_url, prompt, task_type, voice, language))
+        await asyncio.gather(*warmup_tasks)
+        print("  Warmup done.")
+
+    # Build request list
+    request_prompts = [PROMPTS[i % len(PROMPTS)] for i in range(num_prompts)]
+
+    # Run benchmark
+    print(f"  Running {num_prompts} requests with concurrency={max_concurrency}...")
+    semaphore = asyncio.Semaphore(max_concurrency)
+    pbar = tqdm(total=num_prompts, desc=f"  concurrency={max_concurrency}")
+
+    async def limited_request(prompt):
+        async with semaphore:
+            return await send_tts_request(session, api_url, prompt, task_type, voice, language, pbar)
+
+    start_time = time.perf_counter()
+    tasks = [asyncio.create_task(limited_request(p)) for p in request_prompts]
+    results: list[RequestResult] = await asyncio.gather(*tasks)
+    duration = time.perf_counter() - start_time
+    pbar.close()
+
+    await session.close()
+
+    # Compute stats
+    successful = [r for r in results if r.success]
+    failed = [r for r in results if not r.success]
+
+    bench = BenchmarkResult(
+        concurrency=max_concurrency,
+        num_prompts=num_prompts,
+        completed=len(successful),
+        failed=len(failed),
+        duration_s=duration,
+    )
+
+    if successful:
+        ttfps = [r.ttfp * 1000 for r in successful]  # convert to ms
+        e2es = [r.e2e * 1000 for r in successful]
+        rtfs = [r.rtf for r in successful]
+        audio_durs = [r.audio_duration for r in successful]
+
+        bench.mean_ttfp_ms = float(np.mean(ttfps))
+        bench.median_ttfp_ms = float(np.median(ttfps))
+        bench.std_ttfp_ms = float(np.std(ttfps))
+        bench.p90_ttfp_ms = float(np.percentile(ttfps, 90))
+        bench.p95_ttfp_ms = float(np.percentile(ttfps, 95))
+        bench.p99_ttfp_ms = float(np.percentile(ttfps, 99))
+
+        bench.mean_e2e_ms = float(np.mean(e2es))
+        bench.median_e2e_ms = float(np.median(e2es))
+        bench.std_e2e_ms = float(np.std(e2es))
+        bench.p90_e2e_ms = float(np.percentile(e2es, 90))
+        bench.p95_e2e_ms = float(np.percentile(e2es, 95))
+        bench.p99_e2e_ms = float(np.percentile(e2es, 99))
+
+        bench.mean_rtf = float(np.mean(rtfs))
+        bench.median_rtf = float(np.median(rtfs))
+        bench.std_rtf = float(np.std(rtfs))
+        bench.p99_rtf = float(np.percentile(rtfs, 99))
+
+        bench.mean_audio_duration_s = float(np.mean(audio_durs))
+        bench.total_audio_duration_s = float(np.sum(audio_durs))
+        bench.audio_throughput = bench.total_audio_duration_s / duration
+        bench.request_throughput = len(successful) / duration
+
+        bench.per_request = [
+            {
+                "ttfp_ms": r.ttfp * 1000,
+                "e2e_ms": r.e2e * 1000,
+                "rtf": r.rtf,
+                "audio_duration_s": r.audio_duration,
+                "prompt": r.prompt,
+            }
+            for r in successful
+        ]
+
+    # Print summary in standardized performance template
+    W = 50
+    print("")
+    print(f"{'=' * W}")
+    print(f"{'Serving Benchmark Result':^{W}}")
+    print(f"{'=' * W}")
+    print(f"{'Successful requests:':<40}{bench.completed:<10}")
+    print(f"{'Failed requests:':<40}{bench.failed:<10}")
+    print(f"{'Maximum request concurrency:':<40}{max_concurrency:<10}")
+    print(f"{'Benchmark duration (s):':<40}{duration:<10.2f}")
+    print(f"{'Request throughput (req/s):':<40}{bench.request_throughput:<10.2f}")
+    print(f"{'-' * W}")
+    print(f"{'End-to-end Latency':^{W}}")
+    print(f"{'-' * W}")
+    print(f"{'Mean E2EL (ms):':<40}{bench.mean_e2e_ms:<10.2f}")
+    print(f"{'Median E2EL (ms):':<40}{bench.median_e2e_ms:<10.2f}")
+    print(f"{'P99 E2EL (ms):':<40}{bench.p99_e2e_ms:<10.2f}")
+    print(f"{'=' * W}")
+    print(f"{'Audio Result':^{W}}")
+    print(f"{'=' * W}")
+    print(f"{'Total audio duration generated (s):':<40}{bench.total_audio_duration_s:<10.2f}")
+    print(f"{'Audio throughput (audio duration/s):':<40}{bench.audio_throughput:<10.2f}")
+    print(f"{'-' * W}")
+    print(f"{'Time to First Packet':^{W}}")
+    print(f"{'-' * W}")
+    print(f"{'Mean AUDIO_TTFP (ms):':<40}{bench.mean_ttfp_ms:<10.2f}")
+    print(f"{'Median AUDIO_TTFP (ms):':<40}{bench.median_ttfp_ms:<10.2f}")
+    print(f"{'P99 AUDIO_TTFP (ms):':<40}{bench.p99_ttfp_ms:<10.2f}")
+    print(f"{'-' * W}")
+    print(f"{'Real Time Factor':^{W}}")
+    print(f"{'-' * W}")
+    print(f"{'Mean AUDIO_RTF:':<40}{bench.mean_rtf:<10.3f}")
+    print(f"{'Median AUDIO_RTF:':<40}{bench.median_rtf:<10.3f}")
+    print(f"{'P99 AUDIO_RTF:':<40}{bench.p99_rtf:<10.3f}")
+    print(f"{'=' * W}")
+    print("")
+
+    if failed:
+        for r in failed[:3]:
+            print(f"  [ERROR] {r.error[:200]}")
+
+    return bench
+
+
+async def main(args):
+    all_results = []
+
+    for concurrency in args.max_concurrency:
+        result = await run_benchmark(
+            host=args.host,
+            port=args.port,
+            num_prompts=args.num_prompts,
+            max_concurrency=concurrency,
+            num_warmups=args.num_warmups,
+            task_type=args.task_type,
+            voice=args.voice,
+            language=args.language,
+        )
+        result.config_name = args.config_name
+        all_results.append(asdict(result))
+
+    # Save results
+    result_dir = Path(args.result_dir)
+    result_dir.mkdir(parents=True, exist_ok=True)
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    result_file = result_dir / f"bench_{args.config_name}_{timestamp}.json"
+
+    with open(result_file, "w") as f:
+        json.dump(all_results, f, indent=2)
+    print(f"Results saved to {result_file}")
+
+    return all_results
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Qwen3-TTS Benchmark Client")
+    parser.add_argument("--host", type=str, default="127.0.0.1")
+    parser.add_argument("--port", type=int, default=8000)
+    parser.add_argument("--num-prompts", type=int, default=50, help="Number of prompts per concurrency level")
+    parser.add_argument(  # noqa: E501
+        "--max-concurrency", type=int, nargs="+", default=[1, 4, 10], help="Concurrency levels to test"
+    )
+    parser.add_argument("--num-warmups", type=int, default=3)
+    parser.add_argument("--task-type", type=str, default="CustomVoice", choices=["CustomVoice", "VoiceDesign", "Base"])
+    parser.add_argument("--voice", type=str, default="vivian")
+    parser.add_argument("--language", type=str, default="English")
+    parser.add_argument(
+        "--config-name", type=str, default="async_chunk", help="Label for this config (used in filenames)"
+    )
+    parser.add_argument("--result-dir", type=str, default="results")
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    asyncio.run(main(args))

benchmarks/qwen3-tts/vllm_omni/plot_async_chunk.py

@@ -0,0 +1,249 @@
+"""Plot TTFP comparison: async_chunk off vs on.
+
+Generates a bar chart with improvement arrows, matching the Qwen3-Omni
+async_chunk benchmark figure style.
+
+Usage:
+    python plot_async_chunk.py \
+        --off results/bench_async_chunk_off_*.json \
+        --on results/bench_async_chunk_on_*.json \
+        --output results/qwen3_tts_async_chunk_ttfp.png
+
+    # Also supports E2E and RTF metrics:
+    python plot_async_chunk.py \
+        --off results/bench_async_chunk_off_*.json \
+        --on results/bench_async_chunk_on_*.json \
+        --metric e2e \
+        --output results/qwen3_tts_async_chunk_e2e.png
+"""
+
+import argparse
+import json
+from pathlib import Path
+
+import matplotlib.pyplot as plt
+import numpy as np
+
+METRIC_CONFIG = {
+    "ttfp": {
+        "key": "mean_ttfp_ms",
+        "ylabel": "TTFP (s)",
+        "title": "TTFP (Time to First Audio Packet) - Qwen3-TTS, by concurrency",
+        "to_seconds": True,
+    },
+    "e2e": {
+        "key": "mean_e2e_ms",
+        "ylabel": "E2E (s)",
+        "title": "E2E Latency - Qwen3-TTS, by concurrency",
+        "to_seconds": True,
+    },
+    "rtf": {
+        "key": "mean_rtf",
+        "ylabel": "RTF",
+        "title": "Real-Time Factor - Qwen3-TTS, by concurrency",
+        "to_seconds": False,
+    },
+}
+
+
+def load_results(path: str) -> list[dict]:
+    with open(path) as f:
+        return json.load(f)
+
+
+def plot_ttfp_comparison(
+    off_results: list[dict],
+    on_results: list[dict],
+    metric: str,
+    output_path: str,
+    title_override: str | None = None,
+):
+    cfg = METRIC_CONFIG[metric]
+    key = cfg["key"]
+    to_seconds = cfg["to_seconds"]
+
+    off_map = {r["concurrency"]: r for r in off_results}
+    on_map = {r["concurrency"]: r for r in on_results}
+    concurrencies = sorted(set(off_map.keys()) & set(on_map.keys()))
+
+    off_vals = []
+    on_vals = []
+    for c in concurrencies:
+        v_off = off_map[c][key]
+        v_on = on_map[c][key]
+        if to_seconds:
+            v_off /= 1000.0
+            v_on /= 1000.0
+        off_vals.append(v_off)
+        on_vals.append(v_on)
+
+    fig, ax = plt.subplots(figsize=(8, 6))
+
+    x = np.arange(len(concurrencies))
+    width = 0.3
+
+    ax.bar(x - width / 2, off_vals, width, label="async_chunk off", color="#87CEEB", edgecolor="none")
+    ax.bar(x + width / 2, on_vals, width, label="async_chunk on", color="#FFF8DC", edgecolor="#DDD8B8")
+
+    # Draw improvement arrows and labels
+    for i in range(len(concurrencies)):
+        v_off = off_vals[i]
+        v_on = on_vals[i]
+        if v_on > 0:
+            improvement = v_off / v_on
+        else:
+            improvement = float("inf")
+
+        # Arrow from top of off-bar to top of on-bar
+        arrow_start_x = x[i] - width / 2
+        arrow_start_y = v_off * 0.95
+        arrow_end_x = x[i] + width / 2
+        arrow_end_y = v_on * 1.05
+
+        ax.annotate(
+            "",
+            xy=(arrow_end_x, arrow_end_y),
+            xytext=(arrow_start_x, arrow_start_y),
+            arrowprops=dict(arrowstyle="->", color="red", lw=1.5),
+        )
+
+        # Improvement label
+        label_x = (arrow_start_x + arrow_end_x) / 2
+        label_y = arrow_start_y + (v_off - v_on) * 0.15
+        ax.text(
+            label_x,
+            label_y,
+            f"{improvement:.1f}x improvement",
+            ha="center",
+            va="bottom",
+            fontsize=10,
+            color="red",
+            fontweight="bold",
+        )
+
+    title = title_override or cfg["title"]
+    ax.set_title(title, fontsize=13, fontweight="bold")
+    ax.set_ylabel(cfg["ylabel"], fontsize=12)
+    ax.set_xlabel("Max concurrency", fontsize=12)
+    ax.set_xticks(x)
+    ax.set_xticklabels([str(c) for c in concurrencies])
+    ax.set_yscale("log")
+    ax.legend(loc="upper left", fontsize=11)
+    ax.grid(axis="y", alpha=0.3, linestyle="--")
+    ax.set_axisbelow(True)
+
+    plt.tight_layout()
+    Path(output_path).parent.mkdir(parents=True, exist_ok=True)
+    plt.savefig(output_path, dpi=150, bbox_inches="tight")
+    print(f"Plot saved to {output_path}")
+    plt.close()
+
+
+def plot_all_metrics(off_results: list[dict], on_results: list[dict], output_path: str):
+    """Generate a 1x3 subplot with TTFP, E2E, and RTF comparisons."""
+    off_map = {r["concurrency"]: r for r in off_results}
+    on_map = {r["concurrency"]: r for r in on_results}
+    concurrencies = sorted(set(off_map.keys()) & set(on_map.keys()))
+
+    fig, axes = plt.subplots(1, 3, figsize=(18, 6))
+    fig.suptitle("Qwen3-TTS: async_chunk on vs off", fontsize=15, fontweight="bold")
+
+    for ax, metric in zip(axes, ["ttfp", "e2e", "rtf"]):
+        cfg = METRIC_CONFIG[metric]
+        key = cfg["key"]
+        to_seconds = cfg["to_seconds"]
+
+        off_vals = []
+        on_vals = []
+        for c in concurrencies:
+            v_off = off_map[c][key]
+            v_on = on_map[c][key]
+            if to_seconds:
+                v_off /= 1000.0
+                v_on /= 1000.0
+            off_vals.append(v_off)
+            on_vals.append(v_on)
+
+        x = np.arange(len(concurrencies))
+        width = 0.3
+        ax.bar(x - width / 2, off_vals, width, label="async_chunk off", color="#87CEEB")
+        ax.bar(x + width / 2, on_vals, width, label="async_chunk on", color="#FFF8DC", edgecolor="#DDD8B8")
+
+        for i in range(len(concurrencies)):
+            if on_vals[i] > 0:
+                improvement = off_vals[i] / on_vals[i]
+                ax.annotate(
+                    "",
+                    xy=(x[i] + width / 2, on_vals[i] * 1.05),
+                    xytext=(x[i] - width / 2, off_vals[i] * 0.95),
+                    arrowprops=dict(arrowstyle="->", color="red", lw=1.5),
+                )
+                label_y = off_vals[i] * 0.85
+                ax.text(x[i], label_y, f"{improvement:.1f}x", ha="center", fontsize=10, color="red", fontweight="bold")
+
+        ax.set_title(cfg["title"].split(" - ")[0], fontsize=12, fontweight="bold")
+        ax.set_ylabel(cfg["ylabel"], fontsize=11)
+        ax.set_xlabel("Max concurrency", fontsize=11)
+        ax.set_xticks(x)
+        ax.set_xticklabels([str(c) for c in concurrencies])
+        if metric != "rtf":
+            ax.set_yscale("log")
+        ax.legend(fontsize=9)
+        ax.grid(axis="y", alpha=0.3, linestyle="--")
+        ax.set_axisbelow(True)
+
+    plt.tight_layout()
+    Path(output_path).parent.mkdir(parents=True, exist_ok=True)
+    plt.savefig(output_path, dpi=150, bbox_inches="tight")
+    print(f"Plot saved to {output_path}")
+    plt.close()
+
+
+def print_table(off_results: list[dict], on_results: list[dict]):
+    off_map = {r["concurrency"]: r for r in off_results}
+    on_map = {r["concurrency"]: r for r in on_results}
+    concurrencies = sorted(set(off_map.keys()) & set(on_map.keys()))
+
+    print("\n## Benchmark Results: async_chunk off vs on\n")
+    print("| Metric | Concurrency | async_chunk off | async_chunk on | Improvement |")
+    print("| --- | --- | --- | --- | --- |")
+
+    for name, key, fmt in [
+        ("TTFP (ms)", "mean_ttfp_ms", ".1f"),
+        ("E2E (ms)", "mean_e2e_ms", ".1f"),
+        ("RTF", "mean_rtf", ".3f"),
+        ("Throughput", "audio_throughput", ".2f"),
+    ]:
+        for c in concurrencies:
+            v_off = off_map[c].get(key, 0)
+            v_on = on_map[c].get(key, 0)
+            if v_on > 0 and key != "audio_throughput":
+                ratio = f"{v_off / v_on:.1f}x"
+            elif v_off > 0 and key == "audio_throughput":
+                ratio = f"{v_on / v_off:.1f}x"
+            else:
+                ratio = "N/A"
+            print(f"| {name} | {c} | {v_off:{fmt}} | {v_on:{fmt}} | {ratio} |")
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Plot async_chunk comparison for Qwen3-TTS")
+    parser.add_argument("--off", type=str, required=True, help="JSON results for async_chunk off")
+    parser.add_argument("--on", type=str, required=True, help="JSON results for async_chunk on")
+    parser.add_argument("--metric", type=str, default="ttfp", choices=["ttfp", "e2e", "rtf", "all"])
+    parser.add_argument("--output", type=str, default="results/qwen3_tts_async_chunk.png")
+    parser.add_argument("--title", type=str, default=None, help="Custom title override")
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    off_results = load_results(args.off)
+    on_results = load_results(args.on)
+
+    print_table(off_results, on_results)
+
+    if args.metric == "all":
+        plot_all_metrics(off_results, on_results, args.output)
+    else:
+        plot_ttfp_comparison(off_results, on_results, args.metric, args.output, args.title)

benchmarks/qwen3-tts/vllm_omni/run_async_chunk_benchmark.sh

@@ -0,0 +1,169 @@
+#!/bin/bash
+# Qwen3-TTS async_chunk on vs off Benchmark
+#
+# Starts two servers (async_chunk on and off), benchmarks both,
+# and generates comparison plots.
+#
+# Usage:
+#   bash run_async_chunk_benchmark.sh
+#
+# Environment variables:
+#   GPU_DEVICE       - GPU index (default: 0)
+#   MODEL            - Model path (default: Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice)
+#   NUM_PROMPTS      - Prompts per concurrency level (default: 50)
+#   CONCURRENCY      - Space-separated concurrency levels (default: "1 10")
+#   PORT_ON          - Port for async_chunk on server (default: 8000)
+#   PORT_OFF         - Port for async_chunk off server (default: 8001)
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
+cd "$PROJECT_ROOT"
+
+GPU_DEVICE="${GPU_DEVICE:-0}"
+MODEL="${MODEL:-Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice}"
+NUM_PROMPTS="${NUM_PROMPTS:-50}"
+CONCURRENCY="${CONCURRENCY:-1 10}"
+NUM_WARMUPS="${NUM_WARMUPS:-3}"
+PORT_ON="${PORT_ON:-8000}"
+PORT_OFF="${PORT_OFF:-8001}"
+RESULT_DIR="${SCRIPT_DIR}/results"
+TIMESTAMP="$(date +%Y%m%d_%H%M%S)"
+
+# The bundled ``vllm_omni/deploy/qwen3_tts.yaml`` is auto-loaded by the model
+# registry; no ``--deploy-config`` flag needed on the default (ON) path.
+# async_chunk OFF is selected by the ``--no-async-chunk`` CLI flag —
+# the single ``qwen3_tts`` pipeline dispatches to the end-to-end codec
+# processor when ``deploy.async_chunk`` is false.
+
+mkdir -p "${RESULT_DIR}"
+
+echo "============================================================"
+echo " Qwen3-TTS async_chunk Benchmark"
+echo "============================================================"
+echo " GPU:            ${GPU_DEVICE}"
+echo " Model:          ${MODEL}"
+echo " Prompts:        ${NUM_PROMPTS}"
+echo " Concurrency:    ${CONCURRENCY}"
+echo " Port (on/off):  ${PORT_ON} / ${PORT_OFF}"
+echo " Results:        ${RESULT_DIR}"
+echo "============================================================"
+
+cleanup() {
+    echo "Cleaning up servers..."
+    kill "$PID_ON" 2>/dev/null || true
+    kill "$PID_OFF" 2>/dev/null || true
+    wait "$PID_ON" 2>/dev/null || true
+    wait "$PID_OFF" 2>/dev/null || true
+}
+trap cleanup EXIT
+
+wait_for_server() {
+    local port=$1
+    local name=$2
+    local max_wait=300
+    local elapsed=0
+    echo "Waiting for ${name} server on port ${port}..."
+    while ! curl -s "http://localhost:${port}/health" >/dev/null 2>&1; do
+        sleep 5
+        elapsed=$((elapsed + 5))
+        if [ $elapsed -ge $max_wait ]; then
+            echo "ERROR: ${name} server failed to start within ${max_wait}s"
+            exit 1
+        fi
+    done
+    echo "${name} server ready (${elapsed}s)"
+}
+
+# ---- Phase 1: Start async_chunk ON server ----
+echo ""
+echo "[Phase 1] Starting async_chunk ON server on port ${PORT_ON}..."
+CUDA_VISIBLE_DEVICES=${GPU_DEVICE} vllm-omni serve "${MODEL}" \
+    --host 0.0.0.0 --port "${PORT_ON}" \
+    --trust-remote-code --enforce-eager --omni \
+    > "${RESULT_DIR}/server_on_${TIMESTAMP}.log" 2>&1 &
+PID_ON=$!
+
+wait_for_server "${PORT_ON}" "async_chunk_on"
+
+echo "[Phase 1] Benchmarking async_chunk ON..."
+# shellcheck disable=SC2086
+python "${SCRIPT_DIR}/bench_async_chunk.py" \
+    --host 127.0.0.1 --port "${PORT_ON}" \
+    --config-name "async_chunk_on" \
+    --num-prompts "${NUM_PROMPTS}" \
+    --max-concurrency ${CONCURRENCY} \
+    --num-warmups "${NUM_WARMUPS}" \
+    --result-dir "${RESULT_DIR}"
+
+echo "[Phase 1] Stopping async_chunk ON server..."
+kill "$PID_ON" 2>/dev/null || true
+wait "$PID_ON" 2>/dev/null || true
+sleep 5
+
+# ---- Phase 2: Start async_chunk OFF server ----
+echo ""
+echo "[Phase 2] Starting async_chunk OFF server on port ${PORT_OFF}..."
+CUDA_VISIBLE_DEVICES=${GPU_DEVICE} vllm-omni serve "${MODEL}" \
+    --no-async-chunk \
+    --host 0.0.0.0 --port "${PORT_OFF}" \
+    --trust-remote-code --enforce-eager --omni \
+    > "${RESULT_DIR}/server_off_${TIMESTAMP}.log" 2>&1 &
+PID_OFF=$!
+
+wait_for_server "${PORT_OFF}" "async_chunk_off"
+
+echo "[Phase 2] Benchmarking async_chunk OFF (non-streaming)..."
+# shellcheck disable=SC2086
+python "${SCRIPT_DIR}/bench_async_chunk.py" \
+    --host 127.0.0.1 --port "${PORT_OFF}" \
+    --config-name "async_chunk_off" \
+    --num-prompts "${NUM_PROMPTS}" \
+    --max-concurrency ${CONCURRENCY} \
+    --num-warmups "${NUM_WARMUPS}" \
+    --no-stream \
+    --result-dir "${RESULT_DIR}"
+
+echo "[Phase 2] Stopping async_chunk OFF server..."
+kill "$PID_OFF" 2>/dev/null || true
+wait "$PID_OFF" 2>/dev/null || true
+
+# ---- Phase 3: Plot results ----
+echo ""
+echo "[Phase 3] Generating plots..."
+
+# Find the latest result files
+RESULT_ON=$(ls -t "${RESULT_DIR}"/bench_async_chunk_on_*.json 2>/dev/null | head -1)
+RESULT_OFF=$(ls -t "${RESULT_DIR}"/bench_async_chunk_off_*.json 2>/dev/null | head -1)
+
+if [ -z "$RESULT_ON" ] || [ -z "$RESULT_OFF" ]; then
+    echo "ERROR: Could not find result files. Check logs in ${RESULT_DIR}/"
+    exit 1
+fi
+
+echo "  ON results:  ${RESULT_ON}"
+echo "  OFF results: ${RESULT_OFF}"
+
+# TTFP comparison (main figure)
+python "${SCRIPT_DIR}/plot_async_chunk.py" \
+    --off "${RESULT_OFF}" \
+    --on "${RESULT_ON}" \
+    --metric ttfp \
+    --output "${RESULT_DIR}/qwen3_tts_async_chunk_ttfp.png"
+
+# All metrics comparison
+python "${SCRIPT_DIR}/plot_async_chunk.py" \
+    --off "${RESULT_OFF}" \
+    --on "${RESULT_ON}" \
+    --metric all \
+    --output "${RESULT_DIR}/qwen3_tts_async_chunk_all.png"
+
+echo ""
+echo "============================================================"
+echo " Benchmark complete!"
+echo " Results: ${RESULT_DIR}/"
+echo " Plots:"
+echo "   - ${RESULT_DIR}/qwen3_tts_async_chunk_ttfp.png"
+echo "   - ${RESULT_DIR}/qwen3_tts_async_chunk_all.png"
+echo "============================================================"

benchmarks/tts/README.md

@@ -1,227 +0,0 @@
-# TTS Universal Benchmark
-
-A model-agnostic serving benchmark for TTS models in vllm-omni. One CLI
-(`bench_tts.py`) + one YAML registry (`model_configs.yaml`) drive perf and
-quality runs for every registered checkpoint: **Qwen3-TTS** (Base / CustomVoice)
-and **VoxCPM2** today, more to come.
-
-The same three task types — `voice_clone`, `default_voice`, `voice_design` —
-are wired into both the manual CLI and the DFX nightly CI matrix
-(`tests/dfx/perf/tests/test_tts.json`).
-
-## Quick start
-
-### 1. Start the server
-
-```bash
-vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-Base --omni --port 8000
-```
-
-The server auto-loads its Deploy YAML from `vllm_omni/deploy/qwen3_tts.yaml`
-(Pipeline + Deploy schema introduced in #2383). No `--stage-configs-path` or
-`--deploy-config` flag is needed for any registered model.
-
-### 2. Run the benchmark (`vllm bench serve --omni`)
-
-The primary, directly-controllable path. Copy-paste one of these and tweak
-any bench flag (sampling params, endpoint, extra body, warmups, etc.):
-
-#### voice_clone (Qwen3-TTS-Base, seed-tts dataset)
-
-```bash
-vllm bench serve --omni \
-    --host 127.0.0.1 --port 8000 \
-    --model Qwen/Qwen3-TTS-12Hz-1.7B-Base \
-    --backend openai-audio-speech \
-    --endpoint /v1/audio/speech \
-    --dataset-name seed-tts \
-    --dataset-path /path/to/seed-tts-eval \
-    --seed-tts-locale en \
-    --num-prompts 20 --num-warmups 2 \
-    --extra-body '{"task_type":"Base"}' \
-    --max-concurrency 1 --request-rate inf \
-    --percentile-metrics ttft,e2el,audio_rtf,audio_ttfp,audio_duration \
-    --save-result --result-dir ./results
-```
-
-#### default_voice (Qwen3-TTS-CustomVoice, bundled seed_tts_smoke)
-
-```bash
-vllm bench serve --omni \
-    --host 127.0.0.1 --port 8000 \
-    --model Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
-    --backend openai-audio-speech \
-    --endpoint /v1/audio/speech \
-    --dataset-name seed-tts-text \
-    --dataset-path benchmarks/build_dataset/seed_tts_smoke \
-    --seed-tts-locale en \
-    --num-prompts 20 --num-warmups 2 \
-    --extra-body '{"voice":"Vivian","language":"English","task_type":"CustomVoice"}' \
-    --max-concurrency 1 --request-rate inf \
-    --percentile-metrics ttft,e2el,audio_rtf,audio_ttfp,audio_duration \
-    --save-result --result-dir ./results
-```
-
-#### voice_design (Qwen3-TTS-CustomVoice, bundled seed_tts_design)
-
-```bash
-vllm bench serve --omni \
-    --host 127.0.0.1 --port 8000 \
-    --model Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
-    --backend openai-audio-speech \
-    --endpoint /v1/audio/speech \
-    --dataset-name seed-tts-design \
-    --dataset-path benchmarks/build_dataset/seed_tts_design \
-    --seed-tts-locale en \
-    --num-prompts 20 --num-warmups 2 \
-    --extra-body '{"task_type":"VoiceDesign","language":"English"}' \
-    --max-concurrency 1 --request-rate inf \
-    --percentile-metrics ttft,e2el,audio_rtf,audio_ttfp,audio_duration \
-    --save-result --result-dir ./results
-```
-
-#### Add WER / SIM / UTMOS to any of the above
-
-Append `--seed-tts-wer-eval` (and optionally `SEED_TTS_EVAL_DEVICE=cuda:0`
-in the env, per PR #2558). This triggers the seed-tts-eval protocol:
-Whisper-large-v3 ASR → WER, WavLM embeddings → SIM, balacoon/utmos → UTMOS.
-
-### 3. Convenience wrapper (`bench_tts.py`)
-
-If you're running the **canonical** configuration for a registered model,
-`bench_tts.py` loads the right defaults from `model_configs.yaml` and
-emits the exact `vllm bench serve --omni` command above — useful for
-concurrency sweeps and multi-task runs:
-
-```bash
-# Smallest smoke — 5 prompts, concurrency=1
-python benchmarks/tts/bench_tts.py \
-    --model Qwen/Qwen3-TTS-12Hz-1.7B-Base \
-    --task voice_clone \
-    --dataset-path /path/to/seed-tts-eval \
-    --concurrency 1 --num-prompts 5 \
-    --output-dir ./results
-
-# Full concurrency sweep
-python benchmarks/tts/bench_tts.py \
-    --model Qwen/Qwen3-TTS-12Hz-1.7B-Base \
-    --task voice_clone \
-    --dataset-path /path/to/seed-tts-eval \
-    --concurrency 1 2 4 8 16 32 \
-    --num-prompts 20 \
-    --output-dir ./results
-
-# With WER / SIM / UTMOS quality eval (adds ASR + embedding compute)
-python benchmarks/tts/bench_tts.py \
-    --model Qwen/Qwen3-TTS-12Hz-1.7B-Base \
-    --task voice_clone \
-    --dataset-path /path/to/seed-tts-eval \
-    --wer-eval \
-    --concurrency 4 --num-prompts 200 \
-    --output-dir ./results
-```
-
-### 4. Plot a sweep
-
-```bash
-python benchmarks/tts/plot_results.py \
-    --results ./results/*.json \
-    --output ./results/curve.png
-```
-
-Outputs TTFP / RTF / throughput curves (and a markdown table) for every
-`(task, concurrency)` combination in the result set.
-
-## Task types
-
-| Task            | Dataset           | Request body                                        | Checkpoints that support it              |
-|-----------------|-------------------|-----------------------------------------------------|------------------------------------------|
-| `voice_clone`   | `seed-tts`        | `ref_audio` + `ref_text` + `task_type=Base`         | `Qwen3-TTS-*-Base`, `VoxCPM2`            |
-| `default_voice` | `seed-tts-text`   | `voice=Vivian` + `task_type=CustomVoice`            | `Qwen3-TTS-*-CustomVoice`                |
-| `voice_design`  | `seed-tts-design` | `instructions=<natural-language description>` + `task_type=VoiceDesign` | `Qwen3-TTS-*-CustomVoice` |
-
-**`-CustomVoice` checkpoints do NOT ship `speaker_encoder` weights**, so
-voice_clone requests raise `ValueError` at model runtime. Use `-Base` for
-voice_clone.
-
-## Adding a new TTS model
-
-Drop an entry into `model_configs.yaml` — no Python changes required:
-
-```yaml
-models:
-  <org>/<model-id>:
-    supported_tasks: [voice_clone]          # or default_voice / voice_design
-    backend: openai-audio-speech            # vllm bench serve backend
-    endpoint: /v1/audio/speech              # OpenAI-compatible endpoint
-    task_extra_body:                        # merged into every request's body
-      voice_clone:
-        task_type: Base
-```
-
-Then add the model's Deploy YAML under `vllm_omni/deploy/<model>.yaml`
-(Pipeline + Deploy schema) and it's immediately benchable.
-
-## Datasets
-
-| Dataset            | Bundled? | Format            | Source                                                         |
-|--------------------|----------|-------------------|----------------------------------------------------------------|
-| `seed-tts-design`  | ✅       | 5-field meta.lst  | `benchmarks/build_dataset/seed_tts_design/en/meta.lst` (20 prompts) |
-| `seed_tts_smoke`   | ✅       | 4-field meta.lst  | `benchmarks/build_dataset/seed_tts_smoke/en/meta.lst` (20 text-only) |
-| `seed-tts`         | ❌       | 4-field meta.lst + WAVs | Google-Drive: [BytedanceSpeech/seed-tts-eval][seedtts] (~1.2 GB) |
-| `seed-tts-text`    | ❌       | 4-field meta.lst  | Same archive as `seed-tts` (wav column unused)                 |
-
-[seedtts]: https://github.com/BytedanceSpeech/seed-tts-eval
-
-For manual voice_clone / default_voice runs against the full corpus, follow
-`benchmarks/build_dataset/download_process_data_seedtts.md` and point
-`--dataset-path` at the extracted `seedtts_testset` directory.
-
-## DFX nightly CI
-
-`tests/dfx/perf/tests/test_tts.json` wires three perf regimes plus quality:
-
-| eval_phase    | concurrency | purpose                                                 | Baseline metrics                        |
-|---------------|-------------|---------------------------------------------------------|-----------------------------------------|
-| `latency`     | 1           | Single-request TTFP / RTF SLO                           | `median_audio_ttfp_ms`, `median_audio_rtf` |
-| `throughput`  | 8           | Codec-batching cliff sentinel (PDF #272 concurrency≥8)  | `median_audio_ttfp_ms`, `median_audio_rtf` |
-| `quality`     | 4           | WER / SIM / UTMOS regression (disabled in CI by default)| `mean_audio_rtf`                        |
-
-Why `median_*` for latency/throughput and `mean_*` for quality: latency
-distributions have cold-start tails that drag the mean; quality aggregates
-over 200 prompts so single-request outliers don't matter.
-
-Quality entries are `enabled: false` in CI because seed-tts-eval is not
-staged in the Buildkite container (matches the precedent in
-PR #2558 — quality runs are manual / release-validation, not nightly).
-
-## Concurrency cliff regression sentinel
-
-Observed on H20-3e, Qwen3-TTS-1.7B (measured pre-merge on this branch):
-
-| Task          | Model         | c=1    | c=4    | **c=8**    | c=16   | c=32   |
-|---------------|---------------|--------|--------|------------|--------|--------|
-| voice_clone   | 1.7B-Base     | RTF 0.15 / TTFP 165ms | 0.28 / 412ms | **0.49 / 1701ms** | 0.72 / 3355ms | 0.77 / 3772ms |
-| voice_design  | 1.7B-CustomVoice | RTF 0.08 / TTFP 53ms  | 0.11 / 154ms | **0.21 / 872ms**  | 0.33 / 1801ms | 0.38 / 1989ms |
-
-Both models show a **4–6× TTFP jump from c=4 to c=8** while audio throughput
-saturates around c=4–8 — the codec-bs=1 bottleneck documented in
-vllm-project/vllm-omni#272. The `throughput` CI regime at c=8 is the
-sentinel for regressions in this area.
-
-## File layout
-
-```
-benchmarks/tts/
-├── README.md                  (this file)
-├── bench_tts.py               CLI — serve-mode benchmark driver
-├── bench_voxcpm_offline.py    CLI — offline VoxCPM benchmark (sync + streaming)
-├── plot_results.py            Generate per-task / per-concurrency curves
-└── model_configs.yaml         Model registry (supported tasks + extra body)
-```
-
-## Related
-
-- Upstream seed-tts-eval integration: vllm-project/vllm-omni#2558
-- Pipeline + Deploy schema: vllm-project/vllm-omni#2383
-- Concurrency cliff RFC: vllm-project/vllm-omni#272

benchmarks/tts/bench_tts.py

@@ -1,308 +0,0 @@
-#!/usr/bin/env python3
-"""Universal TTS benchmark CLI for vllm-omni.
-
-Runs ``vllm bench serve --omni`` with model-aware defaults loaded from
-``model_configs.yaml``.  Supports Qwen3-TTS, VoxCPM2, and any future TTS
-model registered in the config file -- no code changes needed to add models.
-
-Usage::
-
-    python benchmarks/tts/bench_tts.py \\
-        --model Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \\
-        --task voice_clone \\
-        --locale en \\
-        --concurrency 1 4 \\
-        --num-prompts 20 \\
-        --dataset-path /path/to/seed-tts-eval \\
-        --host localhost --port 8000
-
-See ``--help`` for full option list.
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import math
-import os
-import subprocess
-import sys
-from datetime import datetime
-from pathlib import Path
-from typing import Any
-
-import yaml
-
-
-def _vllm_omni_bin() -> str:
-    """Return the vllm-omni (or vllm) binary co-located with the current Python."""
-    bin_dir = Path(sys.executable).parent
-    for candidate in ("vllm-omni", "vllm"):
-        p = bin_dir / candidate
-        if p.is_file():
-            return str(p)
-    return "vllm-omni"  # fall back and let the shell resolve it
-
-
-_REPO_ROOT = Path(__file__).resolve().parent.parent.parent
-_SCRIPT_DIR = Path(__file__).resolve().parent
-_DEFAULT_MODEL_CONFIGS = _SCRIPT_DIR / "model_configs.yaml"
-
-# Maps task name to the dataset_name used with vllm bench serve
-_TASK_TO_DATASET: dict[str, str] = {
-    "voice_clone": "seed-tts",
-    "default_voice": "seed-tts-text",
-    "voice_design": "seed-tts-design",
-}
-
-# Default design dataset path (bundled with the repo)
-_DEFAULT_DESIGN_DATASET_PATH = str(_REPO_ROOT / "benchmarks" / "build_dataset" / "seed_tts_design")
-
-
-def load_model_configs(path: Path) -> dict[str, Any]:
-    """Load model registry from YAML file."""
-    with open(path, encoding="utf-8") as f:
-        data = yaml.safe_load(f)
-    return data.get("models", {})
-
-
-def build_bench_args(
-    *,
-    host: str,
-    port: int,
-    model: str,
-    task: str,
-    model_cfg: dict[str, Any],
-    locale: str,
-    num_prompts: int,
-    concurrency: int | None,
-    dataset_path: str | None,
-    wer_eval: bool,
-    output_dir: str | None,
-    result_filename: str | None,
-    extra_cli_args: list[str],
-) -> list[str]:
-    """Build the ``vllm bench serve --omni`` command for one (task, concurrency) run."""
-    dataset_name = _TASK_TO_DATASET[task]
-    backend: str = model_cfg["backend"]
-    endpoint: str = model_cfg["endpoint"]
-    task_extra_body: dict[str, Any] = (model_cfg.get("task_extra_body") or {}).get(task) or {}
-
-    # Resolve dataset path
-    if dataset_path:
-        resolved_dataset_path = dataset_path
-    elif task == "voice_design":
-        resolved_dataset_path = _DEFAULT_DESIGN_DATASET_PATH
-    else:
-        resolved_dataset_path = None
-
-    cmd = [
-        _vllm_omni_bin(),
-        "bench",
-        "serve",
-        "--omni",
-        "--host",
-        host,
-        "--port",
-        str(port),
-        "--model",
-        model,
-        "--backend",
-        backend,
-        "--endpoint",
-        endpoint,
-        "--dataset-name",
-        dataset_name,
-        "--num-prompts",
-        str(num_prompts),
-        "--num-warmups",
-        "2",
-        "--percentile-metrics",
-        "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
-    ]
-
-    if resolved_dataset_path:
-        cmd += ["--dataset-path", resolved_dataset_path]
-
-    if locale:
-        cmd += ["--seed-tts-locale", locale]
-
-    if task_extra_body:
-        cmd += ["--extra-body", json.dumps(task_extra_body, separators=(",", ":"))]
-
-    if concurrency is not None:
-        cmd += ["--max-concurrency", str(concurrency), "--request-rate", "inf"]
-
-    if wer_eval:
-        cmd.append("--seed-tts-wer-eval")
-
-    if output_dir or result_filename:
-        out_dir = output_dir or "."
-        os.makedirs(out_dir, exist_ok=True)
-        cmd += ["--save-result", "--result-dir", out_dir]
-        if result_filename:
-            cmd += ["--result-filename", result_filename]
-
-    cmd += extra_cli_args
-    return cmd
-
-
-def run_one_benchmark(cmd: list[str]) -> dict[str, Any] | None:
-    """Run a single benchmark subprocess and return parsed JSON result if available."""
-    print(f"\n{'=' * 60}")
-    print("Running:", " ".join(cmd))
-    print("=" * 60)
-    result = subprocess.run(cmd, check=False)
-    if result.returncode != 0:
-        print(f"[bench_tts] WARNING: benchmark exited with code {result.returncode}")
-        return None
-    # If --save-result was used, find the result file
-    try:
-        result_dir_idx = cmd.index("--result-dir")
-        result_dir = Path(cmd[result_dir_idx + 1])
-        if "--result-filename" in cmd:
-            fname_idx = cmd.index("--result-filename")
-            result_file = result_dir / cmd[fname_idx + 1]
-        else:
-            # find most recently modified json
-            jsons = sorted(result_dir.glob("result_*.json"), key=lambda p: p.stat().st_mtime)
-            result_file = jsons[-1] if jsons else None
-        if result_file and result_file.is_file():
-            return json.loads(result_file.read_text(encoding="utf-8"))
-    except (ValueError, IndexError, OSError):
-        pass
-    return None
-
-
-def print_summary_table(results: list[dict[str, Any]]) -> None:
-    """Print a unified metrics table across all (task, concurrency) runs."""
-    if not results:
-        return
-    header = (
-        f"{'Task':<16} {'Concurrency':>11} {'RTF mean':>10} "
-        f"{'TTFP (ms)':>10} {'Throughput':>12} {'WER':>7} {'SIM':>7} {'UTMOS':>7}"
-    )
-    print(f"\n{'=' * len(header)}")
-    print("BENCHMARK SUMMARY")
-    print("=" * len(header))
-    print(header)
-    print("-" * len(header))
-    for r in results:
-        task = r.get("_task", "?")
-        conc = r.get("_concurrency", "?")
-        rtf = r.get("mean_audio_rtf", float("nan"))
-        ttfp = r.get("mean_audio_ttfp_ms", float("nan"))
-        throughput = r.get("audio_throughput", float("nan"))
-        wer = r.get("seed_tts_mean_wer", float("nan"))
-        sim = r.get("seed_tts_mean_sim", float("nan"))
-        utmos = r.get("seed_tts_mean_utmos", float("nan"))
-
-        def fmt(v: float, digits: int = 3) -> str:
-            return f"{v:.{digits}f}" if not math.isnan(v) else "  n/a"
-
-        print(
-            f"{task:<16} {str(conc):>11} {fmt(rtf):>10} {fmt(ttfp, 0):>10} "
-            f"{fmt(throughput):>12} {fmt(wer):>7} {fmt(sim):>7} {fmt(utmos):>7}"
-        )
-    print("=" * len(header))
-
-
-def main() -> None:
-    """Entry point for the universal TTS benchmark CLI."""
-    parser = argparse.ArgumentParser(
-        description=__doc__,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    parser.add_argument(
-        "--model", required=True, help="HuggingFace model ID (e.g. Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice)"
-    )
-    parser.add_argument("--task", default="all", help="Task type: voice_clone | default_voice | voice_design | all")
-    parser.add_argument("--locale", default="en", choices=["en", "zh"])
-    parser.add_argument("--concurrency", type=int, nargs="+", default=[1, 4], metavar="N")
-    parser.add_argument(
-        "--num-prompts",
-        type=int,
-        nargs="+",
-        default=[20],
-        metavar="N",
-        help="Number of prompts per run. If one value, applied to all concurrency levels.",
-    )
-    parser.add_argument(
-        "--dataset-path", default=None, help="Root of seed-tts-eval dataset (required for voice_clone/default_voice)"
-    )
-    parser.add_argument("--wer-eval", action="store_true", help="Enable WER/SIM/UTMOS quality eval")
-    parser.add_argument("--output-dir", default=None, help="Directory to save result JSON files")
-    parser.add_argument("--host", default="localhost")
-    parser.add_argument("--port", type=int, default=8000)
-    parser.add_argument("--model-configs", default=str(_DEFAULT_MODEL_CONFIGS), help="Path to model_configs.yaml")
-    parser.add_argument("extra", nargs=argparse.REMAINDER, help="Extra args passed directly to vllm bench serve")
-    args = parser.parse_args()
-
-    model_configs = load_model_configs(Path(args.model_configs))
-    if args.model not in model_configs:
-        known = "\n  ".join(model_configs.keys())
-        print(f"[bench_tts] ERROR: model '{args.model}' not in model_configs.yaml.\nKnown models:\n  {known}")
-        sys.exit(1)
-
-    model_cfg = model_configs[args.model]
-    supported_tasks: list[str] = model_cfg.get("supported_tasks", [])
-
-    tasks_to_run: list[str]
-    if args.task == "all":
-        tasks_to_run = supported_tasks
-    elif args.task in supported_tasks:
-        tasks_to_run = [args.task]
-    else:
-        print(
-            f"[bench_tts] ERROR: task '{args.task}' not supported by {args.model}.\nSupported tasks: {supported_tasks}"
-        )
-        sys.exit(1)
-
-    # Align num_prompts list with concurrency list
-    num_prompts_list: list[int] = args.num_prompts
-    if len(num_prompts_list) == 1:
-        num_prompts_list = num_prompts_list * len(args.concurrency)
-    elif len(num_prompts_list) != len(args.concurrency):
-        print(
-            f"[bench_tts] ERROR: --num-prompts ({len(num_prompts_list)} values) must be "
-            f"length 1 or match --concurrency ({len(args.concurrency)} values)."
-        )
-        sys.exit(1)
-
-    all_results: list[dict[str, Any]] = []
-
-    for task in tasks_to_run:
-        for concurrency, num_prompts in zip(args.concurrency, num_prompts_list):
-            ts = datetime.now().strftime("%Y%m%d-%H%M%S")
-            result_filename = f"bench_tts_{args.model.replace('/', '_')}_{task}_c{concurrency}_{ts}.json"
-            cmd = build_bench_args(
-                host=args.host,
-                port=args.port,
-                model=args.model,
-                task=task,
-                model_cfg=model_cfg,
-                locale=args.locale,
-                num_prompts=num_prompts,
-                concurrency=concurrency,
-                dataset_path=args.dataset_path,
-                wer_eval=args.wer_eval,
-                output_dir=args.output_dir,
-                result_filename=result_filename,
-                extra_cli_args=args.extra or [],
-            )
-            result = run_one_benchmark(cmd)
-            if result is not None:
-                result["_task"] = task
-                result["_concurrency"] = concurrency
-                all_results.append(result)
-                # Persist the metadata so plot_results.py can pick it up.
-                if args.output_dir and result_filename:
-                    result_path = Path(args.output_dir) / result_filename
-                    if result_path.is_file():
-                        result_path.write_text(json.dumps(result, indent=2), encoding="utf-8")
-
-    print_summary_table(all_results)
-
-
-if __name__ == "__main__":
-    main()

benchmarks/tts/model_configs.yaml

@@ -1,39 +0,0 @@
-# Universal TTS benchmark model registry.
-# Maps HuggingFace model ID → supported tasks + per-task extra body fields.
-# To add a new TTS model: add an entry here. No code changes required.
-#
-# The server auto-loads its Deploy YAML from vllm_omni/deploy/<model>.yaml via
-# the Pipeline + Deploy schema introduced in #2383, so no stage_config path
-# is tracked here.
-
-models:
-  # -CustomVoice checkpoints lack speaker_encoder weights, so voice_clone is
-  # NOT supported (an attempt raises ValueError from _extract_speaker_embedding
-  # at model runtime).  Use -Base for voice_clone.
-  Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice:
-    supported_tasks: [default_voice, voice_design]
-    backend: openai-audio-speech
-    endpoint: /v1/audio/speech
-    task_extra_body:
-      default_voice:
-        voice: Vivian
-        language: English
-        task_type: CustomVoice
-      voice_design:
-        task_type: VoiceDesign
-        language: English
-
-  Qwen/Qwen3-TTS-12Hz-1.7B-Base:
-    supported_tasks: [voice_clone]
-    backend: openai-audio-speech
-    endpoint: /v1/audio/speech
-    task_extra_body:
-      voice_clone:
-        task_type: Base
-
-  openbmb/VoxCPM2:
-    supported_tasks: [voice_clone]
-    backend: openai-audio-speech
-    endpoint: /v1/audio/speech
-    task_extra_body:
-      voice_clone: {}

benchmarks/tts/plot_results.py

@@ -1,324 +0,0 @@
-"""Plot universal TTS benchmark results.
-
-Reads JSON files saved by ``bench_tts.py`` (via ``vllm bench serve --omni``)
-and generates comparison bar charts grouped by task type.
-
-Metrics plotted:
-- AUDIO_TTFP  (mean audio time-to-first-packet, ms)
-- E2EL        (mean end-to-end latency, ms)
-- Audio RTF   (mean real-time factor)
-- Audio throughput (audio-seconds / wall-second)
-
-Quality metrics (WER / SIM / UTMOS) are printed in a table when present.
-
-Usage::
-
-    # Single run — one JSON per task, all in results/
-    python benchmarks/tts/plot_results.py \\
-        --results results/bench_tts_*.json \\
-        --output results/tts_benchmark.png
-
-    # Compare two runs (e.g. async_chunk on vs off)
-    python benchmarks/tts/plot_results.py \\
-        --results run_a/bench_tts_*.json \\
-        --results run_b/bench_tts_*.json \\
-        --labels "async_chunk_on" "async_chunk_off" \\
-        --output results/comparison.png
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import math
-from pathlib import Path
-
-import matplotlib.pyplot as plt
-import numpy as np
-
-# ---------------------------------------------------------------------------
-# JSON loading
-# ---------------------------------------------------------------------------
-
-
-def load_run(paths: list[str]) -> list[dict]:
-    """Load and merge all JSON files for one run into a flat list of records.
-
-    Each record is expected to have at least ``_concurrency`` (int) and
-    ``_task`` (str) keys injected by ``bench_tts.py``.  Records that come
-    from a file that contains a list are flattened.
-    """
-    records: list[dict] = []
-    for p in paths:
-        raw = json.loads(Path(p).read_text(encoding="utf-8"))
-        if isinstance(raw, list):
-            records.extend(raw)
-        elif isinstance(raw, dict):
-            records.append(raw)
-    return records
-
-
-def _get(record: dict, key: str) -> float:
-    v = record.get(key, float("nan"))
-    if v is None or (isinstance(v, float) and math.isnan(v)):
-        return float("nan")
-    try:
-        return float(v)
-    except (TypeError, ValueError):
-        return float("nan")
-
-
-# ---------------------------------------------------------------------------
-# Plotting helpers
-# ---------------------------------------------------------------------------
-
-
-def _bar_group(
-    ax: plt.Axes,
-    x: np.ndarray,
-    data_per_label: dict[str, list[float]],
-    width: float,
-    colors: list[str],
-    ylabel: str,
-    title: str,
-    concurrency_labels: list[str],
-    fmt: str = ".1f",
-) -> None:
-    n = len(data_per_label)
-    offsets = np.linspace(-(n - 1) * width / 2, (n - 1) * width / 2, n) if n > 1 else [0.0]
-
-    for i, (label, values) in enumerate(data_per_label.items()):
-        plot_vals = [0.0 if math.isnan(v) else v for v in values]
-        bar = ax.bar(x + offsets[i], plot_vals, width, label=label, color=colors[i % len(colors)], alpha=0.85)
-        max_val = max((v for v in values if not math.isnan(v)), default=1.0)
-        for rect, val in zip(bar, values):
-            if not math.isnan(val) and val > 0:
-                ax.text(
-                    rect.get_x() + rect.get_width() / 2,
-                    rect.get_height() + max_val * 0.02,
-                    f"{val:{fmt}}",
-                    ha="center",
-                    va="bottom",
-                    fontsize=8,
-                    fontweight="bold",
-                )
-
-    ax.set_xlabel("Concurrency", fontsize=11)
-    ax.set_ylabel(ylabel, fontsize=11)
-    ax.set_title(title, fontsize=12, fontweight="bold")
-    ax.set_xticks(x)
-    ax.set_xticklabels(concurrency_labels)
-    ax.legend(fontsize=9)
-    ax.grid(axis="y", alpha=0.3)
-    ax.set_axisbelow(True)
-
-
-COLORS = ["#2196F3", "#FF5722", "#4CAF50", "#FFC107", "#9C27B0"]
-
-
-# ---------------------------------------------------------------------------
-# Comparison plot (multiple labels / runs)
-# ---------------------------------------------------------------------------
-
-
-def plot_comparison(
-    all_runs: list[list[dict]],
-    labels: list[str],
-    output_path: str,
-    task_filter: str | None = None,
-    title_prefix: str = "TTS",
-) -> None:
-    """One 2×2 subplot per task found in the data."""
-    # Determine tasks to plot
-    tasks: list[str] = []
-    for run in all_runs:
-        for r in run:
-            t = r.get("_task", "unknown")
-            if t not in tasks:
-                tasks.append(t)
-    if task_filter:
-        tasks = [t for t in tasks if t == task_filter]
-
-    n_tasks = len(tasks)
-    if n_tasks == 0:
-        print("[plot_results] No tasks found in data.")
-        return
-
-    fig, axes_grid = plt.subplots(n_tasks, 4, figsize=(18, 4.5 * n_tasks))
-    fig.suptitle(f"{title_prefix} Benchmark", fontsize=15, fontweight="bold")
-
-    # Ensure axes_grid is always 2D
-    if n_tasks == 1:
-        axes_grid = [axes_grid]
-
-    for row_idx, task in enumerate(tasks):
-        # Collect concurrencies across all runs for this task
-        all_concs: set[int] = set()
-        for run in all_runs:
-            for r in run:
-                if r.get("_task") == task:
-                    c = r.get("_concurrency")
-                    if c is not None:
-                        all_concs.add(int(c))
-        concurrencies = sorted(all_concs)
-        x = np.arange(len(concurrencies))
-        conc_labels = [str(c) for c in concurrencies]
-
-        def _series(run: list[dict], metric_key: str) -> list[float]:
-            conc_map = {int(r["_concurrency"]): r for r in run if r.get("_task") == task and "_concurrency" in r}
-            return [_get(conc_map.get(c, {}), metric_key) for c in concurrencies]
-
-        metrics = [
-            ("mean_audio_ttfp_ms", "TTFP (ms)", "Time-to-First-Packet", ".0f"),
-            ("mean_e2el_ms", "E2E Latency (ms)", "End-to-End Latency", ".0f"),
-            ("mean_audio_rtf", "RTF", "Real-Time Factor (RTF)", ".3f"),
-            ("audio_throughput", "audio-s / wall-s", "Audio Throughput", ".2f"),
-        ]
-
-        axes_row = axes_grid[row_idx]
-        for col_idx, (key, ylabel, subtitle, fmt) in enumerate(metrics):
-            data_per_label = {lbl: _series(run, key) for lbl, run in zip(labels, all_runs)}
-            _bar_group(
-                axes_row[col_idx],
-                x,
-                data_per_label,
-                width=0.3 if len(labels) > 1 else 0.5,
-                colors=COLORS,
-                ylabel=ylabel,
-                title=f"{task} — {subtitle}",
-                concurrency_labels=conc_labels,
-                fmt=fmt,
-            )
-
-    plt.tight_layout()
-    Path(output_path).parent.mkdir(parents=True, exist_ok=True)
-    plt.savefig(output_path, dpi=150, bbox_inches="tight")
-    print(f"Plot saved to {output_path}")
-    plt.close()
-
-
-# ---------------------------------------------------------------------------
-# Markdown comparison table
-# ---------------------------------------------------------------------------
-
-
-def print_comparison_table(all_runs: list[list[dict]], labels: list[str]) -> None:
-    tasks: list[str] = []
-    for run in all_runs:
-        for r in run:
-            t = r.get("_task", "unknown")
-            if t not in tasks:
-                tasks.append(t)
-
-    perf_metrics = [
-        ("TTFP (ms)", "mean_audio_ttfp_ms", ".1f"),
-        ("E2E (ms)", "mean_e2el_ms", ".1f"),
-        ("RTF", "mean_audio_rtf", ".3f"),
-        ("Throughput (a-s/s)", "audio_throughput", ".2f"),
-    ]
-    quality_metrics = [
-        ("WER (%)", "seed_tts_mean_wer", ".1f"),
-        ("SIM", "seed_tts_mean_sim", ".3f"),
-        ("UTMOS", "seed_tts_mean_utmos", ".2f"),
-    ]
-
-    for task in tasks:
-        all_concs: set[int] = set()
-        for run in all_runs:
-            for r in run:
-                if r.get("_task") == task:
-                    c = r.get("_concurrency")
-                    if c is not None:
-                        all_concs.add(int(c))
-        concurrencies = sorted(all_concs)
-
-        print(f"\n## {task}\n")
-        col_header = "| Metric | Concurrency |" + "".join(f" {lbl} |" for lbl in labels)
-        sep = "| --- | --- |" + " --- |" * len(labels)
-        print(col_header)
-        print(sep)
-
-        for metric, key, fmt in perf_metrics + quality_metrics:
-            for c in concurrencies:
-                row = f"| {metric} | {c} |"
-                for run in all_runs:
-                    conc_map = {
-                        int(r["_concurrency"]): r for r in run if r.get("_task") == task and "_concurrency" in r
-                    }
-                    val = _get(conc_map.get(c, {}), key)
-                    row += f" {val:{fmt}} |" if not math.isnan(val) else " n/a |"
-                print(row)
-
-        # Improvement column (2-run comparison only)
-        if len(all_runs) == 2:
-            print(f"\n### Improvement ({labels[0]} vs {labels[1]})\n")
-            print("| Metric | Concurrency | Change |")
-            print("| --- | --- | --- |")
-            for metric, key, _ in perf_metrics:
-                for c in concurrencies:
-                    conc_map0 = {
-                        int(r["_concurrency"]): r for r in all_runs[0] if r.get("_task") == task and "_concurrency" in r
-                    }
-                    conc_map1 = {
-                        int(r["_concurrency"]): r for r in all_runs[1] if r.get("_task") == task and "_concurrency" in r
-                    }
-                    v0 = _get(conc_map0.get(c, {}), key)
-                    v1 = _get(conc_map1.get(c, {}), key)
-                    if not math.isnan(v0) and not math.isnan(v1) and v1 > 0:
-                        pct = (v1 - v0) / v1 * 100
-                        print(f"| {metric} | {c} | {pct:+.1f}% |")
-
-
-# ---------------------------------------------------------------------------
-# CLI
-# ---------------------------------------------------------------------------
-
-
-def parse_args() -> argparse.Namespace:
-    parser = argparse.ArgumentParser(
-        description=__doc__,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    parser.add_argument(
-        "--results",
-        type=str,
-        nargs="+",
-        action="append",
-        required=True,
-        metavar="FILE",
-        help="JSON result file(s) for one run. Repeat --results for multiple runs to compare.",
-    )
-    parser.add_argument(
-        "--labels",
-        type=str,
-        nargs="+",
-        default=None,
-        help="Label for each --results group (must match the number of --results groups).",
-    )
-    parser.add_argument("--output", type=str, default="results/tts_benchmark.png", help="Output image path.")
-    parser.add_argument("--title", type=str, default="TTS", help="Title prefix for the plot.")
-    parser.add_argument("--task", type=str, default=None, help="Filter to a single task (e.g. voice_clone).")
-    return parser.parse_args()
-
-
-def main() -> None:
-    args = parse_args()
-
-    # args.results is a list-of-lists due to action="append"
-    all_runs: list[list[dict]] = [load_run(group) for group in args.results]
-    n_runs = len(all_runs)
-
-    labels: list[str]
-    if args.labels:
-        if len(args.labels) != n_runs:
-            raise SystemExit(f"--labels count ({len(args.labels)}) must match --results groups ({n_runs})")
-        labels = args.labels
-    else:
-        labels = [f"run{i + 1}" for i in range(n_runs)]
-
-    print_comparison_table(all_runs, labels)
-    plot_comparison(all_runs, labels, args.output, task_filter=args.task, title_prefix=args.title)
-
-
-if __name__ == "__main__":
-    main()

benchmarks/voxcpm/README.md

@@ -0,0 +1,119 @@
+# VoxCPM Benchmark
+
+This directory contains both:
+
+- online serving benchmark through the OpenAI-compatible `/v1/audio/speech` API
+- offline benchmark for `Omni` / `AsyncOmni`
+- full offline smoke-matrix orchestration
+
+Both benchmark paths report:
+
+- TTFP: time to first PCM packet
+- E2E latency
+- RTF: real-time factor (`e2e / audio_duration`)
+
+## Offline Benchmark
+
+Single offline benchmark run:
+
+```bash
+python benchmarks/voxcpm/vllm_omni/bench_tts_offline.py \
+    --model /path/to/voxcpm-model \
+    --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm.yaml \
+    --text "This is a split-stage VoxCPM synthesis example running on vLLM Omni." \
+    --warmup-runs 1 \
+    --output-dir benchmarks/voxcpm/results/offline_single
+```
+
+Streaming offline benchmark:
+
+```bash
+python benchmarks/voxcpm/vllm_omni/bench_tts_offline.py \
+    --model /path/to/voxcpm-model \
+    --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \
+    --text "This is a split-stage VoxCPM streaming example running on vLLM Omni." \
+    --warmup-runs 1 \
+    --output-dir benchmarks/voxcpm/results/offline_streaming
+```
+
+Full fixed offline matrix, equivalent to the old `examples/offline_inference/voxcpm/test.py`:
+
+```bash
+python benchmarks/voxcpm/vllm_omni/run_offline_matrix.py \
+    --model /path/to/voxcpm-model \
+    --ref-audio /path/to/reference.wav \
+    --ref-text "The exact transcript spoken in reference.wav." \
+    --output-root benchmarks/voxcpm/results/offline_matrix
+```
+
+The full matrix covers both routes:
+
+- streaming: `voxcpm_async_chunk.yaml`
+- sync: `voxcpm.yaml`
+
+And these six scenarios under each route:
+
+- warmup + single TTS
+- warmup + single voice cloning
+- warmup + batch TTS
+- warmup + batch voice cloning
+- cold single TTS
+- cold single voice cloning
+
+`bench_tts_offline.py` itself no longer writes `summary.json` / `results.json`; it prints TTFP / RTF inline and saves generated WAV files only. The matrix runner keeps only per-case `run.log`.
+
+## Start the Server
+
+Async-chunk:
+
+```bash
+vllm serve /path/to/voxcpm-model \
+    --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \
+    --trust-remote-code \
+    --enforce-eager \
+    --omni \
+    --port 8091
+```
+
+Non-streaming:
+
+```bash
+vllm serve /path/to/voxcpm-model \
+    --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm.yaml \
+    --trust-remote-code \
+    --enforce-eager \
+    --omni \
+    --port 8091
+```
+
+## Run the Benchmark
+
+```bash
+python benchmarks/voxcpm/vllm_omni/bench_tts_serve.py \
+    --host 127.0.0.1 \
+    --port 8091 \
+    --num-prompts 20 \
+    --max-concurrency 1 \
+    --result-dir /tmp/voxcpm_bench
+```
+
+Voice cloning benchmark:
+
+```bash
+python benchmarks/voxcpm/vllm_omni/bench_tts_serve.py \
+    --host 127.0.0.1 \
+    --port 8091 \
+    --num-prompts 10 \
+    --max-concurrency 1 \
+    --ref-audio https://example.com/reference.wav \
+    --ref-text "The exact transcript spoken in the reference audio." \
+    --result-dir /tmp/voxcpm_clone_bench
+```
+
+## Notes
+
+- The benchmark uses `stream=true` and `response_format=pcm` so TTFP is measured from the first audio packet.
+- `RTF < 1.0` means the server generates audio faster than real time.
+- For `voxcpm_async_chunk.yaml`, keep concurrency at `1`. This matches native VoxCPM streaming more closely.
+- Do not benchmark concurrent online streaming on `voxcpm_async_chunk.yaml`; use `voxcpm.yaml` for multi-request throughput runs.
+- For the offline matrix mode, `--ref-audio` and `--ref-text` are required because clone cases are part of the fixed coverage set.

benchmarks/voxcpm/vllm_omni/bench_tts_offline.py

@@ -6,27 +6,6 @@ Supports both:
 - text-only synthesis
 - voice cloning
 - text/clone batch inputs from txt or jsonl
-
-Usage::
-
-    # Sync (default voice)
-    python benchmarks/tts/bench_voxcpm_offline.py \\
-        --model /path/to/VoxCPM \\
-        --text "Hello world" \\
-        --output-dir results/audio/
-
-    # Streaming (async_chunk)
-    python benchmarks/tts/bench_voxcpm_offline.py \\
-        --model /path/to/VoxCPM \\
-        --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \\
-        --txt-prompts prompts.txt \\
-        --output-dir results/audio/
-
-    # Voice cloning batch via JSONL
-    python benchmarks/tts/bench_voxcpm_offline.py \\
-        --model /path/to/VoxCPM \\
-        --jsonl-prompts prompts.jsonl \\
-        --output-dir results/audio/
 """
 
 from __future__ import annotations
@@ -47,23 +26,9 @@ from vllm.utils.argparse_utils import FlexibleArgumentParser
 
 from vllm_omni import AsyncOmni, Omni
 
-
-def _find_repo_root(start: Path) -> Path:
-    """Walk up from ``start`` until a repo marker is found.
-
-    Falls back to ``parents[2]`` for backwards compatibility if no marker hits
-    (which can only happen in unusual checkouts — the tree should always have
-    pyproject.toml + vllm_omni/ at the top level).
-    """
-    for candidate in [start, *start.parents]:
-        if (candidate / "pyproject.toml").is_file() and (candidate / "vllm_omni").is_dir():
-            return candidate
-    return start.parents[2]
-
-
-REPO_ROOT = _find_repo_root(Path(__file__).resolve())
+REPO_ROOT = Path(__file__).resolve().parents[3]
 DEFAULT_STAGE_ASYNC = REPO_ROOT / "vllm_omni" / "model_executor" / "stage_configs" / "voxcpm_async_chunk.yaml"
-DEFAULT_STAGE_SYNC = REPO_ROOT / "vllm_omni" / "deploy" / "voxcpm.yaml"
+DEFAULT_STAGE_SYNC = REPO_ROOT / "vllm_omni" / "model_executor" / "stage_configs" / "voxcpm.yaml"
 
 logger = logging.getLogger(__name__)
 
@@ -508,6 +473,9 @@ def parse_args():
 
 def _is_streaming_stage_config(stage_configs_path: str) -> bool:
     cfg_name = Path(stage_configs_path).name.lower()
+    # Keep routing purely config-path based:
+    # - voxcpm.yaml => sync
+    # - voxcpm_async_chunk.yaml => streaming
     return "async_chunk" in cfg_name
 
 

benchmarks/voxcpm/vllm_omni/bench_tts_serve.py

@@ -0,0 +1,283 @@
+"""Benchmark VoxCPM via /v1/audio/speech.
+
+Reports TTFP (time to first packet), E2E latency, and RTF (real-time factor).
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import time
+from dataclasses import asdict, dataclass, field
+from datetime import datetime
+from pathlib import Path
+
+import aiohttp
+import numpy as np
+from tqdm.asyncio import tqdm
+
+DEFAULT_MODEL = "OpenBMB/VoxCPM1.5"
+DEFAULT_SAMPLE_RATE = 24000
+PROMPTS = [
+    "Hello, welcome to the VoxCPM speech benchmark.",
+    "This is a short benchmark prompt for online text-to-speech generation.",
+    "The quick brown fox jumps over the lazy dog near the riverbank.",
+    "Please remember to bring your identification documents tomorrow morning.",
+    "Learning a new language takes patience, practice, and curiosity.",
+    "This benchmark reports TTFP and RTF for the VoxCPM online serving path.",
+]
+
+
+@dataclass
+class RequestResult:
+    success: bool = False
+    ttfp: float = 0.0
+    e2e: float = 0.0
+    audio_bytes: int = 0
+    audio_duration: float = 0.0
+    rtf: float = 0.0
+    prompt: str = ""
+    error: str = ""
+
+
+@dataclass
+class BenchmarkResult:
+    concurrency: int = 0
+    num_prompts: int = 0
+    completed: int = 0
+    failed: int = 0
+    duration_s: float = 0.0
+    mean_ttfp_ms: float = 0.0
+    median_ttfp_ms: float = 0.0
+    p95_ttfp_ms: float = 0.0
+    mean_e2e_ms: float = 0.0
+    median_e2e_ms: float = 0.0
+    p95_e2e_ms: float = 0.0
+    mean_rtf: float = 0.0
+    median_rtf: float = 0.0
+    p95_rtf: float = 0.0
+    total_audio_duration_s: float = 0.0
+    request_throughput: float = 0.0
+    per_request: list[dict[str, float | str]] = field(default_factory=list)
+
+
+def pcm_bytes_to_duration(num_bytes: int, sample_rate: int = DEFAULT_SAMPLE_RATE, sample_width: int = 2) -> float:
+    num_samples = num_bytes / sample_width
+    return num_samples / sample_rate
+
+
+async def send_tts_request(
+    session: aiohttp.ClientSession,
+    api_url: str,
+    *,
+    model: str,
+    prompt: str,
+    ref_audio: str | None,
+    ref_text: str | None,
+    pbar: tqdm | None = None,
+) -> RequestResult:
+    payload: dict[str, object] = {
+        "model": model,
+        "input": prompt,
+        "stream": True,
+        "response_format": "pcm",
+    }
+    if ref_audio is not None:
+        payload["ref_audio"] = ref_audio
+    if ref_text is not None:
+        payload["ref_text"] = ref_text
+
+    result = RequestResult(prompt=prompt)
+    started_at = time.perf_counter()
+
+    try:
+        async with session.post(api_url, json=payload) as response:
+            if response.status != 200:
+                result.error = f"HTTP {response.status}: {await response.text()}"
+                return result
+
+            first_chunk = True
+            total_bytes = 0
+            async for chunk in response.content.iter_any():
+                if not chunk:
+                    continue
+                if first_chunk:
+                    result.ttfp = time.perf_counter() - started_at
+                    first_chunk = False
+                total_bytes += len(chunk)
+
+            result.e2e = time.perf_counter() - started_at
+            result.audio_bytes = total_bytes
+            result.audio_duration = pcm_bytes_to_duration(total_bytes)
+            if result.audio_duration > 0:
+                result.rtf = result.e2e / result.audio_duration
+            result.success = True
+    except Exception as e:
+        result.error = str(e)
+        result.e2e = time.perf_counter() - started_at
+
+    if pbar is not None:
+        pbar.update(1)
+    return result
+
+
+async def run_benchmark(
+    *,
+    host: str,
+    port: int,
+    model: str,
+    num_prompts: int,
+    max_concurrency: int,
+    num_warmups: int,
+    ref_audio: str | None,
+    ref_text: str | None,
+) -> BenchmarkResult:
+    api_url = f"http://{host}:{port}/v1/audio/speech"
+    connector = aiohttp.TCPConnector(limit=max_concurrency, limit_per_host=max_concurrency, keepalive_timeout=60)
+    timeout = aiohttp.ClientTimeout(total=600)
+
+    async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
+        if num_warmups > 0:
+            print(f"  Warming up with {num_warmups} requests...")
+            warmup_tasks = [
+                send_tts_request(
+                    session,
+                    api_url,
+                    model=model,
+                    prompt=PROMPTS[i % len(PROMPTS)],
+                    ref_audio=ref_audio,
+                    ref_text=ref_text,
+                )
+                for i in range(num_warmups)
+            ]
+            await asyncio.gather(*warmup_tasks)
+            print("  Warmup done.")
+
+        request_prompts = [PROMPTS[i % len(PROMPTS)] for i in range(num_prompts)]
+        semaphore = asyncio.Semaphore(max_concurrency)
+        pbar = tqdm(total=num_prompts, desc=f"  concurrency={max_concurrency}")
+
+        async def limited_request(prompt: str) -> RequestResult:
+            async with semaphore:
+                return await send_tts_request(
+                    session,
+                    api_url,
+                    model=model,
+                    prompt=prompt,
+                    ref_audio=ref_audio,
+                    ref_text=ref_text,
+                    pbar=pbar,
+                )
+
+        started_at = time.perf_counter()
+        results = await asyncio.gather(*[asyncio.create_task(limited_request(prompt)) for prompt in request_prompts])
+        duration = time.perf_counter() - started_at
+        pbar.close()
+
+    succeeded = [result for result in results if result.success]
+    bench = BenchmarkResult(
+        concurrency=max_concurrency,
+        num_prompts=num_prompts,
+        completed=len(succeeded),
+        failed=len(results) - len(succeeded),
+        duration_s=duration,
+    )
+
+    if not succeeded:
+        return bench
+
+    ttfps = np.array([result.ttfp * 1000 for result in succeeded], dtype=np.float64)
+    e2es = np.array([result.e2e * 1000 for result in succeeded], dtype=np.float64)
+    rtfs = np.array([result.rtf for result in succeeded], dtype=np.float64)
+    audio_durations = np.array([result.audio_duration for result in succeeded], dtype=np.float64)
+
+    bench.mean_ttfp_ms = float(np.mean(ttfps))
+    bench.median_ttfp_ms = float(np.median(ttfps))
+    bench.p95_ttfp_ms = float(np.percentile(ttfps, 95))
+    bench.mean_e2e_ms = float(np.mean(e2es))
+    bench.median_e2e_ms = float(np.median(e2es))
+    bench.p95_e2e_ms = float(np.percentile(e2es, 95))
+    bench.mean_rtf = float(np.mean(rtfs))
+    bench.median_rtf = float(np.median(rtfs))
+    bench.p95_rtf = float(np.percentile(rtfs, 95))
+    bench.total_audio_duration_s = float(np.sum(audio_durations))
+    bench.request_throughput = len(succeeded) / duration if duration > 0 else 0.0
+    bench.per_request = [
+        {
+            "prompt": result.prompt,
+            "ttfp_ms": result.ttfp * 1000,
+            "e2e_ms": result.e2e * 1000,
+            "rtf": result.rtf,
+            "audio_duration_s": result.audio_duration,
+        }
+        for result in succeeded
+    ]
+
+    return bench
+
+
+def print_summary(result: BenchmarkResult) -> None:
+    width = 54
+    print("")
+    print("=" * width)
+    print(f"{'VoxCPM Serving Benchmark':^{width}}")
+    print("=" * width)
+    print(f"concurrency         : {result.concurrency}")
+    print(f"requests            : {result.completed}/{result.num_prompts} succeeded")
+    print(f"wall time (s)       : {result.duration_s:.3f}")
+    print(f"mean TTFP (ms)      : {result.mean_ttfp_ms:.2f}")
+    print(f"p95 TTFP (ms)       : {result.p95_ttfp_ms:.2f}")
+    print(f"mean E2E (ms)       : {result.mean_e2e_ms:.2f}")
+    print(f"p95 E2E (ms)        : {result.p95_e2e_ms:.2f}")
+    print(f"mean RTF            : {result.mean_rtf:.3f}")
+    print(f"p95 RTF             : {result.p95_rtf:.3f}")
+    print(f"request throughput  : {result.request_throughput:.2f} req/s")
+    print("=" * width)
+
+
+async def main_async(args) -> None:
+    result_dir = Path(args.result_dir)
+    result_dir.mkdir(parents=True, exist_ok=True)
+
+    all_results: list[BenchmarkResult] = []
+    for concurrency in args.max_concurrency:
+        result = await run_benchmark(
+            host=args.host,
+            port=args.port,
+            model=args.model,
+            num_prompts=args.num_prompts,
+            max_concurrency=concurrency,
+            num_warmups=args.num_warmups,
+            ref_audio=args.ref_audio,
+            ref_text=args.ref_text,
+        )
+        print_summary(result)
+        all_results.append(result)
+
+    payload = {
+        "model": args.model,
+        "created_at": datetime.utcnow().isoformat() + "Z",
+        "results": [asdict(result) for result in all_results],
+    }
+    result_path = result_dir / "bench_tts_serve.json"
+    result_path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
+    print(f"Saved results to: {result_path}")
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Benchmark VoxCPM via /v1/audio/speech")
+    parser.add_argument("--host", default="127.0.0.1", help="Server host")
+    parser.add_argument("--port", type=int, default=8091, help="Server port")
+    parser.add_argument("--model", default=DEFAULT_MODEL, help="Model name or path")
+    parser.add_argument("--num-prompts", type=int, default=20, help="Number of prompts to send")
+    parser.add_argument("--max-concurrency", type=int, nargs="+", default=[1], help="Concurrency levels to benchmark")
+    parser.add_argument("--num-warmups", type=int, default=3, help="Warmup request count")
+    parser.add_argument("--ref-audio", default=None, help="Reference audio URL or data URL for voice cloning")
+    parser.add_argument("--ref-text", default=None, help="Reference audio transcript for voice cloning")
+    parser.add_argument("--result-dir", default="results", help="Directory to save benchmark JSON")
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    asyncio.run(main_async(parse_args()))

benchmarks/voxcpm/vllm_omni/run_offline_matrix.py

@@ -0,0 +1,303 @@
+"""Run the full offline VoxCPM smoke matrix.
+
+This script keeps the old `test.py` coverage, but delegates each case to
+`bench_tts_offline.py` so the benchmark runner itself stays focused on a
+single execution path.
+"""
+
+from __future__ import annotations
+
+import shlex
+import subprocess
+import sys
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from vllm.utils.argparse_utils import FlexibleArgumentParser
+
+REPO_ROOT = Path(__file__).resolve().parents[3]
+BENCH_SCRIPT = Path(__file__).with_name("bench_tts_offline.py")
+DEFAULT_STAGE_ASYNC = REPO_ROOT / "vllm_omni" / "model_executor" / "stage_configs" / "voxcpm_async_chunk.yaml"
+DEFAULT_STAGE_SYNC = REPO_ROOT / "vllm_omni" / "model_executor" / "stage_configs" / "voxcpm.yaml"
+DEFAULT_OUTPUT_ROOT = BENCH_SCRIPT.parents[1] / "results" / "offline_matrix"
+
+SINGLE_TTS_TEXT = "This is a single text-to-speech smoke test for VoxCPM on vLLM Omni."
+SINGLE_CLONE_TEXT = "This sentence is synthesized with the cloned voice for validation."
+BATCH_TTS_TEXTS = [
+    "The first batch text-to-speech sample validates sequential batch execution.",
+    "The second batch text-to-speech sample checks another prompt in the same file.",
+    "The third batch text-to-speech sample completes the sequential batch path.",
+]
+BATCH_CLONE_TEXTS = [
+    "The first cloned sample validates sequential batch voice cloning.",
+    "The second cloned sample checks the same reference voice on another prompt.",
+    "The third cloned sample finishes the shared-reference clone batch path.",
+]
+
+
+@dataclass(frozen=True, slots=True)
+class ModeSpec:
+    name: str
+    stage_config: Path
+
+
+@dataclass(frozen=True, slots=True)
+class CaseSpec:
+    name: str
+    warmup_runs: int
+    prompt_kind: str
+    voice_clone: bool
+
+
+@dataclass(frozen=True, slots=True)
+class CaseResult:
+    mode: str
+    case: str
+    returncode: int
+    elapsed_s: float
+    output_dir: Path
+    log_path: Path
+
+    @property
+    def ok(self) -> bool:
+        return self.returncode == 0
+
+
+MODE_SPECS = [
+    ModeSpec(name="streaming", stage_config=DEFAULT_STAGE_ASYNC),
+    ModeSpec(name="sync", stage_config=DEFAULT_STAGE_SYNC),
+]
+
+CASE_SPECS = [
+    CaseSpec(name="warmup_single_tts", warmup_runs=1, prompt_kind="single", voice_clone=False),
+    CaseSpec(name="warmup_single_clone", warmup_runs=1, prompt_kind="single", voice_clone=True),
+    CaseSpec(name="warmup_batch_tts", warmup_runs=1, prompt_kind="batch", voice_clone=False),
+    CaseSpec(name="warmup_batch_clone", warmup_runs=1, prompt_kind="batch", voice_clone=True),
+    CaseSpec(name="cold_single_tts", warmup_runs=0, prompt_kind="single", voice_clone=False),
+    CaseSpec(name="cold_single_clone", warmup_runs=0, prompt_kind="single", voice_clone=True),
+]
+
+
+def _write_lines(path: Path, lines: list[str]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def _prepare_batch_inputs(output_root: Path) -> tuple[Path, Path]:
+    input_dir = output_root / "inputs"
+    batch_tts_path = input_dir / "batch_tts_prompts.txt"
+    batch_clone_path = input_dir / "batch_clone_prompts.txt"
+    _write_lines(batch_tts_path, BATCH_TTS_TEXTS)
+    _write_lines(batch_clone_path, BATCH_CLONE_TEXTS)
+    return batch_tts_path, batch_clone_path
+
+
+def _base_command(args, mode: ModeSpec, output_dir: Path) -> list[str]:
+    cmd = [
+        args.python,
+        str(BENCH_SCRIPT),
+        "--model",
+        args.model,
+        "--stage-configs-path",
+        str(mode.stage_config),
+        "--output-dir",
+        str(output_dir),
+        "--num-runs",
+        str(args.num_runs),
+        "--stage-init-timeout",
+        str(args.stage_init_timeout),
+    ]
+    cmd.append("--log-stats" if args.log_stats else "--no-log-stats")
+    cmd.extend(["--cfg-value", str(args.cfg_value)])
+    cmd.extend(["--inference-timesteps", str(args.inference_timesteps)])
+    cmd.extend(["--min-len", str(args.min_len)])
+    cmd.extend(["--max-new-tokens", str(args.max_new_tokens)])
+    if args.streaming_prefix_len is not None:
+        cmd.extend(["--streaming-prefix-len", str(args.streaming_prefix_len)])
+    if args.enable_profiler:
+        profiler_dir = Path(args.profiler_dir) if args.profiler_dir is not None else (output_dir / "profiler")
+        cmd.append("--enable-profiler")
+        cmd.extend(["--profiler-dir", str(profiler_dir)])
+        cmd.extend(["--profiler-wait-seconds", str(args.profiler_wait_seconds)])
+        if args.profiler_stages is not None:
+            cmd.append("--profiler-stages")
+            cmd.extend(str(stage_id) for stage_id in args.profiler_stages)
+    return cmd
+
+
+def _build_case_command(
+    args,
+    mode: ModeSpec,
+    case: CaseSpec,
+    *,
+    batch_tts_path: Path,
+    batch_clone_path: Path,
+    output_dir: Path,
+) -> list[str]:
+    cmd = _base_command(args, mode, output_dir)
+    cmd.extend(["--warmup-runs", str(case.warmup_runs)])
+    if case.prompt_kind == "single":
+        cmd.extend(["--text", SINGLE_CLONE_TEXT if case.voice_clone else SINGLE_TTS_TEXT])
+    else:
+        cmd.extend(["--txt-prompts", str(batch_clone_path if case.voice_clone else batch_tts_path)])
+    if case.voice_clone:
+        cmd.extend(["--ref-audio", args.ref_audio, "--ref-text", args.ref_text])
+    return cmd
+
+
+def _run_case(
+    args,
+    mode: ModeSpec,
+    case: CaseSpec,
+    *,
+    batch_tts_path: Path,
+    batch_clone_path: Path,
+    output_root: Path,
+) -> CaseResult:
+    case_output_dir = output_root / mode.name / case.name
+    case_output_dir.mkdir(parents=True, exist_ok=True)
+    case_log_path = case_output_dir / "run.log"
+    cmd = _build_case_command(
+        args,
+        mode,
+        case,
+        batch_tts_path=batch_tts_path,
+        batch_clone_path=batch_clone_path,
+        output_dir=case_output_dir,
+    )
+
+    print()
+    print("=" * 80)
+    print(f"[{mode.name}] {case.name}")
+    print(f"Output directory: {case_output_dir}")
+    print(shlex.join(cmd))
+
+    start = time.perf_counter()
+    with case_log_path.open("w", encoding="utf-8") as log_fp:
+        process = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+            bufsize=1,
+        )
+        assert process.stdout is not None
+        for line in process.stdout:
+            print(line, end="")
+            log_fp.write(line)
+        process.wait()
+
+    elapsed_s = time.perf_counter() - start
+    status = "PASS" if (process.returncode or 0) == 0 else f"FAIL({process.returncode})"
+    print(f"[{mode.name}] {case.name} -> {status} ({elapsed_s:.2f}s)")
+    return CaseResult(
+        mode=mode.name,
+        case=case.name,
+        returncode=int(process.returncode or 0),
+        elapsed_s=elapsed_s,
+        output_dir=case_output_dir,
+        log_path=case_log_path,
+    )
+
+
+def parse_args():
+    parser = FlexibleArgumentParser(description="Run the full offline VoxCPM smoke matrix.")
+    parser.add_argument("--model", type=str, required=True, help="Local VoxCPM model directory.")
+    parser.add_argument("--ref-audio", type=str, required=True, help="Reference audio path for clone cases.")
+    parser.add_argument("--ref-text", type=str, required=True, help="Exact transcript spoken in --ref-audio.")
+    parser.add_argument("--output-root", type=str, default=str(DEFAULT_OUTPUT_ROOT), help="Root directory for outputs.")
+    parser.add_argument("--python", type=str, default=sys.executable, help="Python executable used to launch cases.")
+    parser.add_argument("--stage-init-timeout", type=int, default=600, help="Stage initialization timeout in seconds.")
+    parser.add_argument("--log-stats", dest="log_stats", action="store_true", help="Enable vLLM Omni stats logging.")
+    parser.add_argument(
+        "--no-log-stats",
+        dest="log_stats",
+        action="store_false",
+        help="Disable vLLM Omni stats logging.",
+    )
+    parser.set_defaults(log_stats=True)
+    parser.add_argument("--num-runs", type=int, default=1, help="Number of measured runs per case.")
+    parser.add_argument("--cfg-value", type=float, default=2.0, help="Classifier-free guidance value for VoxCPM.")
+    parser.add_argument("--inference-timesteps", type=int, default=10, help="Number of inference timesteps.")
+    parser.add_argument("--min-len", type=int, default=2, help="Minimum generated token length.")
+    parser.add_argument("--max-new-tokens", type=int, default=4096, help="Maximum generated token length.")
+    parser.add_argument(
+        "--streaming-prefix-len",
+        type=int,
+        default=None,
+        help="Optional VoxCPM streaming window passed to streaming cases.",
+    )
+    parser.add_argument("--enable-profiler", action="store_true", help="Enable torch profiler for each case.")
+    parser.add_argument(
+        "--profiler-dir",
+        type=str,
+        default=None,
+        help="Profiler output root. Defaults to <case-output-dir>/profiler.",
+    )
+    parser.add_argument(
+        "--profiler-stages",
+        type=int,
+        nargs="*",
+        default=None,
+        help="Optional stage ids to profile. Defaults to all configured stages.",
+    )
+    parser.add_argument(
+        "--profiler-wait-seconds",
+        type=float,
+        default=30.0,
+        help="Seconds to wait after stopping profiler for traces to flush.",
+    )
+    args = parser.parse_args()
+    if args.num_runs < 1:
+        parser.error("--num-runs must be >= 1")
+    return args
+
+
+def main(args) -> int:
+    output_root = Path(args.output_root)
+    output_root.mkdir(parents=True, exist_ok=True)
+    batch_tts_path, batch_clone_path = _prepare_batch_inputs(output_root)
+
+    print(f"Model: {args.model}")
+    print(f"Reference audio: {args.ref_audio}")
+    print(f"Reference text: {args.ref_text}")
+    print(f"Python: {args.python}")
+    print(f"Output root: {output_root}")
+    print(f"Cases: {len(MODE_SPECS) * len(CASE_SPECS)}")
+
+    results: list[CaseResult] = []
+    for mode in MODE_SPECS:
+        for case in CASE_SPECS:
+            results.append(
+                _run_case(
+                    args,
+                    mode,
+                    case,
+                    batch_tts_path=batch_tts_path,
+                    batch_clone_path=batch_clone_path,
+                    output_root=output_root,
+                )
+            )
+
+    failed = [result for result in results if not result.ok]
+    print()
+    print("=" * 80)
+    print("Summary:")
+    for result in results:
+        status = "PASS" if result.ok else f"FAIL({result.returncode})"
+        print(f"- [{result.mode}] {result.case}: {status} ({result.elapsed_s:.2f}s)")
+        print(f"  output_dir={result.output_dir}")
+        print(f"  log={result.log_path}")
+
+    print(f"Passed: {len(results) - len(failed)}/{len(results)}")
+    if failed:
+        print("Failed cases:")
+        for result in failed:
+            print(f"- [{result.mode}] {result.case}: see {result.log_path}")
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(parse_args()))

docker/Dockerfile.ci

@@ -1,5 +1,5 @@
 ARG VLLM_BASE_IMAGE=vllm/vllm-openai
-ARG VLLM_BASE_TAG=v0.21.0
+ARG VLLM_BASE_TAG=v0.19.0
 FROM ${VLLM_BASE_IMAGE}:${VLLM_BASE_TAG}
 ARG APP_DIR=/workspace/vllm-omni
 WORKDIR ${APP_DIR}

docker/Dockerfile.cuda

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=vllm/vllm-openai:v0.20.0
+ARG BASE_IMAGE=vllm/vllm-openai:v0.19.0
 FROM ${BASE_IMAGE}
 
 ARG COMMON_WORKDIR=/app

docker/Dockerfile.npu.ci

@@ -1,48 +0,0 @@
-ARG VLLM_ASCEND_IMAGE=quay.io/ascend/vllm-ascend
-ARG VLLM_ASCEND_TAG=nightly-main
-FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
-
-ARG APP_DIR=/vllm-workspace/vllm-omni
-WORKDIR ${APP_DIR}
-COPY . .
-
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-        ffmpeg \
-        libgl1 \
-        libglib2.0-0 \
-        libgomp1 \
-        libsm6 \
-        libxext6 \
-        libxrender1 \
-        libxcb1 \
-        espeak-ng && \
-    rm -rf /var/cache/apt/* && \
-    rm -rf /var/lib/apt/lists/*
-
-RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
-    source /usr/local/Ascend/nnal/atb/set_env.sh && \
-    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
-    python3 -m pip install -U pip && \
-    python3 -m pip install -v \
-      "pytest>=7.0.0" \
-      "pytest-asyncio>=0.21.0" \
-      "pytest-cov>=4.0.0" \
-      "pytest-mock>=3.10.0" \
-      "openpyxl>=3.0.0" \
-      "psutil>=7.2.0" \
-      "soundfile>=0.13.1" \
-      "pyttsx3>=2.99" \
-      "opencv-python>=4.12.0.88" \
-      "imageio[ffmpeg]>=0.6.0" \
-      --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
-      --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
-      --default-timeout=60 && \
-    python3 -m pip install -v -e /vllm-workspace/vllm-omni/ --no-build-isolation \
-      --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
-      --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
-      --default-timeout=60
-
-ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-ENTRYPOINT []

docker/Dockerfile.npu.ci.a3

@@ -1,48 +0,0 @@
-ARG VLLM_ASCEND_IMAGE=quay.io/ascend/vllm-ascend
-ARG VLLM_ASCEND_TAG=nightly-main-a3
-FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
-
-ARG APP_DIR=/vllm-workspace/vllm-omni
-WORKDIR ${APP_DIR}
-COPY . .
-
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-        ffmpeg \
-        libgl1 \
-        libglib2.0-0 \
-        libgomp1 \
-        libsm6 \
-        libxext6 \
-        libxrender1 \
-        libxcb1 \
-        espeak-ng && \
-    rm -rf /var/cache/apt/* && \
-    rm -rf /var/lib/apt/lists/*
-
-RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
-    source /usr/local/Ascend/nnal/atb/set_env.sh && \
-    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
-    python3 -m pip install -U pip && \
-    python3 -m pip install -v \
-      "pytest>=7.0.0" \
-      "pytest-asyncio>=0.21.0" \
-      "pytest-cov>=4.0.0" \
-      "pytest-mock>=3.10.0" \
-      "openpyxl>=3.0.0" \
-      "psutil>=7.2.0" \
-      "soundfile>=0.13.1" \
-      "pyttsx3>=2.99" \
-      "opencv-python>=4.12.0.88" \
-      "imageio[ffmpeg]>=0.6.0" \
-      --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
-      --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
-      --default-timeout=60 && \
-    python3 -m pip install -v -e /vllm-workspace/vllm-omni/ --no-build-isolation \
-      --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
-      --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
-      --default-timeout=60
-
-ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-ENTRYPOINT []

docker/Dockerfile.rocm

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.21.0
+ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.19.0
 FROM ${BASE_IMAGE} AS base
 
 # Declare a variable to know if we want to use the nightly build or the stable build.

docker/Dockerfile.xpu

@@ -14,9 +14,8 @@ RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRO
 RUN apt clean && apt-get update -y && \
     apt-get install -y --no-install-recommends --fix-missing \
     curl \
-    ffmpeg \
+    espeak-ng \
     git \
-    libsndfile1 \
     libsm6 \
     libxext6 \
     libgl1 \
@@ -29,20 +28,8 @@ RUN apt clean && apt-get update -y && \
     python3.12-dev \
     python3-pip
 
-# Add oneAPI repo, pin oneAPI to 2025.3, then install pinned packages in one layer.
-RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \
-    echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list && \
-    printf '%s\n' \
-    'Package: intel-oneapi-* intel-deep-learning-essentials* intel-pti*' \
-    'Pin: version 2025.3*' \
-    'Pin-Priority: 1001' \
-    > /etc/apt/preferences.d/oneapi-2025.3.pref && \
-    apt-get update -y && \
-    apt-get install -y --no-install-recommends \
-    intel-oneapi-compiler-dpcpp-cpp-2025.3 \
-    intel-oneapi-mkl-devel-2025.3 \
-    intel-oneapi-dnnl-devel-2025.3 && \
-    rm -rf /var/lib/apt/lists/*
+RUN apt update && apt upgrade -y && \
+    apt install -y intel-oneapi-compiler-dpcpp-cpp-2025.3
 
 # Install UMD with fixed version
 RUN mkdir neo && \
@@ -65,9 +52,9 @@ RUN curl -LsSf https://astral.sh/uv/install.sh | sh
 RUN uv venv --python ${PYTHON_VERSION} --seed ${VIRTUAL_ENV}
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
-# This oneccl contains the BMG support which is not the case for default version of oneapi 2025.3.
-ARG ONECCL_INSTALLER="intel-oneccl-2021.15.9.14_offline.sh"
-RUN wget "https://github.com/uxlfoundation/oneCCL/releases/download/2021.15.9/${ONECCL_INSTALLER}" && \
+# This oneccl contains the BMG support which is not the case for default version of oneapi 2025.2.
+ARG ONECCL_INSTALLER="intel-oneccl-2021.15.7.8_offline.sh"
+RUN wget "https://github.com/uxlfoundation/oneCCL/releases/download/2021.15.7/${ONECCL_INSTALLER}" && \
     bash "${ONECCL_INSTALLER}" -a --silent --eula accept && \
     rm "${ONECCL_INSTALLER}" && \
     echo "source /opt/intel/oneapi/setvars.sh --force" >> /root/.bashrc && \
@@ -79,7 +66,6 @@ SHELL ["bash", "-c"]
 CMD ["bash", "-c", "source /root/.bashrc && exec bash"]
 
 WORKDIR /workspace/
-
 ENV UV_HTTP_TIMEOUT=500
 
 # Configure package index for XPU
@@ -88,18 +74,24 @@ ENV UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
 ENV UV_INDEX_STRATEGY="unsafe-best-match"
 ENV UV_LINK_MODE="copy"
 
-ARG VLLM_VERSION=v0.21.0
+ARG VLLM_VERSION=v0.19.0
 RUN git clone -b ${VLLM_VERSION} https://github.com/vllm-project/vllm
 WORKDIR /workspace/vllm
 
 RUN --mount=type=cache,target=/root/.cache/uv \
     uv pip install --upgrade pip && \
-    uv pip install -r requirements/xpu.txt && \
-    uv pip install setuptools_scm &&\
-    uv pip install grpcio-tools protobuf nanobind && \
+    uv pip install -r requirements/xpu.txt
+
+ # used for suffix method speculative decoding
+ # build deps for proto + nanobind-based extensions to set up the build environment
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv pip install grpcio-tools protobuf nanobind
+ # arctic-inference is built from source which needs torch-xpu properly installed first
+RUN --mount=type=cache,target=/root/.cache/uv \
     source /opt/intel/oneapi/setvars.sh --force && \
     source /opt/intel/oneapi/ccl/2021.15/env/vars.sh --force && \
-    export CMAKE_PREFIX_PATH="$(python3 -c 'import site; print(site.getsitepackages()[0])'):${CMAKE_PREFIX_PATH}"
+    export CMAKE_PREFIX_PATH="$(python -c 'import site; print(site.getsitepackages()[0])'):${CMAKE_PREFIX_PATH}" && \
+    uv pip install --no-build-isolation arctic-inference==0.1.1
 
 ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib/"
 
@@ -113,9 +105,6 @@ CMD ["/bin/bash"]
 
 FROM vllm-base AS vllm-openai
 
-# install nixl from source code
-RUN uv pip install nixl
-
 # install additional dependencies for openai api server
 RUN --mount=type=cache,target=/root/.cache/uv \
     uv pip install accelerate hf_transfer pytest pytest_asyncio lm_eval[api] modelscope
@@ -123,6 +112,10 @@ RUN --mount=type=cache,target=/root/.cache/uv \
 # install development dependencies (for testing)
 RUN uv pip install -e tests/vllm_test_utils
 
+# install nixl from source code
+ENV NIXL_VERSION=0.7.0
+RUN python /workspace/vllm/tools/install_nixl_from_source_ubuntu.py
+
 # ensure vllm is properly installed
 RUN python -c "import vllm, inspect; print(vllm.__file__)"
 RUN uv pip show vllm
@@ -142,7 +135,7 @@ RUN uv pip install --no-cache-dir ".[dev]" --no-build-isolation
 # FIX triton
 RUN --mount=type=cache,target=/root/.cache/uv \
     uv pip uninstall triton triton-xpu && \
-    uv pip install triton-xpu==3.7.0
+    uv pip install triton-xpu==3.6.0
 
 # remove torch bundled oneccl to avoid conflicts
 RUN --mount=type=cache,target=/root/.cache/uv \

docs/.nav.yml

@@ -6,43 +6,44 @@ nav:
     - getting_started/installation/*
   - Serving:
     - OpenAI-Compatible API:
-      - Chat Completions API: serving/chat_completions_api.md
+      - Diffusion Chat API: serving/diffusion_chat_api.md
       - Image Generation: serving/image_generation_api.md
       - Image Edit: serving/image_edit_api.md
       - Text to Speech: serving/speech_api.md
-      - Videos: serving/videos_api.md
-      - Audio Generation: serving/audio_generate_api.md
       - Streaming Video Input: serving/video_stream_api.md
   - Examples:
     - examples/README.md
     - Offline Inference:
       - BAGEL-7B-MoT: user_guide/examples/offline_inference/bagel.md
+      - CosyVoice3: user_guide/examples/offline_inference/cosyvoice3.md
+      - Fish Speech S2 Pro: user_guide/examples/offline_inference/fish_speech.md
       - GLM-Image Multistage End-to-End Inference: user_guide/examples/offline_inference/glm_image.md
       - Helios Video Generation: user_guide/examples/offline_inference/helios.md
       - HunyuanImage-3.0 Image-to-Text Inference: user_guide/examples/offline_inference/hunyuan_image3.md
       - Image-To-Image: user_guide/examples/offline_inference/image_to_image.md
       - Image-To-Video: user_guide/examples/offline_inference/image_to_video.md
-      - InternVLA-A1: user_guide/examples/offline_inference/internvla_a1.md
       - MammothModa2-Preview: user_guide/examples/offline_inference/mammothmodal2_preview.md
       - MiMo-Audio Offline Inference: user_guide/examples/offline_inference/mimo_audio.md
       - Qwen2.5-Omni: user_guide/examples/offline_inference/qwen2_5_omni.md
       - Qwen3-Omni: user_guide/examples/offline_inference/qwen3_omni.md
+      - Qwen3-TTS: user_guide/examples/offline_inference/qwen3_tts.md
       - Text-To-Audio: user_guide/examples/offline_inference/text_to_audio.md
       - Text-To-Image: user_guide/examples/offline_inference/text_to_image.md
-      - Text-To-Speech: user_guide/examples/offline_inference/text_to_speech.md
       - Text-To-Video: user_guide/examples/offline_inference/text_to_video.md
+      - Voxtral TTS Offline Inference: user_guide/examples/offline_inference/voxtral_tts.md
       - X-To-Video-Audio: user_guide/examples/offline_inference/x_to_video_audio.md
     - Online Serving:
       - BAGEL-7B-MoT: user_guide/examples/online_serving/bagel.md
       - vLLM-Omni Helm Chart: user_guide/examples/online_serving/chart-helm.md
-      - Diffusers Backend Adapter: user_guide/examples/online_serving/diffusers_pipeline_adapter.md
+      - Diffusers Backend Adapter Example: user_guide/examples/online_serving/diffusers_pipeline_adapter.md
+      - Fish Speech S2 Pro: user_guide/examples/online_serving/fish_speech.md
       - GLM-Image Online Serving: user_guide/examples/online_serving/glm_image.md
       - Image-To-Image: user_guide/examples/online_serving/image_to_image.md
       - Image-To-Video: user_guide/examples/online_serving/image_to_video.md
       - Online serving Example of vLLM-Omni for MiMo-Audio: user_guide/examples/online_serving/mimo_audio.md
       - Qwen2.5-Omni: user_guide/examples/online_serving/qwen2_5_omni.md
       - Qwen3-Omni: user_guide/examples/online_serving/qwen3_omni.md
-      - Text-To-Speech: user_guide/examples/online_serving/text_to_speech.md
+      - Qwen3-TTS: user_guide/examples/online_serving/qwen3_tts.md
       - Text-To-Image: user_guide/examples/online_serving/text_to_image.md
       - Text-To-Video: user_guide/examples/online_serving/text_to_video.md
   - General:
@@ -60,6 +61,11 @@ nav:
       - Cache Acceleration:
         - TeaCache: user_guide/diffusion/cache_acceleration/teacache.md
         - Cache-DiT: user_guide/diffusion/cache_acceleration/cache_dit.md
+      - Quantization:
+        - Overview: user_guide/diffusion/quantization/overview.md
+        - FP8: user_guide/diffusion/quantization/fp8.md
+        - Int8: user_guide/diffusion/quantization/int8.md
+        - GGUF: user_guide/diffusion/quantization/gguf.md
       - Attention Backends: user_guide/diffusion/attention_backends.md
       - Frame Interpolation: user_guide/diffusion/frame_interpolation.md
       - Parallelism:
@@ -74,16 +80,6 @@ nav:
       - LoRA: user_guide/diffusion/lora.md
       - Custom Pipeline: features/custom_pipeline.md
       - Step Execution: user_guide/diffusion/step_execution.md
-    - Quantization:
-      - Overview: user_guide/quantization/overview.md
-      - Online Quantization: user_guide/quantization/online.md
-      - Quantized KV Cache: user_guide/quantization/quantized_kvcache.md
-      - FP8 W8A8: user_guide/quantization/fp8.md
-      - Int8 W8A8: user_guide/quantization/int8.md
-      - ModelOpt: user_guide/quantization/modelopt.md
-      - GGUF: user_guide/quantization/gguf.md
-      - AutoRound: user_guide/quantization/autoround.md
-      - msModelSlim: user_guide/quantization/msmodelslim.md
     - ComfyUI: features/comfyui.md
 - Developer Guide:
   - General:
@@ -95,6 +91,7 @@ nav:
     - contributing/model/adding_omni_model.md
     - contributing/model/adding_tts_model.md
     - contributing/model/adding_diffusion_model.md
+    - contributing/model/adding_quantization_model.md
   - CI: contributing/ci
   - Design Documents:
     - design/index.md
@@ -115,7 +112,6 @@ nav:
       - design/feature/async_chunk.md
       - design/feature/vae_parallel.md
       - design/feature/diffusion_step_execution.md
-      - design/feature/diffusion_continuous_batching.md
     - Module Design:
       - design/module/ar_module.md
       - design/module/dit_module.md

docs/api/README.md

@@ -99,6 +99,7 @@ Configuration classes.
 - [vllm_omni.diffusion.cache.teacache.config.TeaCacheConfig][]
 - [vllm_omni.distributed.omni_connectors.utils.config.ConnectorSpec][]
 - [vllm_omni.distributed.omni_connectors.utils.config.OmniTransferConfig][]
+- [vllm_omni.model_executor.models.cosyvoice3.config.CosyVoice3Config][]
 - [vllm_omni.model_executor.models.fish_speech.configuration_fish_speech.FishSpeechConfig][]
 - [vllm_omni.model_executor.models.fish_speech.configuration_fish_speech.FishSpeechFastARConfig][]
 - [vllm_omni.model_executor.models.fish_speech.configuration_fish_speech.FishSpeechSlowARConfig][]
@@ -115,14 +116,10 @@ Configuration classes.
 - [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1DecoderConfig][]
 - [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1DecoderDiTConfig][]
 - [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1EncoderConfig][]
-- [vllm_omni.transformers_utils.configs.cosyvoice3.CosyVoice3Config][]
 - [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Config][]
 - [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Qwen2_5_VLConfig][]
 - [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Qwen2_5_VLTextConfig][]
 - [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Qwen2_5_VLVisionConfig][]
-- [vllm_omni.transformers_utils.configs.omnivoice.OmniVoiceConfig][]
-- [vllm_omni.transformers_utils.configs.voxcpm.VoxCPMConfig][]
-- [vllm_omni.transformers_utils.configs.voxcpm2.VoxCPM2Config][]
 
 ## Workers
 

docs/cli/serve.md

@@ -12,11 +12,9 @@ The stage-based CLI is designed for deployments that require launching each pipe
   `vllm_omni/model_executor/stage_configs/`, the `--stage-configs-path` parameter remains mandatory.
 
 Example: Initializing Stage 0 (Orchestrator and API Server):
-The commands below show a common device mapping where Stage 0 uses GPU 0 and
-worker stages use GPU 1 via `CUDA_VISIBLE_DEVICES`.
 
 ```bash
-CUDA_VISIBLE_DEVICES=0 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
     --port 8091 \
     --stage-id 0 \
     --omni-master-address 127.0.0.1 \
@@ -26,7 +24,7 @@ CUDA_VISIBLE_DEVICES=0 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
 Example: Initializing a Headless Worker Stage (Stage 1):
 
 ```bash
-CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
     --stage-id 1 \
     --headless \
     --omni-master-address 127.0.0.1 \
@@ -48,7 +46,7 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
 the stage-based CLI permits the direct initialization of Stage 1 with explicit parameters:
 
 ```bash
-CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
     --stage-id 1 \
     --headless \
     --gpu-memory-utilization 0.5 \

docs/configuration/stage_configs.md

@@ -19,7 +19,7 @@ The new deploy schema lives under `vllm_omni/deploy/` and is paired with a froze
 | `platforms` | dict | optional | `null` | Keyed by `npu` / `rocm` / `xpu`, each contains a `stages:` list with per-platform overrides applied on top of the CUDA defaults. |
 | `pipeline` | str | optional | `null` | Override the auto-detected pipeline registry key (used for structural variants like `qwen2_5_omni_thinker_only`). |
 | `trust_remote_code` | bool | optional | `true` | **Pipeline-wide.** Trust HF remote code on model load; applies to every stage. |
-| `distributed_executor_backend` | str \| null | optional | `null` | **Pipeline-wide.** Distributed executor backend forwarded to vLLM (`"mp"`, `"ray"`, `"external_launcher"`). If omitted, vLLM auto-selects backend from runtime topology. |
+| `distributed_executor_backend` | str | optional | `"mp"` | **Pipeline-wide.** Executor backend (`"mp"` or `"ray"`). |
 | `dtype` | str \| null | optional | `null` | **Pipeline-wide.** Model dtype for every stage. |
 | `quantization` | str \| null | optional | `null` | **Pipeline-wide.** Quantization method for every stage. |
 | `enable_prefix_caching` | bool | optional | `false` | **Pipeline-wide.** Prefix cache toggle applied to every stage. |
@@ -27,9 +27,6 @@ The new deploy schema lives under `vllm_omni/deploy/` and is paired with a froze
 | `data_parallel_size` | int | optional | `1` | **Pipeline-wide.** DP degree for every stage. |
 | `pipeline_parallel_size` | int | optional | `1` | **Pipeline-wide.** PP degree for every stage. |
 
-Note: for diffusion path, `distributed_executor_backend` currently defaults to
-`mp`, and `ray` / `external_launcher` are not fully supported yet.
-
 ### Stage fields
 
 Each entry under `stages:` accepts any `StageDeployConfig` field directly (no nested `engine_args:`). Only fields whose value legitimately varies across stages live here; pipeline-wide settings (trust_remote_code, distributed_executor_backend, dtype, quantization, prefix/chunked prefill, DP/PP sizes) are declared at the top level and applied to every stage. Unknown keys fall through to `engine_extras:` and are forwarded to the engine.
@@ -102,17 +99,15 @@ The stage-based CLI paradigm facilitates the execution of discrete pipeline stag
 
 For migrated architectures, the system automatically resolves and loads the bundled deployment YAML. Consequently, the primary execution path
 does **not** necessitate the explicit definition of `--deploy-config`:
-the example below uses `CUDA_VISIBLE_DEVICES=0` for Stage 0 and
-`CUDA_VISIBLE_DEVICES=1` for Stage 1.
 
 ```bash
-CUDA_VISIBLE_DEVICES=0 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
     --port 8091 \
     --stage-id 0 \
     --omni-master-address 127.0.0.1 \
     --omni-master-port 26000
 
-CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
     --stage-id 1 \
     --headless \
     --omni-master-address 127.0.0.1 \
@@ -134,7 +129,7 @@ Conversely, in the context of the **stage-based CLI** paradigm, given that each
 can be defined uniformly via explicit CLI flags on the corresponding instantiation command, rendering composite `--stage-overrides` JSON strings unnecessary:
 
 ```bash
-CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
     --stage-id 1 \
     --headless \
     --gpu-memory-utilization 0.5 \
@@ -191,7 +186,7 @@ Within the stage-based CLI paradigm, equivalent configuration parameters can inh
 as command-line arguments to the designated single-stage process instantiation:
 
 ```bash
-CUDA_VISIBLE_DEVICES=0 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
     --stage-id 0 \
     --max-num-seqs 8 \
     --omni-master-address 127.0.0.1 \

docs/contributing/ci/CI_5levels.md

@@ -108,8 +108,7 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
         /tests/dfx/stability/tests/test_qwen3_omni.json<br>
         /tests/dfx/stability/tests/test_wan22.json<br>
         <strong>Reliability:</strong><br>
-        tests/dfx/reliability/test_reliability_{model_key}.py<br>
-        (e.g. <code>test_reliability_qwen3_omni.py</code>, <code>test_reliability_wan22.py</code>)
+        tests/e2e/reliability/test_{model_name}.py
       </td>
       <td>
         <a href="#chapter-4-l5-level-testing-stability-and-reliability-testing">Chapter 4</a><br>
@@ -152,8 +151,6 @@ vllm_omni/                                    tests/
 │   │                                            │   │   └── test_lora_manager.py        ✅
 │   ├── models/                                 │   ├── models/
 │   │   ├── qwen_image/                         │   │   ├── qwen_image/                 (e2e coverage)
-│   │   ├── ovis_image/                         │   │   ├── ovis_image/
-│   │   │   └── ...                             │   │   │   └── test_ovis_image.py     ✅
 │   │   ├── z_image/                            │   │   └── z_image/
 │   │   └── ...                                 │   │       └── test_zimage_tp_constraints.py  ✅
 │   └── worker/                                 │   └── worker/
@@ -234,13 +231,15 @@ vllm_omni/                                    tests/
                                                │   ├── test_qwen3_omni.py
                                                │   ├── test_qwen3_omni_expansion.py
                                                │   ├── test_mimo_audio.py
+                                               │   ├── test_image_gen_edit.py
                                                │   └── test_images_generations_lora.py
                                                └── offline_inference/                  ✅
                                                    ├── test_qwen2_5_omni.py
                                                    ├── test_qwen3_omni.py
                                                    ├── test_bagel_text2img.py
-                                                   ├── test_z_image.py
-                                                   ├── test_wan22.py
+                                                   ├── test_t2i_model.py
+                                                   ├── test_t2v_model.py
+                                                   ├── test_ovis_image.py
                                                    ├── test_zimage_tensor_parallel.py
                                                    ├── test_cache_dit.py
                                                    ├── test_teacache.py
@@ -577,17 +576,12 @@ L5 level testing focuses on the performance of model services under ***long-runn
 ### 4.2 Testing Content and Scope
 
 -   ***Long-term Stability (Stability) Testing***: Uses JSON under `tests/dfx/stability/tests/` (for example `test_qwen3_omni.json` and `test_wan22.json`) to run the service under moderate load for an extended period (e.g., over 12 hours), monitoring whether metrics like memory/VRAM usage, response time, and throughput degrade over time, and whether the service process remains stable.
--   ***Reliability Testing***: Uses pytest suites under `tests/dfx/reliability/` to inject controlled faults against a **live** `vllm_omni serve` instance (same **`omni_server` / `omni_server_function`** fixture style as E2E). Current suites emphasize **GPU memory pressure** (CUDA sidecar “memory hog”), **worker / runtime process kill** (`SIGKILL` on `VLLM::Worker` for Qwen3-Omni or `multiprocessing.spawn` for Wan2.2 video workers), **large multimodal chat** or **`/v1/videos`** jobs under OOM, **`/health` → 503** and **fast-fail / non-hanging concurrent** requests after kill, and **OpenAI-style 5xx error contracts** (e.g. text vs text+audio under OOM). **Post-fault recovery** checks exist where enabled (some cases may be `skip` while issues are tracked). See the Reliability `<details>` block in Section 4.4 for file-level responsibilities and CI markers (`slow`, `hardware_test`, POSIX-only kill).
+-   ***Reliability Testing***: Uses `tests/e2e/reliability/test_{model_name}.py` to actively simulate various fault and abnormal scenarios, such as: dependent service interruption, abnormal input data, network flicker, hardware resource preemption, etc., to verify the system's fault tolerance, self-healing, and graceful degradation capabilities.
 
 ### 4.3 Test Directory and Execution Files
 
 -   ***Stability Test Configuration***: `tests/dfx/stability/tests/test_qwen3_omni.json`, `tests/dfx/stability/tests/test_wan22.json` (one JSON per model / runner family)
--   ***Reliability Test Suite*** (`tests/dfx/reliability/`):
-    -   `test_reliability_qwen3_omni.py` — Qwen3-Omni chat / multimodal reliability (GPU OOM, process kill, recovery, error contract under `--async-chunk` vs default).
-    -   `test_reliability_wan22.py` — Wan2.2 T2V video API reliability (`/v1/videos` under OOM and process kill, recovery).
-    -   `helpers.py` — Shared primitives used by current suites: raw HTTP probes for `/v1/chat/completions` and `/health`, OpenAI-style error parsing, GPU OOM sidecar (`inject_gpu_oom` / `stop_gpu_oom_hogs`), and `pgrep`-based process-kill injector construction (`make_process_kill_fault_injector`).
-    -   `conftest.py` — `fault_injector` and `omni_server_after_fault` / `omni_server_after_fault_function` fixtures to run a callable **after** the server is ready.
-    -   `README.md` — Short local run commands for this directory.
+-   ***Reliability Test Suite***: `tests/e2e/reliability/test_{model_name}.py`
 
 ### 4.4 Execution Method and Example
 
@@ -668,57 +662,8 @@ All other optional parameters follow the same rules as the in Chapter 3.4.
 
 </details>
 
-<details>
-<summary> Reliability test suite (<code>tests/dfx/reliability</code>)</summary>
-
-#### Purpose and relationship to stability
-
-Reliability tests are **short fault-injection** integration runs (L5 **(b)** in `tests/dfx/reliability/README.md`). They complement **stability** JSON-driven long runs: instead of hours of steady traffic, they **perturb** the server (GPU OOM sidecar, fatal signals on selected processes) and check **failure mode** and **latency bounds** (e.g. chat or `/v1/videos` must not hang under concurrent fault-time load).
-
-#### Directory layout
-
-| Path | Responsibility |
-| ---- | -------------- |
-| `helpers.py` | Shared helpers used by current reliability suites: raw `POST`/`GET` probes (`/v1/chat/completions`, `/health`), OpenAI error parsing (`extract_openai_error_contract_from_bytes`), GPU OOM sidecar lifecycle (`inject_gpu_oom`, `stop_gpu_oom_hogs`), and process-kill injector builder (`make_process_kill_fault_injector`). |
-| `conftest.py` | Pytest fixtures: indirect `fault_injector`, `omni_server_after_fault` / `omni_server_after_fault_function` (run injector after server is ready, then yield server). |
-| `test_reliability_qwen3_omni.py` | Qwen3-Omni: OOM vs **text vs text+audio** error contract, large multimodal chat under OOM, concurrent pressure, **SIGKILL** on `VLLM::Worker`, `/health` → 503 + fast-fail + concurrent chat; optional OOM recovery scenario (may be skipped while tracked in issues). |
-| `test_reliability_wan22.py` | Wan2.2 T2V: large `/v1/videos` under OOM, **SIGKILL** on `multiprocessing.spawn` chain, health / fast-fail / concurrent video requests; optional recovery test (may be skipped). |
-| `README.md` | Minimal run / collect examples. |
-
-#### Parametrization and markers
-
-- Each test module defines a **`RELIABILITY_SCENARIOS`** list (`test_name`, `server_params`: model, `stage_config_name` or diffusion `server_args`, etc.). **`create_reliability_omni_server_params()`** in `tests/dfx/conftest.py` resolves stage paths (including XPU substitutions where applicable) and builds **`OmniServerParams`** lists consumed by **`@pytest.mark.parametrize(..., indirect=True)`** on `omni_server` or `omni_server_function`.
-- Cases are tagged **`@pytest.mark.slow`** for weekly / selective CI. GPU-heavy suites use **`@hardware_test(res={"cuda": "H100"}, num_cards=...)`** (Qwen3-Omni paths often require **2** cards; Wan2.2 video paths **1** card).
-- **Process-kill** tests use **`@pytest.mark.skipif(os.name == "nt", ...)`** because injection uses POSIX **`pgrep` / `kill`**.
-
-#### CI trigger
-
-Weekly Buildkite (`.buildkite/test-weekly.yml`) runs, for example:
-
-```bash
-pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m "slow"
-pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m "slow"
-```
-
-#### Local commands
-
-```bash
-pytest --collect-only tests/dfx/reliability
-pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m slow
-pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m slow
-```
-
-#### Adding a new model suite
-
-1. Add `test_reliability_<model>.py` under `tests/dfx/reliability/`.
-2. Define **`RELIABILITY_SCENARIOS`** and pass them through **`create_reliability_omni_server_params()`** with the correct deploy or e2e stage-config directory (same pattern as existing files).
-3. Reuse **`helpers`** for OOM / kill / raw HTTP; prefer **`assert_fault_exception()`** and **`resolve_oom_device_spec()`** from `tests/dfx/conftest.py` for consistent device selection vs stage YAML.
-4. Register **`slow`** (and **`hardware_test`** if needed); extend **`.buildkite/test-weekly.yml`** when the suite should run in weekly L5.
-
-</details>
-
 -   -   ***Stability***: `pytest -s -v tests/dfx/stability/scripts/test_stability_qwen3_omni.py` or `pytest -s -v tests/dfx/stability/scripts/test_stability_wan22.py` (or add `test_stability_<model>.py` alongside a matching JSON config)
-    -   ***Reliability***: `pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m slow` and/or `pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m slow` (add `test_reliability_<suite>.py` for new models)
+    -   ***Reliability***: `pytest -s -v tests/e2e/reliability/test_{model_name}.py`
 
 ## Summary
 

docs/contributing/ci/test_examples/l4_functionality_tests.inc.md

@@ -1,6 +1,6 @@
 **Scope**
 
-For diffusion models, the L4 functionality test covers all or common *diffusion features* that are supported by this model, including several [parallelism acceleration methods](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/parallelism_acceleration/), [CPU offloading](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/cpu_offload_diffusion/), [TeaCache](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/teacache/) and [Cache-DiT](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/cache_dit_acceleration/) cache backends, [quantization methods](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/quantization/overview/).
+For diffusion models, the L4 functionality test covers all or common *diffusion features* that are supported by this model, including several [parallelism acceleration methods](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/parallelism_acceleration/), [CPU offloading](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/cpu_offload_diffusion/), [TeaCache](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/teacache/) and [Cache-DiT](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/cache_dit_acceleration/) cache backends, [quantization methods](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/diffusion/quantization/overview/).
 
 **Test Case Design**
 

docs/contributing/ci/test_guide.md

@@ -80,7 +80,7 @@ Our test scripts use the pytest framework. First, please use `git clone https://
 
     ```bash
     cd tests
-    pytest -s -v -m "full_model and (omni or tts) and H100" --run-level=full_model
+    pytest -s -v -m "full_model and omni and H100" --run-level=full_model
     ```
 
 === "L5 level"
@@ -91,16 +91,10 @@ Our test scripts use the pytest framework. First, please use `git clone https://
     cd tests
 
     # Stability: Qwen3-Omni
-    pytest -s -v dfx/stability/scripts/test_stability_qwen3_omni.py -m slow
+    pytest -s -v dfx/stability/scripts/test_stability_qwen3_omni.py
 
     # Stability: Wan2.2 (v1/videos diffusion benchmark loop)
-    pytest -s -v dfx/stability/scripts/test_stability_wan22.py -m slow
-
-    # Reliability: Qwen3-Omni
-    pytest -s -v dfx/reliability/test_reliability_qwen3_omni.py -m slow
-
-    # Reliability: Wan2.2
-    pytest -s -v dfx/reliability/test_reliability_wan22.py -m slow
+    pytest -s -v dfx/stability/scripts/test_stability_wan22.py
 
     ```
 

docs/contributing/ci/tests_markers.md

@@ -11,8 +11,7 @@ Defined in `pyproject.toml`:
 | `advanced_model`   | L3 tests (run on each merge to main)                 |
 | `full_model`       | L4 tests (run nightly) |
 | `diffusion`        | Diffusion model tests                                     |
-| `omni`             | Omni multimodal model tests |
-| `tts`              | TTS model tests     |
+| `omni`             | Omni model tests                                          |
 | `cache`            | Cache backend tests                                       |
 | `parallel`         | Parallelism/distributed tests                             |
 | `cpu`              | Tests that run on CPU                                     |

docs/contributing/ci/tests_style.md

@@ -54,8 +54,6 @@ vllm_omni/                                    tests/
 │   │                                            │   │   └── test_lora_manager.py        ✅
 │   ├── models/                                 │   ├── models/
 │   │   ├── qwen_image/                         │   │   ├── qwen_image/                 (e2e coverage)
-│   │   ├── ovis_image/                         │   │   ├── ovis_image/
-│   │   │   └── ...                             │   │   │   └── test_ovis_image.py     ✅
 │   │   ├── z_image/                            │   │   └── z_image/
 │   │   └── ...                                 │   │       └── test_zimage_tp_constraints.py  ✅
 │   └── worker/                                 │   └── worker/
@@ -136,13 +134,15 @@ vllm_omni/                                    tests/
                                                │   ├── test_qwen3_omni.py
                                                │   ├── test_qwen3_omni_expansion.py
                                                │   ├── test_mimo_audio.py
+                                               │   ├── test_image_gen_edit.py
                                                │   └── test_images_generations_lora.py
                                                └── offline_inference/                  ✅
                                                    ├── test_qwen2_5_omni.py
                                                    ├── test_qwen3_omni.py
                                                    ├── test_bagel_text2img.py
-                                                   ├── test_z_image.py
-                                                   ├── test_wan22.py
+                                                   ├── test_t2i_model.py
+                                                   ├── test_t2v_model.py
+                                                   ├── test_ovis_image.py
                                                    ├── test_zimage_tensor_parallel.py
                                                    ├── test_cache_dit.py
                                                    ├── test_teacache.py
@@ -213,6 +213,7 @@ E2E Online tests for Qwen3-Omni model with mix input and audio+text output.
 import os
 
 os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
+os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
 
 import threading
 from pathlib import Path

docs/contributing/model/adding_diffusion_model.md

@@ -115,20 +115,17 @@ class YourAttentionBlock(nn.Module):
 from vllm_omni.diffusion.attention.layer import Attention
 from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
 
-class YourSelfAttentionBlock(nn.Module):
+class YourAttentionBlock(nn.Module):
     def __init__(self, ...):
         super().__init__()
 
-        # Initialize vLLM-Omni's Attention layer.
-        # `role` lets users target this site with --diffusion-attention-config
-        # (e.g. --diffusion-attention-config.per_role.self.backend SAGE_ATTN).
+        # Initialize vLLM-Omni's Attention layer
         self.attn = Attention(
             num_heads=self.num_heads,
             head_size=self.head_dim,
             softmax_scale=1.0 / (self.head_dim ** 0.5),
             causal=False,  # Diffusion models typically use bidirectional attention
             num_kv_heads=self.num_kv_heads,
-            role="self",
         )
 
     def forward(self, hidden_states, encoder_hidden_states=None, attention_mask=None, ...):
@@ -136,6 +133,7 @@ class YourSelfAttentionBlock(nn.Module):
         # Create attention metadata
         attn_metadata = AttentionMetadata(attn_mask=attention_mask)
         hidden_states = self.attn(query, key, value, attn_metadata=attn_metadata)
+
 ```
 
 **Key Points:**
@@ -143,35 +141,8 @@ class YourSelfAttentionBlock(nn.Module):
 - **Attention layer initialization:** Done in `__init__`, not per-forward
 - **Tensor shapes:** vLLM-Omni `Attention` expects QKV to have `[B, seq, num_heads, head_dim]` shape
 - **AttentionMetadata:** Wraps attention mask and other metadata
-- **Role:** Tag every `Attention` site with a `role` string so users can configure backends per role (see below)
 
-**Declaring attention roles**
-
-The `role` argument is a free-form string that identifies this attention site. Users can match it from `--diffusion-attention-config.per_role.<role>.*` to swap backends without touching model code. Two conventions cover the common cases:
-
-| Convention | When to use | Example |
-|---|---|---|
-| `"self"` | Q/K/V come from the same hidden state | DiT self-attention block |
-| `"cross"` | K/V come from a separate `encoder_hidden_states` | Text-conditioned cross-attention |
-
-For multi-modal or unusual sites, use a dot-namespaced role and pair it with `role_category` so it can fall back to the generic config when nothing model-specific is set:
-
-```python
-# A model-specific cross-attention site that user config can target
-# either as 'mymodel.audio_to_video' (exact) or as 'cross' (category fallback).
-self.audio_to_video_attn = Attention(
-    num_heads=self.num_heads,
-    head_size=self.head_dim,
-    softmax_scale=1.0 / (self.head_dim ** 0.5),
-    causal=False,
-    role="mymodel.audio_to_video",
-    role_category="cross",
-)
-```
-
-For cross-attention sites whose K/V are replicated across ranks (e.g. text encoder output), pass `skip_sequence_parallel=True` to opt this layer out of sequence-parallel sharding.
-
-**Attention backends:** When the user does not configure a backend, vLLM-Omni asks the current platform for its default (typically `FLASH_ATTN` on CUDA when available). Users override the default via `--diffusion-attention-backend`, the `DIFFUSION_ATTENTION_BACKEND` env var, or finer-grained `--diffusion-attention-config.per_role.*` flags. See [Diffusion Attention Backends](../../user_guide/diffusion/attention_backends.md) for the full configuration surface.
+**Attention backends:** vLLM-Omni automatically selects the attention backend given the environmental variable `DIFFUSION_ATTENTION_BACKEND`. The default attention backend is `FLASH_ATTN` for diffusion models.
 
 #### 1.3: Replace Imports and Utilities
 
@@ -766,16 +737,6 @@ step-scoped phases. The reference implementation is
 Do not enable `step_execution=True` until those four methods are implemented
 and validated against the request-level path.
 
-If you want the pipeline to work with the experimental batched step-wise path
-(`max_num_seqs > 1`), also see:
-[Continuous Batching for Step-Wise Diffusion](../../design/feature/diffusion_continuous_batching.md).
-
-If you expose this in example scripts or recipes, keep it opt-in. Surface
-runtime features like `step_execution` as optional flags instead of silently
-turning them on. For Qwen-Image-style serving examples, document
-`--step-execution` as the feature gate and `--max-num-seqs N` as the
-companion batching knob.
-
 ### Cache Acceleration
 
 #### TeaCache

docs/contributing/model/adding_quantization_model.md

@@ -0,0 +1,478 @@
+# Support FP8 Quantization
+
+This section describes how to add FP8 quantization support to a diffusion model. We use the Qwen-Image pipeline as the reference implementation.
+
+---
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Step-by-Step Implementation](#step-by-step-implementation)
+- [Testing](#testing)
+- [Troubleshooting](#troubleshooting)
+- [Reference Implementations](#reference-implementations)
+- [Summary](#summary)
+
+---
+
+## Overview
+
+### What is FP8 Quantization?
+
+FP8 quantization reduces model memory footprint by storing weights in 8-bit floating point (`float8_e4m3fn`) instead of 16-bit (`bfloat16`/`float16`). This saves ~50% of weight memory.
+
+A diffusion pipeline typically has three quantizable components:
+
+| Component | Layer Types | Quantization Approach |
+|-----------|-----------|----------------------|
+| **DiT (transformer)** | `nn.Linear` | Replace with vLLM quantized linear layers |
+| **Text encoder** | `nn.Linear` | FP8 weight storage via hooks |
+| **VAE** | `nn.Conv2d`, `nn.Conv3d` | FP8 weight storage via hooks |
+
+### Architecture
+
+The quantization system has two mechanisms:
+
+**1. vLLM Quantized Linear Layers (for DiT)**
+
+vLLM provides drop-in replacements for `nn.Linear` that perform FP8 W8A8 computation on supported hardware. These are configured via `quant_config` passed to layer constructors.
+
+Key classes:
+
+| Class | Location | Purpose |
+|-------|----------|---------|
+| `build_quant_config` | `vllm_omni/quantization/__init__.py` | Build quantization config from string/dict/per-component spec |
+| `ComponentQuantizationConfig` | `vllm_omni/quantization/component_config.py` | Per-component routing by layer prefix |
+
+**2. FP8 Weight Storage via Hooks (for text encoder and VAE)**
+
+For models loaded via `from_pretrained()` (outside vLLM's weight pipeline), we use forward hooks to store weights in FP8 and dequantize on-the-fly:
+
+```
+At rest:     weight.data = fp8_tensor (half memory)
+Pre-hook:    weight.data = fp8_weight.to(bf16) * scale
+Forward:     Normal computation in BF16
+Post-hook:   weight.data = fp8_weight (back to FP8)
+```
+
+Key function:
+
+| Function | Location | Purpose |
+|----------|----------|---------|
+| `apply_fp8_weight_storage` | `vllm_omni/diffusion/models/utils.py` | Apply FP8 hooks to Linear/Conv layers (introduced in [#1412](https://github.com/vllm-project/vllm-omni/pull/1412)) |
+
+---
+
+## Step-by-Step Implementation
+
+### Step 1: Add `quant_config` to the Transformer
+
+Pass `quant_config` to vLLM parallel linear layers in the transformer. This enables FP8 W8A8 for DiT linear layers.
+
+#### 1.1: Accept `quant_config` in Constructor
+
+```python
+from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
+
+class YourTransformer2DModel(nn.Module):
+    def __init__(
+        self,
+        *,
+        od_config: OmniDiffusionConfig,
+        quant_config: QuantizationConfig | None = None,
+        # ... other params
+    ):
+        self.quant_config = quant_config
+        # Pass to sub-modules
+        self.blocks = nn.ModuleList([
+            YourTransformerBlock(quant_config=quant_config, ...)
+            for _ in range(num_layers)
+        ])
+```
+
+#### 1.2: Pass `quant_config` to vLLM Linear Layers
+
+Replace `nn.Linear` with vLLM parallel linear layers that accept `quant_config`:
+
+```python
+from vllm.model_executor.layers.linear import (
+    ColumnParallelLinear,
+    QKVParallelLinear,
+    RowParallelLinear,
+)
+
+class YourAttentionBlock(nn.Module):
+    def __init__(self, hidden_size, num_heads, quant_config=None, prefix=""):
+        # QKV projection
+        self.to_qkv = QKVParallelLinear(
+            hidden_size=hidden_size,
+            head_size=hidden_size // num_heads,
+            total_num_heads=num_heads,
+            quant_config=quant_config,
+            prefix=f"{prefix}.to_qkv",
+        )
+        # Output projection
+        self.to_out = RowParallelLinear(
+            input_size=hidden_size,
+            output_size=hidden_size,
+            quant_config=quant_config,
+            prefix=f"{prefix}.to_out",
+        )
+
+class YourFeedForward(nn.Module):
+    def __init__(self, hidden_size, intermediate_size, quant_config=None, prefix=""):
+        self.gate_up_proj = ColumnParallelLinear(
+            input_size=hidden_size,
+            output_size=intermediate_size * 2,
+            quant_config=quant_config,
+            prefix=f"{prefix}.gate_up_proj",
+        )
+        self.down_proj = RowParallelLinear(
+            input_size=intermediate_size,
+            output_size=hidden_size,
+            quant_config=quant_config,
+            prefix=f"{prefix}.down_proj",
+        )
+```
+
+!!! important
+    Always pass the `prefix` parameter. It is used by `ignored_layers` matching to skip sensitive layers.
+
+#### 1.3: Identify Sensitive Layers
+
+Some layers degrade output quality when quantized. Common sensitive layers:
+
+| Layer | Why Sensitive | Typical Models | Quant Methods Affected |
+|-------|-------------|---------------|----------------------|
+| `img_mlp` | Processes denoising latents with shifting dynamic range | Qwen-Image | FP8, Int8 |
+| `feed_forward` | FFN layers in DiT blocks, large dynamic range | Z-Image | Int8 |
+| `proj_out` | Final output projection, small quantization errors amplified | Various | FP8, Int8 |
+| `lm_head` | Output vocabulary projection, precision-critical | Omni (thinker) | NVFP4 |
+| `mlp.gate` | MoE router gates, routing decisions are precision-critical | MoE models | NVFP4 |
+
+**Quantization quality benchmarks** (LPIPS: lower = better, <0.01 imperceptible, >0.1 noticeable):
+
+| Model | Method | All Layers | With `ignored_layers` | Recommendation |
+|-------|--------|-----------|----------------------|----------------|
+| Qwen-Image-2512 | Int8 | LPIPS 0.0197 | 0.0027 (skip `img_mlp`) | Skip `img_mlp` |
+| Z-Image-Turbo | Int8 | LPIPS 0.1597 | 0.0290 (skip `feed_forward`) | Skip `feed_forward` |
+| Z-Image-Turbo | FP8 | LPIPS ~0.005 | — | All layers OK |
+
+To identify sensitive layers for a new model:
+
+1. Run inference with all layers quantized
+2. Compare output quality with BF16 baseline using LPIPS (see `benchmarks/diffusion/quantization_quality.py`)
+3. Selectively disable quantization per layer (via `ignored_layers`) until quality matches
+4. Document the recommended `ignored_layers` in the model's supported models table
+
+### Step 2: Wire Up Quantization in the Pipeline
+
+#### 2.1: Create `quant_config` and Pass to Transformer
+
+In the pipeline's `__init__`, extract the vLLM quant config and pass it to the transformer:
+
+```python
+from vllm_omni.quantization import build_quant_config
+
+class YourPipeline(nn.Module):
+    def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = ""):
+        super().__init__()
+        # ... load scheduler, text_encoder, vae ...
+
+        # Build quant config from OmniDiffusionConfig
+        quant_config = build_quant_config(od_config.quantization_config)
+
+        # Pass to transformer
+        self.transformer = YourTransformer2DModel(
+            od_config=od_config,
+            quant_config=quant_config,
+            # ... other kwargs
+        )
+```
+
+#### 2.2: Apply FP8 Weight Storage to Text Encoder and VAE
+
+For models loaded via `from_pretrained()`, apply hook-based FP8 weight storage.
+
+!!! note
+    The `apply_fp8_weight_storage` function is provided by `vllm_omni/diffusion/models/utils.py`, introduced in [PR #1412](https://github.com/vllm-project/vllm-omni/pull/1412).
+
+```python
+from vllm_omni.diffusion.models.utils import apply_fp8_weight_storage
+
+class YourPipeline(nn.Module):
+    def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = ""):
+        super().__init__()
+
+        self.text_encoder = SomeTextEncoder.from_pretrained(...)
+        self.vae = SomeVAE.from_pretrained(...).to(self.device)
+
+        # Apply FP8 weight storage to text encoder and VAE
+        if (
+            od_config.quantization_config is not None
+            and getattr(od_config.quantization_config, "quant_method", None) == "fp8"
+        ):
+            apply_fp8_weight_storage(self.vae)
+            apply_fp8_weight_storage(self.text_encoder)
+
+        # ... rest of init
+```
+
+`apply_fp8_weight_storage` walks all `nn.Linear`, `nn.Conv2d`, and `nn.Conv3d` modules in the model and applies FP8 hooks to each.
+
+#### 2.3: Fix `load_weights` to Include Pre-loaded Components
+
+Since text encoders and VAEs are loaded via `from_pretrained()` (not through vLLM's weight pipeline), their parameters won't appear in the `loaded_weights` set. Mark them explicitly:
+
+```python
+def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+    loader = AutoWeightsLoader(self)
+    loaded_weights = loader.load_weights(weights)
+    # Mark pre-loaded component weights as loaded
+    loaded_weights |= {f"vae.{name}" for name, _ in self.vae.named_parameters()}
+    loaded_weights |= {f"text_encoder.{name}" for name, _ in self.text_encoder.named_parameters()}
+    return loaded_weights
+```
+
+### Step 3: Add to Supported Models Documentation
+
+Update the FP8 supported models table in `docs/user_guide/diffusion/quantization/fp8.md`:
+
+```markdown
+| Your-Model | `org/Your-Model` | ✅ | ✅ | ✅ | `sensitive_layer` |
+```
+
+---
+
+## Testing
+
+### Quick Verification
+
+```python
+from vllm_omni import Omni
+from vllm_omni.inputs.data import OmniDiffusionSamplingParams
+
+# Test with FP8 enabled
+omni = Omni(model="your-model", quantization="fp8")
+
+outputs = omni.generate(
+    "A cat sitting on a windowsill",
+    OmniDiffusionSamplingParams(num_inference_steps=50),
+)
+```
+
+### Quality Gate Test (LPIPS)
+
+We provide a pytest-based quality gate at `tests/diffusion/quantization/test_quantization_quality.py`.
+It generates outputs with both BF16 and your quantized method using the same seed, computes
+[LPIPS](https://github.com/richzhang/PerceptualSimilarity) perceptual distance, and **fails if it
+exceeds a threshold**. This is the recommended way to validate that a quantization method does not
+unacceptably degrade output quality.
+
+**Step 1: Add your test config** — edit `QUALITY_CONFIGS` in the test file:
+
+```python
+QualityTestConfig(
+    id="int8_z_image",              # pytest ID
+    model="Tongyi-MAI/Z-Image-Turbo",
+    quantization="int8",            # your quantization method
+    task="t2i",                     # "t2i" or "t2v"
+    prompt="a cup of coffee on a wooden table, morning light",
+    max_lpips=0.15,                 # fail threshold (see guide below)
+    num_inference_steps=20,
+),
+```
+
+**Step 2: Run the test:**
+
+```bash
+# Install dependency
+pip install lpips
+
+# Run all quality tests
+pytest tests/diffusion/quantization/test_quantization_quality.py -v -m ""
+
+# Run only your method
+pytest tests/diffusion/quantization/test_quantization_quality.py -v -m "" -k "int8"
+
+# Run a specific model
+pytest tests/diffusion/quantization/test_quantization_quality.py -v -m "" -k "z_image"
+```
+
+**Step 3: Read the output:**
+
+```
+Quantization Quality: int8_z_image
+  Model:         Tongyi-MAI/Z-Image-Turbo
+  Method:        int8
+  LPIPS:         0.0290  (threshold: 0.15)
+  BF16 memory:   19.15 GiB
+  Quant memory:  14.80 GiB  (23% reduction)
+  Result:        PASS
+```
+
+**LPIPS threshold guide:**
+
+| LPIPS Range | Meaning | Recommendation |
+|-------------|---------|----------------|
+| < 0.01 | Imperceptible | Ideal — no visible difference |
+| 0.01 – 0.05 | Minor differences | Good — acceptable for most use cases |
+| 0.05 – 0.15 | Noticeable on close inspection | Acceptable with `ignored_layers` tuning |
+| 0.15 – 0.30 | Clearly visible | Investigate — likely needs `ignored_layers` |
+| > 0.30 | Significant degradation | Quantization method may not suit this model |
+
+!!! tip
+    If your test fails, don't raise the threshold — instead, identify sensitive layers and add
+    them to `ignored_layers` (see next section). Then re-run the test with the lower LPIPS.
+
+### Quality Benchmark (Detailed)
+
+For a full multi-prompt benchmark with Markdown output (suitable for PR descriptions), use the
+standalone benchmark script:
+
+```bash
+python benchmarks/diffusion/quantization_quality.py \
+    --model Tongyi-MAI/Z-Image-Turbo \
+    --quantization fp8 \
+    --prompts \
+        "a cup of coffee on the table" \
+        "an aerial view of a coral reef" \
+    --height 1024 --width 1024 \
+    --num-inference-steps 50 --seed 42
+```
+
+This generates BF16 and quantized outputs, computes per-prompt LPIPS, and prints a Markdown
+summary table ready to paste into your PR.
+
+### Identifying Sensitive Layers with `ignored_layers`
+
+Not all layers tolerate quantization equally. **`ignored_layers`** lets you keep specific layers
+in BF16 while quantizing everything else. This is critical for achieving good quality with
+aggressive quantization methods (Int8, NVFP4).
+
+**Why some layers are sensitive:**
+
+| Layer Type | Why Sensitive | Typical Impact |
+|------------|--------------|----------------|
+| `img_mlp` | Processes denoising latents with shifting dynamic range across timesteps. No built-in normalization to absorb quantization error. | Color shifts, blurriness |
+| `feed_forward` | FFN layers with large dynamic range in DiT blocks. | Artifacts, loss of detail |
+| `proj_out` | Final output projection — small errors get amplified since there are no subsequent layers to correct them. | Overall quality loss |
+| `lm_head` | Vocabulary projection in LLMs — precision-critical for token selection. | Garbage text output |
+| `mlp.gate` | MoE router gates — routing decisions are binary and precision-critical. | Wrong expert selection, quality collapse |
+
+**How to find sensitive layers for your model:**
+
+1. Run the quality gate test with all layers quantized:
+    ```bash
+    pytest tests/diffusion/quantization/test_quantization_quality.py -v -m "" -k "your_model"
+    ```
+
+2. If LPIPS exceeds your threshold, try skipping common sensitive layers one at a time:
+    ```python
+    QualityTestConfig(
+        id="fp8_your_model_skip_mlp",
+        model="your-org/Your-Model",
+        quantization="fp8",
+        task="t2i",
+        prompt="a cup of coffee on a wooden table",
+        max_lpips=0.05,
+    ),
+    ```
+
+    And in the test, modify the Omni call:
+    ```python
+    omni_qt = Omni(
+        model=config.model,
+        quantization_config={
+            "method": config.quantization,
+            "ignored_layers": ["img_mlp"],
+        },
+    )
+    ```
+
+3. Compare LPIPS with and without `ignored_layers` to isolate the problematic layers.
+
+4. Document the recommended `ignored_layers` in the supported models table.
+
+**Real-world examples:**
+
+| Model | Method | All Layers (LPIPS) | With `ignored_layers` | Which Layers to Skip |
+|-------|--------|-------------------|----------------------|---------------------|
+| Qwen-Image-2512 | Int8 | 0.0197 | 0.0027 | `img_mlp` |
+| Z-Image-Turbo | Int8 | 0.1597 | 0.0290 | `feed_forward` |
+| Z-Image-Turbo | FP8 | ~0.005 | — | None needed |
+
+### What to Verify (Checklist)
+
+1. **Model loads without errors** — Check logs for quantization initialization messages
+2. **Quality gate passes** — LPIPS within threshold vs BF16 baseline
+3. **Memory reduction** — Expect ~30-50% savings on quantized components
+4. **Sensitive layers identified** — Document which layers (if any) need `ignored_layers`
+5. **`ignored_layers` pattern works** — Verify that the layer name prefix matches your transformer's module names
+
+---
+
+## Troubleshooting
+
+### Issue: Output Quality Degradation
+
+**Symptoms:** Generated images have visible artifacts, color shifts, or blurriness compared to BF16 baseline.
+
+**Causes & Solutions:**
+
+- **Sensitive layers being quantized:**
+
+    **Solution:** Identify and skip sensitive layers:
+    ```python
+    omni = Omni(
+        model="your-model",
+        quantization_config={
+            "method": "fp8",
+            "ignored_layers": ["img_mlp"],
+        },
+    )
+    ```
+
+- **All layers sensitive (rare):**
+
+    **Solution:** The model may not be suitable for FP8 quantization. Consider weight-only quantization or other methods.
+
+### Issue: Missing Weights Warning
+
+**Symptoms:** Warnings about unloaded weights for `vae.*` or `text_encoder.*` parameters.
+
+**Solution:** Ensure `load_weights` marks pre-loaded component weights. See [Step 2.3](#23-fix-load_weights-to-include-pre-loaded-components).
+
+### Issue: CUDA Out of Memory
+
+**Symptoms:** OOM errors during model loading despite FP8 being enabled.
+
+**Causes & Solutions:**
+
+- **Text encoder/VAE loaded in BF16 before FP8 conversion:**
+
+    FP8 weight storage is applied after `from_pretrained()`, so peak load memory is still BF16. This is a one-time cost during initialization.
+
+---
+
+## Reference Implementations
+
+| Model | Pipeline File | Transformer File | Notes |
+|-------|-------------|-----------------|-------|
+| **Z-Image** | `models/z_image/pipeline_z_image.py` | `models/z_image/z_image_transformer.py` | DiT + text encoder FP8 |
+| **Qwen-Image** | `models/qwen_image/pipeline_qwen_image.py` | `models/qwen_image/qwen_image_transformer.py` | DiT + text encoder + VAE FP8 |
+
+All files are under `vllm_omni/diffusion/`.
+
+---
+
+## Summary
+
+Adding FP8 quantization support to a new model:
+
+1. ✅ **Add `quant_config` to transformer** — Accept and pass to vLLM parallel linear layers with proper `prefix`
+2. ✅ **Identify sensitive layers** — Profile which layers need `ignored_layers` for quality
+3. ✅ **Apply FP8 weight storage** — Call `apply_fp8_weight_storage()` on text encoder and VAE in pipeline `__init__`
+4. ✅ **Fix `load_weights`** — Mark pre-loaded component weights as loaded
+5. ✅ **Update docs** — Add model to the supported models table in `docs/user_guide/diffusion/quantization/fp8.md`
+6. ✅ **Test** — Verify quality, memory savings, and sensitive layer behavior

docs/contributing/model/adding_tts_model.md

@@ -186,17 +186,6 @@ vllm_omni/model_executor/stage_configs/
   your_model_name_async_chunk.yaml   # Streaming mode config
 ```
 
-### Example placement
-
-TTS examples live in the consolidated text-to-speech hub, **not** in their
-own top-level directory. Place per-model scripts under
-`examples/offline_inference/text_to_speech/<your_model>/` and
-`examples/online_serving/text_to_speech/<your_model>/`, and add a section
-to the hub `README.md` files (table row + per-model section) instead of a
-new per-model `README.md`. The mkdocs `generate_examples` hook treats the
-`text_to_speech/` parent as a single example, so per-model READMEs inside
-it would not be picked up — the hub README is the documented surface.
-
 **Qwen3-TTS reference files:**
 
 | File | Purpose |

docs/contributing/profiling.md

@@ -37,8 +37,6 @@ Supported fields:
 | `warmup_iterations` | Torch-profiler warmup iterations. |
 | `active_iterations` | Torch-profiler active iterations. |
 
-For detailed explanations of the fields, please refer to upstream vLLM implementation [vllm/config/profiler.py](https://github.com/vllm-project/vllm/blob/v0.20.1/vllm/config/profiler.py)
-
 ### Minimal configurations by output
 
 Only collect trace output:
@@ -126,7 +124,7 @@ profiler_config = {
     "max_iterations": 0,
     "wait_iterations": 0,
     "warmup_iterations": 0,
-    "active_iterations": 1,
+    "active_iterations": 0,
 }
 ```
 
@@ -222,7 +220,7 @@ vllm serve Wan-AI/Wan2.2-I2V-A14B-Diffusers \
     "max_iterations": 0,
     "wait_iterations": 0,
     "warmup_iterations": 0,
-    "active_iterations": 1
+    "active_iterations": 0
   }'
 ```
 

docs/design/feature/diffusion_continuous_batching.md

@@ -1,156 +0,0 @@
-# Continuous Batching for Step-Wise Diffusion
-
-!!! warning "Experimental Feature"
-    This feature is experimental. It currently applies only to native
-    diffusion pipelines running with `step_execution=True`.
-
-This document describes the batching extension built on top of
-[Diffusion Step Execution](diffusion_step_execution.md). The base
-step-execution contract is unchanged. The batching work is mainly in the
-scheduler and runner layers.
-
-## Why It Helps
-
-Step-wise execution breaks a long denoise loop into scheduler-visible units.
-That gives the runtime a place to admit other compatible requests between
-steps instead of waiting for an entire request to finish.
-
-This matters most in low-MFU or bursty serving scenarios:
-
-- one request's denoise step may not fully saturate the GPU
-- several compatible requests can share the same denoise forward pass
-- throughput and device utilization can improve without changing request-local
-  scheduler state
-
-This is **not** a guaranteed single-request latency win. The main benefit is
-usually higher utilization and better throughput when the workload contains
-multiple in-flight compatible requests.
-
-## Overview
-
-With continuous batching enabled:
-
-- the scheduler may keep multiple compatible requests active at the same time
-- the runner packs request-local step state into one `InputBatch`
-- `denoise_step()` runs on that batch
-- `step_scheduler()` and `post_decode()` still run per request
-
-The current implementation is conservative:
-
-- only compatible requests are batched together
-- request-mode diffusion still runs with `max_num_seqs=1`
-- per-request progress and completion remain independent
-
-## Enablement
-
-Use `--step-execution` as the feature gate, then increase `--max-num-seqs`
-above `1` if you want batching:
-
-```bash
-vllm serve Qwen/Qwen-Image --omni \
-  --port 8091 \
-  --step-execution \
-  --max-num-seqs 8
-```
-
-`--max-num-seqs 1` keeps the step-wise path without enabling batching.
-
-For a reproducible replay flow using the bundled serving benchmark, see the
-Qwen-Image replay commands in
-[`benchmarks/diffusion/README.md`](gh-file:benchmarks/diffusion/README.md)
-and
-[`benchmarks/diffusion/performance_dashboard/qwen_image_serving_performance.md`](gh-file:benchmarks/diffusion/performance_dashboard/qwen_image_serving_performance.md).
-
-## Scheduler
-
-The scheduler derives its batch capacity from `max_num_seqs` through
-`max_num_running_reqs`.
-
-Batch admission is gated by
-[`SamplingParamsKey`](gh-file:vllm_omni/diffusion/sched/interface.py),
-which is built from shape-sensitive and CFG-sensitive sampling fields. This is
-the core correctness rule for batching: requests are only co-batched when they
-share the same denoise tensor contract.
-
-There are two important details:
-
-- `num_inference_steps` is not part of the key, so requests with different
-  total step counts can still share a batch
-- requests also do not need to be at the same current denoise progress; active
-  requests can continue batching even when their current step indices diverge
-- admission is still FIFO, so an incompatible request at the head of the
-  waiting queue blocks later compatible requests
-
-Today that compatibility rule is still shape-sensitive. `height`, `width`,
-`num_frames`, and CFG-related fields remain part of the key, so different
-resolutions or incompatible guidance settings do **not** co-batch yet.
-
-The current batching unit is one `OmniDiffusionRequest`. Requests with
-multiple prompts do not participate in batching today.
-
-## Runner
-
-The runner keeps persistent per-request execution state in
-[`DiffusionRequestState`](gh-file:vllm_omni/diffusion/worker/utils.py),
-while the scheduler owns a separate lightweight request state for queueing and
-lifecycle tracking.
-
-For each step, the runner builds an
-[`InputBatch`](gh-file:vllm_omni/diffusion/worker/input_batch.py) from the
-active request states:
-
-- prompt embeddings and masks are normalized and padded
-- dynamic tensors such as `latents` and `timesteps` are gathered each step
-- buffers are reused when batch composition stays the same
-
-The step-wise batched path is:
-
-1. Run `prepare_encode()` for newly admitted requests.
-2. Build or refresh `InputBatch`.
-3. Run one batched `denoise_step(input_batch)`.
-4. Slice the batched `noise_pred` back per request.
-5. Run per-request `step_scheduler()`.
-6. Run `post_decode()` only for requests that finished denoising.
-7. Scatter updated latents back into persistent request state with
-   [`scatter_latents()`](gh-file:vllm_omni/diffusion/worker/input_batch.py).
-
-This keeps the shared work limited to the denoise forward pass while preserving
-request-local scheduler state and outputs.
-
-## Engine
-
-[`DiffusionEngine`](gh-file:vllm_omni/diffusion/diffusion_engine.py) provides
-the background loop and async add-request path needed for multiple requests to
-accumulate in the scheduler.
-
-This is supporting infrastructure, not the main design point. The batching
-behavior is defined by scheduler-side compatibility gating and runner-side
-batch packing.
-
-## Current Limitations
-
-- Experimental feature; use `max_num_seqs=1` for the older conservative path.
-- Only native pipelines that already support `step_execution=True`.
-- Request-mode diffusion still clamps `max_num_seqs` back to `1`.
-- Only homogeneous batches keyed by `SamplingParamsKey` are supported.
-- Multi-prompt requests are not batched.
-- `cache_backend`, KV transfer, and other request-mode extras are not wired
-  into the batched step-wise path yet.
-- Future work can relax the current same-shape restriction with richer
-  heterogeneous batching policies such as bucketing or padded execution for
-  different resolutions.
-
-## Related Files
-
-- Scheduler base:
-  [`vllm_omni/diffusion/sched/base_scheduler.py`](gh-file:vllm_omni/diffusion/sched/base_scheduler.py)
-- Scheduler interface:
-  [`vllm_omni/diffusion/sched/interface.py`](gh-file:vllm_omni/diffusion/sched/interface.py)
-- Step scheduler:
-  [`vllm_omni/diffusion/sched/step_scheduler.py`](gh-file:vllm_omni/diffusion/sched/step_scheduler.py)
-- Runner:
-  [`vllm_omni/diffusion/worker/diffusion_model_runner.py`](gh-file:vllm_omni/diffusion/worker/diffusion_model_runner.py)
-- Input batch:
-  [`vllm_omni/diffusion/worker/input_batch.py`](gh-file:vllm_omni/diffusion/worker/input_batch.py)
-- Tests:
-  [`tests/diffusion/test_diffusion_scheduler.py`](gh-file:tests/diffusion/test_diffusion_scheduler.py)

docs/design/feature/diffusion_step_execution.md

@@ -1,4 +1,4 @@
-# Adding Step Execution Support for Diffusion Pipelines
+# Diffusion Step Execution
 
 This guide documents vLLM-Omni's stepwise diffusion contract for model authors
 and contributors implementing `step_execution=True` support for a diffusion
@@ -7,20 +7,12 @@ pipeline.
 For end-user enablement, supported models, and current limitations, see
 [Step Execution](../../user_guide/diffusion/step_execution.md).
 
-This document describes the base step-execution contract only. For the
-experimental batching policy layered on top of the step-wise path, see
-[Continuous Batching for Step-Wise Diffusion](diffusion_continuous_batching.md).
-
 ## Current Support Scope
 
 `step_execution` is **not** a generic diffusion toggle. It only works for
 pipelines that implement the segmented stateful contract in
 [`vllm_omni/diffusion/models/interface.py`](gh-file:vllm_omni/diffusion/models/interface.py).
 
-This page is intentionally author-facing. Treat runtime enablement
-(`step_execution=True` in Python or `--step-execution` in serving) as an
-opt-in user knob layered on top of the implementation contract below.
-
 Current in-tree support:
 
 | Pipeline | Example models | Step execution |
@@ -30,9 +22,7 @@ Current in-tree support:
 
 Current engine/runtime limitations:
 
-- Continuous batching with `max_num_seqs > 1` is experimental and documented in
-  [Continuous Batching for Step-Wise Diffusion](diffusion_continuous_batching.md).
-  Keep `max_num_seqs=1` if you want the older conservative behavior.
+- `StepScheduler` only schedules `batch_size=1`.
 - `cache_backend` is not supported in step mode.
 - Request-mode extras such as KV transfer are not wired into step mode yet.
 - Unsupported pipelines now fail early during model loading instead of failing on the first request.

docs/design/feature/pipeline_parallel.md

@@ -1,356 +0,0 @@
-# Pipeline Parallel
-
-This section describes how to add Pipeline Parallelism (PP) to a diffusion pipeline. We use the Wan2.2 text-to-video and
-image-to-video pipelines as the reference implementations.
-
-## Implementation Checklist
-
-Adding Pipeline Parallel support requires:
-
-1. ✅ **Inherit the mixin** - add `PipelineParallelMixin` before `CFGParallelMixin` in the pipeline class
-2. ✅ **Make stage forward resumable** - support `intermediate_tensors` in the transformer forward path
-3. ✅ **Return the right object type** - `IntermediateTensors` on non-last PP ranks, final model output on the last rank
-4. ✅ **Use the standard denoising loop** - call `predict_noise_maybe_with_cfg()` and `scheduler_step_maybe_with_cfg()`
-5. ✅ **Test parity** - compare PP results against the single-GPU baseline
-
----
-
-## Table of Contents
-
-- [Overview](#overview)
-- [Step-by-Step Implementation](#step-by-step-implementation)
-- [Testing](#testing)
-- [Troubleshooting](#troubleshooting)
-- [Reference Implementations](#reference-implementations)
-
----
-
-## Overview
-
-### What is Pipeline Parallelism?
-
-Pipeline Parallelism splits the denoising transformer into multiple sequential stages and places each stage on a
-different rank. Instead of every rank holding the full DiT, each PP rank owns only a slice of the layers.
-
-For each denoising step:
-
-1. Rank 0 starts the forward pass using the current latents.
-2. Each intermediate rank receives hidden states from the previous rank, runs its local layer slice, and forwards the
-   intermediate tensors downstream.
-3. The last PP rank produces the final noise prediction.
-4. The last PP rank applies the scheduler step and sends the updated latents back to rank 0 for the next timestep.
-
-This reduces per-rank model memory and enables larger diffusion transformers to run across multiple GPUs. It can also be
-combined with CFG-Parallel, where each PP pipeline carries one CFG branch.
-
-### Architecture
-
-vLLM-Omni provides `PipelineParallelMixin` to encapsulate the PP communication pattern for diffusion pipelines.
-
-| Method                            | Purpose                       | Automatic Behavior                                                              |
-|-----------------------------------|-------------------------------|---------------------------------------------------------------------------------|
-| `diffuse()`                       | Denoising loop boundary       | Wrapped by `PipelineParallelMixin` to flush pending async PP sends on exit      |
-| `predict_noise_maybe_with_cfg()`  | Predict noise with PP support | Runs partial forwards on non-last PP ranks, combines with CFG logic when needed |
-| `scheduler_step_maybe_with_cfg()` | Step scheduler with PP sync   | Runs scheduler on the last PP rank, returns updated latents to rank 0           |
-| `_sync_pp_send()`                 | Flush pending async sends     | Waits for outstanding `isend` handles before later collectives or decode        |
-
-`PipelineParallelMixin` is intentionally a pipeline-level abstraction. Your model-specific `predict_noise()` still
-defines how a local stage executes.
-
-### How It Works
-
-`predict_noise_maybe_with_cfg()` automatically switches between these modes:
-
-- **PP disabled** (`pipeline_parallel_size == 1`):
-    - Falls back to `CFGParallelMixin.predict_noise_maybe_with_cfg()`
-- **PP only** (`pipeline_parallel_size > 1`, `cfg_parallel_size == 1`):
-    - Rank 0 starts with the input latents
-    - Middle ranks receive `intermediate_tensors`, run their local layer range, and asynchronously send downstream
-    - The last rank returns the final noise prediction
-    - When CFG is enabled in this mode (sequential CFG), both the positive and negative branches run through the same
-      PP pipeline, doubling the communication volume per denoising step. Prefer PP + CFG-Parallel when
-      `cfg_parallel_size > 1` is available to avoid this overhead.
-- **PP + CFG-Parallel** (`pipeline_parallel_size > 1`, `cfg_parallel_size > 1`):
-    - Each PP pipeline carries one CFG branch
-    - The last PP rank all-gathers across the CFG group
-    - CFG combination happens on every last PP rank in the CFG group, matching the non-PP CFG-parallel behavior
-
-`scheduler_step_maybe_with_cfg()` keeps the denoising loop consistent:
-
-- **PP disabled**:
-    - Falls back to `scheduler_step_maybe_with_cfg()`
-- **PP enabled**:
-    - Only the last PP rank has `noise_pred` and runs the scheduler step
-    - The resulting latents are sent back to rank 0
-    - Rank 0 receives an `AsyncLatents` wrapper that resolves only when the tensor is next consumed
-
-This asynchronous design avoids unnecessary blocking between denoising steps.
-
-When a pipeline class is defined, `PipelineParallelMixin` wraps its `diffuse()` method and calls `_sync_pp_send()` in a
-`finally` block after `diffuse()` returns or raises. This keeps model code free of explicit PP cleanup while still
-ensuring that the final non-blocking PP send completes before decode or later collectives.
-
----
-
-## Step-by-Step Implementation
-
-### Step 1: Inherit `PipelineParallelMixin` and `CFGParallelMixin`
-
-`PipelineParallelMixin` **requires** `CFGParallelMixin` and must appear before it in the class MRO. This is enforced at
-class definition time via `__init_subclass__`: defining a pipeline that inherits `PipelineParallelMixin` without
-`CFGParallelMixin`, or after `CFGParallelMixin`, raises a `TypeError` immediately on import.
-
-`PipelineParallelMixin` delegates noise prediction, CFG combination, and scheduler stepping to `CFGParallelMixin`, which
-supplies `predict_noise()`, `predict_noise_maybe_with_cfg()`, `scheduler_step_maybe_with_cfg()`, and
-`combine_cfg_noise()`.
-
-**Example:**
-
-```python
-from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
-from vllm_omni.diffusion.distributed.pipeline_parallel import PipelineParallelMixin
-import torch.nn as nn
-
-
-class YourPipeline(nn.Module, PipelineParallelMixin, CFGParallelMixin):
-    ...
-```
-
-The order matters: `PipelineParallelMixin` must be listed before `CFGParallelMixin` so calls to
-`predict_noise_maybe_with_cfg()` and `scheduler_step_maybe_with_cfg()` resolve to the PP-aware wrappers, while their
-`super()` calls delegate to the CFG implementation when PP is disabled or after the last PP stage.
-
-### Step 2: Make model forward and `predict_noise()` PP-aware
-
-The PP mixin injects `intermediate_tensors` into the normal `predict_noise()` call. Your model forward path must support
-two inputs:
-
-- Normal input from rank 0, typically passed as `hidden_states` or `x`
-- `intermediate_tensors` from upstream PP ranks
-
-The standard model forward pattern is:
-
-1. If `intermediate_tensors` is present, read the local hidden state from it.
-2. Run only this rank's layer slice.
-3. Return `IntermediateTensors(...)` on non-last PP ranks.
-4. Return the final model output on the last PP rank. `CFGParallelMixin.predict_noise()` already follows this contract
-   for common pipelines.
-
-**Minimal example:**
-
-```python
-from vllm.sequence import IntermediateTensors
-from vllm_omni.diffusion.distributed.parallel_state import get_pp_group
-
-
-def forward(self, hidden_states=None, intermediate_tensors=None, **kwargs):
-    if intermediate_tensors is not None:
-        hidden_states = intermediate_tensors["hidden_states"]
-
-    for i in range(self.start_layer, self.end_layer):
-        hidden_states = self.layers[i](hidden_states)
-
-    pp_group = get_pp_group()
-    if not pp_group.is_last_rank:
-        return IntermediateTensors({"hidden_states": hidden_states})
-    return (hidden_states,)
-```
-
-### Step 3: Partition the transformer layers
-
-The local module on each PP rank must expose only that rank's layer slice. In the reference tests and vLLM model
-implementations, this is typically done with vLLM utilities such as `make_layers(...)`, which fill missing ranges with
-`PPMissingLayer`. Besides partitioning the layers themselves, model authors should also wire
-`make_empty_intermediate_tensors_factory(...)` for intermediate tensor allocation and `is_pp_missing_parameter(...)` for
-PP-aware weight loading.
-
-To prepare a transformer for PP, implement the following pieces in order.
-
-#### 3.1 Split the transformer layers across PP ranks
-
-Each PP rank should own only its local layer range, typically exposed as `[start_layer, end_layer)`. In practice this is
-usually done with `make_layers(...)`, which constructs local layers and fills missing ranges with `PPMissingLayer`.
-
-The goal is:
-
-- every PP rank knows its `[start_layer, end_layer)` range
-- non-local layers do not execute on this rank
-- the forward can resume from incoming `intermediate_tensors`
-
-By default, layers are distributed evenly across PP ranks using `get_pp_indices()`. When the number of layers is not
-divisible by the PP size, remaining layers are assigned to middle partitions to balance compute and memory. You can
-override this with the `VLLM_PP_LAYER_PARTITION` environment variable to specify an exact per-rank layer count:
-
-```bash
-# Example: 40 layers across 4 PP ranks, assigning 8 / 12 / 12 / 8 layers
-export VLLM_PP_LAYER_PARTITION=8,12,12,8
-```
-
-The value must be a comma-separated list of integers whose length equals `pipeline_parallel_size` and whose sum equals
-the total number of transformer layers.
-
-#### 3.2 Expose `make_empty_intermediate_tensors`
-
-Your transformer module should expose `self.make_empty_intermediate_tensors`, typically created with
-`make_empty_intermediate_tensors_factory(...)`.
-
-This is important because PP ranks need a consistent way to allocate placeholder `IntermediateTensors` with the expected
-keys and hidden dimension.
-
-**Example:**
-
-```python
-from vllm.model_executor.models.utils import make_empty_intermediate_tensors_factory
-
-self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
-    ["hidden_states"],
-    inner_dim,
-)
-```
-
-For Wan2.2, the intermediate payload between PP stages is the token sequence stored under `"hidden_states"`, so the
-factory is created with that key and the transformer hidden size.
-
-#### 3.3 Return `IntermediateTensors` on non-last PP ranks
-
-Your model `forward()` or custom `predict_noise()` implementation should consume `intermediate_tensors` on non-first
-ranks and return `IntermediateTensors(...)` on non-last ranks.
-
-This lets each PP stage resume from the upstream hidden states and pass the local result to the next stage.
-
-#### 3.4 Skip non-local weights during `load_weights()`
-
-When a model is PP-partitioned, many parameters in the checkpoint belong to layers that do not exist on the current
-rank. `load_weights()` must therefore skip parameters for missing PP stages using `is_pp_missing_parameter(...)`.
-
-If this is not done, weight loading will either fail or incorrectly try to load tensors into `PPMissingLayer`
-placeholders.
-
-The Wan2.2 transformer is the best reference here: it uses `is_pp_missing_parameter(...)` before loading both remapped
-and fused parameters.
-
-If your model has multiple transformer variants, PP still works as long as each selected transformer obeys the same
-contract.
-
-### Step 4: Use the standard denoising contract
-
-vLLM-Omni diffusion pipelines already route denoising through `diffuse()`, `predict_noise_maybe_with_cfg()`, and
-`scheduler_step_maybe_with_cfg()`. `PipelineParallelMixin` overrides those standard helpers when
-`pipeline_parallel_size > 1`, so model integrations should not add separate PP-specific helper names or manual post-loop
-synchronization.
-
-In PP mode:
-
-- Non-last PP ranks return `None` from `predict_noise_maybe_with_cfg()` after sending `IntermediateTensors` downstream.
-- The last PP rank returns the final prediction, and with CFG-Parallel every last PP rank in the CFG group receives the
-  combined result.
-- Rank 0 receives `AsyncLatents` from `scheduler_step_maybe_with_cfg()`, which resolves only when the latents are
-  consumed.
-- Pending non-blocking sends are flushed automatically when `diffuse()` exits.
-
----
-
-## Testing
-
-Use an offline inference script with `pipeline_parallel_size > 1`.
-
-```bash
-python examples/offline_inference/text_to_video/text_to_video.py \
---model=Wan-AI/Wan2.2-TI2V-5B-Diffusers \
---width=1280 \
---height=704 \
---guidance-scale=5.0 \
---prompt="Two anthropomorphic cats in comfy boxing gear and bright gloves fight intensely on a spotlighted stage" \
---output=t2v_5B_pp2.mp4 \
---pipeline-parallel-size=2
-```
-
-For PP + CFG-Parallel together:
-
-```bash
-python examples/offline_inference/text_to_video/text_to_video.py \
---model=Wan-AI/Wan2.2-TI2V-5B-Diffusers \
---width=1280 \
---height=704 \
---guidance-scale=5.0 \
---prompt="Two anthropomorphic cats in comfy boxing gear and bright gloves fight intensely on a spotlighted stage" \
---output=t2v_5B_pp2_cfg2.mp4 \
---pipeline-parallel-size=2 \
---cfg-parallel-size=2
-```
-
-**Verify:**
-
-1. The run completes without hangs at the PP stage boundary
-2. Output quality matches the non-PP baseline within normal numerical variance
-3. Peak memory per GPU drops compared with the single-rank model
-4. There are no outstanding communication errors before decode
-
----
-
-## Troubleshooting
-
-### Issue: `TypeError` on import — `CFGParallelMixin` missing
-
-**Symptoms:** Importing a pipeline that inherits `PipelineParallelMixin` raises:
-
-```
-TypeError: YourPipeline inherits PipelineParallelMixin but not CFGParallelMixin.
-```
-
-or:
-
-```
-TypeError: YourPipeline must inherit PipelineParallelMixin before CFGParallelMixin ...
-```
-
-**Cause:** `PipelineParallelMixin` enforces via `__init_subclass__` that the subclass also inherits `CFGParallelMixin`
-and lists `PipelineParallelMixin` first in the MRO.
-
-**Solution:** Add `CFGParallelMixin` to the pipeline's base classes after `PipelineParallelMixin`:
-
-```python
-from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
-from vllm_omni.diffusion.distributed.pipeline_parallel import PipelineParallelMixin
-
-
-class YourPipeline(nn.Module, PipelineParallelMixin, CFGParallelMixin):
-    ...
-```
-
-### Issue: Non-last PP ranks crash when calling `predict_noise`
-
-**Symptoms:** Shape or missing-input errors on ranks other than the first or last PP rank.
-
-**Cause:** The model forward path assumes direct input tensors and ignores `intermediate_tensors`.
-
-**Solution:** Update the transformer `forward()` or custom `predict_noise()` path to load hidden states from
-`intermediate_tensors` when present.
-
-### Issue: PP outputs differ from single-GPU baseline
-
-**Symptoms:** The PP run finishes but produces numerically inconsistent outputs.
-
-**Causes & Solutions:**
-
-- **Local layer partitioning is wrong**
-    - Verify each rank runs only its `[start_layer, end_layer)` slice
-- **Non-last ranks return a plain tensor instead of `IntermediateTensors`**
-    - Return `IntermediateTensors({...})` until the last PP stage
-- **CFG branch wiring is incorrect**
-    - When CFG is enabled, confirm positive and negative kwargs are passed exactly as in the non-PP path
-
----
-
-## Reference Implementations
-
-Complete examples in the codebase:
-
-| Component               | Path                                                       | Notes                                                                                                                   |
-|-------------------------|------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------|
-| `PipelineParallelMixin` | `vllm_omni/diffusion/distributed/pipeline_parallel.py`     | Core PP communication and scheduler helpers                                                                             |
-| `CFGParallelMixin`      | `vllm_omni/diffusion/distributed/cfg_parallel.py`          | Default `predict_noise()` tuple normalization and CFG helper fallback                                                   |
-| Wan2.2 transformer      | `vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py`  | Reference for layer partitioning, `IntermediateTensors`, `make_empty_intermediate_tensors`, and PP-aware weight loading |
-| Wan2.2 T2V pipeline     | `vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py`     | Reference PP + CFG integration for text-to-video                                                                        |
-| Wan2.2 I2V pipeline     | `vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py` | Reference PP + CFG integration for image-to-video                                                                       |
-| PP tests                | `tests/diffusion/distributed/test_pipeline_parallel.py`    | Baseline parity and async communication tests                                                                           |

docs/design/index.md

@@ -10,8 +10,6 @@ This section contains design documents and architecture specifications for vLLM-
 
 - [Disaggregated Inference](feature/disaggregated_inference.md)
 - [Ray-based Execution](feature/ray_based_execution.md)
-- [Adding Step Execution Support for Diffusion Pipelines](feature/diffusion_step_execution.md)
-- [Continuous Batching for Step-Wise Diffusion](feature/diffusion_continuous_batching.md)
 
 ## Module Design Documents
 

docs/design/module/dit_module.md

@@ -201,7 +201,7 @@ class _BaseScheduler(SchedulerInterface):
 
 - **Shared cleanup logic**: Request-id registration, finish handling, and state removal are centralized instead of duplicated in each policy.
 
-- **Current constraint boundary**: `_BaseScheduler` derives `max_num_running_reqs` from `max_num_seqs`, but request-mode diffusion is still clamped back to `1` by the engine. The step-wise path can keep this above `1` for compatible-request batching.
+- **Current constraint**: `max_num_running_reqs` remains `1` because the current engine path is still synchronous request-mode execution.
 
 #### 2.4 Current `RequestScheduler` Policy
 
@@ -217,7 +217,7 @@ class RequestScheduler(_BaseScheduler):
 
 - **FIFO request scheduling**: Waiting requests are promoted in queue order.
 
-- **Single-request admission**: `RequestScheduler` still admits one active request at a time because request-mode execution completes a whole request per dispatch.
+- **Single-request admission**: The current policy only admits one active request at a time.
 
 - **Executor result feedback**: `update_from_output()` converts executor output into `FINISHED_COMPLETED` or `FINISHED_ERROR` and returns finished scheduler ids.
 
@@ -237,7 +237,7 @@ while True:
 
 - **No scheduler-owned IPC**: Scheduler no longer talks to workers directly.
 
-- **Split concurrency model**: Request-mode diffusion remains single-active-request, while the step-wise path can keep multiple compatible requests running and advance them independently between denoise steps.
+- **Conservative concurrency**: The current request-mode implementation still allows only one active request at a time.
 
 ---
 
@@ -455,7 +455,7 @@ To learn more about the diffusion pipeline and how to add a new diffusion pipeli
 
 #### Architecture
 
-The attention system uses a **role-aware backend selector pattern**. Each `Attention` site declares a semantic `role` (`"self"`, `"cross"`, or a model-specific string). The selector consults the user's `AttentionConfig` to pick a backend per role and falls back to the platform default when nothing is configured.
+The attention system uses a **backend selector pattern** that automatically chooses the optimal attention implementation based on hardware and model configuration.
 
 #### Backend Selection
 
@@ -463,90 +463,92 @@ The attention system uses a **role-aware backend selector pattern**. Each `Atten
 
 ```python
 class Attention(nn.Module):
-    def __init__(self, num_heads, head_size, causal, softmax_scale, *,
-                 role="self", role_category=None, ...):
-        # Resolve backend for this role from the active OmniDiffusionConfig
-        config = get_current_diffusion_config_or_none()
-        attention_config = config.attention_config if config is not None else None
-        attn_backend_cls, spec = get_attn_backend_for_role(
-            role=role,
-            head_size=head_size,
-            attention_config=attention_config,
-            role_category=role_category,
-        )
-        self.attn_impl_cls = attn_backend_cls.get_impl_cls()
-        self.attention = self.attn_impl_cls(..., backend_kwargs=spec.extra if spec else None)
+    def __init__(self, num_heads, head_size, causal, softmax_scale, ...):
+        # Auto-select backend
+        self.attn_backend = get_attn_backend(-1)
+        self.attn_impl_cls = self.attn_backend.get_impl_cls()
+        self.attention = self.attn_impl_cls(...)
 ```
 
 **Available Backends**:
 
-- **FlashAttention** (`FLASH_ATTN`): Memory-efficient kernel that dispatches to FA2/FA3 on CUDA, AITER on ROCm, and `mindiesd` on Ascend NPU.
+- **FlashAttention**: Optimized CUDA kernel (FA2/FA3) - memory efficient via tiling
 
-- **SDPA** (`TORCH_SDPA`): PyTorch's scaled dot-product attention — cross-platform fallback.
+- **SDPA**: PyTorch's scaled dot-product attention - default, cross-platform
 
-- **SageAttention** (`SAGE_ATTN`): Quantized attention implementation from the SageAttention library.
+- **SageAttention**: Sparse attention implementation from SageAttention library
+
+- **AscendAttention**: NPU-optimized attention for Ascend hardware
 
 These backends provide the **kernel implementations** for attention computation. For attention-level sequence parallelism strategies (Ring Attention, Ulysses), see [Parallel Attention](#52-parallel-attention).
 
 #### Backend Selection Mechanism
 
-Selection is driven by `AttentionConfig` on `OmniDiffusionConfig`, which carries a global `default` plus a `per_role` map of `AttentionSpec` entries. A role string is resolved in this order:
-
 ```python
-def get_attn_backend_for_role(role, head_size, attention_config=None, role_category=None):
-    # 1. attention_config.per_role[role]            — exact match
-    # 2. attention_config.per_role[role_category]   — category fallback
-    # 3. attention_config.default                   — global default
-    # 4. Platform default                           — hardware-specific
-    if attention_config is not None:
-        spec, source = attention_config.resolve_with_source(
-            role=role, role_category=role_category,
-        )
-        if spec is not None:
-            return load_backend(spec.backend), spec
-    return current_omni_platform.get_diffusion_attn_backend_cls(...), None
+def get_attn_backend(head_size: int) -> type[AttentionBackend]:
+    # Check environment variable
+    backend_name = os.environ.get("DIFFUSION_ATTENTION_BACKEND")
+
+    if backend_name:
+        return load_backend(backend_name.upper())
+
+    # Default to SDPA
+    return SDPABackend
 ```
 
 **Selection Priority**:
 
-1. **Per-role override** (`--diffusion-attention-config.per_role.<role>.backend`) — finest control; matched on the layer's exact `role` string.
+1. **Environment Variable**: `DIFFUSION_ATTENTION_BACKEND` for manual override
 
-2. **Role category fallback** (`--diffusion-attention-config.per_role.<category>.backend`) — used when an exact match is missing and the layer declared `role_category` (e.g. a `"mymodel.audio_to_video"` site can fall back to `"cross"`).
+    - Valid values: `FLASH_ATTN`, `TORCH_SDPA`, `SAGE_ATTN`, `ASCEND`
 
-3. **Global default** (`--diffusion-attention-backend FLASH_ATTN`, or `--diffusion-attention-config.default.backend`, or `DIFFUSION_ATTENTION_BACKEND` env var).
+    - Example: `export DIFFUSION_ATTENTION_BACKEND=SAGE_ATTN`
 
-4. **Platform default** — `current_omni_platform.get_diffusion_attn_backend_cls(...)` picks the best-available kernel for the hardware.
+2. **Automatic Fallback**: Falls back to SDPA if selected backend unavailable
 
-`AttentionSpec.extra` is forwarded to the backend constructor as `backend_kwargs`, so backend-specific parameters (e.g. SparseBlock `block_size`) can be set without changing model code.
-
-For the user-facing CLI surface, see [Diffusion Attention Backends](../../user_guide/diffusion/attention_backends.md). For the role declaration contract on the model side, see [Adding a Diffusion Model](../../contributing/model/adding_diffusion_model.md).
+3. **Hardware Detection**: Can select based on device type (NPU, CUDA, etc.)
 
 **Backend Availability**:
 
 - **SDPA**: Always available (PyTorch built-in)
 
-- **FlashAttention**: Requires `flash-attn` on CUDA, `aiter` on ROCm, or `mindiesd` on Ascend NPU
+- **FlashAttention**: Requires `flash-attn` package installed
 
 - **SageAttention**: Requires `sage-attention` package (from THU-ML GitHub)
 
+- **AscendAttention**: Only available on Ascend NPU hardware
+
 #### Attention Backend Registry
 
-**Location**: `vllm_omni/diffusion/attention/selector.py` + `vllm_omni/platforms/<device>/platform.py`
+**Location**: `vllm_omni/diffusion/attention/selector.py`
 
-Backend resolution is delegated to the active platform: `current_omni_platform.get_diffusion_attn_backend_cls(selected_backend, head_size)` returns a fully-qualified class path that `_load_backend_cls` imports lazily, with the result cached by `(backend_name, head_size)`.
+The attention system uses a **registry pattern** to manage and dynamically load attention backends. This allows for easy extension and runtime selection of backends.
+
+
+**Registry Structure**:
 
 ```python
-@cache
-def _cached_get_backend_cls(backend_name: str | None, head_size: int) -> type[AttentionBackend]:
-    backend_cls_path = current_omni_platform.get_diffusion_attn_backend_cls(
-        selected_backend=backend_name,
-        head_size=head_size,
-    )
-    return _load_backend_cls(backend_cls_path)
+# Registry mapping backend names to their module paths and class names
+_BACKEND_CONFIG = {
+    "FLASH_ATTN": {
+        "module": "vllm_omni.diffusion.attention.backends.flash_attn",
+        "class": "FlashAttentionBackend",
+    },
+    "TORCH_SDPA": {
+        "module": "vllm_omni.diffusion.attention.backends.sdpa",
+        "class": "SDPABackend",
+    },
+    "SAGE_ATTN": {
+        "module": "vllm_omni.diffusion.attention.backends.sage_attn",
+        "class": "SageAttentionBackend",
+    },
+    "ASCEND": {
+        "module": "vllm_omni.diffusion.attention.backends.ascend_attn",
+        "class": "AscendAttentionBackend",
+    },
+}
 ```
 
-Each platform (`cuda`, `rocm`, `xpu`, `musa`, `npu`) maps backend names like `"FLASH_ATTN"`, `"TORCH_SDPA"`, `"SAGE_ATTN"` to the right backend class path for that hardware, including head-size compatibility checks. Passing `selected_backend=None` lets the platform pick its own default.
-
 #### Attention Backend Integration
 
 The `Attention` layer integrates backends through a unified interface. Here's how **FlashAttentionBackend** is integrated as an example:

docs/getting_started/installation/gpu.md

@@ -34,7 +34,7 @@ vLLM-Omni is a Python library that supports the following GPU variants. The libr
 
 ### Pre-built wheels
 
-Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, 0.18.0, 0.20.0, and 0.21.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).
+Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, and 0.18.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).
 
 === "NVIDIA CUDA"
 

docs/getting_started/installation/gpu/cuda.inc.md

@@ -20,7 +20,7 @@ Therefore, it is recommended to install vLLM and vLLM-Omni with a **fresh new**
 
 vLLM-Omni is built based on vLLM. Please install it with command below.
 ```bash
-uv pip install vllm==0.21.0 --torch-backend=auto
+uv pip install vllm --torch-backend=auto
 ```
 
 #### Installation of vLLM-Omni
@@ -39,13 +39,13 @@ uv pip install 'vllm-omni[demo]'
 # --8<-- [start:build-wheel-from-source]
 
 #### Installation of vLLM
-If you do not need to modify source code of vLLM, you can directly install the stable 0.21.0 release version of the library
+If you do not need to modify source code of vLLM, you can directly install the stable 0.19.0 release version of the library
 
 ```bash
-uv pip install vllm==0.21.0 --torch-backend=auto
+uv pip install vllm==0.19.0 --torch-backend=auto
 ```
 
-The 0.21.0 release of vLLM ships CUDA 13.0-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.
+The 0.19.0 release of vLLM ships CUDA 13.0-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.
 
 #### Installation of vLLM-Omni
 Since vllm-omni is rapidly evolving, it's recommended to install it from source
@@ -66,12 +66,12 @@ If you want to check, modify or debug with source code of vLLM, install the libr
 ```bash
 git clone https://github.com/vllm-project/vllm.git
 cd vllm
-git checkout v0.21.0
+git checkout v0.19.0
 ```
 Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set `VLLM_PRECOMPILED_WHEEL_LOCATION` as your local absolute path of whl file.
 ```bash
-#For CUDA 13.0 (the default for v0.21.0; the wheel filename has no `+cu130` suffix)
-export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.21.0/vllm-0.21.0-cp38-abi3-manylinux_2_35_x86_64.whl
+#For CUDA 13.0
+export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.19.0/vllm-0.19.0+cu130-cp38-abi3-manylinux_2_35_x86_64.whl
 ```
 Install vllm with command below (If you have no existing PyTorch).
 ```bash
@@ -80,7 +80,7 @@ uv pip install --editable .
 Install vllm with command below (If you already have PyTorch).
 ```bash
 python use_existing_torch.py
-uv pip install -r requirements/build/cuda.txt
+uv pip install -r requirements/build.txt
 uv pip install --no-build-isolation --editable .
 ```
 </details>
@@ -102,7 +102,7 @@ docker run --runtime nvidia --gpus 2 \
     --env "HF_TOKEN=$HF_TOKEN" \
     -p 8091:8091 \
     --ipc=host \
-    vllm/vllm-omni:v0.21.0 \
+    vllm/vllm-omni:v0.18.0 \
     --model Qwen/Qwen3-Omni-30B-A3B-Instruct --port 8091
 ```
 

docs/getting_started/installation/gpu/rocm.inc.md

@@ -13,7 +13,7 @@ vLLM-Omni current recommends the steps in under setup through Docker Images.
 
 vLLM-Omni is built based on vLLM. Please install it with command below.
 ```bash
-uv pip install vllm==0.21.0+rocm722 --extra-index-url https://wheels.vllm.ai/rocm/0.21.0/rocm722
+uv pip install vllm==0.19.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.19.0/rocm700
 ```
 
 #### Installation of vLLM-Omni
@@ -34,13 +34,13 @@ uv pip install onnxruntime-rocm
 # --8<-- [start:build-wheel-from-source]
 
 #### Installation of vLLM
-If you do not need to modify source code of vLLM, you can directly install the stable 0.21.0 release version of the library
+If you do not need to modify source code of vLLM, you can directly install the stable 0.19.0 release version of the library
 
 ```bash
-uv pip install vllm==0.21.0+rocm722 --extra-index-url https://wheels.vllm.ai/rocm/0.21.0/rocm722
+uv pip install vllm==0.19.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.19.0/rocm700
 ```
 
-The pre-built 0.21.0 vLLM wheel targets ROCm 7.2.2. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
+The pre-built 0.19.0 vLLM wheel targets ROCm 7.0. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
 
 #### Installation of vLLM-Omni
 Since vllm-omni is rapidly evolving, it's recommended to install it from source
@@ -58,7 +58,7 @@ If you want to check, modify or debug with source code of vLLM, install the libr
 ```bash
 git clone https://github.com/vllm-project/vllm.git
 cd vllm
-git checkout v0.21.0
+git checkout v0.19.0
 python3 -m pip install -r requirements/rocm.txt
 python3 setup.py develop
 ```
@@ -130,7 +130,7 @@ docker run --rm \
   -v ~/.cache/huggingface:/root/.cache/huggingface \
   --env "HF_TOKEN=$HF_TOKEN" \
   -p 8091:8091 \
-  vllm/vllm-omni-rocm:v0.21.0 \
+  vllm/vllm-omni-rocm:v0.19.0 \
   --model Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
 ```
 
@@ -149,7 +149,7 @@ docker run --rm -it \
   -v ~/.cache/huggingface:/root/.cache/huggingface \
   --env "HF_TOKEN=$HF_TOKEN" \
   --entrypoint bash \
-  vllm/vllm-omni-rocm:v0.21.0
+  vllm/vllm-omni-rocm:v0.19.0
 ```
 
 # --8<-- [end:pre-built-images]

docs/getting_started/quickstart.md

@@ -19,10 +19,10 @@ uv venv --python 3.12 --seed
 source .venv/bin/activate
 
 # On CUDA
-uv pip install vllm==0.21.0 --torch-backend=auto
+uv pip install vllm==0.19.0 --torch-backend=auto
 
 # On ROCm
-uv pip install vllm==0.21.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.21.0/rocm721
+uv pip install vllm==0.19.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.19.0/rocm700
 
 git clone https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni
@@ -31,12 +31,6 @@ uv pip install -e .
 
 For additional installation methods — please see the [installation guide](installation/README.md).
 
-
-!!! note
-    It is important to install the same major & minor version of vLLM and vLLM Omni, otherwise things may not work as expected. If the versions are misaligned, you will see a warning when you import vLLM Omni.
-
-    If you are seeing strange behavior with the `vllm` command not handling the `--omni` flag correctly, you most likely have a version mismatch with vLLM < `0.21.0` and vLLM Omni `0.21.0`, as vLLM Omni no longer hijacks the vLLM entrypoint. Updating vLLM should resolve this issue.
-
 ## Offline Inference
 
 Text-to-image generation quickstart with vLLM-Omni:

docs/models/supported_models.md

@@ -20,7 +20,6 @@ th {
 | `Qwen3OmniMoeForConditionalGeneration` | Qwen3-Omni | `Qwen/Qwen3-Omni-30B-A3B-Instruct` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `Qwen2_5OmniForConditionalGeneration` | Qwen2.5-Omni | `Qwen/Qwen2.5-Omni-7B`, `Qwen/Qwen2.5-Omni-3B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `BagelForConditionalGeneration` | BAGEL (DiT-only) | `ByteDance-Seed/BAGEL-7B-MoT` | ✅︎ | ✅︎ | | ✅︎ |
-| `InternVLAA1Pipeline` | InternVLA-A1 | `InternRobotics/InternVLA-A1-3B` | ✅︎ | ✅︎ | | |
 | `HunyuanImage3ForCausalMM` | HunyuanImage3.0 (DiT-only) | `tencent/HunyuanImage-3.0`, `tencent/HunyuanImage-3.0-Instruct` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `QwenImagePipeline` | Qwen-Image | `Qwen/Qwen-Image` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `QwenImagePipeline` | Qwen-Image-2512 | `Qwen/Qwen-Image-2512` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
@@ -32,7 +31,6 @@ th {
 | `ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `WanPipeline` | Wan2.1-T2V, Wan2.2-T2V, Wan2.2-TI2V | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`, `Wan-AI/Wan2.1-T2V-14B-Diffusers`, `Wan-AI/Wan2.2-T2V-A14B-Diffusers`, `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `WanImageToVideoPipeline` | Wan2.2-I2V | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
-| `WanSpeechToVideoPipeline` | Wan2.2-S2V | `Wan-AI/Wan2.2-S2V-14B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `Wan22VACEPipeline` | Wan2.1-VACE | `Wan-AI/Wan2.1-VACE-1.3B-diffusers`, `Wan-AI/Wan2.1-VACE-14B-diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `LTX2Pipeline` | LTX-2-T2V | `Lightricks/LTX-2` | ✅︎ | ✅︎ | | |
 | `LTX2ImageToVideoPipeline` | LTX-2-I2V | `Lightricks/LTX-2` | ✅︎ | ✅︎ | | |
@@ -54,7 +52,6 @@ th {
 | `FluxPipeline` | FLUX.1-schnell | `black-forest-labs/FLUX.1-schnell` | ✅︎ | ✅︎ | | ✅︎ |
 | `OmniGen2Pipeline` | OmniGen2 | `OmniGen2/OmniGen2` | ✅︎ | ✅︎ | | ✅︎ |
 | `StableAudioPipeline` | Stable-Audio-Open | `stabilityai/stable-audio-open-1.0` | ✅︎ | ✅︎ | | ✅︎ |
-| `AudioXPipeline` | AudioX | `zhangj1an/AudioX` | ✅︎ | ✅︎ | | |
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-CustomVoice | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-VoiceDesign | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-Base | `Qwen/Qwen3-TTS-12Hz-0.6B-Base` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
@@ -64,11 +61,9 @@ th {
 | `Flux2Pipeline` | FLUX.2-dev | `black-forest-labs/FLUX.2-dev` | ✅︎ | ✅︎ | | |
 | `FishSpeechSlowARForConditionalGeneration` | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✅︎ | ✅︎ | | |
 | `DreamIDOmniPipeline` | DreamID-Omni | `XuGuo699/DreamID-Omni` | ✅︎ | ✅︎ | | |
-| `SenseNovaU1Pipeline` | SenseNova-U1 (DiT-only) | `SenseNova/SenseNova-U1-8B-MoT` | ✅︎ | | | |
 | `HunyuanVideo15Pipeline` | HunyuanVideo-1.5-T2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | ✅︎ | ✅︎ | | |
 | `HunyuanVideo15ImageToVideoPipeline` | HunyuanVideo-1.5-I2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_i2v` | ✅︎ | ✅︎ | | |
 | `VoxtralTTSForConditionalGeneration` | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✅︎ | ✅︎ | | |
-| `CovoAudioForConditionalGeneration` | Covo-Audio-Chat | `tencent/Covo-Audio-Chat` | ✅︎ | | | |
 |`DyninOmniForConditionalGeneration` | Dynin-Omni | `snu-aidas/Dynin-Omni` | ✅︎ | | | |
-| `ErnieImagePipeline` | ERNIE-Image | `baidu/ERNIE-Image`, `baidu/ERNIE-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
+
 ✅︎ indicates the model is supported on that backend. Empty cells mean not listed as supported on that backend.