Merge branch 'main' into docs/add-new-committers

Correct yongxiang mapping in the diffusion cache ownership list.

Signed-off-by: hsliuustc0106 <liuhongsheng4@huawei.com>
Co-authored-by: Cursor <cursoragent@cursor.com>

.buildkite/nightly-release-pipeline.yaml

@@ -1,20 +0,0 @@
-steps:
-  - label: "Build and upload wheel"
-    key: "build-wheel"
-    agents:
-      queue: cpu_queue_release
-    commands:
-      - "curl -LsSf https://astral.sh/uv/install.sh | sh"
-      - 'export PATH="$HOME/.local/bin:$PATH"'
-      - "uv venv --python=3.12 && source .venv/bin/activate"
-      - "uv pip install --upgrade build"
-      - "python3 -m build"
-      - "bash .buildkite/scripts/upload-nightly-wheels.sh"
-
-  - label: "Generate and upload wheel indices"
-    depends_on: "build-wheel"
-    allow_dependency_failure: true
-    agents:
-      queue: small_cpu_queue_release
-    commands:
-      - "bash .buildkite/scripts/generate-and-upload-nightly-index.sh"

.buildkite/pipeline-intel.yaml

@@ -10,7 +10,7 @@ steps:
           DOCKER_BUILDKIT: "1"
           # Buildkite will automatically replace this with the actual commit hash
           VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
-          VLLM_VERSION: "v0.20.0"
+          VLLM_VERSION: "v0.21.0"
         priority: 100
         timeout_in_minutes: 60
         soft_fail: false

.buildkite/pipeline-npu.yaml

@@ -78,15 +78,19 @@ steps:
       echo "--- Image pushed successfully"
       echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VLLM_IMAGE_TAG}"
 
-  # - label: "Upload Ready Pipeline"
-  #   depends_on: image-build
-  #   key: upload-ready-pipeline
-  #   if: build.branch != "main" && build.pull_request.labels includes "npu-test"
-  #   agents:
-  #     queue: "ascend-a2b3"
-  #     resource_class: "npu-1"
-  #   command: |
-  #     buildkite-agent pipeline upload .buildkite/test-npu-ready.yaml
+  - label: "Upload Ready Pipeline"
+    depends_on: image-build
+    key: upload-ready-pipeline
+    if: >-
+      build.branch != "main" && (
+        build.pull_request.labels includes "npu-test" ||
+        build.pull_request.labels includes "ready"
+      )
+    agents:
+      queue: "ascend-a2b3"
+      resource_class: "npu-1"
+    command: |
+      buildkite-agent pipeline upload .buildkite/test-npu-ready.yaml
 
   # L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
   - label: "Upload Nightly Pipeline"

.buildkite/rebase-pipeline.yaml

@@ -0,0 +1,38 @@
+steps:
+  - label: ":docker: Build image"
+    key: image-build
+    commands:
+      - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
+      - "docker build --progress=plain --file docker/Dockerfile.ci -t vllm-omni-ci ."
+      - "docker tag vllm-omni-ci public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
+      - "docker push public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
+    agents:
+      queue: "cpu_queue_premerge"
+
+  # L2 Test
+  - label: "Upload Ready Pipeline"
+    depends_on: image-build
+    key: upload-ready-pipeline
+    commands:
+      - buildkite-agent pipeline upload .buildkite/test-ready.yml
+    agents:
+      queue: "cpu_queue_premerge"
+
+  # L3 Test
+  - label: "Upload Merge Pipeline"
+    depends_on: image-build
+    key: upload-merge-pipeline
+    commands:
+      - buildkite-agent pipeline upload .buildkite/test-merge.yml
+    agents:
+      queue: "cpu_queue_premerge"
+
+  # L4 Test
+  - label: "Upload Nightly Pipeline"
+    depends_on: image-build
+    key: upload-nightly-pipeline
+    if: build.env("NIGHTLY") == "1"
+    commands:
+      - buildkite-agent pipeline upload .buildkite/test-nightly.yml
+    agents:
+      queue: "cpu_queue_premerge"

.buildkite/release-pipeline.yaml

@@ -1,38 +1,110 @@
 steps:
-  - label: ":docker: Build image"
-    key: image-build
-    commands:
-      - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
-      - "docker build --progress=plain --file docker/Dockerfile.ci -t vllm-omni-ci ."
-      - "docker tag vllm-omni-ci public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
-      - "docker push public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
-    agents:
-      queue: "cpu_queue_premerge"
+  - input: "Provide Release version here"
+    id: input-release-version
+    fields:
+      - text: "What is the release version?"
+        key: release-version
 
-  # L2 Test
-  - label: "Upload Ready Pipeline"
-    depends_on: image-build
-    key: upload-ready-pipeline
-    commands:
-      - buildkite-agent pipeline upload .buildkite/test-ready.yml
+  - label: "Build and upload wheel"
+    depends_on: ~
+    key: "build-wheel"
     agents:
-      queue: "cpu_queue_premerge"
+      queue: cpu_queue_release
+    commands:
+      - "curl -LsSf https://astral.sh/uv/install.sh | sh"
+      - 'export PATH="$HOME/.local/bin:$PATH"'
+      - "uv venv --python=3.12 && source .venv/bin/activate"
+      - "uv pip install --upgrade build"
+      - "python3 -m build"
+      - "bash .buildkite/scripts/upload-nightly-wheels.sh"
 
-  # L3 Test
-  - label: "Upload Merge Pipeline"
-    depends_on: image-build
-    key: upload-merge-pipeline
-    commands:
-      - buildkite-agent pipeline upload .buildkite/test-merge.yml
+  - label: "Generate and upload wheel indices"
+    depends_on: "build-wheel"
+    allow_dependency_failure: true
     agents:
-      queue: "cpu_queue_premerge"
+      queue: small_cpu_queue_release
+    commands:
+      - "bash .buildkite/scripts/generate-and-upload-nightly-index.sh"
 
-  # L4 Test
-  - label: "Upload Nightly Pipeline"
-    depends_on: image-build
-    key: upload-nightly-pipeline
-    if: build.env("NIGHTLY") == "1"
-    commands:
-      - buildkite-agent pipeline upload .buildkite/test-nightly.yml
+  - block: "Upload release wheels to PyPI"
+    depends_on: "build-wheel"
+    key: block-upload-release-wheels
+
+  - label: "Upload release wheels to PyPI"
+    depends_on:
+      - block-upload-release-wheels
+      - input-release-version
+    key: upload-release-wheels
     agents:
-      queue: "cpu_queue_premerge"
+      queue: small_cpu_queue_release
+    commands:
+      - "bash .buildkite/scripts/publish-release-wheels-pypi.sh"
+
+  - block: "Unblock to build release Docker images"
+    depends_on: ~
+    key: block-build-release-images
+
+  - group: "Build release Docker images"
+    key: "build-release-images"
+    depends_on: block-build-release-images
+    allow_dependency_failure: true
+    steps:
+      - label: "Build release image - x86_64"
+        key: build-release-image-x86
+        depends_on: ~
+        agents:
+          queue: cpu_queue_release
+        commands:
+          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
+          - |
+            DOCKER_BUILDKIT=1 docker build \
+              --build-arg max_jobs=16 \
+              --tag public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m) \
+              --progress plain \
+              -f docker/Dockerfile.cuda .
+          - "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m)"
+          # re-tag to default image tag and push, just in case arm64 build fails
+          - "docker tag public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m) public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT"
+          - "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT"
+        env:
+          DOCKER_BUILDKIT: "1"
+
+      - label: "Build release image - aarch64"
+        key: build-release-image-arm64
+        depends_on: ~
+        agents:
+          queue: arm64_cpu_queue_release
+        commands:
+          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
+          - |
+            DOCKER_BUILDKIT=1 docker build \
+              --build-arg max_jobs=16 \
+              --tag public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m) \
+              --progress plain \
+              -f docker/Dockerfile.cuda .
+          - "docker push public.ecr.aws/q9t5s3a7/vllm-omni-release-repo:$BUILDKITE_COMMIT-$(uname -m)"
+        env:
+          DOCKER_BUILDKIT: "1"
+
+  - block: "Publish release images to DockerHub"
+    key: block-publish-release-images
+    depends_on:
+      - build-release-image-x86
+      - build-release-image-arm64
+      - input-release-version
+
+  - label: "Publish release images to DockerHub"
+    depends_on:
+      - block-publish-release-images
+    key: publish-release-images-dockerhub
+    agents:
+      queue: small_cpu_queue_release
+    commands:
+      - "bash .buildkite/scripts/publish-release-images.sh"
+    plugins:
+      - docker-login#v3.0.0:
+          username: vllmbot
+          password-env: DOCKERHUB_TOKEN
+    env:
+      DOCKER_BUILDKIT: "1"
+      DOCKERHUB_USERNAME: "vllmbot"

.buildkite/scripts/hardware_ci/run-xpu-test.sh

@@ -52,5 +52,6 @@ time timeout -k 30 30m docker run \
     pip install tblib==3.1.0
     cd /workspace/vllm-omni
     pytest -v -s -m "core_model and xpu and B60"
+    export VLLM_XPU_USE_SAMPLER_KERNEL=0    # NOTE: Remove this after vLLM v0.21.1 is merged. Fixes Qwen2-5 omni-expansion tests.
     pytest -v -s -m "advanced_model and xpu and B60"
 '

.buildkite/scripts/publish-release-images.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+# SPDX-License-Identifier: Apache-2.0
+#
+# Publish release Docker images from ECR to DockerHub.
+# Pulls per-arch images, tags with `latest` and versioned tags, pushes them,
+# then creates and pushes a multi-arch manifest.
+
+set -euo pipefail
+
+RELEASE_VERSION=$(buildkite-agent meta-data get release-version --default "" | sed 's/^v//')
+if [ -z "${RELEASE_VERSION}" ]; then
+  echo "ERROR: release-version metadata not set"
+  exit 1
+fi
+
+COMMIT="$BUILDKITE_COMMIT"
+DOCKERHUB_REPO="vllm/vllm-omni"
+ECR_REPO="public.ecr.aws/q9t5s3a7/vllm-omni-release-repo"
+
+echo "========================================"
+echo "Publishing release images v${RELEASE_VERSION}"
+echo "  Commit: ${COMMIT}"
+echo "========================================"
+
+# Login to ECR to pull staging images
+aws ecr-public get-login-password --region us-east-1 | \
+  docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
+
+docker pull "${ECR_REPO}:${COMMIT}-x86_64"
+docker pull "${ECR_REPO}:${COMMIT}-aarch64"
+
+docker tag "${ECR_REPO}:${COMMIT}-x86_64" "${DOCKERHUB_REPO}:latest-x86_64"
+docker tag "${ECR_REPO}:${COMMIT}-x86_64" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-x86_64"
+docker push "${DOCKERHUB_REPO}:latest-x86_64"
+docker push "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-x86_64"
+
+docker tag "${ECR_REPO}:${COMMIT}-aarch64" "${DOCKERHUB_REPO}:latest-aarch64"
+docker tag "${ECR_REPO}:${COMMIT}-aarch64" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-aarch64"
+docker push "${DOCKERHUB_REPO}:latest-aarch64"
+docker push "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-aarch64"
+
+docker manifest rm "${DOCKERHUB_REPO}:latest" || true
+docker manifest rm "${DOCKERHUB_REPO}:v${RELEASE_VERSION}" || true
+docker manifest create "${DOCKERHUB_REPO}:latest" "${DOCKERHUB_REPO}:latest-x86_64" "${DOCKERHUB_REPO}:latest-aarch64"
+docker manifest create "${DOCKERHUB_REPO}:v${RELEASE_VERSION}" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-x86_64" "${DOCKERHUB_REPO}:v${RELEASE_VERSION}-aarch64"
+docker manifest push "${DOCKERHUB_REPO}:latest"
+docker manifest push "${DOCKERHUB_REPO}:v${RELEASE_VERSION}"
+
+echo ""
+echo "Successfully published release images for v${RELEASE_VERSION}"

.buildkite/scripts/publish-release-wheels-pypi.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: Apache-2.0
+
+set -e
+
+BUCKET="vllm-wheels"
+SUBPATH="omni/$BUILDKITE_COMMIT"
+S3_COMMIT_PREFIX="s3://$BUCKET/$SUBPATH/"
+
+RELEASE_VERSION=$(buildkite-agent meta-data get release-version | sed 's/^v//')
+if [[ -z "$RELEASE_VERSION" ]]; then
+  echo "[FATAL] release-version metadata not set."
+  exit 1
+fi
+echo "Release version: $RELEASE_VERSION"
+
+if [[ -z "$PYPI_TOKEN" ]]; then
+  echo "[FATAL] PYPI_TOKEN is not set."
+  exit 1
+else
+  export TWINE_USERNAME="__token__"
+  export TWINE_PASSWORD="$PYPI_TOKEN"
+fi
+
+set -x
+
+if ! command -v uv &> /dev/null; then
+  curl -LsSf https://astral.sh/uv/install.sh | sh
+  export PATH="$HOME/.local/bin:$PATH"
+fi
+
+uv venv --python 3.12 /tmp/vllm-omni-release-env
+source /tmp/vllm-omni-release-env/bin/activate
+uv pip install twine
+
+DIST_DIR=/tmp/vllm-omni-release-dist
+mkdir -p "$DIST_DIR"
+
+echo "Downloading wheels from S3:"
+aws s3 ls "$S3_COMMIT_PREFIX"
+aws s3 cp --recursive --exclude "*" --include "vllm_omni-${RELEASE_VERSION}*.whl" --exclude "*dev*" "$S3_COMMIT_PREFIX" "$DIST_DIR"
+ls -la "$DIST_DIR"
+
+PYPI_WHEEL_FILES=$(find "$DIST_DIR" -name "vllm_omni-${RELEASE_VERSION}*.whl" -not -name "*+*")
+if [[ -z "$PYPI_WHEEL_FILES" ]]; then
+  echo "[FATAL] No wheels found for version ${RELEASE_VERSION}"
+  exit 1
+fi
+
+python3 -m twine check $PYPI_WHEEL_FILES
+python3 -m twine upload --non-interactive --verbose $PYPI_WHEEL_FILES
+echo "Wheels uploaded to PyPI"

.buildkite/test-merge.yml

@@ -1,5 +1,6 @@
 env:
   VLLM_WORKER_MULTIPROC_METHOD: spawn
+  VLLM_USE_FLASHINFER_MOE_FP16: 0
   HF_HUB_DOWNLOAD_TIMEOUT: 300
   HF_HUB_ETAG_TIMEOUT: 60
 
@@ -172,7 +173,7 @@ steps:
               pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m 'advanced_model and cuda' --run-level 'advanced_model'
             "
         agents:
-          queue: "gpu_1_queue"
+          queue: "gpu_4_queue"
         plugins:
           - docker#v5.2.0:
               image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
@@ -193,7 +194,7 @@ steps:
               pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model'
             "
         agents:
-          queue: "gpu_1_queue"
+          queue: "gpu_4_queue"
         plugins:
           - docker#v5.2.0:
               image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT

.buildkite/test-nightly.yml

@@ -1,5 +1,6 @@
 env:
   VLLM_WORKER_MULTIPROC_METHOD: spawn
+  VLLM_USE_FLASHINFER_MOE_FP16: 0
   HF_HUB_DOWNLOAD_TIMEOUT: 300
   HF_HUB_ETAG_TIMEOUT: 60
 
@@ -388,7 +389,7 @@ steps:
                   - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                     resources:
                       limits:
-                        nvidia.com/gpu: 1
+                        nvidia.com/gpu: 2
                     volumeMounts:
                       - name: devshm
                         mountPath: /dev/shm
@@ -659,87 +660,6 @@ steps:
                       path: /mnt/hf-cache
                       type: DirectoryOrCreate
 
-      - label: ":full_moon: Diffusion X2I(&A&T) · GEBench Accuracy Test"
-        timeout_in_minutes: 60
-        commands:
-          - pytest -s -v tests/e2e/accuracy/test_gebench_h100_smoke.py --run-level full_model --gebench-model Qwen/Qwen-Image-2512 --accuracy-judge-model QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ --accuracy-gpu 0 --gebench-port 8093 --accuracy-workers 1
-          - buildkite-agent artifact upload "tests/e2e/accuracy/artifacts/gebench_qwen-image-2512/summary*.json"
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
-      - label: ":full_moon: Diffusion X2I(&A&T) · GEdit-Bench Accuracy Test"
-        timeout_in_minutes: 60
-        commands:
-          - pytest -s -v tests/e2e/accuracy/test_gedit_bench_h100_smoke.py --run-level full_model --gedit-model Qwen/Qwen-Image-Edit --accuracy-judge-model QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ --accuracy-gpu 0 --gedit-port 8093 --gedit-samples-per-group 20 --accuracy-workers 1
-          - buildkite-agent artifact upload "tests/e2e/accuracy/artifacts/gedit_scores_qwen-image-edit/qwen-image-edit_all_all_vie_score_*.csv"
-          - buildkite-agent artifact upload "tests/e2e/accuracy/artifacts/gedit_scores_qwen-image-edit/qwen-image-edit_all_all_summary_*.json"
-        agents:
-          queue: "mithril-h100-pool"
-        plugins:
-          - kubernetes:
-              podSpec:
-                containers:
-                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                    resources:
-                      limits:
-                        nvidia.com/gpu: 1
-                    volumeMounts:
-                      - name: devshm
-                        mountPath: /dev/shm
-                      - name: hf-cache
-                        mountPath: /root/.cache/huggingface
-                    env:
-                      - name: HF_HOME
-                        value: /root/.cache/huggingface
-                      - name: VLLM_HTTP_TIMEOUT_KEEP_ALIVE
-                        value: "120"
-                      - name: HF_TOKEN
-                        valueFrom:
-                          secretKeyRef:
-                            name: hf-token-secret
-                            key: token
-                nodeSelector:
-                  node.kubernetes.io/instance-type: gpu-h100-sxm
-                volumes:
-                  - name: devshm
-                    emptyDir:
-                      medium: Memory
-                  - name: hf-cache
-                    hostPath:
-                      path: /mnt/hf-cache
-                      type: DirectoryOrCreate
-
       - label: ":full_moon: Diffusion X2I(&A&T) · Accuracy Test"
         timeout_in_minutes: 180
         commands:
@@ -1039,7 +959,7 @@ steps:
       - label: ":full_moon: Diffusion X2V · Function Test"
         timeout_in_minutes: 90
         commands:
-          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_hunyuan_video_15_expansion.py  --run-level "full_model"
+          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_hunyuan_video_15_expansion.py  -m "full_model and cuda" --run-level "full_model"
         agents:
           queue: "mithril-h100-pool"
         plugins:

.buildkite/test-npu-ready.yaml

@@ -0,0 +1,21 @@
+steps:
+  - group: ":card_index_dividers: TTS Model Test"
+    key: ready-tts-group
+    depends_on: upload-ready-pipeline
+    steps:
+      - label: ":sound: TTS · Qwen3-TTS CustomVoice Test"
+        agents:
+          queue: "ascend-a2b3"
+          resource_class: "npu-1"
+        image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}"
+        plugins:
+          - kubernetes:
+              podSpecPatch:
+                imagePullSecrets:
+                  - name: swr-secret
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'

.buildkite/test-ready.yml

@@ -1,5 +1,6 @@
 env:
   VLLM_WORKER_MULTIPROC_METHOD: spawn
+  VLLM_USE_FLASHINFER_MOE_FP16: 0
   HF_HUB_DOWNLOAD_TIMEOUT: 300
   HF_HUB_ETAG_TIMEOUT: 60
 
@@ -303,7 +304,7 @@ steps:
               pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'
             "
         agents:
-          queue: "gpu_1_queue"
+          queue: "gpu_4_queue"
         plugins:
           - docker#v5.2.0:
               image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
@@ -326,7 +327,7 @@ steps:
               pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py -m 'core_model' --run-level 'core_model'
             "
         agents:
-          queue: "gpu_1_queue"
+          queue: "gpu_4_queue"
         plugins:
           - docker#v5.2.0:
               image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT

.buildkite/test-weekly.yml

@@ -1,5 +1,6 @@
 env:
   VLLM_WORKER_MULTIPROC_METHOD: spawn
+  VLLM_USE_FLASHINFER_MOE_FP16: 0
   HF_HUB_DOWNLOAD_TIMEOUT: 300
   HF_HUB_ETAG_TIMEOUT: 60
 
@@ -83,3 +84,65 @@ steps:
                 hostPath:
                   path: /mnt/hf-cache
                   type: DirectoryOrCreate
+
+  # Invalid HTTP / WS payloads vs live Omni servers (Issue #3649 tracks known skips).
+  - group: "Reliability Test - Invalid parameters Test"
+    depends_on: upload-weekly-pipeline
+    if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
+    steps:
+      - label: "Invalid parameters Test · H100"
+        timeout_in_minutes: 120
+        commands:
+          - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100"
+        agents:
+          queue: "mithril-h100-pool"
+        plugins:
+          - kubernetes:
+              podSpec:
+                containers:
+                  - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                    resources:
+                      limits:
+                        nvidia.com/gpu: 2
+                    volumeMounts:
+                      - name: devshm
+                        mountPath: /dev/shm
+                      - name: hf-cache
+                        mountPath: /root/.cache/huggingface
+                    env:
+                      - name: HF_HOME
+                        value: /root/.cache/huggingface
+                      - name: HF_TOKEN
+                        valueFrom:
+                          secretKeyRef:
+                            name: hf-token-secret
+                            key: token
+                nodeSelector:
+                  node.kubernetes.io/instance-type: gpu-h100-sxm
+                volumes:
+                  - name: devshm
+                    emptyDir:
+                      medium: Memory
+                  - name: hf-cache
+                    hostPath:
+                      path: /mnt/hf-cache
+                      type: DirectoryOrCreate
+
+      # Stable Audio Open, Qwen3-TTS speech/batch/voices paths marked ``L4``.
+      - label: "Invalid parameters Test · L4"
+        timeout_in_minutes: 120
+        commands:
+          - pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and L4"
+        agents:
+          queue: "gpu_1_queue"
+        plugins:
+          - docker#v5.2.0:
+              image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+              always-pull: true
+              propagate-environment: true
+              shm-size: "8gb"
+              environment:
+                - "HF_HOME=/fsx/hf_cache"
+                - "HF_TOKEN"
+              volumes:
+                - "/fsx/hf_cache:/fsx/hf_cache"

.claude/skills/diffusion-perf-opt/SKILL.md

@@ -0,0 +1,488 @@
+---
+name: diffusion-perf-opt
+description: Diagnose and optimize vLLM Omni diffusion workloads, especially Wan/Qwen/Flux-style image and video generation. Use when Codex is asked to analyze profiling traces, choose parallel strategies, inspect torch profiler trace.json or trace.json.gz timelines, estimate optimization ROI, investigate GPU idle/free bubbles, compare USP/CFG/HSDP/VAE parallelism, or design operator/host/quantization optimizations for vLLM Omni.
+---
+
+# vLLM Omni Performance Optimization
+
+Use this skill to run a disciplined optimization loop for vLLM Omni diffusion workloads. Keep two ideas separate: real performance baselines are collected with low overhead, while torch profiler traces are diagnostic artifacts and may distort latency.
+
+## First Questions
+
+Before proposing changes, ask for the optimization scene if it is not already known:
+
+- GPU model, card count, topology, and whether NVLink is present.
+- Model and pipeline, for example Wan2.2 I2V A14B.
+- User workload: resolution, frames, steps, batch/concurrency, CFG scales, prompt/image inputs.
+- Three runnable commands for each model/workload, or scripts that generate them:
+  - **Server startup command**: exact `vllm serve` command, environment variables, model path, port, parallelism flags, profiler flags, and precision/compile settings.
+  - **User request command**: exact single-request client command (for example curl or Python client) with fixed prompt/media/seed/size/steps. Use this to validate correctness and collect per-request stage timings.
+  - **User benchmark command**: exact repeatable benchmark command or script with warmup count, measured iteration count, concurrency/batch policy, output directory, and summary format.
+- Current enabled strategies: USP/SP, CFG parallel, HSDP/FSDP, VAE patch parallel, torch.compile, profiler options.
+- Optimization target: latency, throughput, memory, cost, or quality-preserving speed.
+- Precision/quality tolerance: bf16/fp8/quantization/sparsity/approximate attention allowed or not.
+
+## Workflow
+
+1. **Freeze the measurement protocol and commands.**
+   - Before analyzing parallel strategies or traces, establish the three commands for the target model/workload: server startup, single user request, and user benchmark.
+   - Prefer checked-in scripts or one-off shell scripts over free-form commands in chat. The scripts should make workload variables explicit, including model path, port/base URL, prompt/media inputs, resolution, frames, steps, seed, warmup, measured iterations, concurrency, and output path.
+   - Measurement reports must preserve the concrete process, not just the final numbers. For each tested configuration, write down the server startup command, the user request command, the user benchmark command or polling loop, the final server response metrics, and the client-side timing/HTTP result.
+   - Present all duration and latency values in **milliseconds (ms)** in tables and summaries. If an API returns seconds, convert to ms before reporting. Keep raw units only inside quoted raw logs or source snippets.
+   - Keep profiler modes separate in the command set:
+     - Baseline/benchmark commands should avoid torch profiler and stack collection.
+     - Diagnostic commands may enable torch profiler after a baseline identifies a bottleneck.
+   - If any of the three commands is missing, ask for it or create a proposed script before proceeding.
+   - Make clear which command is authoritative for correctness validation and which command is authoritative for performance numbers.
+
+2. **Collect a real baseline.**
+   - Disable PyTorch profiler and stack collection.
+   - Disable or fix torch.compile state so A/B is fair.
+   - Avoid `--enforce-eager` for production-speed baselines unless eager is the target.
+   - Prefer `--log-stats` and `--enable-diffusion-pipeline-profiler` for low-overhead stage timing. PR 3069 has the relevant metrics/log-stats changes; if local code does not include them, fetch or cherry-pick the minimal metrics changes rather than merging unrelated PR drift.
+   - Run warmup requests and exclude first-request lazy init.
+
+3. **Model the parallel strategy before testing.**
+   - Estimate compute and communication for self-attention, cross-attention, FFN, CFG, VAE encode/decode, and HSDP.
+   - Select a small candidate matrix rather than exhaustively testing everything.
+   - Typical 2-card candidates: `USP=2`, `CFG=2`, `USP=1/HSDP on-off`, VAE parallel on-off if memory allows.
+   - Typical 4-card candidates: `USP=4`, `CFG=2 x USP=2`, `USP=2 x HSDP`, VAE parallel on-off.
+   - Typical 8-card candidates for official CFG-enabled video diffusion models:
+     - Primary candidate: `CFG=2 x USP=4` with VAE patch parallel across all 8 ranks.
+     - Compare against: `CFG=1 x USP=8` to test whether larger sequence parallel groups beat CFG branch parallelism.
+     - Isolate VAE parallelism: keep the DiT strategy fixed and compare VAE patch parallel on/off or different VAE patch sizes.
+     - If long-video `diffuse` remains dominant and Ulysses all-to-all is suspected, test a hybrid sequence strategy such as `CFG=2 x USP=2 x Ring=2`.
+     - Test HSDP off only if the model fits without it; treat HSDP primarily as a memory strategy until A/B proves otherwise.
+   - Prefer USP/SP for long video token sequences; prefer CFG parallel when CFG doubles transformer forwards and sequence length is modest.
+   - Convert workload shape to latent/patch token counts before choosing candidates. For video models, record frames, latent frames, latent HxW, patch size, approximate token count, and which stages should scale with the token count.
+
+4. **Search for the best parallel configuration.**
+   - Start with a small matrix that answers one question per comparison:
+     - `CFG parallel vs larger USP`: compare `CFG=2 x USP=world/2` against `CFG=1 x USP=world` for CFG-enabled workloads.
+     - `VAE patch parallel value`: compare the best DiT strategy with VAE patch parallel enabled and disabled.
+     - `HSDP cost`: compare HSDP on/off only when both configurations fit memory.
+     - `Ulysses vs Ring`: test a Ring/Ulysses hybrid only after long-sequence `diffuse` is confirmed dominant.
+   - For each configuration, create or record a stable config id, for example `A_cfg2_usp4_vaepp8_hsdp_tiling`.
+   - For each config id, capture:
+     - exact server command and environment variables,
+     - observed distributed setup from logs, such as SP groups, CFG groups, HSDP shard/replicate sizes, VAE patch size,
+     - exact request command for every scenario,
+     - final server response metrics and client-side elapsed time,
+     - output artifact paths and any failed/empty responses.
+   - Use one warmup request per scenario, then at least three measured repeats for the shortlisted config. For exploratory matrix pruning, one measured request is acceptable only if the margin is large; label it as one-shot.
+   - Compare configurations by stage, not only end-to-end latency. A configuration can improve `diffuse` while hurting `vae.decode`; record both effects.
+   - Select the best config only after checking the target metric, dominant stages, memory headroom, and output correctness.
+
+5. **Run targeted A/B tests.**
+   - Change one variable per test.
+   - Keep model, input, seed, request parameters, GPU placement, and warmup policy fixed.
+   - Record latency, stage timings, memory, output quality, and logs.
+   - Report comparison tables in ms. Include at least end-to-end client time, server `inference_time`, server stage generation time if available, `vae.encode`, `diffuse`, `vae.decode`, and peak memory.
+
+6. **Collect diagnostic trace only after narrowing hypotheses.**
+   - Use torch profiler for a small number of requests.
+   - Run two separate diagnostic traces instead of mixing concerns:
+     - **Operator/shape trace**: enable `torch_profiler_record_shapes=True` and keep stack collection disabled. Use this to rank CUDA kernels, NCCL collectives, attention/MLP/norm/RoPE work, and shape-specific hot operators.
+     - **Host-stack trace**: enable `torch_profiler_with_stack=True` and normally keep shape collection disabled. Use this to map CPU/Python host gaps, synchronization points, scheduler paths, and request handling overhead.
+   - Keep both trace commands and reports separate from baseline/benchmark commands. Torch profiler latency is diagnostic only and must not be used as the final latency claim.
+   - Prefer profiling only the narrowed dominant scenario, for example the highest-resolution/video-length case where `diffuse` dominates.
+   - If profiler endpoints are available, run one warmup request first, then call `/start_profile`, run one profiled request, and call `/stop_profile`. This keeps model initialization and warmup out of the diagnostic trace.
+   - Start by analyzing rank 0 only. Expand to more ranks only if rank 0 suggests imbalance, unclear GPU idle/free bubbles, high NCCL wait, server timing mismatch, CFG branch imbalance, or USP group stragglers.
+   - Diagnostic reports must be written to disk and preserve: server command, profiler config, warmup/request/polling commands, trace artifact paths, rank analyzed, analyzer output or summary, and the decision about whether additional ranks are necessary.
+   - Analyze both rank-level balance and device-level free bubbles when additional ranks are opened.
+
+7. **Analyze host, communication, and operators.**
+   - Find GPU idle/free intervals and map each large gap to the enclosing CPU/Python code.
+   - Separate real GPU idle from profiler overhead such as CUPTI `Command Buffer Full`.
+   - Compare NCCL kernel time to user annotations; annotations can overcount nested intervals.
+   - Rank operator work by total CUDA time and by repeated small-kernel launch count.
+
+8. **Produce an optimization plan.**
+   - Classify candidates as P0/P1/P2.
+   - For each candidate, state necessity, expected benefit, implementation path, validation plan, and quality risk.
+   - Do not implement high-risk operator rewrites before proving the operator is a bottleneck for the target shapes.
+   - End the plan with a user-facing candidate selection table. The assistant
+     should not automatically choose a risky optimization just because it is
+     technically possible. Present the options clearly and let the user decide
+     which item is worth implementing based on latency target, engineering
+     budget, memory headroom, and quality tolerance.
+   - Organize the plan by optimization layer, not by a flat list of ideas:
+     - host/runtime optimization,
+     - measurement/benchmark reliability,
+     - parallelism and communication,
+     - VAE encode/decode and media pre/post processing,
+     - operator fusion and layout cleanup,
+     - attention main-path optimization,
+     - algorithmic, precision, or approximation changes.
+   - For each layer, tie every candidate back to evidence from baseline
+     metrics, diagnostic trace, source code, or output quality requirements.
+     Do not list generic optimizations without a trace or workload reason.
+
+## Priority Rules
+
+- **P0:** low risk, likely useful, or required for trustworthy measurement. Examples: real baseline, warmup, targeted parallel A/B, disabling avoidable `empty_cache`, scheduler coefficient caching.
+- **P1:** meaningful code changes with contained risk. Examples: cross-attention KV caching, VAE gather/broadcast reduction, AdaLayerNorm/RMSNorm/RoPE fusion after trace evidence.
+- **P2:** high implementation or quality risk. Examples: FA to LA replacement, custom Triton/CUDA fused kernels, FP8/quantization, sparsity/Rainfusion-style acceleration.
+
+Every implemented optimization needs A/B validation and quality regression. A/B means same workload and hardware before/after. Quality regression means checking generated image/video stability, artifacts, temporal flicker, and seed behavior when precision or approximate kernels change.
+
+## Optimization Layers
+
+After baseline, parallel-search, and diagnostic traces, summarize optimization
+opportunities by layer. This is the core of the performance analysis: the goal
+is to connect evidence to a scoped implementation and a validation plan.
+
+### Host and Runtime Optimization
+
+Purpose: remove CPU/Python stalls, synchronization points, allocator overhead,
+and request-path overhead that leave GPU lanes empty.
+
+Evidence to look for:
+
+- High `idle_pct` or large `GAP` blocks in `trace_analyzer.py`.
+- Host-stack trace lines such as `torch.cuda.empty_cache`,
+  `cudaStreamSynchronize`, `cudaDeviceSynchronize`, Python locks, scheduler
+  waits, image/video preprocessing, or repeated small allocation paths.
+- Difference between client wall-clock time and server `inference_time_s`.
+
+Typical candidates:
+
+- Make avoidable `torch.cuda.empty_cache()` optional or guard it by memory
+  headroom.
+- Cache scheduler coefficients, timesteps, masks, or other tiny repeated CPU
+  computations when the request shape/steps are fixed.
+- Remove avoidable host-device synchronizations and blocking logging/stat calls.
+- Move expensive preprocessing out of the critical path or cache fixed prompt,
+  image, and transform work for benchmark scenarios.
+- Ensure benchmark scripts record client-side elapsed time, HTTP status, output
+  path, and server response metrics.
+
+Priority guidance:
+
+- Usually P0 when the change is measurement reliability or an obvious removable
+  synchronization.
+- Usually P1 when it changes scheduling, memory lifetime, or request execution
+  order.
+
+Validation:
+
+- Re-run non-profiler baseline with same workload and seed.
+- Confirm peak memory headroom if disabling cache cleanup.
+- Confirm generated output exists and quality/seed behavior is unchanged.
+
+### Parallelism and Communication Optimization
+
+Purpose: choose the right decomposition for CFG branches, sequence tokens, model
+weights, VAE tiles, and rank topology.
+
+Evidence to look for:
+
+- Baseline A/B across `CFG`, `USP/SP`, `Ring`, `HSDP/FSDP`, and VAE patch
+  parallelism.
+- Stage timing shifts: `diffuse`, `vae.encode`, `vae.decode`, and server
+  end-to-end.
+- NCCL kernel time from trace, not only `user_annotation` time.
+- Rank imbalance across SP group ranks or CFG branch ranks.
+- Memory headroom and OOM risk.
+
+Typical candidates:
+
+- `CFG=2 x USP=world/2` versus `CFG=1 x USP=world` for CFG-enabled models.
+- VAE patch parallel on/off or patch size tuning.
+- HSDP/FSDP on/off only if both configurations fit memory.
+- Ulysses versus Ulysses+Ring only after long-sequence `diffuse` is confirmed
+  dominant and all-to-all is suspected.
+- Rank mapping/topology changes if all-rank traces show stragglers or NCCL wait.
+- Buffer reuse or preallocation for FSDP/HSDP all-gather paths.
+
+Priority guidance:
+
+- P0/P1 for configuration-only changes with strong measured wins.
+- P1 for buffer reuse or rank mapping changes.
+- P2 for invasive distributed algorithm changes.
+
+Validation:
+
+- Measure by stage and memory, not only end-to-end.
+- Use one-variable A/B with identical prompt/media/seed/shape/steps.
+- When communication is suspected, compare rank0-3 in one USP group and rank0
+  versus rank4 across CFG branches for `CFG=2 x USP=4`.
+
+### VAE and Media Pipeline Optimization
+
+Purpose: reduce encode/decode, tiling, split/gather, and media conversion time.
+
+Evidence to look for:
+
+- Large `vae.encode` or `vae.decode` in low-overhead stage timings.
+- Host-stack gaps in VAE tile split, gather, merge, broadcast, or image/video
+  transforms.
+- VAE kernels or cuDNN convolution in operator trace.
+- Whether every rank needs the final decoded tensor.
+
+Typical candidates:
+
+- Keep VAE patch parallel enabled when it has clear measured benefit.
+- Reduce VAE gather/broadcast to only ranks that need the final media output.
+- Reuse tile metadata, split buffers, or gather buffers.
+- Evaluate bf16/autocast behavior for VAE only with visual quality checks.
+- Avoid redundant image conversion, resize, or tensor construction in repeated
+  benchmark runs.
+
+Priority guidance:
+
+- P0/P1 if VAE is a large share of the target workload or if a host gap is
+  obvious and low risk.
+- Lower priority when `diffuse` dominates and VAE is already patch-parallelized.
+
+Validation:
+
+- Compare `vae.encode`, `vae.decode`, server end-to-end, and peak memory.
+- Check output video integrity, artifacts, flicker, and seed stability.
+
+### Operator Fusion and Layout Cleanup
+
+Purpose: reduce high-frequency small kernels, memory bandwidth pressure, layout
+conversions, and launch overhead in transformer and VAE blocks.
+
+Evidence to look for:
+
+- Top operator tables showing many `aten::copy_`, `aten::cat`,
+  `split_with_sizes_copy`, `aten::add`, `aten::mul`, `aten::div`, norm,
+  activation, RoPE, or reshape/layout kernels.
+- `ops_rankN.xlsx` `by_shape` sheet showing repeated small shapes inside the
+  same block path.
+- Trace lanes showing many short kernels between larger GEMM/attention kernels.
+- Source code patterns with repeated elementwise chains or layout conversions.
+
+Typical fusion targets:
+
+- AdaLayerNorm / RMSNorm / LayerNorm plus scale/shift fusion.
+- RoPE fusion with Q/K layout preparation when shapes are stable.
+- Residual add, scale, gate, and elementwise chains.
+- MLP gate/up/down cleanup, such as fusing activation and multiply around
+  GEMM outputs when feasible.
+- QKV projection and reshape/split/cat path cleanup.
+- Attention pre/post layout cleanup to avoid unnecessary copies, cats, and
+  splits around sequence parallel all-to-all.
+
+Priority guidance:
+
+- P1 when implemented with existing PyTorch/Triton/local helper patterns and
+  validated against exact outputs or tolerances.
+- P2 when it requires custom CUDA/Triton kernels, changes numerics, or touches
+  attention math directly.
+
+Validation:
+
+- First prove the operator family is material for the target shape.
+- Use non-profiler A/B for latency and stage timing.
+- Use quality regression checks for generated video stability.
+- Check compile behavior and graph breaks if using `torch.compile`.
+
+### Attention Main-Path Optimization
+
+Purpose: address the dominant self-attention cost when FlashAttention or other
+attention kernels dominate CUDA time.
+
+Evidence to look for:
+
+- Operator trace where FlashAttention/SDPA kernels dominate total CUDA time.
+- Attention shape from `ops_rankN.xlsx` `by_shape`, model code, or trace
+  metadata.
+- Whether attention cost scales with latent frames, latent H/W, patch size, or
+  CFG duplication.
+- Layout/copy/all-to-all work around attention.
+
+Typical candidates:
+
+- Verify the attention backend and shape are on the intended fast kernel path.
+- Compare supported attention backends only with identical workload and quality
+  settings.
+- Reduce attention input size by safe model/config choices when allowed:
+  latent resolution, frame count, patching, boundary ratio, or windowing.
+- Remove avoidable layout conversions before/after attention.
+- Reuse condition-side KV or other static inputs if the model structure allows.
+- Consider custom kernels, sparse/window/linear attention, or approximation only
+  after quality risk is accepted.
+
+Priority guidance:
+
+- P1 for backend/config/layout changes with preserved math.
+- P2 for approximate attention, sparsity, custom kernels, or any change that can
+  alter quality/temporal consistency.
+
+Validation:
+
+- Always include output quality and seed behavior checks.
+- Compare `diffuse`, server end-to-end, peak memory, and attention kernel time
+  in diagnostic traces if needed.
+
+### Algorithmic, Precision, and Approximation Optimization
+
+Purpose: reduce mathematical work or precision cost beyond local code cleanup.
+
+Evidence to look for:
+
+- A single operator family dominates even after low-risk runtime, parallel, and
+  fusion work.
+- Memory bandwidth or compute utilization suggests precision or quantization
+  could matter.
+- The user explicitly allows quality-preserving or approximate methods.
+
+Typical candidates:
+
+- FP8/quantization for transformer or selected projections.
+- Sparsity or Rainfusion-style acceleration.
+- Reduced steps, scheduler changes, distillation, or caching across frames.
+- Approximate attention or linear attention.
+
+Priority guidance:
+
+- Usually P2 because quality, numerics, and implementation risk are high.
+
+Validation:
+
+- Requires strict A/B, visual quality review, temporal flicker checks, seed
+  stability, and possibly human evaluation.
+
+### Interpolation, Super-Resolution, and E2E Pipeline Optimization
+
+Purpose: optimize the whole user-visible video product, not only the base
+diffusion invocation. Some deployments trade base-model latency against
+post-processing, interpolation, or super-resolution stages.
+
+Evidence to look for:
+
+- E2E latency breakdown across base generation, interpolation, super-resolution,
+  encoding, storage, and response streaming.
+- Fast/slow GPU or fast/slow stage analysis across multiple cards and pipeline
+  stages.
+- User quality target: resolution, FPS, temporal smoothness, and acceptable
+  post-processing artifacts.
+
+Typical candidates:
+
+- Add or optimize a frame interpolation stage when it reduces required base
+  model frames for the same perceived FPS.
+- Add or optimize a super-resolution model when generating lower base
+  resolution plus SR is faster for the target quality.
+- Analyze E2E2 pipeline behavior: client request, service scheduling, diffusion,
+  VAE/media, post-process, file write, and response.
+- Identify fast/slow cards or stages and rebalance pipeline placement.
+
+Priority guidance:
+
+- P1 when using proven interpolation/SR components without changing diffusion
+  math.
+- P2 when quality risk is high or the pipeline adds significant operational
+  complexity.
+
+Validation:
+
+- Measure E2E wall-clock, per-stage server timings, output FPS/resolution,
+  artifacts, flicker, and user-visible quality.
+
+### Optimization Candidate Library
+
+Use this table as a compact menu, not as automatic recommendations. Pick items
+only when baseline metrics, trace evidence, source inspection, or quality
+tolerance supports them.
+
+| Layer | Candidate | Evidence | Priority | Validation focus |
+|---|---|---|---|---|
+| Measurement | Freeze server/request/benchmark commands | Missing or drifting commands | P0 | Repeatable non-profiler A/B |
+| Measurement | Separate baseline and diagnostic profiler runs | Profiler used for latency claims | P0 | Low-overhead stage timings |
+| Host/runtime | Guard or remove avoidable `empty_cache` | Host-stack gaps or sync stalls | P0/P1 | Latency, peak memory, OOM safety |
+| Host/runtime | Cache scheduler coefficients/timesteps | Repeated tiny CPU/GPU work | P0/P1 | Same seed/output, stage timing |
+| Host/runtime | Reduce framework scheduling overhead | Client time exceeds server time | P1 | E2E latency, throughput |
+| Parallel | `CFG=2 x USP=world/2` vs `USP=world` | CFG doubles forward work | P0/P1 | `diffuse`, NCCL, memory |
+| Parallel | Tune VAE patch parallelism | VAE encode/decode is material | P0/P1 | VAE time, output correctness |
+| Parallel | HSDP on/off or buffer reuse | HSDP affects memory/all-gather | P1 | Memory, latency, OOM risk |
+| Parallel | Ulysses vs Ulysses+Ring | Long sequence all-to-all suspected | P1/P2 | Rank balance, NCCL kernels |
+| Cross-attn | Disable SP for short condition tokens | Cross-attn comm exceeds compute | P1 | `diffuse`, correctness |
+| VAE/media | Reduce VAE gather/broadcast | Rank traces show VAE wait | P1 | Rank balance, output file |
+| VAE/media | Reuse tile metadata/buffers | Tile split/merge host gaps | P1 | `vae.encode/decode`, memory |
+| VAE/media | VAE bf16/autocast | VAE float kernels are slow | P1/P2 | Artifacts, flicker, seed stability |
+| Operator fusion | AdaLayerNorm/LayerNorm fusion | Norm plus scale/shift kernels | P1 | Numeric tolerance, latency |
+| Operator fusion | RMSNorm fusion | Many small RMSNorm kernels | P1 | Numeric tolerance, latency |
+| Operator fusion | RoPE cache/fuse/layout cleanup | RoPE copy/reshape kernels | P1/P2 | Kernel count, correctness |
+| Operator layout | QKV or attention layout cleanup | Copy/cat/split around attention | P1 | Copy kernels, compile behavior |
+| Attention | Verify backend fast path | FA/SDPA dominates trace | P1 | `diffuse`, attention kernels |
+| Attention | FA to LA or selected-head LA | Attention remains dominant | P2 | Quality, temporal stability |
+| Precision | Transformer FP8/quantization | Compute/bandwidth bound and allowed | P2 | Quality, speed, stability |
+| Sparsity | Rainfusion-style acceleration | DiT compute remains dominant | P2 | Prompt diversity, quality |
+| Pipeline | Frame interpolation | Fewer base frames can meet FPS | P1/P2 | E2E latency, motion artifacts |
+| Pipeline | Super-resolution | Lower base res plus SR may win | P1/P2 | Detail quality, artifacts |
+| E2E | Fast/slow-card analysis | Multi-card stragglers | P0/P1 | Per-rank/stage wall-clock |
+
+### Optimization Plan Template
+
+Use this table shape when reporting the next work items:
+
+| Priority | Layer | Candidate | Evidence | Expected benefit | Implementation path | Validation | Quality risk |
+|---|---|---|---|---|---|---|---|
+| P0 | Host/runtime | Guard `empty_cache` | Host-stack gap points to `torch.cuda.empty_cache` | Small latency reduction, less idle | Add config/env guard | Non-profiler A/B, memory check | Low |
+| P1 | Operator fusion | RMSNorm/AdaLayerNorm fusion | High-frequency norm/elementwise kernels | Lower launch/bandwidth overhead | Use existing fusion helper or targeted Triton | A/B + output check | Medium |
+| P1/P2 | Attention | Attention layout/backend investigation | FA kernel dominates CUDA time | Potentially large | Inspect shapes/backend and remove layout copies | A/B + trace + quality | Medium/high |
+
+Then present a short selection prompt using the same rows:
+
+```text
+Which candidate should we implement next?
+
+1. P0 Host/runtime: guard empty_cache
+   - Expected benefit: small but low-risk latency reduction.
+   - Risk: possible memory increase/OOM if memory headroom is insufficient.
+
+2. P1 Operator fusion: inspect by_shape and implement first norm/RoPE/layout fusion
+   - Expected benefit: medium if high-frequency small kernels are confirmed.
+   - Risk: numerical/compile/quality validation needed.
+
+3. P1/P2 Attention: FA/LA/backend/layout investigation
+   - Expected benefit: potentially large.
+   - Risk: high quality and implementation risk.
+```
+
+If the user has not chosen an item, default to explaining tradeoffs and asking
+which candidate to execute. Only proceed autonomously on low-risk P0 measurement
+or instrumentation fixes.
+
+## Analysis Helpers
+
+PyTorch profiler traces are Chrome/Perfetto-compatible JSON files, usually
+`trace_rankN.json` or `trace_rankN.json.gz`. They normally contain a top-level
+`traceEvents` list, though some exporters emit the raw event list directly.
+
+Use the checked-in analyzer from the repository root:
+
+```bash
+python3 .claude/skills/diffusion-perf-opt/scripts/trace_analyzer.py \
+  vllm_profile/.../trace_rank0.json.gz \
+  --min-gap-ms 5 \
+  --topn 20
+```
+
+For rank imbalance or communication questions, pass all relevant rank traces in
+one command. For host gaps, lower `--min-gap-ms` to `1` and use a host-stack
+trace. Read `gpu_span_s`, `busy_union_s`, `idle_union_s`, `idle_pct`, `GAP`
+blocks, `Top GPU/operator events by total duration`, and `Top NCCL-like events
+by category`. Treat `cat=user_annotation` NCCL ranges as enclosing annotations;
+prefer `cat=kernel` or `cat=gpu_user_annotation` for real device work.
+
+The analyzer summarizes timing only. It does not parse tensor shapes, attribute
+overlap to individual streams, prove quality, or provide final latency claims.
+Use `ops_rankN.xlsx` or PyTorch key averages for shape analysis, and re-test any
+optimization with non-profiler baseline commands.
+
+Read `references/optimization-playbook.md` when drafting the optimization table or comparing candidate techniques.
+
+## vLLM Omni Heuristics
+
+- Cross-attention usually should not use USP/SP when text/image condition token count is much smaller than latent video tokens. Confirm via trace; in Wan2.2 I2V, self-attention dominates cross-attention.
+- VAE bf16/autocast is often worthwhile but requires visual quality checks.
+- VAE patch parallel can help decode/encode but may add gather/merge/broadcast overhead. Check whether all ranks need the final decoded tensor.
+- HSDP/FSDP is primarily a memory strategy. If the model fits without it, run an on/off latency comparison.
+- Scheduler work can create small host/device gaps; cache tiny solve coefficients when timesteps/order are known.
+- `torch.cuda.empty_cache()` can prevent OOM but creates synchronization/idle. Make it optional if memory headroom is sufficient.
+- `Command Buffer Full` in profiler output is profiler overhead, not a model optimization target.

.claude/skills/diffusion-perf-opt/agents/openai.yaml

@@ -0,0 +1,4 @@
+interface:
+  display_name: "vLLM Omni Perf Opt"
+  short_description: "Profile and optimize vLLM Omni diffusion performance."
+  default_prompt: "Analyze and optimize a vLLM Omni diffusion workload using baselines, traces, parallel strategy modeling, and prioritized optimization plans."

.claude/skills/diffusion-perf-opt/references/optimization-playbook.md

@@ -0,0 +1,82 @@
+# Optimization Playbook
+
+Use this reference when turning trace evidence into prioritized vLLM Omni optimizations.
+
+## Parallel Strategy
+
+Start from workload shape:
+
+- Long video tokens, high resolution, many frames: self-attention and FFN dominate. USP/SP is usually valuable.
+- CFG enabled with positive and negative prompts: transformer forward count doubles. CFG parallel may beat USP on small/moderate sequence lengths.
+- Cross-attention: skip SP/USP when condition tokens are much smaller than latent tokens; extra all-to-all can exceed compute saved.
+- HSDP/FSDP: use to fit memory first. If memory allows, compare without HSDP because all-gathers and lazy init add overhead.
+- VAE parallel: useful when encode/decode is material. Check gather/merge/broadcast and whether every rank needs final decoded output.
+
+Candidate matrices:
+
+- 2 cards: `USP=2`, `CFG=2`, HSDP on/off, VAE PP on/off.
+- 4 cards: `USP=4`, `CFG=2 x USP=2`, `USP=2 x HSDP`, VAE PP on/off.
+- 8 cards: add pipeline/stage parallel only after measuring orchestration and inter-stage transfer overhead.
+
+## Host And Free Bubble Analysis
+
+Look for device intervals where no kernel, memcpy, or memset is active.
+
+Common causes:
+
+- First request FSDP/HSDP lazy init.
+- Python preprocessing: image load, resize, `to_tensor`, tokenizer, request marshaling.
+- `torch.cuda.empty_cache()` or explicit synchronizations.
+- Scheduler small linear algebra such as tiny `torch.linalg.solve`.
+- VAE tile split/merge Python loops.
+- Profiler overhead such as CUPTI command buffer flush/full.
+
+Also inspect trace-external time:
+
+- API request parsing and multipart upload.
+- Engine/orchestrator dispatch.
+- Worker RPC and queue wait.
+- Output serialization, video write, base64, HTTP response.
+
+## Operator Optimization Candidates
+
+Evaluate necessity before implementation:
+
+- **FA to LA / LA for head:** high potential only if attention is the main CUDA hotspot and target shapes fit LA well.
+- **Cross-attention BSND/no transpose:** do only if transpose/copy around cross-attn is measurable.
+- **AdaLayerNorm fusion:** fuse norm, scale/shift, gate, residual, and dtype conversion if repeated elementwise kernels are significant.
+- **RMSNorm fusion:** useful for Q/K norms and VAE norms when small-kernel launch count is high.
+- **FP32 LayerNorm to bf16:** quality-sensitive; run visual regression.
+- **LA preprocess:** integrate only if preprocess overhead is cached or amortized.
+- **RoPE move/cache/fuse:** useful when RoPE small kernels are visible around every self-attn block.
+- **VAE RMSNorm replacement:** only after VAE internal trace shows norm as a real component of decode/encode.
+- **Scheduler fusion/cache:** cache coefficient solves and fuse CFG combine with scheduler arithmetic when possible.
+
+Before writing a custom kernel, check:
+
+- Local vLLM Omni implementations.
+- Upstream vLLM kernels and attention backends.
+- SGLang, FlashInfer, FlashAttention, and Triton examples.
+- Shape constraints and layout expectations.
+
+## Quantization And Sparsity
+
+Treat these as high-risk optimizations:
+
+- Rainfusion/sparse diffusion can be high reward but needs model-specific quality validation.
+- VAE decode quantization can reduce decode time but usually has smaller end-to-end upside than transformer attention/FFN work.
+- FP8 or approximate kernels require seed-stable visual regression, temporal flicker checks, and prompt diversity.
+
+## Output Format For Recommendations
+
+Return a table with:
+
+- Priority: P0/P1/P2.
+- Optimization.
+- Evidence from trace or code.
+- Expected benefit.
+- Risk/quality impact.
+- Implementation path.
+- Validation command or A/B test.
+
+State clearly when a recommendation is a hypothesis requiring new trace with shapes.

.claude/skills/diffusion-perf-opt/scripts/trace_analyzer.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+"""Summarize torch profiler Chrome traces for vLLM Omni diffusion optimization."""
+
+from __future__ import annotations
+
+import argparse
+import collections
+import gzip
+import json
+from pathlib import Path
+from typing import Any
+
+GPU_CATS = {"kernel", "gpu_memcpy", "gpu_memset"}
+CPU_CATS = {"python_function", "user_annotation", "cpu_op", "cuda_runtime", "cuda_driver"}
+
+
+def open_trace(path: Path):
+    if path.suffix == ".gz":
+        return gzip.open(path, "rt")
+    return path.open("rt")
+
+
+def event_name(event: dict[str, Any]) -> str:
+    return str(event.get("name", ""))
+
+
+def summarize_trace(path: Path, min_gap_us: float, topn: int) -> None:
+    with open_trace(path) as f:
+        data = json.load(f)
+
+    events = data if isinstance(data, list) else data.get("traceEvents", [])
+    gpu: list[tuple[float, float, float, str, str, Any, Any]] = []
+    cpu: list[tuple[float, float, float, str, str, Any, Any]] = []
+    by_gpu_name: dict[str, list[float]] = collections.defaultdict(lambda: [0, 0.0, 0.0])
+    by_name: dict[str, list[float]] = collections.defaultdict(lambda: [0, 0.0, 0.0])
+    nccl: dict[tuple[str, str], list[float]] = collections.defaultdict(lambda: [0, 0.0, 0.0])
+
+    for event in events:
+        dur = event.get("dur")
+        ts = event.get("ts")
+        if dur is None or ts is None or dur <= 0:
+            continue
+        cat = str(event.get("cat", ""))
+        name = event_name(event)
+        row = (float(ts), float(ts + dur), float(dur), name, cat, event.get("pid"), event.get("tid"))
+        stat = by_name[name]
+        stat[0] += 1
+        stat[1] += dur
+        stat[2] = max(stat[2], dur)
+        if cat in GPU_CATS:
+            gpu.append(row)
+            gstat = by_gpu_name[name]
+            gstat[0] += 1
+            gstat[1] += dur
+            gstat[2] = max(gstat[2], dur)
+        elif cat in CPU_CATS:
+            cpu.append(row)
+        if "nccl" in name.lower():
+            nstat = nccl[(cat, name)]
+            nstat[0] += 1
+            nstat[1] += dur
+            nstat[2] = max(nstat[2], dur)
+
+    print(f"\n== {path}")
+    print(f"events={len(events)} gpu_events={len(gpu)} cpu_events={len(cpu)}")
+    if not gpu:
+        print("No GPU events found.")
+        return
+
+    gpu.sort()
+    merged: list[list[Any]] = []
+    for start, end, dur, name, cat, pid, tid in gpu:
+        if not merged or start > merged[-1][1]:
+            merged.append([start, end, [(start, end, dur, name, cat, pid, tid)]])
+        else:
+            merged[-1][1] = max(merged[-1][1], end)
+            merged[-1][2].append((start, end, dur, name, cat, pid, tid))
+
+    span = max(end for _, end, *_ in gpu) - min(start for start, *_ in gpu)
+    busy = sum(end - start for start, end, _ in merged)
+    idle = span - busy
+    print(
+        f"gpu_span_s={span / 1e6:.3f} "
+        f"busy_union_s={busy / 1e6:.3f} "
+        f"idle_union_s={idle / 1e6:.3f} "
+        f"idle_pct={idle / span * 100:.2f}"
+    )
+
+    interesting_cpu = [
+        row
+        for row in cpu
+        if row[2] >= 1000
+        and (
+            row[4] in {"python_function", "user_annotation"}
+            or "cudaStreamSynchronize" in row[3]
+            or "cudaDeviceSynchronize" in row[3]
+            or "cudaLaunch" in row[3]
+            or "cudaMemcpy" in row[3]
+        )
+    ]
+
+    gaps = []
+    for idx in range(1, len(merged)):
+        gap_start = merged[idx - 1][1]
+        gap_end = merged[idx][0]
+        gap_dur = gap_end - gap_start
+        if gap_dur >= min_gap_us:
+            prev_event = max(merged[idx - 1][2], key=lambda x: x[1])
+            next_event = min(merged[idx][2], key=lambda x: x[0])
+            mid = (gap_start + gap_end) / 2
+            containers = [row for row in interesting_cpu if row[0] <= mid <= row[1]]
+            containers = sorted(containers, key=lambda x: x[2])[:8]
+            gaps.append((gap_dur, gap_start, gap_end, prev_event, next_event, containers))
+
+    print(f"gaps_ge_{min_gap_us / 1000:.3f}ms count={len(gaps)} sum_s={sum(g[0] for g in gaps) / 1e6:.3f}")
+    for gap_dur, gap_start, gap_end, prev_event, next_event, containers in sorted(gaps, reverse=True)[:topn]:
+        print(f"\nGAP {gap_dur / 1000:.3f} ms ts={gap_start:.0f}->{gap_end:.0f}")
+        print(f"  prev {prev_event[4]} {prev_event[2] / 1000:.3f} ms {prev_event[3][:160]}")
+        print(f"  next {next_event[4]} {next_event[2] / 1000:.3f} ms {next_event[3][:160]}")
+        for row in containers:
+            print(f"  in   {row[4]} {row[2] / 1000:.3f} ms {row[3][:180]}")
+
+    print("\nTop GPU/operator events by total duration:")
+    for name, (count, total, max_dur) in sorted(by_gpu_name.items(), key=lambda kv: kv[1][1], reverse=True)[:topn]:
+        print(f"  {int(count):8d} total={total / 1e6:9.3f}s max={max_dur / 1000:9.3f}ms {name[:180]}")
+
+    print("\nTop NCCL-like events by category:")
+    for (cat, name), (count, total, max_dur) in sorted(nccl.items(), key=lambda kv: kv[1][1], reverse=True)[:topn]:
+        print(f"  {int(count):8d} total={total / 1e6:9.3f}s max={max_dur / 1000:9.3f}ms cat={cat} {name[:160]}")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("traces", nargs="+", type=Path, help="trace.json or trace.json.gz files")
+    parser.add_argument("--min-gap-ms", type=float, default=5.0, help="minimum GPU idle gap to print")
+    parser.add_argument("--topn", type=int, default=20, help="number of gaps/hotspots to print")
+    args = parser.parse_args()
+
+    for trace in args.traces:
+        summarize_trace(trace, args.min_gap_ms * 1000.0, args.topn)
+
+
+if __name__ == "__main__":
+    main()

.claude/skills/readme.md

@@ -18,6 +18,9 @@ include:
 
 - `add-diffusion-model`: guides integration of a new diffusion model into
   `vllm-omni`
+- `diffusion-perf-opt`: guides diffusion model performance optimization,
+  including profiling traces, parallel strategies, stage timing analysis, and
+  benchmark-driven tuning
 - `add-omni-model`: covers addition of new omni-modality model support
 - `add-tts-model`: covers integration of new TTS models and related serving
   workflows

README.md

@@ -9,19 +9,19 @@ Easy, fast, and cheap omni-modality model serving for everyone
 </h3>
 
 <p align="center">
-| <a href="https://vllm-omni.readthedocs.io/en/latest/"><b>Documentation</b></a> | <a href="https://discuss.vllm.ai"><b>User Forum</b></a> | <a href="https://slack.vllm.ai"><b>Developer Slack</b></a> | <a href="docs/assets/WeChat.jpg"><b>WeChat</b></a> | <a href="https://arxiv.org/abs/2602.02204"><b>Paper</b></a> | <a href="https://docs.google.com/presentation/d/1XJWgv79lORl8rbaVvp2d5Sqs6ZEBgAgj/edit?slide=id.p1#slide=id.p1"><b>Slides</b></a> |
+| <a href="https://vllm-omni.readthedocs.io/en/latest/"><b>Documentation</b></a> | <a href="https://deepwiki.com/vllm-project/vllm-omni"><b>DeepWiki</b></a> | <a href="https://discuss.vllm.ai"><b>User Forum</b></a> | <a href="https://slack.vllm.ai"><b>Developer Slack</b></a> | <a href="docs/assets/WeChat.jpg"><b>WeChat</b></a> | <a href="https://arxiv.org/abs/2602.02204"><b>Paper</b></a> | <a href="https://docs.google.com/presentation/d/111-L8zF7A1j_YI_cR8JsblofdScdRr2f/edit?usp=sharing&ouid=110473603432222024453&rtpof=true&sd=true"><b>Slides</b></a> |
 </p>
 
 
 ---
 
 *Latest News* 🔥
+- [2026/05] We released [0.20.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.20.0) - refreshes the serving/runtime stack for large-scale omni workloads, and improves diffusion model performance, quantization, and hardware readiness across CUDA, ROCm, MUSA, NPU, and XPU backends.
 - [2026/03] We released [0.18.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.18.0) - strengthens the core runtime through a large entrypoint refactor and scheduler/runtime cleanups, expands unified quantization and diffusion execution, broadens multimodal model coverage, and improves production readiness across audio, omni, image, video, RL, and multi-platform deployments.
 - [2026/03] Check out our first public [project deepdive](https://youtu.be/sgwNfsNnR9I) at the vLLM Hong Kong Meetup!
 - [2026/03] **[vllm-omni-skills](https://github.com/hsliuustc0106/vllm-omni-skills)** is a community-driven collection of AI assistant skills that help developers work with vLLM-Omni more effectively. These skills can be used with popular agentic AI coding assistants like **Cursor IDE**, **Claude**, **Codex**, and more.
 - [2026/02] We released [0.16.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.16.0) - A major alignment + capability release that rebases onto **upstream vLLM v0.16.0** and significantly expands performance, distributed execution, and production readiness across **Qwen3-Omni / Qwen3-TTS**, **Bagel**, **MiMo-Audio**, **GLM-Image** and the **Diffusion (DiT) image/video stack**—while also improving platform coverage (CUDA / ROCm / NPU / XPU), CI quality, and documentation.
 - [2026/02] We released [0.14.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.14.0) - This is the first **stable release** of vLLM-Omni that expands Omni’s diffusion / image-video generation and audio / TTS stack, improves distributed execution and memory efficiency, and broadens platform/backend coverage (GPU/ROCm/NPU/XPU). It also brings meaningful upgrades to serving APIs, profiling & benchmarking, and overall stability. Please check our latest [paper](https://arxiv.org/abs/2602.02204) for architecture design and performance results.
-- [2026/01] We released [0.12.0rc1](https://github.com/vllm-project/vllm-omni/releases/tag/v0.12.0rc1) - a major RC milestone focused on maturing the diffusion stack, strengthening OpenAI-compatible serving, expanding omni-model coverage, and improving stability across platforms (GPU/NPU/ROCm).
 - [2025/11] vLLM community officially released [vllm-project/vllm-omni](https://github.com/vllm-project/vllm-omni) in order to support omni-modality models serving.
 
 ---
@@ -56,9 +56,8 @@ vLLM-Omni is flexible and easy to use with:
 
 vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:
 
-- **Omni-modality models** (e.g. Qwen3-Omni, Cosmos, HunyuanImage, BAGEL)
-- **TTS models** (e.g. Qwen3-TTS, VoxCPM2, Ming-Omni-TTS, CosyVoice3)
-- **Diffusion models** — image, video, and audio generation (e.g. Qwen-Image, Wan2.2, FLUX)
+- Omni-modality models (e.g. Qwen-Omni)
+- Multi-modality generation models (e.g. Qwen-Image)
 
 ## Getting Started
 

benchmarks/build_dataset/seed_tts_design/en/meta.lst

@@ -18,3 +18,13 @@ vd017|||The quarterly results exceed expectations across all major metrics.|A sh
 vd018|||Chapter one. The morning sun filtered gently through the forest canopy.|A smooth, rich male audiobook narrator voice, expressive and engaging.
 vd019|||To be or not to be, that is the question.|A theatrical female voice, dramatic and expressive, stage projection quality.
 vd020|||And that is all for tonight. Stay well out there, everyone.|A warm, velvety male late-night radio DJ voice, smooth and intimate.
+vd021|||Please fasten your seatbelts; we will be experiencing some turbulence shortly.|A calm, professional female flight attendant voice, reassuring and clear.
+vd022|||The fossil record suggests these creatures lived more than sixty million years ago.|A measured male documentary narrator voice, scientific and curious.
+vd023|||Hey kiddo, did you finish your homework before dinner today?|A gentle, caring father voice, warm and slightly tired after a long day.
+vd024|||Live from courtside, the home team has just taken a commanding lead.|A high-energy male sports broadcaster, urgent and engaged.
+vd025|||Please remain on the line while I transfer you to the next available agent.|A polite, neutral female call-center voice, professional and patient.
+vd026|||I have a dream that one day this nation will rise up and live out the true meaning of its creed.|A powerful, resonant male orator voice, slow and impassioned.
+vd027|||Mama always said life was like a box of chocolates; you never know what you're gonna get.|A folksy male voice with a Southern American accent, warm and reflective.
+vd028|||Ahoy there, matey! Hoist the colors and prepare to set sail.|A theatrical pirate male voice, gruff and boisterous.
+vd029|||Ladies and gentlemen, please welcome to the stage our keynote speaker.|A polished female event MC voice, gracious and articulate.
+vd030|||Once you've completed the form, click submit and we'll review it within two business days.|A friendly female product-onboarding voice, encouraging and clear.

benchmarks/build_dataset/seed_tts_smoke/en/meta.lst

@@ -18,3 +18,13 @@ smoke017|||In the distance, the mountains were shrouded in thick morning fog.
 smoke018|||Our company reported record revenue for the fourth quarter of the fiscal year.
 smoke019|||She explained the new policy in detail during the staff meeting this morning.
 smoke020|||The children laughed and played in the garden until the sun began to set.
+smoke021|||A long line of cars stretched along the coastal highway after the lighthouse opened.
+smoke022|||The chef recommended the seared scallops with a lemon butter sauce as the special.
+smoke023|||Our quarterly engineering review begins promptly at two in the afternoon on Thursday.
+smoke024|||Despite the heavy rain, the marathon runners kept a steady pace through downtown.
+smoke025|||She finished the crossword puzzle in under fifteen minutes while sipping her tea.
+smoke026|||The new release adds streaming output, faster startup, and better error reporting.
+smoke027|||His grandfather taught him how to whittle small wooden birds from cedar offcuts.
+smoke028|||Travelers were advised to check the timetable since trains were running on a holiday schedule.
+smoke029|||The startup launched a public beta after eighteen months of closed development.
+smoke030|||At night the desert sky filled with stars so bright the Milky Way was clearly visible.

benchmarks/diffusion/README.md

@@ -30,7 +30,7 @@ python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
 
 **Notes**
 
-- The benchmark talks to `http://<host>:<port>/v1/chat/completions`.
+- By default, image tasks talk to `http://<host>:<port>/v1/chat/completions`; video tasks talk to `/v1/videos`.
 - If you run the server on another host or port, pass `--base-url` accordingly.
 
 ## 2. Supported Datasets
@@ -95,8 +95,9 @@ You can point to your own trace using `--dataset-path`.
 
 ### Basic flags
 
-- `--base-url`: Server address (the script calls `.../v1/chat/completions`).
+- `--base-url`: Server address; `--endpoint` selects the path appended to this base URL.
 - `--model`: The OpenAI-compatible `model` field.
+- `--endpoint`: API endpoint path. Leading `/` is optional, e.g. `/v1/videos` or `v1/videos`.
 - `--task`: Task type (e.g., `t2i`, `t2v`, `i2i`, `i2v`).
 - `--dataset`: Dataset mode (`vbench` / `trace` / `random`).
 - `--num-prompts`: Number of requests to send.

benchmarks/diffusion/backends.py

@@ -151,7 +151,7 @@ async def async_request_chat_completions(
     return output
 
 
-async def async_request_openai_images(
+async def async_request_openai_image_generations(
     input: RequestFuncInput,
     session: aiohttp.ClientSession,
     pbar: tqdm | None = None,
@@ -348,12 +348,38 @@ async def async_request_v1_videos(
     return output
 
 
+LEGACY_BACKEND_ENDPOINT_ALIASES = {
+    "vllm-omni": "/v1/chat/completions",
+    "openai": "/v1/images/generations",
+}
+
+
+def normalize_endpoint(value: str) -> str:
+    endpoint = str(value).strip()
+    if not endpoint:
+        raise ValueError("endpoint must not be empty.")
+    endpoint = LEGACY_BACKEND_ENDPOINT_ALIASES.get(
+        endpoint,
+        LEGACY_BACKEND_ENDPOINT_ALIASES.get(endpoint.lstrip("/"), endpoint),
+    )
+    if not endpoint.startswith("/"):
+        endpoint = f"/{endpoint}"
+    return endpoint
+
+
+def endpoint_filename_token(value: str) -> str:
+    token = normalize_endpoint(value).lstrip("/")
+    for bad in ("/", "\\", ":", "*", "?", '"', "<", ">", "|"):
+        token = token.replace(bad, "_")
+    return token or "endpoint"
+
+
 backends_function_mapping = {
     "2i": {
-        "vllm-omni": (async_request_chat_completions, "/v1/chat/completions"),
-        "openai": (async_request_openai_images, "/v1/images/generations"),
+        "/v1/chat/completions": (async_request_chat_completions, "/v1/chat/completions"),
+        "/v1/images/generations": (async_request_openai_image_generations, "/v1/images/generations"),
     },
     "2v": {
-        "v1/videos": (async_request_v1_videos, "/v1/videos"),
+        "/v1/videos": (async_request_v1_videos, "/v1/videos"),
     },
 }

benchmarks/diffusion/diffusion_benchmark_serving.py

@@ -6,31 +6,31 @@
 Benchmark online serving for diffusion models (Image/Video Generation).
 If you want to use i2v, i2i dataset, you should `uv pip install gdown` first
 
-Supports multiple backends:
-    - vllm-omni: Uses /v1/chat/completions endpoint (default)
-    - openai: Uses /v1/images/generations endpoint
-    - v1/videos: Use /v1/videos endpoint
+Supports multiple endpoints:
+    - /v1/chat/completions: OpenAI chat-compatible image requests
+    - /v1/images/generations: OpenAI image generation requests
+    - /v1/videos: Async video jobs
 
 Usage:
-    # Video (v1/videos backend)
+    # Video (/v1/videos endpoint)
     t2v:
     python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --backend v1/videos --dataset vbench --task t2v --num-prompts 10 \
+        --endpoint /v1/videos --dataset vbench --task t2v --num-prompts 10 \
         --height 480 --width 640 --fps 16 --num-frames 80
 
     i2v:
     python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --backend v1/videos --dataset vbench --task i2v --num-prompts 10
+        --endpoint /v1/videos --dataset vbench --task i2v --num-prompts 10
 
 
-    # Image (vllm-omni backend)
+    # Image (/v1/chat/completions endpoint)
     t2i:
     python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --backend vllm-omni --dataset vbench --task t2i --num-prompts 10 \
+        --endpoint /v1/chat/completions --dataset vbench --task t2i --num-prompts 10 \
         --height 1024 --width 1024
 
     python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --backend vllm-omni --dataset random --task t2i --num-prompts 1 \
+        --endpoint /v1/chat/completions --dataset random --task t2i --num-prompts 1 \
         --max-concurrency 1 --enable-negative-prompt \
         --random-request-config '[
             {"width":512,"height":512,"num_inference_steps":20,"weight":0.15},
@@ -41,18 +41,18 @@ Usage:
 
     i2i:
     python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --backend vllm-omni --dataset vbench --task i2i --num-prompts 10
+        --endpoint /v1/chat/completions --dataset vbench --task i2i --num-prompts 10
 
-    # Image (openai backend)
+    # Image (/v1/images/generations endpoint)
     t2i:
     python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --backend openai --dataset vbench --task t2i --num-prompts 10 \
+        --endpoint /v1/images/generations --dataset vbench --task t2i --num-prompts 10 \
         --height 1024 --width 1024 --port 3000
 
     # Video (v1/videos)
     t2v:
     python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --backend v1/videos --dataset random --task t2v --num-prompts 1 \
+        --endpoint /v1/videos --dataset random --task t2v --num-prompts 1 \
         --max-concurrency 1 --enable-negative-prompt \
         --random-request-config '[
             {"width":854,"height":480,"num_inference_steps":18,"num_frames":120,"fps":24,"weight":1}
@@ -80,7 +80,12 @@ from typing import Any
 import aiohttp
 import numpy as np
 import requests
-from backends import RequestFuncInput, RequestFuncOutput, backends_function_mapping
+from backends import (
+    RequestFuncInput,
+    RequestFuncOutput,
+    backends_function_mapping,
+    normalize_endpoint,
+)
 from PIL import Image
 from tqdm.asyncio import tqdm
 
@@ -883,6 +888,14 @@ def wait_for_service(base_url: str, timeout: int = 120) -> None:
         time.sleep(1)
 
 
+def _default_endpoint_for_task(task: str) -> str:
+    if task in {"t2v", "i2v", "ti2v"}:
+        return "/v1/videos"
+    if task in {"t2i", "i2i", "ti2i"}:
+        return "/v1/chat/completions"
+    raise ValueError(f"Unsupported task for endpoint resolution: {task}")
+
+
 async def benchmark(args):
     # Construct base_url if not provided
     if args.base_url is None:
@@ -902,18 +915,23 @@ async def benchmark(args):
             f"Valid image tasks: {sorted(IMAGE_TASKS)}"
         )
 
-    valid_backends = sorted(backends_function_mapping[task_type].keys())
+    raw_endpoint = args.endpoint if args.endpoint is not None else args.backend
+    if raw_endpoint is None:
+        raw_endpoint = _default_endpoint_for_task(args.task)
+    args.endpoint = normalize_endpoint(raw_endpoint)
 
-    if args.backend not in valid_backends:
+    valid_endpoints = sorted(backends_function_mapping[task_type].keys())
+
+    if args.endpoint not in valid_endpoints:
         logger.error(
-            f"Invalid backend '{args.backend}' for task '{args.task}' (task type: '{task_type}').\n"
-            f"Valid backends for this task type: {valid_backends}\n"
-            f"Example usage: --task {args.task} --backend {valid_backends[0]}"
+            f"Invalid endpoint '{args.endpoint}' for task '{args.task}' (task type: '{task_type}').\n"
+            f"Valid endpoints for this task type: {valid_endpoints}\n"
+            f"Example usage: --task {args.task} --endpoint {valid_endpoints[0]}"
         )
-        raise ValueError("Backend validation failed. See log above for valid options.")
+        raise ValueError("Endpoint validation failed. See log above for valid options.")
 
-    # Setup API URL and request function based on backend
-    request_func, api_url = backends_function_mapping[task_type][args.backend]
+    # Setup API URL and request function based on endpoint.
+    request_func, api_url = backends_function_mapping[task_type][args.endpoint]
     api_url = f"{args.base_url}{api_url}"
 
     if args.dataset == "vbench":
@@ -976,15 +994,15 @@ async def benchmark(args):
     metrics = calculate_metrics(outputs, total_duration, requests_list, args, args.slo)
 
     # Add configuration info to metrics for JSON output
-    metrics["backend"] = args.backend
+    metrics["endpoint"] = args.endpoint
     metrics["model"] = args.model
     metrics["dataset"] = args.dataset
     metrics["task"] = args.task
 
-    print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=60, c="="))
+    print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="="))
 
     # Section 1: Configuration
-    print("{:<40} {:<15}".format("Backend:", args.backend))
+    print("{:<40} {:<15}".format("Endpoint:", args.endpoint))
     print("{:<40} {:<15}".format("Model:", args.model))
     print("{:<40} {:<15}".format("Dataset:", args.dataset))
     print("{:<40} {:<15}".format("Task:", args.task))
@@ -1047,12 +1065,17 @@ if __name__ == "__main__":
     parser.add_argument("--host", type=str, default="localhost", help="Server host.")
     parser.add_argument("--port", type=int, default=8091, help="Server port.")
     parser.add_argument("--model", type=str, default="default", help="Model name.")
+    parser.add_argument(
+        "--endpoint",
+        type=str,
+        default=None,
+        help=("Endpoint path to target. Leading '/' is optional, e.g. /v1/videos or v1/videos."),
+    )
     parser.add_argument(
         "--backend",
         type=str,
-        default="vllm-omni",
-        choices=["vllm-omni", "openai", "v1/videos"],
-        help="Backend to target the benchmark to.",
+        default=None,
+        help=argparse.SUPPRESS,
     )
     parser.add_argument(
         "--dataset",

benchmarks/diffusion/performance_dashboard/qwen_image_serving_performance.md

@@ -58,7 +58,7 @@ Record these parameters when reporting performance results.
 
 ```bash
 python benchmarks/diffusion/diffusion_benchmark_serving.py \
-    --backend vllm-omni \
+    --endpoint /v1/chat/completions \
     --dataset <DATASET_NAME> \
     --task t2i \
     --num-prompts <N> \
@@ -71,7 +71,7 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \
 
 | Parameter              | Description                       |
 | ---------------------- | --------------------------------- |
-| `--backend`            | Serving backend (use `vllm-omni`) |
+| `--endpoint`           | API endpoint (use `/v1/chat/completions`; leading `/` is optional) |
 | `--dataset`            | Dataset name (`random` or custom) |
 | `--task`               | Task type (e.g., `t2i`)           |
 | `--num-prompts`        | Total number of requests          |
@@ -129,7 +129,7 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \
 
 ```bash
 python benchmarks/diffusion/diffusion_benchmark_serving.py \
-    --backend vllm-omni \
+    --endpoint /v1/chat/completions \
     --dataset random \
     --task t2i \
     --num-prompts 1 \

benchmarks/diffusion/performance_dashboard/wan_2_2_serving_performance.md

@@ -53,7 +53,7 @@ Record these parameters when reporting performance results.
 
 ```bash
 python benchmarks/diffusion/diffusion_benchmark_serving.py \
-    --backend v1/videos \
+    --endpoint /v1/videos \
     --dataset <DATASET_NAME> \
     --task t2v \
     --num-prompts <N> \
@@ -66,7 +66,7 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \
 
 | Parameter              | Description                       |
 | ---------------------- | --------------------------------- |
-| `--backend`            | Serving backend (use `v1/videos`) |
+| `--endpoint`           | API endpoint (use `/v1/videos`; leading `/` is optional) |
 | `--dataset`            | Dataset name (`random` or custom) |
 | `--task`               | Task type (e.g., `t2v`)           |
 | `--num-prompts`        | Total number of requests          |
@@ -120,7 +120,7 @@ python benchmarks/diffusion/diffusion_benchmark_serving.py \
 
 ```bash
 python benchmarks/diffusion/diffusion_benchmark_serving.py \
-    --backend v1/videos \
+    --endpoint /v1/videos \
     --dataset random \
     --task t2v \
     --num-prompts 1 \

benchmarks/diffusion/quantization_quality.py

@@ -148,6 +148,7 @@ def _build_omni_kwargs(args, quantization=None):
 def _generate_image(omni, args, prompt, seed):
     """Generate a single image and return (PIL.Image, time_seconds, memory_gib)."""
     from vllm_omni.inputs.data import OmniDiffusionSamplingParams
+    from vllm_omni.outputs import OmniRequestOutput
     from vllm_omni.platforms import current_omni_platform
 
     generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(seed)
@@ -165,8 +166,9 @@ def _generate_image(omni, args, prompt, seed):
     elapsed = time.perf_counter() - start
     peak_mem = torch.accelerator.max_memory_allocated() / (1024**3)
 
-    first = outputs[0]
-    req_out = first.request_output[0] if hasattr(first, "request_output") else first
+    req_out = OmniRequestOutput.unwrap_result(outputs)
+    if not req_out.images:
+        raise ValueError("Could not extract image output from result.")
     img = req_out.images[0]
     return img, elapsed, peak_mem
 

docker/Dockerfile.ci

@@ -1,5 +1,5 @@
 ARG VLLM_BASE_IMAGE=vllm/vllm-openai
-ARG VLLM_BASE_TAG=v0.20.0
+ARG VLLM_BASE_TAG=v0.21.0
 FROM ${VLLM_BASE_IMAGE}:${VLLM_BASE_TAG}
 ARG APP_DIR=/workspace/vllm-omni
 WORKDIR ${APP_DIR}

docker/Dockerfile.npu.ci

@@ -15,7 +15,8 @@ RUN apt-get update && \
         libsm6 \
         libxext6 \
         libxrender1 \
-        libxcb1 && \
+        libxcb1 \
+        espeak-ng && \
     rm -rf /var/cache/apt/* && \
     rm -rf /var/lib/apt/lists/*
 
@@ -29,6 +30,11 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
       "pytest-cov>=4.0.0" \
       "pytest-mock>=3.10.0" \
       "openpyxl>=3.0.0" \
+      "psutil>=7.2.0" \
+      "soundfile>=0.13.1" \
+      "pyttsx3>=2.99" \
+      "opencv-python>=4.12.0.88" \
+      "imageio[ffmpeg]>=0.6.0" \
       --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
       --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
       --default-timeout=60 && \

docker/Dockerfile.npu.ci.a3

@@ -15,7 +15,8 @@ RUN apt-get update && \
         libsm6 \
         libxext6 \
         libxrender1 \
-        libxcb1 && \
+        libxcb1 \
+        espeak-ng && \
     rm -rf /var/cache/apt/* && \
     rm -rf /var/lib/apt/lists/*
 
@@ -29,6 +30,11 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
       "pytest-cov>=4.0.0" \
       "pytest-mock>=3.10.0" \
       "openpyxl>=3.0.0" \
+      "psutil>=7.2.0" \
+      "soundfile>=0.13.1" \
+      "pyttsx3>=2.99" \
+      "opencv-python>=4.12.0.88" \
+      "imageio[ffmpeg]>=0.6.0" \
       --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
       --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
       --default-timeout=60 && \

docker/Dockerfile.rocm

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.20.0
+ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.21.0
 FROM ${BASE_IMAGE} AS base
 
 # Declare a variable to know if we want to use the nightly build or the stable build.

docker/Dockerfile.xpu

@@ -88,7 +88,7 @@ ENV UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
 ENV UV_INDEX_STRATEGY="unsafe-best-match"
 ENV UV_LINK_MODE="copy"
 
-ARG VLLM_VERSION=v0.20.0
+ARG VLLM_VERSION=v0.21.0
 RUN git clone -b ${VLLM_VERSION} https://github.com/vllm-project/vllm
 WORKDIR /workspace/vllm
 

docs/.nav.yml

@@ -6,10 +6,11 @@ nav:
     - getting_started/installation/*
   - Serving:
     - OpenAI-Compatible API:
-      - Diffusion Chat API: serving/diffusion_chat_api.md
+      - Chat Completions API: serving/chat_completions_api.md
       - Image Generation: serving/image_generation_api.md
       - Image Edit: serving/image_edit_api.md
       - Text to Speech: serving/speech_api.md
+      - Videos: serving/videos_api.md
       - Audio Generation: serving/audio_generate_api.md
       - Streaming Video Input: serving/video_stream_api.md
   - Examples:

docs/api/README.md

@@ -99,7 +99,6 @@ Configuration classes.
 - [vllm_omni.diffusion.cache.teacache.config.TeaCacheConfig][]
 - [vllm_omni.distributed.omni_connectors.utils.config.ConnectorSpec][]
 - [vllm_omni.distributed.omni_connectors.utils.config.OmniTransferConfig][]
-- [vllm_omni.model_executor.models.cosyvoice3.config.CosyVoice3Config][]
 - [vllm_omni.model_executor.models.fish_speech.configuration_fish_speech.FishSpeechConfig][]
 - [vllm_omni.model_executor.models.fish_speech.configuration_fish_speech.FishSpeechFastARConfig][]
 - [vllm_omni.model_executor.models.fish_speech.configuration_fish_speech.FishSpeechSlowARConfig][]
@@ -116,10 +115,14 @@ Configuration classes.
 - [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1DecoderConfig][]
 - [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1DecoderDiTConfig][]
 - [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1EncoderConfig][]
+- [vllm_omni.transformers_utils.configs.cosyvoice3.CosyVoice3Config][]
 - [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Config][]
 - [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Qwen2_5_VLConfig][]
 - [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Qwen2_5_VLTextConfig][]
 - [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Qwen2_5_VLVisionConfig][]
+- [vllm_omni.transformers_utils.configs.omnivoice.OmniVoiceConfig][]
+- [vllm_omni.transformers_utils.configs.voxcpm.VoxCPMConfig][]
+- [vllm_omni.transformers_utils.configs.voxcpm2.VoxCPM2Config][]
 
 ## Workers
 

docs/community/governance.md

@@ -39,11 +39,34 @@ Committers have write access and merge rights. They typically have deep expertis
 - [@linyueqian](https://github.com/linyueqian): TTS and Omni Support
 - [@lishunyang12](https://github.com/lishunyang12): Quantization and Configuration
 - [@princepride](https://github.com/princepride): Diffusion and Omni Support
-- [@SamitHuang](https://github.com/SamitHuang): RL and Diffusion
+- [@RuixiangMa](https://github.com/RuixiangMa): Diffusion models, parallel, cache, and docs
+- [@SamitHuang](https://github.com/SamitHuang): RL, Diffusion, and cache
 - [@tzhouam](https://github.com/tzhouam): Engine and New Model Support
-- [@wtomin](https://github.com/wtomin): Diffusion and Parallelism
+- [@wtomin](https://github.com/wtomin): Diffusion models, parallel, and docs
 - [@ZeldaHuang](https://github.com/ZeldaHuang): Omni Support
-- [@ZJY0516](https://github.com/ZJY0516): Diffusion and CustomOp
+- [@ZJY0516](https://github.com/ZJY0516): Diffusion attention backend, kernel fusion, and CustomOp
+- [@yuanheng-zhao](https://github.com/yuanheng-zhao): Diffusion cache, offload, and Omni Support
+
+## Diffusion Workload Division
+
+This section breaks down diffusion-related responsibilities. If you have PRs touching these areas, please ping the listed owners for review.
+
+### Runtime-related
+
+- Diffusion models: @RuixiangMa, @wtomin
+
+### Optimization
+
+- Parallel: @RuixiangMa, @wtomin
+- Attention backend: @ZJY0516
+- Cache: @yuanheng-zhao, @RuixiangMa, @SamitHuang
+- Offload: @yuanheng-zhao
+- Quantization: @lishunyang12, @david6666666
+- Kernel fusion / communication-computation: @ZJY0516
+
+### Docs & Test
+
+- Diffusion docs: @RuixiangMa, @wtomin
 
 ## Meetings
 

docs/design/feature/pipeline_parallel.md

@@ -0,0 +1,356 @@
+# Pipeline Parallel
+
+This section describes how to add Pipeline Parallelism (PP) to a diffusion pipeline. We use the Wan2.2 text-to-video and
+image-to-video pipelines as the reference implementations.
+
+## Implementation Checklist
+
+Adding Pipeline Parallel support requires:
+
+1. ✅ **Inherit the mixin** - add `PipelineParallelMixin` before `CFGParallelMixin` in the pipeline class
+2. ✅ **Make stage forward resumable** - support `intermediate_tensors` in the transformer forward path
+3. ✅ **Return the right object type** - `IntermediateTensors` on non-last PP ranks, final model output on the last rank
+4. ✅ **Use the standard denoising loop** - call `predict_noise_maybe_with_cfg()` and `scheduler_step_maybe_with_cfg()`
+5. ✅ **Test parity** - compare PP results against the single-GPU baseline
+
+---
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Step-by-Step Implementation](#step-by-step-implementation)
+- [Testing](#testing)
+- [Troubleshooting](#troubleshooting)
+- [Reference Implementations](#reference-implementations)
+
+---
+
+## Overview
+
+### What is Pipeline Parallelism?
+
+Pipeline Parallelism splits the denoising transformer into multiple sequential stages and places each stage on a
+different rank. Instead of every rank holding the full DiT, each PP rank owns only a slice of the layers.
+
+For each denoising step:
+
+1. Rank 0 starts the forward pass using the current latents.
+2. Each intermediate rank receives hidden states from the previous rank, runs its local layer slice, and forwards the
+   intermediate tensors downstream.
+3. The last PP rank produces the final noise prediction.
+4. The last PP rank applies the scheduler step and sends the updated latents back to rank 0 for the next timestep.
+
+This reduces per-rank model memory and enables larger diffusion transformers to run across multiple GPUs. It can also be
+combined with CFG-Parallel, where each PP pipeline carries one CFG branch.
+
+### Architecture
+
+vLLM-Omni provides `PipelineParallelMixin` to encapsulate the PP communication pattern for diffusion pipelines.
+
+| Method                            | Purpose                       | Automatic Behavior                                                              |
+|-----------------------------------|-------------------------------|---------------------------------------------------------------------------------|
+| `diffuse()`                       | Denoising loop boundary       | Wrapped by `PipelineParallelMixin` to flush pending async PP sends on exit      |
+| `predict_noise_maybe_with_cfg()`  | Predict noise with PP support | Runs partial forwards on non-last PP ranks, combines with CFG logic when needed |
+| `scheduler_step_maybe_with_cfg()` | Step scheduler with PP sync   | Runs scheduler on the last PP rank, returns updated latents to rank 0           |
+| `_sync_pp_send()`                 | Flush pending async sends     | Waits for outstanding `isend` handles before later collectives or decode        |
+
+`PipelineParallelMixin` is intentionally a pipeline-level abstraction. Your model-specific `predict_noise()` still
+defines how a local stage executes.
+
+### How It Works
+
+`predict_noise_maybe_with_cfg()` automatically switches between these modes:
+
+- **PP disabled** (`pipeline_parallel_size == 1`):
+    - Falls back to `CFGParallelMixin.predict_noise_maybe_with_cfg()`
+- **PP only** (`pipeline_parallel_size > 1`, `cfg_parallel_size == 1`):
+    - Rank 0 starts with the input latents
+    - Middle ranks receive `intermediate_tensors`, run their local layer range, and asynchronously send downstream
+    - The last rank returns the final noise prediction
+    - When CFG is enabled in this mode (sequential CFG), both the positive and negative branches run through the same
+      PP pipeline, doubling the communication volume per denoising step. Prefer PP + CFG-Parallel when
+      `cfg_parallel_size > 1` is available to avoid this overhead.
+- **PP + CFG-Parallel** (`pipeline_parallel_size > 1`, `cfg_parallel_size > 1`):
+    - Each PP pipeline carries one CFG branch
+    - The last PP rank all-gathers across the CFG group
+    - CFG combination happens on every last PP rank in the CFG group, matching the non-PP CFG-parallel behavior
+
+`scheduler_step_maybe_with_cfg()` keeps the denoising loop consistent:
+
+- **PP disabled**:
+    - Falls back to `scheduler_step_maybe_with_cfg()`
+- **PP enabled**:
+    - Only the last PP rank has `noise_pred` and runs the scheduler step
+    - The resulting latents are sent back to rank 0
+    - Rank 0 receives an `AsyncLatents` wrapper that resolves only when the tensor is next consumed
+
+This asynchronous design avoids unnecessary blocking between denoising steps.
+
+When a pipeline class is defined, `PipelineParallelMixin` wraps its `diffuse()` method and calls `_sync_pp_send()` in a
+`finally` block after `diffuse()` returns or raises. This keeps model code free of explicit PP cleanup while still
+ensuring that the final non-blocking PP send completes before decode or later collectives.
+
+---
+
+## Step-by-Step Implementation
+
+### Step 1: Inherit `PipelineParallelMixin` and `CFGParallelMixin`
+
+`PipelineParallelMixin` **requires** `CFGParallelMixin` and must appear before it in the class MRO. This is enforced at
+class definition time via `__init_subclass__`: defining a pipeline that inherits `PipelineParallelMixin` without
+`CFGParallelMixin`, or after `CFGParallelMixin`, raises a `TypeError` immediately on import.
+
+`PipelineParallelMixin` delegates noise prediction, CFG combination, and scheduler stepping to `CFGParallelMixin`, which
+supplies `predict_noise()`, `predict_noise_maybe_with_cfg()`, `scheduler_step_maybe_with_cfg()`, and
+`combine_cfg_noise()`.
+
+**Example:**
+
+```python
+from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
+from vllm_omni.diffusion.distributed.pipeline_parallel import PipelineParallelMixin
+import torch.nn as nn
+
+
+class YourPipeline(nn.Module, PipelineParallelMixin, CFGParallelMixin):
+    ...
+```
+
+The order matters: `PipelineParallelMixin` must be listed before `CFGParallelMixin` so calls to
+`predict_noise_maybe_with_cfg()` and `scheduler_step_maybe_with_cfg()` resolve to the PP-aware wrappers, while their
+`super()` calls delegate to the CFG implementation when PP is disabled or after the last PP stage.
+
+### Step 2: Make model forward and `predict_noise()` PP-aware
+
+The PP mixin injects `intermediate_tensors` into the normal `predict_noise()` call. Your model forward path must support
+two inputs:
+
+- Normal input from rank 0, typically passed as `hidden_states` or `x`
+- `intermediate_tensors` from upstream PP ranks
+
+The standard model forward pattern is:
+
+1. If `intermediate_tensors` is present, read the local hidden state from it.
+2. Run only this rank's layer slice.
+3. Return `IntermediateTensors(...)` on non-last PP ranks.
+4. Return the final model output on the last PP rank. `CFGParallelMixin.predict_noise()` already follows this contract
+   for common pipelines.
+
+**Minimal example:**
+
+```python
+from vllm.sequence import IntermediateTensors
+from vllm_omni.diffusion.distributed.parallel_state import get_pp_group
+
+
+def forward(self, hidden_states=None, intermediate_tensors=None, **kwargs):
+    if intermediate_tensors is not None:
+        hidden_states = intermediate_tensors["hidden_states"]
+
+    for i in range(self.start_layer, self.end_layer):
+        hidden_states = self.layers[i](hidden_states)
+
+    pp_group = get_pp_group()
+    if not pp_group.is_last_rank:
+        return IntermediateTensors({"hidden_states": hidden_states})
+    return (hidden_states,)
+```
+
+### Step 3: Partition the transformer layers
+
+The local module on each PP rank must expose only that rank's layer slice. In the reference tests and vLLM model
+implementations, this is typically done with vLLM utilities such as `make_layers(...)`, which fill missing ranges with
+`PPMissingLayer`. Besides partitioning the layers themselves, model authors should also wire
+`make_empty_intermediate_tensors_factory(...)` for intermediate tensor allocation and `is_pp_missing_parameter(...)` for
+PP-aware weight loading.
+
+To prepare a transformer for PP, implement the following pieces in order.
+
+#### 3.1 Split the transformer layers across PP ranks
+
+Each PP rank should own only its local layer range, typically exposed as `[start_layer, end_layer)`. In practice this is
+usually done with `make_layers(...)`, which constructs local layers and fills missing ranges with `PPMissingLayer`.
+
+The goal is:
+
+- every PP rank knows its `[start_layer, end_layer)` range
+- non-local layers do not execute on this rank
+- the forward can resume from incoming `intermediate_tensors`
+
+By default, layers are distributed evenly across PP ranks using `get_pp_indices()`. When the number of layers is not
+divisible by the PP size, remaining layers are assigned to middle partitions to balance compute and memory. You can
+override this with the `VLLM_PP_LAYER_PARTITION` environment variable to specify an exact per-rank layer count:
+
+```bash
+# Example: 40 layers across 4 PP ranks, assigning 8 / 12 / 12 / 8 layers
+export VLLM_PP_LAYER_PARTITION=8,12,12,8
+```
+
+The value must be a comma-separated list of integers whose length equals `pipeline_parallel_size` and whose sum equals
+the total number of transformer layers.
+
+#### 3.2 Expose `make_empty_intermediate_tensors`
+
+Your transformer module should expose `self.make_empty_intermediate_tensors`, typically created with
+`make_empty_intermediate_tensors_factory(...)`.
+
+This is important because PP ranks need a consistent way to allocate placeholder `IntermediateTensors` with the expected
+keys and hidden dimension.
+
+**Example:**
+
+```python
+from vllm.model_executor.models.utils import make_empty_intermediate_tensors_factory
+
+self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
+    ["hidden_states"],
+    inner_dim,
+)
+```
+
+For Wan2.2, the intermediate payload between PP stages is the token sequence stored under `"hidden_states"`, so the
+factory is created with that key and the transformer hidden size.
+
+#### 3.3 Return `IntermediateTensors` on non-last PP ranks
+
+Your model `forward()` or custom `predict_noise()` implementation should consume `intermediate_tensors` on non-first
+ranks and return `IntermediateTensors(...)` on non-last ranks.
+
+This lets each PP stage resume from the upstream hidden states and pass the local result to the next stage.
+
+#### 3.4 Skip non-local weights during `load_weights()`
+
+When a model is PP-partitioned, many parameters in the checkpoint belong to layers that do not exist on the current
+rank. `load_weights()` must therefore skip parameters for missing PP stages using `is_pp_missing_parameter(...)`.
+
+If this is not done, weight loading will either fail or incorrectly try to load tensors into `PPMissingLayer`
+placeholders.
+
+The Wan2.2 transformer is the best reference here: it uses `is_pp_missing_parameter(...)` before loading both remapped
+and fused parameters.
+
+If your model has multiple transformer variants, PP still works as long as each selected transformer obeys the same
+contract.
+
+### Step 4: Use the standard denoising contract
+
+vLLM-Omni diffusion pipelines already route denoising through `diffuse()`, `predict_noise_maybe_with_cfg()`, and
+`scheduler_step_maybe_with_cfg()`. `PipelineParallelMixin` overrides those standard helpers when
+`pipeline_parallel_size > 1`, so model integrations should not add separate PP-specific helper names or manual post-loop
+synchronization.
+
+In PP mode:
+
+- Non-last PP ranks return `None` from `predict_noise_maybe_with_cfg()` after sending `IntermediateTensors` downstream.
+- The last PP rank returns the final prediction, and with CFG-Parallel every last PP rank in the CFG group receives the
+  combined result.
+- Rank 0 receives `AsyncLatents` from `scheduler_step_maybe_with_cfg()`, which resolves only when the latents are
+  consumed.
+- Pending non-blocking sends are flushed automatically when `diffuse()` exits.
+
+---
+
+## Testing
+
+Use an offline inference script with `pipeline_parallel_size > 1`.
+
+```bash
+python examples/offline_inference/text_to_video/text_to_video.py \
+--model=Wan-AI/Wan2.2-TI2V-5B-Diffusers \
+--width=1280 \
+--height=704 \
+--guidance-scale=5.0 \
+--prompt="Two anthropomorphic cats in comfy boxing gear and bright gloves fight intensely on a spotlighted stage" \
+--output=t2v_5B_pp2.mp4 \
+--pipeline-parallel-size=2
+```
+
+For PP + CFG-Parallel together:
+
+```bash
+python examples/offline_inference/text_to_video/text_to_video.py \
+--model=Wan-AI/Wan2.2-TI2V-5B-Diffusers \
+--width=1280 \
+--height=704 \
+--guidance-scale=5.0 \
+--prompt="Two anthropomorphic cats in comfy boxing gear and bright gloves fight intensely on a spotlighted stage" \
+--output=t2v_5B_pp2_cfg2.mp4 \
+--pipeline-parallel-size=2 \
+--cfg-parallel-size=2
+```
+
+**Verify:**
+
+1. The run completes without hangs at the PP stage boundary
+2. Output quality matches the non-PP baseline within normal numerical variance
+3. Peak memory per GPU drops compared with the single-rank model
+4. There are no outstanding communication errors before decode
+
+---
+
+## Troubleshooting
+
+### Issue: `TypeError` on import — `CFGParallelMixin` missing
+
+**Symptoms:** Importing a pipeline that inherits `PipelineParallelMixin` raises:
+
+```
+TypeError: YourPipeline inherits PipelineParallelMixin but not CFGParallelMixin.
+```
+
+or:
+
+```
+TypeError: YourPipeline must inherit PipelineParallelMixin before CFGParallelMixin ...
+```
+
+**Cause:** `PipelineParallelMixin` enforces via `__init_subclass__` that the subclass also inherits `CFGParallelMixin`
+and lists `PipelineParallelMixin` first in the MRO.
+
+**Solution:** Add `CFGParallelMixin` to the pipeline's base classes after `PipelineParallelMixin`:
+
+```python
+from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
+from vllm_omni.diffusion.distributed.pipeline_parallel import PipelineParallelMixin
+
+
+class YourPipeline(nn.Module, PipelineParallelMixin, CFGParallelMixin):
+    ...
+```
+
+### Issue: Non-last PP ranks crash when calling `predict_noise`
+
+**Symptoms:** Shape or missing-input errors on ranks other than the first or last PP rank.
+
+**Cause:** The model forward path assumes direct input tensors and ignores `intermediate_tensors`.
+
+**Solution:** Update the transformer `forward()` or custom `predict_noise()` path to load hidden states from
+`intermediate_tensors` when present.
+
+### Issue: PP outputs differ from single-GPU baseline
+
+**Symptoms:** The PP run finishes but produces numerically inconsistent outputs.
+
+**Causes & Solutions:**
+
+- **Local layer partitioning is wrong**
+    - Verify each rank runs only its `[start_layer, end_layer)` slice
+- **Non-last ranks return a plain tensor instead of `IntermediateTensors`**
+    - Return `IntermediateTensors({...})` until the last PP stage
+- **CFG branch wiring is incorrect**
+    - When CFG is enabled, confirm positive and negative kwargs are passed exactly as in the non-PP path
+
+---
+
+## Reference Implementations
+
+Complete examples in the codebase:
+
+| Component               | Path                                                       | Notes                                                                                                                   |
+|-------------------------|------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------|
+| `PipelineParallelMixin` | `vllm_omni/diffusion/distributed/pipeline_parallel.py`     | Core PP communication and scheduler helpers                                                                             |
+| `CFGParallelMixin`      | `vllm_omni/diffusion/distributed/cfg_parallel.py`          | Default `predict_noise()` tuple normalization and CFG helper fallback                                                   |
+| Wan2.2 transformer      | `vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py`  | Reference for layer partitioning, `IntermediateTensors`, `make_empty_intermediate_tensors`, and PP-aware weight loading |
+| Wan2.2 T2V pipeline     | `vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py`     | Reference PP + CFG integration for text-to-video                                                                        |
+| Wan2.2 I2V pipeline     | `vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2_i2v.py` | Reference PP + CFG integration for image-to-video                                                                       |
+| PP tests                | `tests/diffusion/distributed/test_pipeline_parallel.py`    | Baseline parity and async communication tests                                                                           |

docs/getting_started/installation/gpu.md

@@ -34,7 +34,7 @@ vLLM-Omni is a Python library that supports the following GPU variants. The libr
 
 ### Pre-built wheels
 
-Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, and 0.18.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).
+Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, 0.18.0, 0.20.0, and 0.21.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).
 
 === "NVIDIA CUDA"
 

docs/getting_started/installation/gpu/cuda.inc.md

@@ -20,7 +20,7 @@ Therefore, it is recommended to install vLLM and vLLM-Omni with a **fresh new**
 
 vLLM-Omni is built based on vLLM. Please install it with command below.
 ```bash
-uv pip install vllm --torch-backend=auto
+uv pip install vllm==0.21.0 --torch-backend=auto
 ```
 
 #### Installation of vLLM-Omni
@@ -39,13 +39,13 @@ uv pip install 'vllm-omni[demo]'
 # --8<-- [start:build-wheel-from-source]
 
 #### Installation of vLLM
-If you do not need to modify source code of vLLM, you can directly install the stable 0.20.0 release version of the library
+If you do not need to modify source code of vLLM, you can directly install the stable 0.21.0 release version of the library
 
 ```bash
-uv pip install vllm==0.20.0 --torch-backend=auto
+uv pip install vllm==0.21.0 --torch-backend=auto
 ```
 
-The 0.20.0 release of vLLM ships CUDA 13.0-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.
+The 0.21.0 release of vLLM ships CUDA 13.0-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.
 
 #### Installation of vLLM-Omni
 Since vllm-omni is rapidly evolving, it's recommended to install it from source
@@ -66,12 +66,12 @@ If you want to check, modify or debug with source code of vLLM, install the libr
 ```bash
 git clone https://github.com/vllm-project/vllm.git
 cd vllm
-git checkout v0.20.0
+git checkout v0.21.0
 ```
 Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set `VLLM_PRECOMPILED_WHEEL_LOCATION` as your local absolute path of whl file.
 ```bash
-#For CUDA 13.0 (the default for v0.20.0; the wheel filename has no `+cu130` suffix)
-export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.20.0/vllm-0.20.0-cp38-abi3-manylinux_2_35_x86_64.whl
+#For CUDA 13.0 (the default for v0.21.0; the wheel filename has no `+cu130` suffix)
+export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.21.0/vllm-0.21.0-cp38-abi3-manylinux_2_35_x86_64.whl
 ```
 Install vllm with command below (If you have no existing PyTorch).
 ```bash
@@ -102,7 +102,7 @@ docker run --runtime nvidia --gpus 2 \
     --env "HF_TOKEN=$HF_TOKEN" \
     -p 8091:8091 \
     --ipc=host \
-    vllm/vllm-omni:v0.18.0 \
+    vllm/vllm-omni:v0.21.0 \
     --model Qwen/Qwen3-Omni-30B-A3B-Instruct --port 8091
 ```
 

docs/getting_started/installation/gpu/rocm.inc.md

@@ -13,7 +13,7 @@ vLLM-Omni current recommends the steps in under setup through Docker Images.
 
 vLLM-Omni is built based on vLLM. Please install it with command below.
 ```bash
-uv pip install vllm==0.20.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm721
+uv pip install vllm==0.21.0+rocm722 --extra-index-url https://wheels.vllm.ai/rocm/0.21.0/rocm722
 ```
 
 #### Installation of vLLM-Omni
@@ -34,13 +34,13 @@ uv pip install onnxruntime-rocm
 # --8<-- [start:build-wheel-from-source]
 
 #### Installation of vLLM
-If you do not need to modify source code of vLLM, you can directly install the stable 0.20.0 release version of the library
+If you do not need to modify source code of vLLM, you can directly install the stable 0.21.0 release version of the library
 
 ```bash
-uv pip install vllm==0.20.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm721
+uv pip install vllm==0.21.0+rocm722 --extra-index-url https://wheels.vllm.ai/rocm/0.21.0/rocm722
 ```
 
-The pre-built 0.20.0 vLLM wheel targets ROCm 7.2.1. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
+The pre-built 0.21.0 vLLM wheel targets ROCm 7.2.2. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
 
 #### Installation of vLLM-Omni
 Since vllm-omni is rapidly evolving, it's recommended to install it from source
@@ -58,7 +58,7 @@ If you want to check, modify or debug with source code of vLLM, install the libr
 ```bash
 git clone https://github.com/vllm-project/vllm.git
 cd vllm
-git checkout v0.20.0
+git checkout v0.21.0
 python3 -m pip install -r requirements/rocm.txt
 python3 setup.py develop
 ```
@@ -130,7 +130,7 @@ docker run --rm \
   -v ~/.cache/huggingface:/root/.cache/huggingface \
   --env "HF_TOKEN=$HF_TOKEN" \
   -p 8091:8091 \
-  vllm/vllm-omni-rocm:v0.20.0 \
+  vllm/vllm-omni-rocm:v0.21.0 \
   --model Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
 ```
 
@@ -149,7 +149,7 @@ docker run --rm -it \
   -v ~/.cache/huggingface:/root/.cache/huggingface \
   --env "HF_TOKEN=$HF_TOKEN" \
   --entrypoint bash \
-  vllm/vllm-omni-rocm:v0.20.0
+  vllm/vllm-omni-rocm:v0.21.0
 ```
 
 # --8<-- [end:pre-built-images]

docs/getting_started/quickstart.md

@@ -19,10 +19,10 @@ uv venv --python 3.12 --seed
 source .venv/bin/activate
 
 # On CUDA
-uv pip install vllm==0.20.0 --torch-backend=auto
+uv pip install vllm==0.21.0 --torch-backend=auto
 
 # On ROCm
-uv pip install vllm==0.20.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm721
+uv pip install vllm==0.21.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.21.0/rocm721
 
 git clone https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni
@@ -35,7 +35,7 @@ For additional installation methods — please see the [installation guide](inst
 !!! note
     It is important to install the same major & minor version of vLLM and vLLM Omni, otherwise things may not work as expected. If the versions are misaligned, you will see a warning when you import vLLM Omni.
 
-    If you are seeing strange behavior with the `vllm` command not handling the `--omni` flag correctly, you most likely have a version mismatch with vLLM < `0.20.0` and vLLM Omni `0.20.0`, as vLLM Omni no longer hijacks the vLLM entrypoint. Updating vLLM should resolve this issue.
+    If you are seeing strange behavior with the `vllm` command not handling the `--omni` flag correctly, you most likely have a version mismatch with vLLM < `0.21.0` and vLLM Omni `0.21.0`, as vLLM Omni no longer hijacks the vLLM entrypoint. Updating vLLM should resolve this issue.
 
 ## Offline Inference
 

docs/serving/chat_completions_api.md

@@ -1,4 +1,4 @@
-# Diffusion Chat Completions API
+# Chat Completions API
 
 vLLM-Omni supports generating and editing images via the `/v1/chat/completions`
 endpoint using diffusion models. This page explains how to pass generation

docs/serving/videos_api.md

@@ -0,0 +1,152 @@
+# Videos API
+
+vLLM-Omni provides an OpenAI-compatible video generation API for diffusion
+video models. The API supports asynchronous video jobs through `/v1/videos` and
+a synchronous benchmark-oriented endpoint through `/v1/videos/sync`.
+
+Each server instance runs a single model specified at startup with
+`vllm serve <model> --omni`.
+
+## Quick Start
+
+### Start the Server
+
+```bash
+vllm serve Wan-AI/Wan2.2-T2V-A14B-Diffusers --omni --port 8091
+```
+
+### Create a Video Job
+
+```bash
+create_response=$(curl -s http://localhost:8091/v1/videos \
+  -F "prompt=A cinematic tracking shot of a mountain lake at sunrise" \
+  -F "width=1280" \
+  -F "height=720" \
+  -F "num_frames=80" \
+  -F "fps=16" \
+  -F "num_inference_steps=40")
+
+video_id=$(echo "${create_response}" | jq -r '.id')
+```
+
+### Poll and Download
+
+```bash
+curl -s "http://localhost:8091/v1/videos/${video_id}" | jq .
+curl -L "http://localhost:8091/v1/videos/${video_id}/content" -o output.mp4
+```
+
+## API Reference
+
+### Endpoints
+
+| Endpoint | Method | Description |
+|----------|--------|-------------|
+| `/v1/videos` | `POST` | Create an asynchronous video generation job |
+| `/v1/videos/sync` | `POST` | Generate a video synchronously and return raw video bytes |
+| `/v1/videos/{video_id}` | `GET` | Retrieve job status and metadata |
+| `/v1/videos` | `GET` | List stored video jobs |
+| `/v1/videos/{video_id}/content` | `GET` | Download generated video content |
+| `/v1/videos/{video_id}` | `DELETE` | Delete a video job and stored output |
+
+### Request Parameters
+
+`POST /v1/videos` and `POST /v1/videos/sync` accept `multipart/form-data`.
+
+#### OpenAI-style fields
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `prompt` | string | **required** | Text prompt for video generation |
+| `model` | string | server's model | Optional model name |
+| `seconds` | string | null | Requested clip duration in seconds |
+| `size` | string | null | Requested output size in `WIDTHxHEIGHT` format |
+| `user` | string | null | Optional user identifier |
+
+#### vLLM-Omni extension fields
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `input_reference` | file | null | Uploaded reference image for image-to-video requests |
+| `image_reference` | string | null | JSON-encoded reference image payload; do not combine with `input_reference` |
+| `width` | integer | model default | Output video width |
+| `height` | integer | model default | Output video height |
+| `num_frames` | integer | model default | Number of generated frames |
+| `fps` | integer | model default | Output frames per second |
+| `num_inference_steps` | integer | model default | Number of diffusion steps |
+| `guidance_scale` | number | null | CFG guidance scale for the low-noise stage |
+| `guidance_scale_2` | number | null | CFG guidance scale for the high-noise stage |
+| `boundary_ratio` | number | null | Boundary split ratio for multi-stage denoising |
+| `flow_shift` | number | null | Scheduler flow-shift value |
+| `true_cfg_scale` | number | null | True CFG scale when supported by the model |
+| `seed` | integer | null | Random seed for reproducibility |
+| `negative_prompt` | string | null | Text describing what to avoid in the generated video |
+| `enable_frame_interpolation` | boolean | null | Enable post-generation frame interpolation |
+| `frame_interpolation_exp` | integer | null | Interpolation exponent; `1=2x`, `2=4x`, and so on |
+| `frame_interpolation_scale` | number | null | RIFE inference scale |
+| `frame_interpolation_model_path` | string | null | Local path or Hugging Face repo for the interpolation model |
+| `lora` | string | null | JSON-encoded LoRA configuration object |
+| `extra_params` | string | null | JSON-encoded object for additional model-specific parameters |
+
+### Create Response
+
+`POST /v1/videos` returns a job record:
+
+```json
+{
+  "id": "video-123",
+  "status": "queued",
+  "created_at": 1701234567
+}
+```
+
+The final content is available from `/v1/videos/{video_id}/content` after the
+job status becomes `completed`.
+
+### Synchronous Response
+
+`POST /v1/videos/sync` blocks until generation finishes and returns raw video
+bytes. It is useful for benchmarks and simple scripts that do not need job
+storage or polling.
+
+## Examples
+
+### Image-to-Video
+
+```bash
+curl -s http://localhost:8091/v1/videos \
+  -F "prompt=animate this image with subtle camera movement" \
+  -F "input_reference=@input.png" \
+  -F "width=1280" \
+  -F "height=720" \
+  -F "num_frames=80" \
+  -F "fps=16"
+```
+
+### Synchronous Generation
+
+```bash
+curl -X POST http://localhost:8091/v1/videos/sync \
+  -F "prompt=A small robot walking through a neon city" \
+  -F "width=854" \
+  -F "height=480" \
+  -F "num_frames=80" \
+  -F "fps=16" \
+  -o output.mp4
+```
+
+## Storage
+
+Set `VLLM_OMNI_STORAGE_PATH` to control where asynchronous video outputs are
+stored:
+
+```bash
+export VLLM_OMNI_STORAGE_PATH=/var/tmp/vllm-omni-videos
+```
+
+## Model-Specific Examples
+
+For complete text-to-video and image-to-video walkthroughs, see:
+
+- [Text-to-Video](../user_guide/examples/online_serving/text_to_video.md)
+- [Image-to-Video](../user_guide/examples/online_serving/image_to_video.md)

docs/user_guide/diffusion/frame_interpolation.md

@@ -28,7 +28,6 @@ Frame interpolation is currently supported for:
 
 - `WanPipeline` (Wan2.2 text-to-video)
 - `WanImageToVideoPipeline`
-- `Wan22TI2VPipeline`
 
 ## Request Parameters
 

docs/user_guide/diffusion/parallelism/overview.md

@@ -4,13 +4,14 @@ This guide covers the parallelism methods in vLLM-Omni for speeding up diffusion
 
 ## Supported Methods
 
-| Method | Description |
-|--------|-------------|
-| **[Tensor Parallelism](tensor_parallel.md)** | Shards DiT weights across GPUs to reduce per-GPU memory |
-| **[Sequence Parallelism](sequence_parallel.md)** | Splits sequence dimension across GPUs (Ulysses-SP, Ring-Attention, or hybrid) for high-resolution images and videos |
-| **[CFG-Parallel](cfg_parallel.md)** | Runs CFG positive/negative branches on separate GPUs for ~1.8x speedup on guided generation |
-| **[VAE Patch Parallelism](vae_patch_parallel.md)** | Distributes VAE decode spatially across GPUs to reduce peak VAE memory |
-| **[HSDP](hsdp.md)** | Shards full model weights via PyTorch FSDP2 to enable large-model inference on memory-constrained GPUs |
-| **[Expert Parallelism](expert_parallel.md)** | Shards MoE expert blocks across GPUs for MoE models (e.g. HunyuanImage3.0) |
+| Method                                             | Description                                                                                                         |
+|----------------------------------------------------|---------------------------------------------------------------------------------------------------------------------|
+| **[Tensor Parallelism](tensor_parallel.md)**       | Shards DiT weights across GPUs to reduce per-GPU memory                                                             |
+| **[Sequence Parallelism](sequence_parallel.md)**   | Splits sequence dimension across GPUs (Ulysses-SP, Ring-Attention, or hybrid) for high-resolution images and videos |
+| **[CFG-Parallel](cfg_parallel.md)**                | Runs CFG positive/negative branches on separate GPUs for ~1.8x speedup on guided generation                         |
+| **[Pipeline Parallelism](pipeline_parallel.md)**   | Splits the denoising transformer block-wise across sequential GPU stages to reduce per-GPU model memory             |
+| **[VAE Patch Parallelism](vae_patch_parallel.md)** | Distributes VAE decode spatially across GPUs to reduce peak VAE memory                                              |
+| **[HSDP](hsdp.md)**                                | Shards full model weights via PyTorch FSDP2 to enable large-model inference on memory-constrained GPUs              |
+| **[Expert Parallelism](expert_parallel.md)**       | Shards MoE expert blocks across GPUs for MoE models (e.g. HunyuanImage3.0)                                          |
 
 See [Supported Models](../../diffusion_features.md#supported-models) for per-model compatibility.

docs/user_guide/diffusion/parallelism/pipeline_parallel.md

@@ -0,0 +1,205 @@
+# Pipeline Parallelism Guide
+
+## Table of Content
+
+- [Overview](#overview)
+- [Quick Start](#quick-start)
+- [Example Script](#example-script)
+- [Configuration Parameters](#configuration-parameters)
+- [Best Practices](#best-practices)
+- [Troubleshooting](#troubleshooting)
+- [Summary](#summary)
+
+---
+
+## Overview
+
+Pipeline Parallelism splits the denoising transformer block-wise into sequential stages across GPUs. Each rank owns only
+part of the transformer, which reduces per-GPU model memory and enables larger diffusion models to run across multiple
+devices.
+
+It can also be combined with other distributed methods such as CFG-Parallel, Tensor Parallelism, and Sequence
+Parallelism.
+
+See supported models list in [Supported Models](../../diffusion_features.md#supported-models).
+
+---
+
+## Quick Start
+
+### Basic Usage
+
+Simplest working example:
+
+```python
+from vllm_omni import Omni
+from vllm_omni.diffusion.data import DiffusionParallelConfig
+from vllm_omni.inputs.data import OmniDiffusionSamplingParams
+
+omni = Omni(
+    model="Wan-AI/Wan2.2-TI2V-5B-Diffusers",
+    parallel_config=DiffusionParallelConfig(
+        pipeline_parallel_size=2,
+    ),
+)
+
+outputs = omni.generate(
+    {"prompt": "A cinematic drone shot over snowy mountains"},
+    OmniDiffusionSamplingParams(
+        num_inference_steps=40,
+        num_frames=81,
+        height=704,
+        width=1280,
+    ),
+)
+```
+
+---
+
+## Example Script
+
+### Offline Inference
+
+Use python scripts under:
+
+- `examples/offline_inference/text_to_video/text_to_video.py`
+- `examples/offline_inference/image_to_video/image_to_video.py`
+
+Text-to-video example:
+
+```bash
+python examples/offline_inference/text_to_video/text_to_video.py \
+--model=Wan-AI/Wan2.2-TI2V-5B-Diffusers \
+--width=1280 \
+--height=704 \
+--guidance-scale=5.0 \
+--prompt="Two anthropomorphic cats in comfy boxing gear and bright gloves fight intensely on a spotlighted stage" \
+--output=t2v_5B_pp2.mp4 \
+--pipeline-parallel-size=2
+```
+
+Pipeline Parallelism can also be combined with CFG-Parallel:
+
+```bash
+python examples/offline_inference/text_to_video/text_to_video.py \
+--model=Wan-AI/Wan2.2-TI2V-5B-Diffusers \
+--width=1280 \
+--height=704 \
+--guidance-scale=5.0 \
+--prompt="Two anthropomorphic cats in comfy boxing gear and bright gloves fight intensely on a spotlighted stage" \
+--output=t2v_5B_pp2_cfg2.mp4 \
+--pipeline-parallel-size=2 \
+--cfg-parallel-size=2
+```
+
+### Online Serving
+
+Enable Pipeline Parallelism in online serving:
+
+```bash
+# Default PP configuration
+vllm serve Wan-AI/Wan2.2-TI2V-5B-Diffusers --omni --port 8091 --pipeline-parallel-size 2
+
+# PP + CFG-Parallel
+vllm serve Wan-AI/Wan2.2-TI2V-5B-Diffusers --omni --port 8091 \
+  --pipeline-parallel-size 2 \
+  --cfg-parallel-size 2
+```
+
+---
+
+## Configuration Parameters
+
+In `DiffusionParallelConfig`
+
+| Parameter                | Type | Default | Description                                                                                                      |
+|--------------------------|------|---------|------------------------------------------------------------------------------------------------------------------|
+| `pipeline_parallel_size` | int  | 1       | Number of pipeline-parallel stages. Set to a value greater than 1 to split the denoising transformer across GPUs |
+
+> [!NOTE]
+> Total GPU count is the product of all enabled distributed dimensions, for example
+`pipeline_parallel_size * cfg_parallel_size * tensor_parallel_size * ulysses_degree * ring_degree`.
+
+### Manual Layer Partitioning
+
+By default, transformer layers are distributed evenly across PP ranks. You can override this with the
+`VLLM_PP_LAYER_PARTITION` environment variable to assign a specific number of layers to each rank:
+
+```bash
+# Example: 40 layers across 4 PP ranks, assigning 8 / 12 / 12 / 8 layers
+export VLLM_PP_LAYER_PARTITION=8,12,12,8
+```
+
+The value must be a comma-separated list of integers whose length equals `pipeline_parallel_size` and whose sum equals
+the total number of transformer layers. This is useful when you want to balance memory or compute asymmetrically across
+ranks.
+
+
+---
+
+## Best Practices
+
+### When to Use
+
+**Good for:**
+
+- Large diffusion transformers that do not fit comfortably on one GPU
+- Multi-GPU setups where reducing per-GPU model memory is more important than minimizing communication
+- Combining with CFG-Parallel or other distributed methods on supported models
+
+**Not for:**
+
+- Single GPU setups
+- Models that do not support Pipeline Parallelism
+  (check [supported models](../../diffusion_features.md#supported-models))
+- Very small models where inter-stage communication overhead may outweigh the benefit
+
+### Expected Behavior
+
+Pipeline Parallelism primarily reduces per-GPU memory usage by splitting the transformer into stage-local blocks.
+Depending on the model, topology, and resolution, it may also help execution fit into available hardware, but it is not
+primarily a latency optimization.
+
+---
+
+## Troubleshooting
+
+### Common Issue 1: No benefit from Pipeline Parallelism
+
+**Symptoms**: PP is enabled, but latency does not improve or becomes slightly worse.
+
+**Solutions**:
+
+1. **Check your goal:**
+
+```python
+# PP is mainly for memory scaling, not guaranteed latency speedup
+parallel_config = DiffusionParallelConfig(pipeline_parallel_size=2)
+```
+
+2. **Check model support:**
+    - Verify your model in [supported models](../../diffusion_features.md#supported-models)
+    - PP is currently validated only on selected pipelines
+
+3. **Combine with other methods when appropriate:**
+    - PP can be combined with CFG-Parallel, Tensor Parallelism, or Sequence Parallelism on supported models
+
+### Common Issue 2: PP pipeline fails at import
+
+**Symptoms**: Importing a custom pipeline raises a `TypeError` about `CFGParallelMixin` or mixin order.
+
+**Solutions**:
+
+1. Inherit both mixins.
+2. Put `PipelineParallelMixin` before `CFGParallelMixin` in the class base list.
+3. Use `class YourPipeline(nn.Module, PipelineParallelMixin, CFGParallelMixin): ...` as the reference pattern.
+
+---
+
+## Summary
+
+1. ✅ **Enable Pipeline Parallelism** - Set `pipeline_parallel_size > 1` in `DiffusionParallelConfig`
+2. ✅ **Use Supported Models** - Verify your model supports PP in
+   [supported models](../../diffusion_features.md#supported-models)
+3. ✅ **Combine When Needed** - PP can be combined with CFG-Parallel and other distributed methods on supported pipelines
+4. ✅ **Scale for Memory** - Use PP primarily to reduce per-GPU model memory and fit larger transformers

docs/user_guide/diffusion_features.md

@@ -35,14 +35,15 @@ Cache methods trade minimal quality for significant speedup. Quality loss is typ
 
 Parallelism methods distribute computation across GPUs without quality loss (mathematically equivalent to single-GPU).
 
-| Method | Description | Best For |
-|--------|-------------|----------|
-| **[Ulysses-SP](diffusion/parallelism/sequence_parallel.md)** | Sequence parallelism via all-to-all communication | High-resolution images (>1536px) or long videos with 2-8 GPUs |
-| **[Ring-Attention](diffusion/parallelism/sequence_parallel.md)** | Sequence parallelism via ring-based communication | Videos, very long sequences, memory-constrained, with 2-8 GPUs |
-| **[CFG-Parallel](diffusion/parallelism/cfg_parallel.md)** | Splits CFG positive/negative branches across devices | Image editing with CFG guidance (true_cfg_scale > 1) on 2 GPUs |
-| **[Tensor Parallelism](diffusion/parallelism/tensor_parallel.md)** | Shards model weights across devices | Large models that don't fit in single GPU, with 2+ GPUs |
-| **[HSDP](diffusion/parallelism/hsdp.md)** | Weight sharding via FSDP2, redistributed on-demand at runtime | Very large models (14B+) on limited VRAM, combinable with SP |
-| **[Expert Parallelism](diffusion/parallelism/expert_parallel.md)** | Shards MoE expert MLP blocks across devices | MoE diffusion models (e.g., HunyuanImage3.0) |
+| Method                                                                 | Description                                                              | Best For                                                          |
+|------------------------------------------------------------------------|--------------------------------------------------------------------------|-------------------------------------------------------------------|
+| **[Ulysses-SP](diffusion/parallelism/sequence_parallel.md)**           | Sequence parallelism via all-to-all communication                        | High-resolution images (>1536px) or long videos with 2-8 GPUs     |
+| **[Ring-Attention](diffusion/parallelism/sequence_parallel.md)**       | Sequence parallelism via ring-based communication                        | Videos, very long sequences, memory-constrained, with 2-8 GPUs    |
+| **[CFG-Parallel](diffusion/parallelism/cfg_parallel.md)**              | Splits CFG positive/negative branches across devices                     | Image editing with CFG guidance (true_cfg_scale > 1) on 2 GPUs    |
+| **[Tensor Parallelism](diffusion/parallelism/tensor_parallel.md)**     | Shards model weights across devices                                      | Large models that don't fit in single GPU, with 2+ GPUs           |
+| **[Pipeline Parallelism](diffusion/parallelism/pipeline_parallel.md)** | Splits the denoising transformer block-wise across sequential GPU stages | Large diffusion transformers that need lower per-GPU model memory |
+| **[HSDP](diffusion/parallelism/hsdp.md)**                              | Weight sharding via FSDP2, redistributed on-demand at runtime            | Very large models (14B+) on limited VRAM, combinable with SP      |
+| **[Expert Parallelism](diffusion/parallelism/expert_parallel.md)**     | Shards MoE expert MLP blocks across devices                              | MoE diffusion models (e.g., HunyuanImage3.0)                      |
 
 #### Startup Optimization
 
@@ -105,32 +106,32 @@ The following tables show which models support each feature:
 
 ### ImageGen
 
-| Model | ⚡TeaCache | ⚡Cache-DiT | 🔀SP (Ulysses & Ring) | 🔀CFG-Parallel | 🔀Tensor-Parallel | 🔀HSDP | 💾CPU Offload (Layerwise) | 💾VAE-Patch-Parallel | 💾Quantization | 🔄Step Execution |
-|-------|:----------:|:-----------:|:---------------------:|:--------------:|:-----------------:|:------:|:------------------------:|:--------------------:|:--------------:|:----------------:|
-| **Bagel** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅  | ❌ | ❌ | ❌ |
-| **FLUX.1-dev** | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ |
-| **FLUX.1-schnell** | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ |
-| **FLUX.2-klein** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ |
-| **FLUX.1-Kontext-dev** | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ |
-| **FLUX.2-dev** | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ |
-| **GLM-Image** | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ |
-| **HunyuanImage3** | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ |
-| **LongCat-Image** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ |
-| **LongCat-Image-Edit** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ |
-| **MagiHuman** | ❌ | ❌ | ❌ | ❓ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ |
-| **MammothModa2(T2I)** | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
-| **Nextstep_1(T2I)** | ❓ | ❓ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ |
-| **OmniGen2** | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
-| **Ovis-Image** | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
-| **Qwen-Image** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ✅ | ✅ |
-| **Qwen-Image-2512** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ✅ | ✅ |
-| **Qwen-Image-Edit** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ |
-| **Qwen-Image-Edit-2509** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ✅ | ❌ | ❌ |
-| **Qwen-Image-Layered** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ |
-| **SenseNova-U1** | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ |
-| **Stable-Diffusion3.5** | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ (decode) | ❌ | ❌ |
-| **Z-Image** | ✅ | ✅ | ✅ | ❓ | ✅ (TP=2 only) | ✅ | ❌ | ✅ (decode) | ✅ | ❌ |
-| **ERNIE-Image** | ❌ | ✅ | ✅ | ❓ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
+| Model                    | ⚡TeaCache | ⚡Cache-DiT | 🔀SP (Ulysses & Ring) | 🔀CFG-Parallel | 🔀Tensor-Parallel | 🔀Pipeline-Parallel | 🔀HSDP | 💾CPU Offload (Layerwise) | 💾VAE-Patch-Parallel | 💾Quantization | 🔄Step Execution |
+|--------------------------|:---------:|:----------:|:---------------------:|:--------------:|:-----------------:|:-------------------:|:------:|:-------------------------:|:--------------------:|:--------------:|:----------------:|
+| **Bagel**                |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |          ❌           |       ❌        |        ❌         |
+| **FLUX.1-dev**           |     ✅     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ✅    |             ❌             |          ❌           |       ✅        |        ❌         |
+| **FLUX.1-schnell**       |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ✅    |             ❌             |          ❌           |       ✅        |        ❌         |
+| **FLUX.2-klein**         |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ❌             |          ❌           |       ✅        |        ❌         |
+| **FLUX.1-Kontext-dev**   |     ❌     |     ❌      |           ❌           |       ❌        |         ✅         |          ❌          |   ✅    |             ❌             |          ❌           |       ❌        |        ❌         |
+| **FLUX.2-dev**           |     ✅     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ✅    |             ❌             |          ❌           |       ❌        |        ❌         |
+| **GLM-Image**            |     ❌     |     ❌      |           ❌           |       ✅        |         ✅         |          ❌          |   ✅    |             ❌             |          ❌           |       ❌        |        ❌         |
+| **HunyuanImage3**        |     ❌     |     ✅      |           ❌           |       ❌        |         ✅         |          ❌          |   ❌    |             ❌             |          ❌           |       ✅        |        ❌         |
+| **LongCat-Image**        |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
+| **LongCat-Image-Edit**   |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
+| **MagiHuman**            |     ❌     |     ❌      |           ❌           |       ❓        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
+| **MammothModa2(T2I)**    |     ❌     |     ❌      |           ❌           |       ❌        |         ❌         |          ❌          |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
+| **Nextstep_1(T2I)**      |     ❓     |     ❓      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
+| **OmniGen2**             |     ❌     |     ✅      |           ❌           |       ❌        |         ✅         |          ❌          |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
+| **Ovis-Image**           |     ❌     |     ✅      |           ❌           |       ✅        |         ❌         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
+| **Qwen-Image**           |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ✅        |        ✅         |
+| **Qwen-Image-2512**      |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ✅        |        ✅         |
+| **Qwen-Image-Edit**      |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
+| **Qwen-Image-Edit-2509** |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |        ✅ (decode)         |          ✅           |       ❌        |        ❌         |
+| **Qwen-Image-Layered**   |     ✅     |     ✅      |           ✅           |       ✅        |         ✅         |          ❌          |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
+| **SenseNova-U1**         |     ❌     |     ❌      |           ❌           |       ❌        |         ✅         |          ❌          |   ❌    |             ✅             |          ❌           |       ❌        |        ❌         |
+| **Stable-Diffusion3.5**  |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |          ❌          |   ❌    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
+| **Z-Image**              |     ✅     |     ✅      |           ✅           |       ❓        |   ✅ (TP=2 only)   |          ❌          |   ✅    |             ❌             |      ✅ (decode)      |       ✅        |        ❌         |
+| **ERNIE-Image**          |     ❌     |     ✅      |           ✅           |       ❓        |         ✅         |          ❌          |   ✅    |             ✅             |          ❌           |       ❌        |        ❌         |
 
 > Notes:
 > 1. Nextstep_1(T2I) does not support cache acceleration methods such as TeaCache or Cache-DiT.
@@ -138,15 +139,15 @@ The following tables show which models support each feature:
 
 ### VideoGen
 
-| Model | ⚡TeaCache | ⚡Cache-DiT | 🔀SP (Ulysses & Ring) | 🔀CFG-Parallel | 🔀Tensor-Parallel | 🔀HSDP | 💾CPU Offload (Layerwise) | 💾VAE-Patch-Parallel | 💾Quantization | 🔄Step Execution |
-|-------|:----------:|:-----------:|:---------------------:|:--------------:|:-----------------:|:------:|:------------------------:|:--------------------:|:--------------:|:----------------:|
-| **Wan2.2** | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (encode/decode) | ❌ | ❌ |
-| **Wan2.1-VACE** | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ |
-| **LTX-2** | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
-| **LTX-2.3** | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
-| **Helios** | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
-| **HunyuanVideo-1.5 T2V I2V** | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ✅ | ❌ |
-| **DreamID-Omni** | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
+| Model                        | ⚡TeaCache | ⚡Cache-DiT | 🔀SP (Ulysses & Ring) | 🔀CFG-Parallel | 🔀Tensor-Parallel | Pipeline-Parallel | 🔀HSDP | 💾CPU Offload (Layerwise) | 💾VAE-Patch-Parallel | 💾Quantization | 🔄Step Execution |
+|------------------------------|:---------:|:----------:|:---------------------:|:--------------:|:-----------------:|:-----------------:|:------:|:-------------------------:|:--------------------:|:--------------:|:----------------:|
+| **Wan2.2**                   |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ✅         |   ✅    |             ✅             |  ✅ (encode/decode)   |       ❌        |        ❌         |
+| **Wan2.1-VACE**              |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |      ✅ (decode)      |       ❌        |        ❌         |
+| **LTX-2**                    |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |          ❌           |       ❌        |        ❌         |
+| **LTX-2.3**                  |     ❌     |     ✅      |           ✅           |       ❌        |         ✅         |         ❌         |   ❌    |             ❌             |          ❌           |       ❌        |        ❌         |
+| **Helios**                   |     ❌     |     ✅      |           ✅           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |          ❌           |       ❌        |        ❌         |
+| **HunyuanVideo-1.5 T2V I2V** |     ❌     |     ✅      |           ❌           |       ✅        |         ✅         |         ❌         |   ✅    |             ✅             |      ✅ (decode)      |       ✅        |        ❌         |
+| **DreamID-Omni**             |     ❌     |     ❌      |           ❌           |       ✅        |         ❌         |         ❌         |   ✅    |             ✅             |          ❌           |       ❌        |        ❌         |
 
 **Frame Interpolation Support**
 
@@ -155,9 +156,9 @@ The following tables show which models support each feature:
 
 ### AudioGen
 
-| Model | ⚡TeaCache | ⚡Cache-DiT | 🔀SP (Ulysses & Ring) | 🔀CFG-Parallel | 🔀Tensor-Parallel | 🔀HSDP | 💾CPU Offload (Layerwise) | 💾VAE-Patch-Parallel | 💾Quantization | 🔄Step Execution |
-|-------|:----------:|:-----------:|:---------------------:|:--------------:|:-----------------:|:------:|:------------------------:|:--------------------:|:--------------:|:----------------:|
-| **Stable-Audio-Open** | ✅ | ❌ | ❓ | ❓ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ |
+| Model                 | ⚡TeaCache | ⚡Cache-DiT | 🔀SP (Ulysses & Ring) | 🔀CFG-Parallel | 🔀Tensor-Parallel | 🔀Pipeline-Parallel | 🔀HSDP | 💾CPU Offload (Layerwise) | 💾VAE-Patch-Parallel | 💾Quantization | 🔄Step Execution |
+|-----------------------|:---------:|:----------:|:---------------------:|:--------------:|:-----------------:|:-------------------:|:------:|:-------------------------:|:--------------------:|:--------------:|:----------------:|
+| **Stable-Audio-Open** |     ✅     |     ❌      |           ❓           |       ❓        |         ❌         |          ❌          |   ✅    |             ✅             |          ❌           |       ✅        |        ❌         |
 
 
 ## Feature Compatibility
@@ -257,7 +258,7 @@ Measured on NVIDIA H800:
 
 **Parallelism Methods:**
 
-- **[Parallelism Overview](diffusion/parallelism/overview.md)** - Tensor Parallelism, Sequence Parallelism, CFG Parallelism, HSDP, and Expert Parallelism
+- **[Parallelism Overview](diffusion/parallelism/overview.md)** - Tensor Parallelism, Sequence Parallelism, CFG Parallelism, Pipeline Parallelism, HSDP, and Expert Parallelism
 
 **Memory Optimization:**
 

docs/user_guide/examples/online_serving/glm_image.md

@@ -101,7 +101,7 @@ guides.
 
 When using `/v1/chat/completions`, pass these inside `extra_body` in the curl
 JSON, or via the `extra_body` keyword argument in the OpenAI Python SDK (see the
-[Diffusion Chat API guide](../../../serving/diffusion_chat_api.md)).
+[Chat Completions API guide](../../../serving/chat_completions_api.md)).
 When using the dedicated [`/v1/images/generations`](../../../serving/image_generation_api.md)
 or [`/v1/images/edits`](../../../serving/image_edit_api.md) endpoints, pass
 the supported generation controls as top-level fields directly. For image

docs/user_guide/examples/online_serving/image_to_image.md

@@ -208,7 +208,7 @@ Wrap generation parameters inside `extra_body` in the request JSON:
     ```
 
     For details on how generation parameters are handled across different clients, see the
-    [Diffusion Chat API guide](../../../../serving/diffusion_chat_api.md).
+    [Chat Completions API guide](../../../../serving/chat_completions_api.md).
 
 ### Layered Image Generation (Qwen-Image-Layered)
 
@@ -361,7 +361,7 @@ Provide multiple images in `content` (order matters):
 
 When using `/v1/chat/completions`, pass these inside `extra_body` in the curl
 JSON, or via the `extra_body` keyword argument in the OpenAI Python SDK (see the
-[Diffusion Chat API guide](../../../../serving/diffusion_chat_api.md)).
+[Chat Completions API guide](../../../../serving/chat_completions_api.md)).
 When using the dedicated [`/v1/images/edits`](../../../../serving/image_edit_api.md)
 endpoint, pass the supported generation controls as top-level form fields
 directly. For image dimensions and count, use `size` and `n` rather than

docs/user_guide/examples/online_serving/text_to_image.md

@@ -216,7 +216,7 @@ Wrap generation parameters inside `extra_body` in the request JSON:
     ```
 
     For details on how generation parameters are handled across different clients, see the
-    [Diffusion Chat API guide](../../../../serving/diffusion_chat_api.md).
+    [Chat Completions API guide](../../../../serving/chat_completions_api.md).
 
 ### Multimodal Input (Text + Structured Content)
 
@@ -237,7 +237,7 @@ Wrap generation parameters inside `extra_body` in the request JSON:
 
 When using `/v1/chat/completions`, pass these inside `extra_body` in the curl
 JSON, or via the `extra_body` keyword argument in the OpenAI Python SDK (see the
-[Diffusion Chat API guide](../../../../serving/diffusion_chat_api.md)).
+[Chat Completions API guide](../../../../serving/chat_completions_api.md)).
 When using the dedicated [`/v1/images/generations`](../../../../serving/image_generation_api.md)
 endpoint, pass the supported generation controls as top-level JSON fields
 directly. For image dimensions and count, use `size` and `n` rather than

docs/user_guide/quantization/overview.md

@@ -138,3 +138,98 @@ build_quant_config({"method": "gguf", "gguf_model": "/path/to/model.gguf"})
 build_quant_config({"transformer": {"method": "fp8"}, "vae": None})
 build_quant_config(None)
 ```
+
+## Output Similarity Comparison Tool
+
+Use `vllm_omni.quantization.tools.compare_diffusion_trajectory_similarity`
+to compare a reference diffusion run with a quantized candidate run using the
+same prompt, seed, resolution, scheduler settings, and inference steps. The
+tool compares final decoded images or video frames, and also reports generation
+latency and worker-reported peak memory when available.
+
+This is useful when validating whether online quantization, an offline
+pre-quantized checkpoint, or a new `ignored_layers` choice keeps generation
+quality close to the BF16 reference.
+
+### Online Quantization Example
+
+```bash
+python -m vllm_omni.quantization.tools.compare_diffusion_trajectory_similarity \
+  --task t2i \
+  --model Qwen/Qwen-Image \
+  --candidate-quantization fp8 \
+  --ignored-layers img_mlp \
+  --prompt "a cup of coffee on the table" \
+  --height 512 --width 512 \
+  --num-inference-steps 20 \
+  --seed 142 \
+  --output-json /tmp/qwen_image_fp8_similarity/result.json \
+  --save-output-dir /tmp/qwen_image_fp8_similarity/images \
+  --enforce-eager
+```
+
+### Offline Checkpoint Example
+
+Use `--candidate-model` when the candidate is already quantized or lives at a
+different model path:
+
+```bash
+python -m vllm_omni.quantization.tools.compare_diffusion_trajectory_similarity \
+  --task t2i \
+  --reference-model Qwen/Qwen-Image \
+  --candidate-model /path/to/qwen-image-fp8-checkpoint \
+  --prompt "a cup of coffee on the table" \
+  --height 512 --width 512 \
+  --num-inference-steps 20 \
+  --seed 142 \
+  --output-json /tmp/qwen_image_fp8_checkpoint_similarity/result.json
+```
+
+If the checkpoint does not include a loadable quantization config, pass one
+explicitly:
+
+```bash
+--candidate-quantization-config-json '{"method":"fp8"}'
+```
+
+### Output Metrics
+
+The output JSON includes `output_metrics`, `reference_generation`, and
+`candidate_generation`.
+
+| Metric | Direction | Meaning |
+|--------|-----------|---------|
+| `cosine_similarity` | Higher is better | Vector direction similarity between output pixels or frames. Useful as a broad sanity check. |
+| `mae` | Lower is better | Mean absolute pixel or frame error. For decoded outputs, values are in uint8 pixel units. |
+| `mse` / `rmse` | Lower is better | Squared error and its square root. These penalize localized large differences more than `mae`. |
+| `max_abs` | Lower is better | Worst single-element absolute error. Treat it as an outlier/debug signal, not as a release gate. |
+| `l2` / `relative_l2` | Lower is better | Absolute and reference-normalized L2 distance. `relative_l2` is easier to compare across resolutions. |
+| `psnr_db` | Higher is better | Pixel-space signal-to-noise ratio in dB for uint8 images or frames. |
+| `avg_generation_time_s` | Lower is better | Average wall-clock generation time across measured runs. |
+| `max_peak_memory_mb` | Lower is better | Maximum worker-reported peak device memory across measured runs, when the worker reports it. |
+
+Recommended starting thresholds for same-seed diffusion comparisons:
+
+| Metric | Smoke threshold | Stricter target | Notes |
+|--------|-----------------|-----------------|-------|
+| `psnr_db` | `>= 20.0` | `>= 25.0` | Good for quick image or frame regression checks. |
+| `mae` | `<= 12.0` | `<= 6.0` | Interpreted in decoded uint8 pixel units. |
+| `cosine_similarity` | `>= 0.98` | `>= 0.995` | Less sensitive to global scale than L2-style metrics. |
+| `relative_l2` | `<= 0.20` | `<= 0.08` | Useful when comparing across prompts or resolutions. |
+
+These thresholds are heuristics. Tune them by model family, task, resolution,
+quantization method, and deployment tolerance. For release gating, pair the
+numeric report with visual inspection of saved reference and candidate outputs.
+
+The tool intentionally reports separate quality, latency, and memory metrics
+instead of a single consolidated similarity score. A single score can hide
+important tradeoffs, for example a candidate with good PSNR but a meaningful
+memory regression, or a candidate with low average error but localized visual
+artifacts. If you need a project-specific pass/fail gate, define it as an
+explicit policy over the individual metrics.
+
+Pixel-level metrics do not measure semantic consistency. For higher-cost
+evaluation, you can complement this report with a vision-language judge that
+describes the reference and candidate outputs and compares those descriptions.
+Keep that semantic check separate from this lightweight tool so users can
+choose whether the additional model cost and latency are appropriate.

docs/user_guide/quantization/quantized_kvcache.md

@@ -8,15 +8,20 @@ vLLM-Omni supports online FP8 quantization for eligible diffusion Flash
 Attention (FA) to reduce FA latency while keeping model weights in their
 original dtype.
 
-This feature is configured through `kv_cache_dtype`, matching the option name
-used by vLLM's language-model KV-cache quantization. In vLLM-Omni diffusion
-pipelines, however, it is a runtime FA path: Q/K/V tensors are dynamically
-quantized before the attention operator. It does not quantize model weights and
-is separate from [FP8 W8A8](fp8.md), [Int8 W8A8](int8.md), or pre-quantized
-checkpoint formats.
+This feature is configured through `diffusion_kv_cache_dtype` on
+`OmniDiffusionConfig` (CLI: `--diffusion-kv-cache-dtype`). It is intentionally
+**not** the same as vLLM's `--kv-cache-dtype`, which controls autoregressive
+language-model KV cache storage and defaults to `"auto"`. Diffusion FA
+quantization uses the dedicated diffusion flags so omni serve does not inherit
+that default.
 
-If `kv_cache_dtype` is not set, behavior is unchanged and attention runs in the
-native dtype.
+In vLLM-Omni diffusion pipelines, this is a runtime FA path: Q/K/V tensors are
+dynamically quantized before the attention operator. It does not quantize model
+weights and is separate from [FP8 W8A8](fp8.md), [Int8 W8A8](int8.md), or
+pre-quantized checkpoint formats.
+
+If `diffusion_kv_cache_dtype` is not set, behavior is unchanged and attention
+runs in the native dtype.
 
 ## Hardware Support
 
@@ -30,8 +35,8 @@ native dtype.
 Legend: `✅` supported, `❌` unsupported.
 
 FP8 FA is currently implemented only for the NPU Flash Attention backend. Other
-backends do not support `kv_cache_dtype="fp8"` for diffusion attention and fall
-back to native dtype execution.
+backends do not support `diffusion_kv_cache_dtype="fp8"` for diffusion attention
+and fall back to native dtype execution.
 
 ## Model Type Support
 
@@ -40,7 +45,7 @@ back to native dtype execution.
 | Model | Scope | Status | Notes |
 |-------|-------|--------|-------|
 | Wan2.2 | Eligible DiT full-attention FA on Ascend NPU | Tested | Compare quality and latency against a BF16 baseline before production use |
-| Other diffusion models | Eligible DiT full-attention FA on Ascend NPU | Not tested | You can try `kv_cache_dtype="fp8"`; tune `kv_cache_skip_steps` and `kv_cache_skip_layers` when higher precision is needed |
+| Other diffusion models | Eligible DiT full-attention FA on Ascend NPU | Not tested | You can try `diffusion_kv_cache_dtype="fp8"`; tune `diffusion_kv_cache_skip_steps` and `diffusion_kv_cache_skip_layers` when higher precision is needed |
 
 ### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS)
 
@@ -50,8 +55,8 @@ guide documents support.
 ### Multi-Stage Diffusion Model (BAGEL, GLM-Image)
 
 Not tested. If the diffusion stage uses the same NPU Flash Attention backend,
-`kv_cache_dtype` may apply in theory; validate quality and latency for each
-stage and model.
+`diffusion_kv_cache_dtype` may apply in theory; validate quality and latency for
+each stage and model.
 
 ## Configuration
 
@@ -67,15 +72,15 @@ python examples/offline_inference/image_to_video/image_to_video.py \
     --num-inference-steps 4 \
     --ulysses-degree 4 \
     --vae-patch-parallel-size 4 \
-    --kv-cache-dtype fp8 \
-    --kv-cache-skip-steps "0,1" \
-    --kv-cache-skip-layers "0-2"
+    --diffusion-kv-cache-dtype fp8 \
+    --diffusion-kv-cache-skip-steps "0,1" \
+    --diffusion-kv-cache-skip-layers "0-2"
 ```
 
 Online serving:
 
 ```bash
-vllm serve <your-model> --omni --kv-cache-dtype fp8
+vllm serve <your-model> --omni --diffusion-kv-cache-dtype fp8
 ```
 
 Stage config:
@@ -86,18 +91,23 @@ stage_args:
     stage_type: diffusion
     engine_args:
       model_stage: dit
-      kv_cache_dtype: "fp8"
-      kv_cache_skip_steps: "0,1"
-      kv_cache_skip_layers: "0-2"
+      diffusion_kv_cache_dtype: "fp8"
+      diffusion_kv_cache_skip_steps: "0,1"
+      diffusion_kv_cache_skip_layers: "0-2"
 ```
 
+Legacy YAML keys `kv_cache_dtype`, `kv_cache_skip_steps`, and
+`kv_cache_skip_layers` are still accepted when constructing
+`OmniDiffusionConfig` (for example via `from_kwargs`); prefer the `diffusion_*`
+names for new configs.
+
 ## Parameters
 
 | Parameter | Type | Default | Description |
 |-----------|------|---------|-------------|
-| `kv_cache_dtype` | str \| None | `None` | Set to `"fp8"` to enable dynamic FP8 FA on supported attention backends |
-| `kv_cache_skip_steps` | str \| None | `None` | Denoising step selector to keep in native dtype, for example `"0,1,4-6"` |
-| `kv_cache_skip_layers` | str \| None | `None` | Transformer layer selector to keep in native dtype, for example `"0-2,10"` |
+| `diffusion_kv_cache_dtype` | str \| None | `None` | Set to `"fp8"` to enable dynamic FP8 FA on supported attention backends |
+| `diffusion_kv_cache_skip_steps` | str \| None | `None` | Denoising step selector to keep in native dtype, for example `"0,1,4-6"` |
+| `diffusion_kv_cache_skip_layers` | str \| None | `None` | Transformer layer selector to keep in native dtype, for example `"0-2,10"` |
 
 Selectors use comma-separated integers and inclusive ranges. Listed steps or
 layers skip FP8 FA; all other eligible full-attention forwards use the FP8 path.
@@ -106,9 +116,9 @@ layers skip FP8 FA; all other eligible full-attention forwards use the FP8 path.
 
 1. Compare generated images or videos against a BF16 baseline with the same
    seed, prompt, resolution, frame count, and denoising steps.
-2. Use `kv_cache_skip_steps` for denoising steps where quality is more
+2. Use `diffusion_kv_cache_skip_steps` for denoising steps where quality is more
    sensitive.
-3. Use `kv_cache_skip_layers` for transformer layers that show visible quality
+3. Use `diffusion_kv_cache_skip_layers` for transformer layers that show visible quality
    regressions.
 4. Report both latency and quality results when enabling this option for a new
    model. For image or video models, include visual comparison and quantitative

examples/offline_inference/hunyuan_image3/README.md

@@ -112,6 +112,7 @@ python end2end.py --modality text2img \
                   --additional-config '{"torchair_graph_config":{"enabled":true}}'
 ```
 
+
 ## Key Arguments
 
 | Argument | Description |
@@ -123,16 +124,15 @@ python end2end.py --modality text2img \
 | `--steps` | Number of diffusion inference steps for image generation. |
 | `--guidance-scale` | Classifier-free guidance scale for image generation. |
 | `--height`, `--width` | Output image size for `text2img`. |
-| `--bot-task` | Prompt behavior. `auto` selects the default from `--modality`; `think` adds `<think>`; `recaption` adds `<recaption>`; `vanilla` uses the text-to-image pretrain template. |
+| `--bot-task` | Override prompt mode. `none`, `think`, `recaption`, `think_recaption`, or `vanilla`. |
 | `--sys-type` | Override the system prompt type, for example `en_unified` or `en_vanilla`. |
 | `--vae-use-tiling` | Enable VAE tiling for memory reduction. |
 
 ## Notes
 
-- `hunyuan_image3_ar.yaml` is a 4-card AR-only text/comprehension deploy. It sets `engine_output_type: text`, `final_output_type: text`, and text sampling defaults.
-- `hunyuan_image3_dit.yaml` is a single-stage DiT deploy with `stage_id: 0`; it does not require stage 1 or a running AR stage.
+- `hunyuan_image3_ar.yaml` is a 4-card AR-only text/comprehension deploy.
+- `hunyuan_image3_dit.yaml` is a single-stage DiT deploy with `stage_id: 0`.
 - The old HunyuanImage3 YAMLs under `model_executor/stage_configs/` and `platforms/*/stage_configs/` have been folded into the deploy YAMLs.
-- This PR does not keep the HunyuanImage3 AR-to-DiT KV reuse wiring. The deploy YAMLs describe the topology and platform settings only.
 
 ## Prompt Format
 
@@ -148,22 +148,8 @@ Assistant: {trigger_tag?}
 
 - `<img>`: Placeholder for each input image (single token; expanded by the multimodal pipeline).
 - Trigger tags: `<think>` for CoT and `<recaption>` for recaptioning, placed after `Assistant: `.
-- System prompt: Auto-selected based on task.
-- `t2i_vanilla` is the only task that uses the bare pretrain template without chat structure.
-- The example composes the internal prompt task from `--modality` and `--bot-task`
-  before calling `prompt_utils`; for example, `img2text + think` becomes
-  `i2t_think` for prompt and stop-token lookup.
+- System prompt: Auto-selected from `task` and `bot_task`.
+- `bot_task='vanilla'` with `task='t2i'` uses the bare pretrain template.
 
 The shared `vllm_omni.diffusion.models.hunyuan_image3.prompt_utils.build_prompt_tokens()`
 helper handles segment-by-segment tokenization and matches HF `apply_chat_template`.
-
-## FAQ
-
-- **OOM errors**: Decrease `gpu_memory_utilization` in the deploy YAML, use a smaller `max_num_batched_tokens`, or enable VAE tiling with `--vae-use-tiling`.
-- **Custom image sizes**: Use `--height` and `--width` flags (multiples of 16 recommended).
-
-| Stage | VRAM (approx) |
-| :--- | :--- |
-| Stage 0 (AR) | ~15 GiB + KV Cache |
-| Stage 1 (DiT) | ~30 GiB |
-| Total (8-GPU) | ~45 GiB + KV Cache |

examples/offline_inference/hunyuan_image3/end2end.py

@@ -1,16 +1,5 @@
 """
 HunyuanImage-3.0-Instruct unified end-to-end inference script.
-
-Supports all modalities through a single entry point:
-  - text2img:  Text → AR → DiT → Image
-  - img2img:   Text+Image → AR → DiT → Edited Image (IT2I)
-  - img2text:  Image+Text → AR → Text description (I2T)
-  - text2text: Text → AR → Text (comprehension, no image)
-
-Usage:
-    python end2end.py --modality text2img --prompts "A cute cat"
-    python end2end.py --modality img2img --image-path input.png --prompts "Make it snowy"
-    python end2end.py --modality img2text --image-path input.png --prompts "Describe this image"
 """
 
 import argparse
@@ -19,18 +8,25 @@ import os
 from pathlib import Path
 
 from vllm_omni.diffusion.models.hunyuan_image3.prompt_utils import (
-    _TASK_PRESETS,
+    MAX_IMAGES_PER_REQUEST,
     build_prompt_tokens,
     resolve_stop_token_ids,
+    resolve_sys_type,
 )
 from vllm_omni.entrypoints.omni import Omni
 from vllm_omni.inputs.data import OmniPromptType
 
-# Default deploy configs are absolute so this example works from any cwd.
 _REPO_ROOT = Path(__file__).resolve().parents[3]
 _DEFAULT_DEPLOY_CONFIG = str(_REPO_ROOT / "vllm_omni" / "deploy" / "hunyuan_image3.yaml")
 _DEFAULT_AR_DEPLOY_CONFIG = str(_REPO_ROOT / "vllm_omni" / "deploy" / "hunyuan_image3_ar.yaml")
 
+_MODALITY_TASK_MAP: dict[str, tuple[str, str | None]] = {
+    "text2img": ("t2i", "think"),
+    "img2img": ("it2i", "think"),
+    "img2text": ("i2t", None),
+    "text2text": ("t2t", None),
+}
+
 _MODALITY_DEFAULT_DEPLOY_CONFIG = {
     "text2img": _DEFAULT_DEPLOY_CONFIG,
     "img2img": _DEFAULT_DEPLOY_CONFIG,
@@ -45,78 +41,60 @@ _MODALITY_MODE = {
     "text2text": "text-to-text",
 }
 
-_MODALITY_TASK_MAP = {
-    "text2img": "t2i",
-    "img2img": "it2i",
-    "img2text": "i2t",
-    "text2text": "t2t",
-}
-
 
 def parse_args():
     parser = argparse.ArgumentParser(description="HunyuanImage-3.0-Instruct end-to-end inference.")
-    parser.add_argument(
-        "--model",
-        default="tencent/HunyuanImage-3.0-Instruct",
-        help="Model name or local path.",
-    )
+    parser.add_argument("--model", default="tencent/HunyuanImage-3.0-Instruct", help="Model name or local path.")
     parser.add_argument(
         "--modality",
         default="text2img",
-        choices=["text2img", "img2img", "img2text", "text2text"],
-        help="Modality mode to control stage execution.",
+        choices=list(_MODALITY_TASK_MAP),
     )
     parser.add_argument("--prompts", nargs="+", default=None, help="Input text prompts.")
     parser.add_argument(
         "--image-path",
         type=str,
         default=None,
-        help="Path to input image (for img2img/img2text).",
+        help="Input image path(s) for img2img/img2text. Comma-separated for multi-image (up to 3).",
     )
-    parser.add_argument(
-        "--output",
-        type=str,
-        default=".",
-        help="Output directory to save results.",
-    )
-
-    # Generation parameters
+    parser.add_argument("--output", type=str, default=".", help="Output directory to save results.")
     parser.add_argument("--steps", type=int, default=50, help="Number of inference steps.")
     parser.add_argument("--guidance-scale", type=float, default=5.0, help="Classifier-free guidance scale.")
     parser.add_argument("--seed", type=int, default=42, help="Random seed.")
     parser.add_argument("--height", type=int, default=1024, help="Output image height.")
     parser.add_argument("--width", type=int, default=1024, help="Output image width.")
-    parser.add_argument(
-        "--vae-use-tiling",
-        action="store_true",
-        help="Enable VAE tiling for memory optimization.",
-    )
-
-    # Prompt configuration
+    parser.add_argument("--vae-use-tiling", action="store_true", help="Enable VAE tiling.")
     parser.add_argument(
         "--bot-task",
         type=str,
-        default="auto",
-        choices=["auto", "think", "recaption", "think_recaption", "vanilla"],
-        help=(
-            "Prompt behavior. 'auto' selects the default for the modality; "
-            "'think' adds <think>; 'recaption' adds <recaption>; "
-            "'vanilla' uses the t2i pretrain template."
-        ),
-    )
-    parser.add_argument(
-        "--sys-type",
-        type=str,
         default=None,
-        help="Override system prompt type (e.g. en_unified, en_vanilla).",
+        choices=["none", "think", "recaption", "think_recaption", "vanilla"],
+        help="Override prompt mode. Default: auto from --modality.",
     )
-
-    # Omni init args
+    parser.add_argument("--sys-type", type=str, default=None, help="Override system prompt type.")
     parser.add_argument("--deploy-config", type=str, default=None, help="Custom deploy YAML path.")
     parser.add_argument("--stage-configs-path", type=str, default=None, help="Custom legacy stage config YAML path.")
     parser.add_argument("--log-stats", action="store_true", default=False)
     parser.add_argument("--init-timeout", type=int, default=300, help="Initialization timeout in seconds.")
     parser.add_argument("--enforce-eager", action="store_true", help="Disable torch.compile.")
+    parser.add_argument(
+        "--diffusion-kv-cache-dtype",
+        type=str,
+        default=None,
+        help="Diffusion attention KV cache dtype, for example 'fp8'. Separate from vLLM --kv-cache-dtype.",
+    )
+    parser.add_argument(
+        "--diffusion-kv-cache-skip-steps",
+        type=str,
+        default=None,
+        help="Denoising step selector to keep diffusion KV cache in native dtype, for example '0,1,4-6'.",
+    )
+    parser.add_argument(
+        "--diffusion-kv-cache-skip-layers",
+        type=str,
+        default=None,
+        help="Transformer layer selector to keep diffusion KV cache in native dtype, for example '0-2,10'.",
+    )
     parser.add_argument(
         "--additional-config",
         type=str,
@@ -157,22 +135,13 @@ def main():
     os.makedirs(args.output, exist_ok=True)
     additional_config = parse_additional_config(args.additional_config)
 
-    # Determine task for prompt formatting from modality + bot behavior.
-    task = _MODALITY_TASK_MAP[args.modality]
-    assert task is not None
-    bot_task = args.bot_task
-    if bot_task != "auto":
-        task = task + "_" + bot_task
-    if task not in _TASK_PRESETS:
-        valid_bot_tasks = {
-            "text2img": ["think", "recaption", "vanilla"],
-            "img2img": ["think", "recaption", "think_recaption"],
-            "img2text": ["auto"],
-            "text2text": ["auto"],
-        }[args.modality]
-        raise ValueError(
-            f"--bot-task {bot_task!r} is not supported for {args.modality}. Choose from: {valid_bot_tasks}"
-        )
+    task, default_bot_task = _MODALITY_TASK_MAP[args.modality]
+    if args.bot_task is None:
+        bot_task: str | None = default_bot_task
+    elif args.bot_task == "none":
+        bot_task = None
+    else:
+        bot_task = args.bot_task
 
     if args.deploy_config is not None and args.stage_configs_path is not None:
         raise ValueError("--deploy-config and --stage-configs-path are mutually exclusive.")
@@ -182,13 +151,16 @@ def main():
     if deploy_config is None and stage_configs_path is None:
         deploy_config = _MODALITY_DEFAULT_DEPLOY_CONFIG[args.modality]
 
-    # Build Omni
     omni_kwargs = {
         "model": args.model,
         "vae_use_tiling": args.vae_use_tiling,
         "log_stats": args.log_stats,
         "init_timeout": args.init_timeout,
         "enforce_eager": args.enforce_eager,
+        "mode": _MODALITY_MODE[args.modality],
+        "diffusion_kv_cache_dtype": args.diffusion_kv_cache_dtype,
+        "diffusion_kv_cache_skip_steps": args.diffusion_kv_cache_skip_steps,
+        "diffusion_kv_cache_skip_layers": args.diffusion_kv_cache_skip_layers,
     }
 
     if additional_config is not None:
@@ -197,71 +169,67 @@ def main():
         omni_kwargs["deploy_config"] = deploy_config
     else:
         omni_kwargs["stage_configs_path"] = stage_configs_path
-    omni_kwargs["mode"] = _MODALITY_MODE[args.modality]
 
     omni = Omni(**omni_kwargs)
 
-    # Prepare prompts
     prompts = args.prompts or ["A cute cat"]
-    if not prompts:
-        print("[Info] No prompts provided, using default.")
-        prompts = ["A cute cat"]
-
-    # Load image if needed
-    input_image = None
+    input_images: list = []
     if args.modality in ("img2img", "img2text"):
-        if not args.image_path or not os.path.exists(args.image_path):
+        if not args.image_path:
             raise ValueError(f"--image-path required for {args.modality}, got: {args.image_path}")
         from PIL import Image
 
-        input_image = Image.open(args.image_path).convert("RGB")
+        image_paths = [p.strip() for p in args.image_path.split(",") if p.strip()]
+        if len(image_paths) > MAX_IMAGES_PER_REQUEST:
+            raise ValueError(
+                f"--image-path accepts at most {MAX_IMAGES_PER_REQUEST} images for "
+                f"HunyuanImage-3.0 IT2I, got {len(image_paths)}: {args.image_path}"
+            )
+        for image_path in image_paths:
+            if not os.path.exists(image_path):
+                raise ValueError(f"Image path does not exist: {image_path}")
+            input_images.append(Image.open(image_path).convert("RGB"))
+        if not input_images:
+            raise ValueError(f"--image-path produced no usable paths: {args.image_path!r}")
 
-    # Load tokenizer for segment-wise prompt tokenization (matches HF
-    # apply_chat_template byte-for-byte; see build_prompt_tokens docstring).
     from transformers import AutoTokenizer
 
     tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
+    mm_image_payload = (input_images[0] if len(input_images) == 1 else input_images) if input_images else None
 
-    # Format prompts
     formatted_prompts: list[OmniPromptType] = []
-    for p in prompts:
-        result = build_prompt_tokens(p, tokenizer, task=task, sys_type=args.sys_type)
+    for prompt in prompts:
+        build_kwargs: dict = {"task": task, "bot_task": bot_task, "sys_type": args.sys_type}
+        if input_images:
+            build_kwargs["num_images"] = len(input_images)
+        result = build_prompt_tokens(prompt, tokenizer, **build_kwargs)
         token_ids = result.token_ids
-        effective_sys_type = result.system_prompt_type
+        effective_sys_type = args.sys_type or resolve_sys_type(bot_task)
 
-        # `prompt_token_ids` drives the AR stage (matches HF byte-for-byte).
-        # `prompt` and `use_system_prompt` are forwarded by ar2diffusion to
-        # the DiT stage so the diffusion pipeline can rebuild the same
-        # system prefix when constructing its model inputs.
         prompt_dict: dict = {
             "prompt_token_ids": token_ids,
-            "prompt": p,
+            "prompt": prompt,
             "use_system_prompt": effective_sys_type,
         }
-
         if args.modality == "text2img":
             prompt_dict["modalities"] = ["image"]
         elif args.modality == "img2img":
             prompt_dict["modalities"] = ["image"]
-            prompt_dict["multi_modal_data"] = {"image": input_image}
-            prompt_dict["height"] = input_image.height
-            prompt_dict["width"] = input_image.width
+            prompt_dict["multi_modal_data"] = {"image": mm_image_payload}
+            prompt_dict["height"] = input_images[0].height
+            prompt_dict["width"] = input_images[0].width
         elif args.modality == "img2text":
             prompt_dict["modalities"] = ["text"]
-            prompt_dict["multi_modal_data"] = {"image": input_image}
-        elif args.modality == "text2text":
+            prompt_dict["multi_modal_data"] = {"image": mm_image_payload}
+        else:
             prompt_dict["modalities"] = ["text"]
-
         formatted_prompts.append(prompt_dict)
 
-    # Build sampling params from defaults
     params_list = list(omni.default_sampling_params_list)
 
-    # Override diffusion params if applicable
     from vllm_omni.inputs.data import OmniDiffusionSamplingParams
 
     ar_stop_token_ids = resolve_stop_token_ids(task=task, bot_task=bot_task, tokenizer=tokenizer)
-    assert ar_stop_token_ids is not None
     for sp in params_list:
         if isinstance(sp, OmniDiffusionSamplingParams):
             sp.num_inference_steps = args.steps
@@ -269,13 +237,12 @@ def main():
             sp.guidance_scale_provided = True
             if args.seed is not None:
                 sp.seed = args.seed
-            if args.modality in ("text2img",):
+            if args.modality == "text2img":
                 sp.height = args.height
                 sp.width = args.width
         elif hasattr(sp, "stop_token_ids"):
             sp.stop_token_ids = ar_stop_token_ids
 
-    # Print configuration
     print(f"\n{'=' * 60}")
     print("HunyuanImage-3.0 Generation Configuration:")
     print(f"  Model: {args.model}")
@@ -291,6 +258,9 @@ def main():
         print(f"  Inference steps: {args.steps}")
         print(f"  Guidance scale: {args.guidance_scale}")
         print(f"  Seed: {args.seed}")
+        print(f"  diffusion_kv_cache_dtype: {args.diffusion_kv_cache_dtype}")
+        print(f"  diffusion_kv_cache_skip_steps: {args.diffusion_kv_cache_skip_steps}")
+        print(f"  diffusion_kv_cache_skip_layers: {args.diffusion_kv_cache_skip_layers}")
     if args.modality == "text2img":
         print(f"  Output size: {args.width}x{args.height}")
     if args.image_path:
@@ -300,13 +270,9 @@ def main():
     print(f"  Prompts: {prompts}")
     print(f"{'=' * 60}\n")
 
-    # Generate
     omni_outputs = list(omni.generate(prompts=formatted_prompts, sampling_params_list=params_list))
-
-    # Process outputs
     img_idx = 0
     for req_output in omni_outputs:
-        # Text output (AR stage or text-only)
         ro = getattr(req_output, "request_output", None)
         txt = ""
         if ro and getattr(ro, "outputs", None):
@@ -320,11 +286,9 @@ def main():
         if txt:
             print(f"[Output] Text:\n{txt}")
 
-        # Image output (DiT stage)
         images = getattr(req_output, "images", None)
         if not images and ro and hasattr(ro, "images"):
             images = ro.images
-
         if images:
             for j, img in enumerate(images):
                 save_path = os.path.join(args.output, f"output_{img_idx}_{j}.png")

examples/offline_inference/image_to_video/image_to_video.py

@@ -103,22 +103,22 @@ def parse_args() -> argparse.Namespace:
         help="Sampling solver for Wan2.2 pipelines. Use 'euler' for Lightning/Distill setups.",
     )
     parser.add_argument(
-        "--kv-cache-dtype",
+        "--diffusion-kv-cache-dtype",
         type=str,
         default=None,
-        help="Config-level KV cache dtype (e.g. float8_e4m3fn).",
+        help="Diffusion attention KV cache dtype (e.g. float8_e4m3fn). Separate from vLLM --kv-cache-dtype.",
     )
     parser.add_argument(
-        "--kv-cache-skip-steps",
+        "--diffusion-kv-cache-skip-steps",
         type=str,
         default=None,
-        help="Config-level KV-cache quantization skip-step selector, e.g. '0-9,20,25-30'.",
+        help="Diffusion KV-cache quantization skip-step selector, e.g. '0-9,20,25-30'.",
     )
     parser.add_argument(
-        "--kv-cache-skip-layers",
+        "--diffusion-kv-cache-skip-layers",
         type=str,
         default=None,
-        help="Config-level KV-cache quantization skip-layer selector, e.g. '0,1,4-8'.",
+        help="Diffusion KV-cache quantization skip-layer selector, e.g. '0,1,4-8'.",
     )
     parser.add_argument("--output", type=str, default="i2v_output.mp4", help="Path to save the video (mp4).")
     parser.add_argument("--fps", type=int, default=None, help="Frames per second for the output video.")
@@ -142,6 +142,42 @@ def parse_args() -> argparse.Namespace:
         action="store_true",
         help="Enable layerwise (blockwise) offloading on DiT modules.",
     )
+    parser.add_argument(
+        "--enforce-eager",
+        action="store_true",
+        help="Disable torch.compile and force eager execution.",
+    )
+    parser.add_argument(
+        "--audio-sample-rate",
+        type=int,
+        default=24000,
+        help="Sample rate for audio output when saved (default: 24000).",
+    )
+    parser.add_argument(
+        "--cache-backend",
+        type=str,
+        default=None,
+        choices=["cache_dit", "tea_cache"],
+        help=(
+            "Cache backend to use for acceleration. "
+            "Options: 'cache_dit' (DBCache + SCM + TaylorSeer), 'tea_cache' (Timestep Embedding Aware Cache). "
+            "Default: None (no cache acceleration)."
+        ),
+    )
+    parser.add_argument(
+        "--enable-diffusion-pipeline-profiler",
+        action="store_true",
+        help="Enable diffusion pipeline profiler to display stage durations.",
+    )
+    parser.add_argument(
+        "--quantization",
+        type=str,
+        default=None,
+        choices=["fp8", "mxfp8", "int8", "gguf"],
+        help="Quantization method for the transformer. mxfp8: W8A8 MXFP8 online quant (NPU). fp8: online FP8 (GPU).",
+    )
+
+    # Distributed and parallel execution
     parser.add_argument(
         "--ulysses-degree",
         type=int,
@@ -173,28 +209,6 @@ def parse_args() -> argparse.Namespace:
         default=1,
         help="Number of GPUs used for VAE patch/tile parallelism (decode).",
     )
-    parser.add_argument(
-        "--enforce-eager",
-        action="store_true",
-        help="Disable torch.compile and force eager execution.",
-    )
-    parser.add_argument(
-        "--audio-sample-rate",
-        type=int,
-        default=24000,
-        help="Sample rate for audio output when saved (default: 24000).",
-    )
-    parser.add_argument(
-        "--cache-backend",
-        type=str,
-        default=None,
-        choices=["cache_dit", "tea_cache"],
-        help=(
-            "Cache backend to use for acceleration. "
-            "Options: 'cache_dit' (DBCache + SCM + TaylorSeer), 'tea_cache' (Timestep Embedding Aware Cache). "
-            "Default: None (no cache acceleration)."
-        ),
-    )
     parser.add_argument(
         "--use-hsdp",
         action="store_true",
@@ -219,16 +233,10 @@ def parse_args() -> argparse.Namespace:
         ),
     )
     parser.add_argument(
-        "--quantization",
-        type=str,
-        default=None,
-        choices=["fp8", "mxfp8", "int8", "gguf"],
-        help="Quantization method for the transformer. mxfp8: W8A8 MXFP8 online quant (NPU). fp8: online FP8 (GPU).",
-    )
-    parser.add_argument(
-        "--enable-diffusion-pipeline-profiler",
-        action="store_true",
-        help="Enable diffusion pipeline profiler to display stage durations.",
+        "--pipeline-parallel-size",
+        type=int,
+        default=1,
+        help="Number of pipeline parallel stages.",
     )
     parser.add_argument(
         "--profiler-config",
@@ -326,6 +334,7 @@ def main():
         use_hsdp=args.use_hsdp,
         hsdp_shard_size=args.hsdp_shard_size,
         hsdp_replicate_size=args.hsdp_replicate_size,
+        pipeline_parallel_size=args.pipeline_parallel_size,
     )
     omni_kwargs = dict(
         model=args.model,
@@ -334,9 +343,9 @@ def main():
         vae_use_tiling=args.vae_use_tiling,
         boundary_ratio=args.boundary_ratio,
         flow_shift=args.flow_shift,
-        kv_cache_dtype=args.kv_cache_dtype,
-        kv_cache_skip_steps=args.kv_cache_skip_steps,
-        kv_cache_skip_layers=args.kv_cache_skip_layers,
+        diffusion_kv_cache_dtype=args.diffusion_kv_cache_dtype,
+        diffusion_kv_cache_skip_steps=args.diffusion_kv_cache_skip_steps,
+        diffusion_kv_cache_skip_layers=args.diffusion_kv_cache_skip_layers,
         enable_cpu_offload=args.enable_cpu_offload,
         parallel_config=parallel_config,
         enforce_eager=args.enforce_eager,
@@ -361,12 +370,13 @@ def main():
     print(f"  Inference steps: {args.num_inference_steps}")
     print(f"  Frames: {args.num_frames}")
     print(f"  Solver: {args.sample_solver}")
-    print(f"  kv_cache_dtype(config): {args.kv_cache_dtype}")
-    print(f"  kv_cache_skip_steps(config): {args.kv_cache_skip_steps}")
-    print(f"  kv_cache_skip_layers(config): {args.kv_cache_skip_layers}")
+    print(f"  diffusion_kv_cache_dtype(config): {args.diffusion_kv_cache_dtype}")
+    print(f"  diffusion_kv_cache_skip_steps(config): {args.diffusion_kv_cache_skip_steps}")
+    print(f"  diffusion_kv_cache_skip_layers(config): {args.diffusion_kv_cache_skip_layers}")
     print(
         f"  Parallel configuration: cfg_parallel_size={args.cfg_parallel_size},"
-        f" tensor_parallel_size={args.tensor_parallel_size}, vae_patch_parallel_size={args.vae_patch_parallel_size}"
+        f" tensor_parallel_size={args.tensor_parallel_size}, vae_patch_parallel_size={args.vae_patch_parallel_size},"
+        f" pipeline_parallel_size={args.pipeline_parallel_size}"
     )
     print(f"  Video size: {args.width}x{args.height}")
     print(f"{'=' * 60}\n")

examples/offline_inference/text_to_speech/README.md

@@ -73,13 +73,15 @@ python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
 ```
 
 ### Voice cloning
-Pass a reference audio. Note that CosyVoice3's `--prompt-text` is a system-style prompt for the GPT stage, not a reference transcript:
+If `--ref-audio` is omitted, the script downloads the upstream
+[`zero_shot_prompt.wav`](https://github.com/FunAudioLLM/CosyVoice/blob/main/asset/zero_shot_prompt.wav) from the CosyVoice repo into the current directory.
+To use your own clip, pass `--ref-audio /path/to/reference.wav`, and modify `--prompt-text` correspondingly.
 ```bash
 python examples/offline_inference/text_to_speech/cosyvoice3/end2end.py \
     --model pretrained_models/Fun-CosyVoice3-0.5B \
     --tokenizer pretrained_models/Fun-CosyVoice3-0.5B/CosyVoice-BlankEN \
-    --ref-audio prompt.wav \
-    --prompt-text "You are a helpful assistant.<|endofprompt|>Testing my voices. Why should I not?"
+    --ref-audio /path/to/reference.wav \
+    --prompt-text "You are a helpful assistant.<|endofprompt|>Trascript in your ref audio clip"
 ```
 
 ### Streaming
@@ -317,44 +319,39 @@ pip install voxcpm soundfile
 export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/VoxCPM/src
 ```
 
-If the native VoxCPM `config.json` does not contain HF metadata such as `model_type`, prepare a persistent HF-compatible config directory and point the stage configs to it via `VLLM_OMNI_VOXCPM_HF_CONFIG_PATH`:
-
-```bash
-export VOXCPM_MODEL=/path/to/voxcpm-model
-export VLLM_OMNI_VOXCPM_HF_CONFIG_PATH=/tmp/voxcpm_hf_config
-mkdir -p "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"
-cp "$VOXCPM_MODEL/config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/config.json"
-cp "$VOXCPM_MODEL/generation_config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/generation_config.json" 2>/dev/null || true
-python3 -c 'import json, os; p=os.path.join(os.environ["VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"], "config.json"); cfg=json.load(open(p, "r", encoding="utf-8")); cfg["model_type"]="voxcpm"; cfg.setdefault("architectures", ["VoxCPMForConditionalGeneration"]); json.dump(cfg, open(p, "w", encoding="utf-8"), indent=2, ensure_ascii=False)'
-```
-
 ### Quick start
 ```bash
 python examples/offline_inference/text_to_speech/voxcpm/end2end.py \
-    --model "$VOXCPM_MODEL" \
+    --model openbmb/VoxCPM-0.5B \
     --text "This is a split-stage VoxCPM synthesis example running on vLLM Omni."
 ```
 
 ### Voice cloning
 ```bash
 python examples/offline_inference/text_to_speech/voxcpm/end2end.py \
-    --model "$VOXCPM_MODEL" \
+    --model openbmb/VoxCPM-0.5B \
     --text "This sentence is synthesized with a cloned voice." \
     --ref-audio /path/to/reference.wav \
     --ref-text  "The exact transcript spoken in reference.wav."
 ```
 
 ### Streaming
-Pass the async-chunk stage config:
+Pass `--streaming` together with the legacy async-chunk stage config:
 ```bash
 python examples/offline_inference/text_to_speech/voxcpm/end2end.py \
-    --model "$VOXCPM_MODEL" \
+    --model openbmb/VoxCPM-0.5B \
+    --streaming \
     --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \
     --text "This is a split-stage VoxCPM streaming example running on vLLM Omni."
 ```
 
+### Persistent HF config (optional)
+The engine auto-patches a missing `model_type` in the checkpoint's `config.json` in-memory at startup (see `vllm_omni/engine/arg_utils.py`), so no manual setup is required.
+
+If you'd rather maintain a persistent patched config dir (e.g. to share across runs or to add fields the auto-patcher doesn't set), export `VLLM_OMNI_VOXCPM_HF_CONFIG_PATH` to its location — both the bundled deploy and async-chunk yamls read it via `${oc.env:VLLM_OMNI_VOXCPM_HF_CONFIG_PATH,}`.
+
 ### Notes
-- `voxcpm.yaml` is the default non-streaming stage config; `voxcpm_async_chunk.yaml` enables streaming.
+- Non-streaming auto-loads `vllm_omni/deploy/voxcpm.yaml` from the model directory name / HF `model_type`. Streaming uses the legacy `voxcpm_async_chunk.yaml` passed via `--stage-configs-path`.
 - Streaming is currently single-request oriented.
 - `--ref-text` must be the real transcript of `--ref-audio`; mismatched text degrades quality.
 - For online serving, see the [VoxCPM section in the online hub](../../online_serving/text_to_speech/README.md#voxcpm). For benchmark reporting, see [`benchmarks/voxcpm`](../../../benchmarks/voxcpm/README.md).

examples/offline_inference/text_to_speech/cosyvoice3/end2end.py

@@ -2,18 +2,32 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 import argparse
 import os
+import urllib.request
+from pathlib import Path
 
 import numpy as np
 import soundfile as sf
 from vllm import SamplingParams
-from vllm.assets.audio import AudioAsset
 from vllm.multimodal.media.audio import load_audio
 
 from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
 from vllm_omni.entrypoints.omni import Omni
-from vllm_omni.model_executor.models.cosyvoice3.config import CosyVoice3Config
 from vllm_omni.model_executor.models.cosyvoice3.tokenizer import get_qwen_tokenizer
 from vllm_omni.model_executor.models.cosyvoice3.utils import extract_text_token
+from vllm_omni.transformers_utils.configs.cosyvoice3 import CosyVoice3Config
+
+# Upstream zero-shot reference clip
+ZERO_SHOT_PROMPT_URL = "https://raw.githubusercontent.com/FunAudioLLM/CosyVoice/main/asset/zero_shot_prompt.wav"
+
+
+def _default_ref_audio() -> str:
+    # Download the upstream zero_shot_prompt.wav into the current dir
+    dest = Path("zero_shot_prompt.wav")
+    if not dest.exists() or dest.stat().st_size == 0:
+        print(f"Downloading default reference audio to {dest}")
+        urllib.request.urlretrieve(ZERO_SHOT_PROMPT_URL, dest)
+
+    return str(dest)
 
 
 def run_e2e():
@@ -36,9 +50,15 @@ def run_e2e():
     parser.add_argument(
         "--prompt-text",
         type=str,
-        default="You are a helpful assistant.<|endofprompt|>Testing my voices. Why should I not?",
+        default="You are a helpful assistant.<|endofprompt|>希望你以后，能够做的比我还好呦!",
+    )
+    parser.add_argument(
+        "--ref-audio",
+        type=str,
+        default=None,
+        help="Path to reference audio for voice cloning. "
+        "If unset, downloads the upstream CosyVoice3 zero-shot prompt audio clip",
     )
-    parser.add_argument("--ref-audio", type=str, default="prompt.wav")
     parser.add_argument(
         "--tokenizer",
         type=str,
@@ -66,24 +86,22 @@ def run_e2e():
     sampling_cfg = {"top_p": 0.8, "top_k": 25, "eos_token_id": 6561 + 1}
 
     print("Model initialized. Preparing inputs...")
-    if args.ref_audio:
-        if not os.path.exists(args.ref_audio):
-            raise FileNotFoundError(f"Audio file not found: {args.ref_audio}")
-        # Load at native sample rate
-        audio_signal, sr = load_audio(args.ref_audio, sr=None)
+    ref_audio_path = args.ref_audio or _default_ref_audio()
+    if not os.path.exists(ref_audio_path):
+        raise FileNotFoundError(f"Audio file not found: {ref_audio_path}")
+    # Load at native sample rate
+    audio_signal, sr = load_audio(ref_audio_path, sr=None)
 
-        # Validate sample rate before processing (similar to original CosyVoice)
-        min_sr = 16000
-        if sr < min_sr:
-            raise ValueError(
-                f"Audio sample rate {sr} Hz is too low. "
-                f"Minimum required: {min_sr} Hz. "
-                f"Please provide audio with sample rate >= {min_sr} Hz."
-            )
+    # Validate sample rate before processing (similar to original CosyVoice)
+    min_sr = 16000
+    if sr < min_sr:
+        raise ValueError(
+            f"Audio sample rate {sr} Hz is too low. "
+            f"Minimum required: {min_sr} Hz. "
+            f"Please provide audio with sample rate >= {min_sr} Hz."
+        )
 
-        audio_data = (audio_signal.astype(np.float32), sr)
-    else:
-        audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate
+    audio_data = (audio_signal.astype(np.float32), sr)
 
     prompts = {
         "prompt": args.text,

examples/offline_inference/text_to_speech/voxcpm/end2end.py

@@ -11,12 +11,9 @@ import soundfile as sf
 import torch
 from vllm.utils.argparse_utils import FlexibleArgumentParser
 
-from tests.helpers.stage_config import get_deploy_config_path
 from vllm_omni import AsyncOmni, Omni
 from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
 
-DEFAULT_SYNC_STAGE_CONFIG = get_deploy_config_path("voxcpm.yaml")
-
 
 def _build_prompt(args) -> dict[str, Any]:
     additional_information: dict[str, list[Any]] = {
@@ -59,10 +56,6 @@ def _extract_sample_rate(mm: dict[str, Any]) -> int:
     return int(sr_raw)
 
 
-def _is_streaming_stage_config(stage_config_path: str) -> bool:
-    return "async_chunk" in Path(stage_config_path).stem
-
-
 def _save_audio(audio: torch.Tensor, sample_rate: int, output_dir: Path, request_id: str) -> Path:
     output_dir.mkdir(parents=True, exist_ok=True)
     output_path = output_dir / f"output_{request_id}.wav"
@@ -87,6 +80,7 @@ async def _run_streaming(args) -> Path:
     omni = AsyncOmni(
         model=args.model,
         stage_configs_path=args.stage_configs_path,
+        deploy_config=args.deploy_config,
         log_stats=args.log_stats,
         stage_init_timeout=args.stage_init_timeout,
     )
@@ -133,6 +127,7 @@ def _run_sync(args) -> Path:
     omni = Omni(
         model=args.model,
         stage_configs_path=args.stage_configs_path,
+        deploy_config=args.deploy_config,
         log_stats=args.log_stats,
         stage_init_timeout=args.stage_init_timeout,
     )
@@ -164,11 +159,27 @@ def _run_sync(args) -> Path:
 def parse_args():
     parser = FlexibleArgumentParser(description="Minimal offline VoxCPM example for vLLM Omni.")
     parser.add_argument("--model", type=str, required=True, help="Local VoxCPM model directory.")
+    parser.add_argument(
+        "--deploy-config",
+        type=str,
+        default=None,
+        help=("Override the deploy config path."),
+    )
     parser.add_argument(
         "--stage-configs-path",
         type=str,
-        default=DEFAULT_SYNC_STAGE_CONFIG,
-        help=("Stage config path. Use voxcpm.yaml for non-streaming or voxcpm_async_chunk.yaml for streaming."),
+        default=None,
+        help=(
+            "Legacy stage_args yaml path. Required for streaming "
+            "(vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml); "
+            "leave unset for non-streaming to use the auto-resolved deploy config."
+        ),
+    )
+    parser.add_argument(
+        "--streaming",
+        action="store_true",
+        default=False,
+        help="Stream audio chunks as they arrive via AsyncOmni.",
     )
     parser.add_argument("--text", type=str, required=True, help="Input text for synthesis.")
     parser.add_argument("--ref-audio", type=str, default=None, help="Reference audio path for voice cloning.")
@@ -194,9 +205,10 @@ def parse_args():
 
 
 def main(args) -> None:
-    route = "streaming" if _is_streaming_stage_config(args.stage_configs_path) else "sync"
+    route = "streaming" if args.streaming else "sync"
     print(f"Model: {args.model}")
-    print(f"Stage config: {args.stage_configs_path}")
+    print(f"Deploy config: {args.deploy_config or '<auto from HF model_type>'}")
+    print(f"Stage configs path: {args.stage_configs_path or '<unused>'}")
     print(f"Route: {route}")
     if route == "streaming":
         asyncio.run(_run_streaming(args))

examples/offline_inference/text_to_video/text_to_video.py

@@ -142,6 +142,32 @@ def parse_args() -> argparse.Namespace:
         action="store_true",
         help="Enable layerwise (blockwise) offloading on DiT modules.",
     )
+    parser.add_argument(
+        "--audio-sample-rate",
+        type=int,
+        default=24000,
+        help="Sample rate for audio output when saved (default: 24000).",
+    )
+    parser.add_argument(
+        "--enable-diffusion-pipeline-profiler",
+        action="store_true",
+        help="Enable diffusion pipeline profiler to display stage durations.",
+    )
+    parser.add_argument(
+        "--profiler-config",
+        type=parse_profiler_config,
+        default=None,
+        help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
+    )
+    parser.add_argument(
+        "--quantization",
+        type=str,
+        default=None,
+        choices=["fp8", "mxfp8", "int8", "gguf"],
+        help="Quantization method for the transformer. mxfp8: W8A8 MXFP8 online quant (NPU). fp8: online FP8 (GPU).",
+    )
+
+    # Distributed and parallel execution
     parser.add_argument(
         "--ulysses-mode",
         type=str,
@@ -174,41 +200,23 @@ def parse_args() -> argparse.Namespace:
         default=1,
         help="Number of GPUs used for tensor parallelism (TP) inside the DiT.",
     )
-    parser.add_argument(
-        "--audio-sample-rate",
-        type=int,
-        default=24000,
-        help="Sample rate for audio output when saved (default: 24000).",
-    )
     parser.add_argument(
         "--vae-patch-parallel-size",
         type=int,
         default=1,
         help="Number of GPUs used for VAE patch/tile parallelism (decode).",
     )
+    parser.add_argument(
+        "--pipeline-parallel-size",
+        type=int,
+        default=1,
+        help="Number of pipeline parallel stages.",
+    )
     parser.add_argument(
         "--enable-expert-parallel",
         action="store_true",
         help="Enable expert parallelism for MoE layers.",
     )
-    parser.add_argument(
-        "--enable-diffusion-pipeline-profiler",
-        action="store_true",
-        help="Enable diffusion pipeline profiler to display stage durations.",
-    )
-    parser.add_argument(
-        "--profiler-config",
-        type=parse_profiler_config,
-        default=None,
-        help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
-    )
-    parser.add_argument(
-        "--quantization",
-        type=str,
-        default=None,
-        choices=["fp8", "mxfp8", "int8", "gguf"],
-        help="Quantization method for the transformer. mxfp8: W8A8 MXFP8 online quant (NPU). fp8: online FP8 (GPU).",
-    )
     parser.add_argument(
         "--use-hsdp",
         action="store_true",
@@ -262,6 +270,7 @@ def main():
         cfg_parallel_size=args.cfg_parallel_size,
         tensor_parallel_size=args.tensor_parallel_size,
         vae_patch_parallel_size=args.vae_patch_parallel_size,
+        pipeline_parallel_size=args.pipeline_parallel_size,
         enable_expert_parallel=args.enable_expert_parallel,
     )
 
@@ -307,7 +316,8 @@ def main():
     print(
         f"  Parallel configuration: ulysses_degree={args.ulysses_degree}, ring_degree={args.ring_degree},"
         f" cfg_parallel_size={args.cfg_parallel_size}, tensor_parallel_size={args.tensor_parallel_size},"
-        f" vae_patch_parallel_size={args.vae_patch_parallel_size}, enable_expert_parallel={args.enable_expert_parallel}"
+        f" vae_patch_parallel_size={args.vae_patch_parallel_size}, pipeline_parallel_size={args.pipeline_parallel_size},"
+        f" enable_expert_parallel={args.enable_expert_parallel}"
     )
     print(f"  Video size: {args.width}x{args.height}")
     print(f"{'=' * 60}\n")

examples/online_serving/text_to_speech/qwen3_tts/streaming_speech_client.py

@@ -7,6 +7,12 @@ Usage:
     # Send full text at once
     python streaming_speech_client.py --text "Hello world. How are you? I am fine."
 
+    # Pick a built-in speaker for CustomVoice models
+    python streaming_speech_client.py \
+        --text "打开电子枪，关闭扫描，切换到低倍模式。" \
+        --speaker "Serena" \
+        --language "Chinese"
+
     # Simulate STT: send text word-by-word with delay
     python streaming_speech_client.py \
         --text "Hello world. How are you? I am fine." \
@@ -211,7 +217,9 @@ def main():
 
     args = parser.parse_args()
 
-    # Build session config (only include non-None values)
+    # Build session config (only include non-None values).
+    # Server canonical field is `voice`; `speaker` is accepted as an alias
+    # (see StreamingSpeechSessionConfig.voice in protocol/audio.py).
     config = {}
     for key in [
         "model",

pyproject.toml

@@ -252,3 +252,4 @@ fullset = "fullset"
 Vai = "Vai"
 tockens = "tockens"
 CANN = "CANN"
+ND = "ND"

recipes/Baidu/ERNIE-Image.md

@@ -1,4 +1,6 @@
-# ERNIE-Image for text-to-image generation
+# ERNIE-Image
+
+> Text-to-image online serving (ERNIE-Image 8B)
 
 ## Summary
 

recipes/LTX/LTX-2.3.md

@@ -1,4 +1,4 @@
-# LTX-2.3 Text-to-Video with Audio on 1x GPU (96GB VRAM)
+# LTX-2.3
 
 > 22B parameter text-to-video + audio generation model served via vLLM-Omni
 

recipes/LTX/LTX-2.md

@@ -1,6 +1,6 @@
-# LTX-2 T2V and I2V
+# LTX-2
 
-> LTX-2 for text-to-video & image-to-video on 1x H200 141GB
+> Text-to-video and image-to-video serving
 
 ## Summary
 

recipes/Qwen/Qwen-Image-Edit.md

@@ -0,0 +1,171 @@
+# Qwen-Image-Edit
+
+> Image editing serving
+
+## Summary
+
+- Vendor: Qwen
+- Model: `Qwen/Qwen-Image-Edit`
+- Task: Text-guided single-image editing
+- Mode: Online serving with the OpenAI-compatible multimodal endpoint
+- Maintainer: Community
+
+## When to use this recipe
+
+Use this recipe as a baseline for serving `Qwen/Qwen-Image-Edit` for single-image editing on H200 141GB. Two configurations are validated below, sharing one software environment:
+
+| Configuration | Parallelism | Notes |
+| --- | --- | --- |
+| `1x H200` | none | Single-card baseline; ~60 GiB peak VRAM on a 141 GB card |
+| `2x H200` | TP=2 | Lower single-request latency (≈ 1.60× speedup) and lower per-GPU VRAM |
+
+## References
+
+- Model: <https://huggingface.co/Qwen/Qwen-Image-Edit>
+- Example guide: [`examples/online_serving/image_to_image/README.md`](../../examples/online_serving/image_to_image/README.md)
+- User guide: [`docs/user_guide/examples/online_serving/image_to_image.md`](../../docs/user_guide/examples/online_serving/image_to_image.md)
+
+## Hardware Support
+
+### GPU
+
+The two hardware configurations below share the same software environment.
+
+#### Environment
+
+- OS: Linux
+- Python: 3.12+
+- Driver / runtime: CUDA-capable runtime matching the repository build
+- vLLM: use versions from your current checkout, >=0.20.0
+- vLLM-Omni: use versions from your current checkout
+
+#### 1x H200 141GB
+
+##### Command
+
+```bash
+CUDA_VISIBLE_DEVICES=0 vllm serve Qwen/Qwen-Image-Edit --omni \
+  --port 8092
+```
+
+The bundled launcher is equivalent:
+
+```bash
+bash examples/online_serving/image_to_image/run_server.sh
+```
+
+The server is ready when the log shows `INFO:     Application startup complete.` (preceded by `Started server process [PID]` and `Waiting for application startup.`).
+
+##### Verification
+
+Run a single-image edit against the public example asset:
+
+```bash
+wget https://vllm-public-assets.s3.us-west-2.amazonaws.com/omni-assets/qwen-bear.png
+
+PROMPT="Let this mascot dance under the moon, surrounded by floating stars and poetic bubbles such as 'Be Kind'"
+IMG_B64=$(base64 -w0 qwen-bear.png)
+
+curl -s http://localhost:8092/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -d "$(jq -n --arg prompt "$PROMPT" --arg img "$IMG_B64" '{
+    messages: [{
+      role: "user",
+      content: [
+        {type: "text", text: $prompt},
+        {type: "image_url", image_url: {url: ("data:image/png;base64," + $img)}}
+      ]
+    }],
+    extra_body: {
+      height: 1024,
+      width: 1024,
+      num_inference_steps: 50,
+      guidance_scale: 1.0,
+      seed: 42
+    }
+  }')" \
+  | jq -r '.choices[0].message.content[0].image_url.url' \
+  | cut -d',' -f2- | base64 -d > qwen_image_edit_1card.png
+```
+
+The decoded PNG is a 1024×1024 RGB image of the bear mascot dancing under stylized moonlight, surrounded by floating stars and a "Be Kind" text bubble.
+
+You can also run the offline CLI without standing up a server:
+
+```bash
+python examples/offline_inference/image_to_image/image_edit.py \
+  --image qwen-bear.png \
+  --prompt "Let this mascot dance under the moon, ..." \
+  --output qwen_image_edit_offline.png \
+  --num-inference-steps 50 \
+  --cfg-scale 4.0
+```
+
+Validated reference numbers for 1024×1024, 50 denoising steps, batch size 1, `guidance_scale=1.0`, seed 42 (`Cold` = first request after startup, `Warm` = identical subsequent request):
+
+| Cold latency | Warm latency | Peak GPU memory |
+| ---:         | ---:         | ---:            |
+| 17.80 s      | 17.78 s      | 61,091 MiB (~59.66 GiB) |
+
+##### Notes
+
+- Generation parameters live under `extra_body` for `/v1/chat/completions`, or as top-level form fields when using `/v1/images/edits`. See [`examples/online_serving/image_to_image/README.md`](../../examples/online_serving/image_to_image/README.md) for the full request / response schema.
+- Memory: a 1024×1024 / 50-step edit uses ~60 GiB on a single H200, well within the 141 GB headroom.
+- Multi-image variants of Qwen-Image-Edit (e.g. `Qwen-Image-Edit-2509`) are intentionally out of scope for this recipe and are not validated here.
+
+#### 2x H200 141GB (TP=2)
+
+##### Command
+
+```bash
+CUDA_VISIBLE_DEVICES=0,1 vllm serve Qwen/Qwen-Image-Edit --omni \
+  --port 8092 \
+  --tensor-parallel-size 2
+```
+
+In addition to the `Application startup complete.` line, the TP=2 server's `non-default args` log line echoes `'tensor_parallel_size': 2`, and a separate `Worker N: Initialized device and distributed environment` line is emitted for each rank — one per visible GPU.
+
+##### Verification
+
+Run the same `curl` command as in the **1x H200** Verification section above against the TP=2 server (same prompt, same seed, same image). Save the result to `qwen_image_edit_2card_tp2.png` so the two outputs sit side by side for comparison. During TP=2 inference, `nvidia-smi` shows sustained `~99 %` utilization on both GPUs, which is the expected signal that TP wiring is active.
+
+Same workload as the 1x table above (1024×1024, 50 steps, `guidance_scale=1.0`, seed 42):
+
+| Configuration | Cold latency | Warm latency | Peak per-GPU memory |
+| ---           | ---:         | ---:         | ---:                |
+| 1x H200       | 17.80 s      | 17.78 s      | 61,091 MiB (~59.66 GiB) |
+| 2x H200 TP=2  | 11.16 s      | 11.09 s      | 49,377 MiB (~48.22 GiB) |
+
+TP=2 reduces single-request warm latency from `17.78 s` to `11.09 s` (≈ **1.60×**) and drops per-GPU peak VRAM by ~11.4 GiB.
+
+##### Notes
+
+- TP=2 shards model weights across both cards, which is what produces both the latency reduction and the per-GPU VRAM drop above.
+- CFG-Parallel (`--cfg-parallel-size 2`) is supported on the Qwen-Image family but only when `cfg_scale > 1`. The verification command above uses `guidance_scale=1.0` to match the public example image, so it does not exercise CFG-parallel; a separate run with `cfg_scale > 1` is needed if you want to benchmark that path.
+- For deeper acceleration knobs (Cache-DiT, sequence parallel, HSDP) see [`docs/user_guide/diffusion/parallelism/overview.md`](../../docs/user_guide/diffusion/parallelism/overview.md); this recipe intentionally documents only the validated TP=2 baseline.
+
+#### Profiling
+
+The numbers below are from a warm request at 1024×1024, 50 steps, and `guidance_scale=1.0`, measured with `--enable-diffusion-pipeline-profiler` and a 250 ms `nvidia-smi --query-gpu=memory.used` sampler. For TP=2, latency uses rank max, and VRAM is per GPU.
+
+Per-stage latency:
+
+| Stage | TP=1 | TP=2 | Notes |
+| --- | ---: | ---: | --- |
+| `text_encoder.forward` | 0.132 s | 0.135 s | ~1.0× |
+| `vae.encode` | 0.040 s | 0.041 s | ~1.0× |
+| `diffuse` | 17.105 s | 10.382 s | **1.65×** speedup |
+| `vae.decode` | 0.064 s | 0.066 s | ~1.0× |
+| `forward` | 17.372 s | 10.652 s | 1.63× speedup |
+
+Per-GPU VRAM:
+
+| Snapshot | TP=1 | TP=2 | Notes |
+| --- | ---: | ---: | --- |
+| Static VRAM | 58,777 MiB | 47,065 MiB | after model load, before request |
+| Peak VRAM | 61,091 MiB | 49,377 MiB | during inference |
+| Request-time increment | 2,314 MiB | 2,312 MiB | peak minus static |
+
+The latency gain mainly comes from the DiT denoising stage (`diffuse` in the table above). The other measured stages, such as text encoder and VAE encode/decode, stay essentially unchanged between TP=1 and TP=2, which is consistent with them not benefiting meaningfully from TP in this pipeline. The `forward` row's 1.63× speedup tracks `diffuse`, since `diffuse` accounts for about 98% of total pipeline time.
+
+For memory, TP=2 saves about 11.4 GiB per GPU, mostly from static model-state memory, likely dominated by tensor-parallel sharding of the DiT weights. The request-time increment is almost unchanged, so TP=2 does not materially reduce transient inference-time memory for this single-image, batch-1 workload.

recipes/Qwen/Qwen-Image.md

@@ -1,4 +1,6 @@
-# Qwen-Image for text-to-image serving with optional continuous batching on 1x A100 80GB
+# Qwen-Image
+
+> Text-to-image serving with optional step-wise continuous batching
 
 ## Summary
 
@@ -102,7 +104,7 @@ To replay the batching benefit with matched warmup:
 
 ```bash
 python benchmarks/diffusion/diffusion_benchmark_serving.py \
-  --backend vllm-omni \
+  --endpoint /v1/chat/completions \
   --dataset vbench \
   --task t2i \
   --model Qwen/Qwen-Image \

recipes/Qwen/Qwen3-Omni.md

@@ -1,5 +1,7 @@
 # Qwen3-Omni
 
+> Online serving for multimodal chat
+
 ## Summary
 
 - Vendor: Qwen

recipes/Qwen/Qwen3-TTS.md

@@ -1,4 +1,6 @@
-# Qwen3-TTS for text-to-speech
+# Qwen3-TTS
+
+> Text-to-speech serving (CustomVoice / VoiceDesign / Base)
 
 ## Summary
 

recipes/README.md

@@ -19,34 +19,29 @@ recipes/
   Qwen/
     Qwen3-Omni.md
     Qwen3-TTS.md
-  Tencent-Hunyuan/
-    HunyuanVideo.md
+  Tencent/
+    Covo-Audio-Chat.md
 ```
 
 ## Available Recipes
 
-- [`Qwen/Qwen-Image.md`](./Qwen/Qwen-Image.md): text-to-image serving recipe for
-  Qwen-Image on `1x A100 80GB`, including optional step-wise continuous batching replay
-- [`Qwen/Qwen3-Omni.md`](./Qwen/Qwen3-Omni.md): online serving recipe for
-  multimodal chat on `1x A100 80GB`
-- [`Tencent/Covo-Audio-Chat.md`](./Tencent/Covo-Audio-Chat.md): online
-  serving recipe for audio chat on `1x A100 80GB`
-- [`Qwen/Qwen3-TTS.md`](./Qwen/Qwen3-TTS.md): text-to-speech serving recipe
-  for Qwen3-TTS (CustomVoice / VoiceDesign / Base) on `1x H100/A100 80GB`
-- [`LTX/LTX-2.md`](./LTX/LTX-2.md): text-to-video and image-to-video serving
-  recipe for LTX-2 on `1x H200 141GB`
-- [`Wan-AI/Wan2.2-I2V.md`](./Wan-AI/Wan2.2-I2V.md): image-to-video serving
-  recipe for Wan2.2 14B on `8x Ascend NPU (A2/A3)`
-- [`Tencent-Hunyuan/HunyuanImage-3.0-Instruct.md`](./Tencent-Hunyuan/HunyuanImage-3.0-Instruct.md):
-  DiT-only text-to-image serving and benchmark recipe for HunyuanImage-3.0-Instruct
-  on `4x H100/H800 80GB`
-- [`inclusionAI/Ming-flash-omni-2.0.md`](./inclusionAI/Ming-flash-omni-2.0.md):
-  online serving recipe for multimodal chat (`4x H100 80GB`) and standalone TTS (`1x H100 80GB`)
-- [`Baidu/ERNIE-Image.md`](./Baidu/ERNIE-Image.md): text-to-image serving
-  online serving recipe for ERNIE-Image 8B on `1x RTX 4090 24GB` or `2x RTX 4090 24GB`
-- [`fishaudio/Fish-Speech-S2-Pro.md`](./fishaudio/Fish-Speech-S2-Pro.md): online serving recipe for TTS on `1x A800 80GB`
-- [`audiox/AudioX.md`](./audiox/AudioX.md): offline + online recipe for AudioX
-  unified text/video→audio diffusion on `1x L4 24GB`
+| Recipe | Task | Hardware |
+|--------|------|----------|
+| [`audiox/AudioX.md`](./audiox/AudioX.md) | Offline + online unified text/video→audio diffusion | 1x L4 24GB |
+| [`Baidu/ERNIE-Image.md`](./Baidu/ERNIE-Image.md) | Text-to-image online serving (ERNIE-Image 8B) | 1x or 2x RTX 4090 24GB |
+| [`fishaudio/Fish-Speech-S2-Pro.md`](./fishaudio/Fish-Speech-S2-Pro.md) | Online serving for TTS | 1x A800 80GB |
+| [`inclusionAI/Ming-flash-omni-2.0.md`](./inclusionAI/Ming-flash-omni-2.0.md) | Online serving for multimodal chat + standalone TTS | 4x H100 / 1x H100 80GB |
+| [`LTX/LTX-2.md`](./LTX/LTX-2.md) | Text-to-video and image-to-video serving | 1x H200 141GB |
+| [`LTX/LTX-2.3.md`](./LTX/LTX-2.3.md) | Text-to-video with audio generation (22B) | 1x GPU (96GB VRAM) |
+| [`Qwen/Qwen-Image.md`](./Qwen/Qwen-Image.md) | Text-to-image serving with step-wise continuous batching replay | 1x A100 80GB |
+| [`Qwen/Qwen-Image-Edit.md`](./Qwen/Qwen-Image-Edit.md) | Text-guided single-image editing | 1x or 2x H200 141GB |
+| [`Qwen/Qwen3-Omni.md`](./Qwen/Qwen3-Omni.md) | Online serving for multimodal chat | 1x A100 80GB |
+| [`Qwen/Qwen3-TTS.md`](./Qwen/Qwen3-TTS.md) | Text-to-speech serving (CustomVoice / VoiceDesign / Base) | 1x H100/A100 80GB |
+| [`SenseNova/SenseNova-U1.md`](./SenseNova/SenseNova-U1.md) | Unified image generation and understanding | 1x H200 (144GB) |
+| [`Tencent/Covo-Audio-Chat.md`](./Tencent/Covo-Audio-Chat.md) | Online serving for audio chat | 1x A100 80GB |
+| [`Tencent/HunyuanImage-3.0-Instruct.md`](./Tencent/HunyuanImage-3.0-Instruct.md) | DiT-only text-to-image serving and benchmark | 4x H100/H800 80GB |
+| [`Wan-AI/Wan2.2-I2V.md`](./Wan-AI/Wan2.2-I2V.md) | Image-to-video serving (Wan2.2 14B) | 8x Ascend NPU (A2/A3) |
+| [`Wan-AI/Wan2.2-S2V.md`](./Wan-AI/Wan2.2-S2V.md) | Speech-to-video serving (Wan2.2 14B) | 2x A100/H100 80GB |
 
 Within a single recipe file, include different hardware support sections such
 as `GPU`, `ROCm`, and `NPU`, and add concrete tested configurations like

recipes/SenseNova/SenseNova-U1.md

@@ -1,4 +1,6 @@
-# SenseNova-U1 for unified image generation and understanding
+# SenseNova-U1
+
+> Unified image generation and understanding
 
 ## Summary
 

recipes/TEMPLATE.md

@@ -1,6 +1,6 @@
 # Recipe Title
 
-> Example: Qwen3-Omni for speech chat on 1x A100 80GB
+> Example: Qwen3-Omni for speech chat
 
 ## Summary
 

recipes/Tencent/Covo-Audio-Chat.md

@@ -1,4 +1,6 @@
-# Covo-Audio-Chat for audio chat on 1x A100 80GB
+# Covo-Audio-Chat
+
+> Online serving for audio chat
 
 ## Summary
 

recipes/Tencent/HunyuanImage-3.0-Instruct.md

@@ -1,7 +1,6 @@
-# HunyuanImage-3.0-Instruct DiT Image Generation on 4x GPU
+# HunyuanImage-3.0-Instruct
 
-> DiT-only text-to-image recipe for HunyuanImage-3.0-Instruct with FP8,
-> tensor parallelism, sequence parallelism, and CFG parallelism.
+> DiT-only text-to-image serving and benchmark
 
 ## Summary
 

recipes/Wan-AI/Wan2.2-I2V.md

@@ -1,4 +1,6 @@
-# Wan2.2 Image To Video
+# Wan2.2 Image-to-Video
+
+> Image-to-video serving (Wan2.2 14B)
 
 ## Summary
 

recipes/Wan-AI/Wan2.2-S2V.md

@@ -1,4 +1,6 @@
-# Wan2.2 Speech To Video
+# Wan2.2 Speech-to-Video
+
+> Speech-to-video serving (Wan2.2 14B)
 
 ## Summary
 

recipes/fishaudio/Fish-Speech-S2-Pro.md

@@ -1,5 +1,7 @@
 # Fish Speech S2 Pro
 
+> Online serving for TTS
+
 ## Summary
 
 - Vendor: FishAudio

recipes/inclusionAI/Ming-flash-omni-2.0.md

@@ -1,4 +1,6 @@
-# Ming-flash-omni 2.0 for omni-speech chat and standalone TTS
+# Ming-flash-omni 2.0
+
+> Online serving for multimodal chat + standalone TTS
 
 ## Summary
 

tests/benchmarks/metrics/test_metrics.py

@@ -63,5 +63,96 @@ def test_total_input_aggregated_from_output_prompt_len():
     )
 
 
+# ============================================================================
+# TTFT suppression for pure-audio (TTS) benchmarks
+# ============================================================================
+
+
+class _EmptyAwareTokenizer:
+    """Minimal tokenizer stub: token count == len(text), so '' -> 0 tokens.
+
+    Mirrors production where a TTS speech endpoint returns empty generated_text,
+    making total_output == 0 (the real CI path uses a real tokenizer, not None).
+    """
+
+    def __call__(self, text, add_special_tokens=False):
+        class _R:
+            pass
+
+        r = _R()
+        r.input_ids = [0] * len(text)
+        return r
+
+
+def _make_tts_output(prompt_len: int) -> MixRequestFuncOutput:
+    """Pure-TTS output: no text tokens, only audio. ttft is left unset (0.0)."""
+    output = MixRequestFuncOutput()
+    output.success = True
+    output.prompt_len = prompt_len
+    output.output_tokens = 0
+    output.generated_text = ""
+    output.ttft = 0.0
+    output.text_latency = 1.0
+    output.latency = 1.0
+    output.start_time = 0.0
+    output.itl = []
+    output.audio_ttfp = 0.05
+    output.audio_rtf = 0.2
+    output.audio_duration = 5.0
+    output.audio_frames = 120000
+    output.input_audio_duration = 0.0
+    output.error = ""
+    return output
+
+
+_TTS_PERCENTILE_METRICS = ["ttft", "e2el", "audio_rtf", "audio_ttfp", "audio_duration"]
+
+
+def test_tts_benchmark_omits_ttft(capsys):
+    """Pure-TTS run (total_output == 0) must not print a Time to First Token section."""
+    outputs = [_make_tts_output(100), _make_tts_output(120)]
+
+    calculate_metrics(
+        input_requests=[],
+        outputs=outputs,
+        dur_s=10.0,
+        tokenizer=_EmptyAwareTokenizer(),
+        selected_percentiles=[99.0],
+        goodput_config_dict={},
+        task_type=TaskType.GENERATION,
+        selected_percentile_metrics=_TTS_PERCENTILE_METRICS,
+        max_concurrency=None,
+        request_rate=float("inf"),
+        benchmark_duration=10.0,
+    )
+
+    out = capsys.readouterr().out
+    assert "Time to First Token" not in out, "TTS bench must not surface a meaningless TTFT"
+    assert "Time to First Packet" in out, "audio TTFP must still be reported"
+    assert "End-to-end Latency" in out, "e2el must still be reported"
+
+
+def test_text_benchmark_still_reports_ttft(capsys):
+    """Regression guard: real text generation (total_output > 0) keeps TTFT."""
+    outputs = [_make_output(100, output_tokens=10), _make_output(120, output_tokens=10)]
+
+    calculate_metrics(
+        input_requests=[],
+        outputs=outputs,
+        dur_s=10.0,
+        tokenizer=_EmptyAwareTokenizer(),
+        selected_percentiles=[99.0],
+        goodput_config_dict={},
+        task_type=TaskType.GENERATION,
+        selected_percentile_metrics=_TTS_PERCENTILE_METRICS,
+        max_concurrency=None,
+        request_rate=float("inf"),
+        benchmark_duration=10.0,
+    )
+
+    out = capsys.readouterr().out
+    assert "Time to First Token" in out, "text benchmarks must keep TTFT"
+
+
 if __name__ == "__main__":
     pytest.main([__file__, "-v", "-s"])

tests/benchmarks/test_diffusion_backends_metrics.py

@@ -1,6 +1,11 @@
 import pytest
 
-from benchmarks.diffusion.backends import RequestFuncInput, async_request_chat_completions
+from benchmarks.diffusion.backends import (
+    RequestFuncInput,
+    async_request_chat_completions,
+    endpoint_filename_token,
+    normalize_endpoint,
+)
 
 
 class _MockResponse:
@@ -29,6 +34,32 @@ class _MockSession:
         return _MockResponse(self._payload)
 
 
+@pytest.mark.core_model
+@pytest.mark.benchmark
+@pytest.mark.cpu
+def test_endpoint_normalization_accepts_optional_leading_slash():
+    assert normalize_endpoint("v1/videos") == "/v1/videos"
+    assert normalize_endpoint("/v1/videos") == "/v1/videos"
+    assert normalize_endpoint("v1/chat/completions") == "/v1/chat/completions"
+    assert normalize_endpoint("v1/images/generations") == "/v1/images/generations"
+
+
+@pytest.mark.core_model
+@pytest.mark.benchmark
+@pytest.mark.cpu
+def test_endpoint_normalization_accepts_legacy_backend_aliases():
+    assert normalize_endpoint("vllm-omni") == "/v1/chat/completions"
+    assert normalize_endpoint("openai") == "/v1/images/generations"
+
+
+@pytest.mark.core_model
+@pytest.mark.benchmark
+@pytest.mark.cpu
+def test_endpoint_filename_token_drops_leading_slash():
+    assert endpoint_filename_token("/v1/videos") == "v1_videos"
+    assert endpoint_filename_token("v1/chat/completions") == "v1_chat_completions"
+
+
 @pytest.mark.core_model
 @pytest.mark.benchmark
 @pytest.mark.cpu

tests/conftest.py

@@ -36,6 +36,7 @@ def pytest_terminal_summary(terminalreporter, exitstatus, config):
 _ASSERTION_EXPORT_NAMES = (
     "assert_audio_speech_response",
     "assert_diffusion_response",
+    "assert_http_error",
     "assert_image_diffusion_response",
     "assert_image_valid",
     "assert_omni_response",
@@ -54,6 +55,7 @@ _MEDIA_EXPORT_NAMES = (
 _STAGE_CONFIG_EXPORT_NAMES = ("modify_stage_config",)
 _RUNTIME_EXPORT_NAMES = (
     "DiffusionResponse",
+    "HttpResponse",
     "OmniResponse",
     "OmniRunner",
     "OmniRunnerHandler",

tests/core/sched/test_omni_scheduler_input_coordinator_cleanup.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+from vllm.v1.request import RequestStatus
+
+import vllm_omni.core.sched.omni_ar_scheduler as ar_sched_mod
+import vllm_omni.core.sched.omni_generation_scheduler as gen_sched_mod
+from vllm_omni.core.sched.omni_ar_scheduler import OmniARScheduler
+from vllm_omni.core.sched.omni_generation_scheduler import OmniGenerationScheduler
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+
+class FakeInputCoordinator:
+    def __init__(self) -> None:
+        self.freed: list[str] = []
+
+    def free_finished_request(self, request_id: str) -> None:
+        self.freed.append(request_id)
+
+
+class DummyRequest:
+    request_id = "req-free"
+    client_index = 0
+    additional_information = None
+
+    def is_finished(self) -> bool:
+        return True
+
+
+@pytest.mark.parametrize(
+    ("scheduler_cls", "scheduler_mod"),
+    [
+        (OmniARScheduler, ar_sched_mod),
+        (OmniGenerationScheduler, gen_sched_mod),
+    ],
+)
+def test_finish_requests_cleans_input_coordinator_for_finished_ids(
+    monkeypatch: pytest.MonkeyPatch,
+    scheduler_cls,
+    scheduler_mod,
+) -> None:
+    coordinator = FakeInputCoordinator()
+    scheduler = scheduler_cls.__new__(scheduler_cls)
+    scheduler.chunk_transfer_adapter = None
+    scheduler.input_coordinator = coordinator
+
+    def fake_finish_requests(self, request_ids, finished_status):
+        assert request_ids == ["req-a", "req-b"]
+        assert finished_status == RequestStatus.FINISHED_STOPPED
+        return [("req-a", 0), ("req-b", 1)]
+
+    monkeypatch.setattr(scheduler_mod.VLLMScheduler, "finish_requests", fake_finish_requests)
+
+    result = scheduler_cls.finish_requests(
+        scheduler,
+        ["req-a", "req-b"],
+        RequestStatus.FINISHED_STOPPED,
+    )
+
+    assert result == [("req-a", 0), ("req-b", 1)]
+    assert coordinator.freed == ["req-a", "req-b"]
+
+
+def test_ar_free_request_cleans_input_coordinator_on_normal_free() -> None:
+    coordinator = FakeInputCoordinator()
+    scheduler = OmniARScheduler.__new__(OmniARScheduler)
+    scheduler.input_coordinator = coordinator
+    scheduler._omits_kv_transfer_cache = {"req-free": True}
+    scheduler.encoder_cache_manager = SimpleNamespace(free=lambda request: None)
+    scheduler.finished_req_ids = set()
+    scheduler.finished_req_ids_dict = None
+    scheduler._new_prompt_len_snapshot = {"req-free": 3}
+    scheduler._connector_finished = lambda request: (False, None)
+    scheduler._should_transfer_kv_for_request = lambda request_id: False
+    scheduler._free_blocks = lambda request: None
+
+    result = OmniARScheduler._free_request(scheduler, DummyRequest())
+
+    assert result is None
+    assert coordinator.freed == ["req-free"]
+    assert scheduler._omits_kv_transfer_cache == {}
+    assert scheduler._new_prompt_len_snapshot == {}
+
+
+def test_generation_free_request_cleans_input_coordinator(monkeypatch: pytest.MonkeyPatch) -> None:
+    coordinator = FakeInputCoordinator()
+    scheduler = OmniGenerationScheduler.__new__(OmniGenerationScheduler)
+    scheduler.input_coordinator = coordinator
+
+    def fake_free_request(self, request, delay_free_blocks=False):
+        assert delay_free_blocks is True

tests/core/sched/test_omni_scheduling_coordinator.py

@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""Unit tests for OmniSchedulingCoordinator (formerly ChunkSchedulingCoordinator).
+"""Unit tests for OmniSchedulingCoordinator.
 
 These tests use mock request objects and mock queues.  They do not require
 GPU, vLLM runtime, or any connector.
@@ -11,10 +11,12 @@ from __future__ import annotations
 import unittest
 from types import SimpleNamespace
 
+import torch
+
 import vllm_omni.core.sched.omni_scheduling_coordinator as coord_mod
 from vllm_omni.core.sched.omni_scheduling_coordinator import (
-    ChunkSchedulingCoordinator,
     OmniSchedulingCoordinator,
+    uses_qwen3_omni_full_payload_input_coordinator,
 )
 
 # ------------------------------------------------------------------ #
@@ -89,11 +91,55 @@ class MockQueue:
 # ------------------------------------------------------------------ #
 
 
+class TestFullPayloadCoordinatorSelection(unittest.TestCase):
+    def test_qwen3_omni_talker_and_code2wav_use_full_payload_input_coordinator(self):
+        for model_stage in ("talker", "code2wav"):
+            model_config = SimpleNamespace(
+                stage_id=1,
+                async_chunk=False,
+                model_arch="Qwen3OmniMoeForConditionalGeneration",
+                model_stage=model_stage,
+            )
+
+            self.assertTrue(uses_qwen3_omni_full_payload_input_coordinator(model_config))
+
+    def test_async_chunk_and_non_qwen3_omni_do_not_use_full_payload_input_coordinator(self):
+        cases = [
+            SimpleNamespace(
+                stage_id=1,
+                async_chunk=True,
+                model_arch="Qwen3OmniMoeForConditionalGeneration",
+                model_stage="talker",
+            ),
+            SimpleNamespace(
+                stage_id=1,
+                async_chunk=False,
+                model_arch="Qwen3TTSForConditionalGeneration",
+                model_stage="code2wav",
+            ),
+            SimpleNamespace(
+                stage_id=1,
+                async_chunk=False,
+                model_arch="Qwen2_5OmniForConditionalGeneration",
+                model_stage="talker",
+            ),
+            SimpleNamespace(
+                stage_id=0,
+                async_chunk=False,
+                model_arch="Qwen3OmniMoeForConditionalGeneration",
+                model_stage="thinker",
+            ),
+        ]
+
+        for model_config in cases:
+            self.assertFalse(uses_qwen3_omni_full_payload_input_coordinator(model_config))
+
+
 class TestChunkCoordinatorStateTransition(unittest.TestCase):
     """Test 5: process_pending_chunks transitions WAITING_FOR_CHUNK → target."""
 
     def test_ready_request_transitions_to_waiting(self):
-        coord = ChunkSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1, async_chunk=True)
+        coord = OmniSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1, async_chunk=True)
 
         req = _make_request("r1", status=RequestStatus.WAITING_FOR_CHUNK)
         waiting = MockQueue([req])
@@ -110,7 +156,7 @@ class TestChunkCoordinatorStateTransition(unittest.TestCase):
         self.assertIn("r1", coord.requests_with_ready_chunks)
 
     def test_non_ready_stays_waiting_for_chunk(self):
-        coord = ChunkSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1, async_chunk=True)
+        coord = OmniSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1, async_chunk=True)
 
         req = _make_request("r1", status=RequestStatus.WAITING_FOR_CHUNK)
         waiting = MockQueue([req])
@@ -126,7 +172,7 @@ class TestChunkCoordinatorStateTransition(unittest.TestCase):
         self.assertEqual(req.status, RequestStatus.WAITING_FOR_CHUNK)
 
     def test_stage_0_is_noop(self):
-        coord = ChunkSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=0)
+        coord = OmniSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=0)
         req = _make_request("r1")
         waiting = MockQueue([req])
         running: list = []
@@ -144,7 +190,7 @@ class TestChunkCoordinatorRestoreQueues(unittest.TestCase):
     """Test 6: restore_queues returns waiting-for-chunk requests."""
 
     def test_restore(self):
-        coord = ChunkSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1)
+        coord = OmniSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1)
 
         r1 = _make_request("r1")
         r2 = _make_request("r2")
@@ -166,7 +212,7 @@ class TestChunkCoordinatorFinishedSignal(unittest.TestCase):
     """Test 8: chunk_finished_req_ids → finished_requests."""
 
     def test_finished_signal(self):
-        coord = ChunkSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1, async_chunk=True)
+        coord = OmniSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1, async_chunk=True)
 
         req = _make_request("r1", status=RequestStatus.WAITING_FOR_CHUNK)
         waiting = MockQueue([req])
@@ -187,7 +233,7 @@ class TestChunkCoordinatorUpdateRequestMetadata(unittest.TestCase):
 
     def test_ar_mode_no_longer_sets_additional_information(self):
         """AR mode only processes scheduling metadata, not full payloads."""
-        coord = ChunkSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1)
+        coord = OmniSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1)
 
         req = _make_request("r1")
         requests = {"r1": req}
@@ -204,10 +250,14 @@ class TestChunkCoordinatorUpdateRequestMetadata(unittest.TestCase):
         self.assertIsNone(getattr(req, "additional_information", None))
 
     def test_generation_mode(self):
-        coord = ChunkSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1)
+        coord = OmniSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1)
 
         req = _make_request("r1")
         req.prompt_token_ids = [0, 0, 0]
+        req.num_prompt_tokens = 3
+        req.num_computed_tokens = 3
+        req._all_token_ids = [0, 0, 0, 99]
+        req._output_token_ids = [99]
         requests = {"r1": req}
 
         request_metadata = {
@@ -220,16 +270,61 @@ class TestChunkCoordinatorUpdateRequestMetadata(unittest.TestCase):
         coord.update_request_metadata(requests, request_metadata, model_mode="generation")
 
         self.assertEqual(req.prompt_token_ids, [10, 20, 30])
+        self.assertEqual(req.num_prompt_tokens, 3)
         self.assertEqual(req.num_computed_tokens, 0)
+        self.assertEqual(req._all_token_ids, [10, 20, 30])
+        self.assertEqual(req._output_token_ids, [])
         self.assertIsNone(req.additional_information)
         self.assertEqual(req._omni_initial_model_buffer, {"meta": {"left_context_size": 25}})
 
+    def test_generation_mode_flattens_tensor_code_predictor_codes(self):
+        coord = OmniSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1)
+
+        req = _make_request("r1")
+        req.prompt_token_ids = [9]
+        req.num_prompt_tokens = 1
+        req._all_token_ids = [9, 8]
+        req._output_token_ids = [8]
+        requests = {"r1": req}
+
+        coord.update_request_metadata(
+            requests,
+            {"r1": {"code_predictor_codes": torch.tensor([[1, 2, 3]], dtype=torch.long)}},
+            model_mode="generation",
+        )
+
+        self.assertEqual(req.prompt_token_ids, [1, 2, 3])
+        self.assertEqual(req.num_prompt_tokens, 3)
+        self.assertEqual(req._all_token_ids, [1, 2, 3])
+        self.assertEqual(req._output_token_ids, [])
+
+    def test_generation_mode_flattens_nested_code_predictor_codes(self):
+        coord = OmniSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1)
+
+        req = _make_request("r1")
+        req.prompt_token_ids = [9]
+        req.num_prompt_tokens = 1
+        req._all_token_ids = [9, 8]
+        req._output_token_ids = [8]
+        requests = {"r1": req}
+
+        coord.update_request_metadata(
+            requests,
+            {"r1": {"code_predictor_codes": [[1, 2], [3, 4]]}},
+            model_mode="generation",
+        )
+
+        self.assertEqual(req.prompt_token_ids, [1, 2, 3, 4])
+        self.assertEqual(req.num_prompt_tokens, 4)
+        self.assertEqual(req._all_token_ids, [1, 2, 3, 4])
+        self.assertEqual(req._output_token_ids, [])
+
 
 class TestChunkCoordinatorPostprocess(unittest.TestCase):
     """Test postprocess_scheduler_output clears ready chunks."""
 
     def test_clear_ready(self):
-        coord = ChunkSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1)
+        coord = OmniSchedulingCoordinator(scheduler_max_num_seqs=10, stage_id=1)
         coord.requests_with_ready_chunks = {"r1", "r2"}
 
         new_req = SimpleNamespace(req_id="r1")
@@ -366,6 +461,33 @@ class TestWaitingForInputTransition(unittest.TestCase):
         self.assertEqual(len(coord.pending_input_registrations), 1)
         self.assertEqual(coord.pending_input_registrations[0].request_id, "r1")
 
+    def test_idle_cycles_retain_received_marker_before_request_appears(self):
+        coord = OmniSchedulingCoordinator(
+            scheduler_max_num_seqs=10,
+            stage_id=1,
+            async_chunk=False,
+        )
+        coord._full_payload_input_received.add("late")
+        coord.finished_requests.add("late")
+
+        waiting = MockQueue()
+        running: list = []
+
+        coord.process_pending_full_payload_inputs(waiting, running, stage_recv_req_ids=set())
+
+        self.assertIn("late", coord._full_payload_input_received)
+        self.assertIn("late", coord.finished_requests)
+
+        late_req = _make_request("late", status=RequestStatus.WAITING)
+        waiting.add_request(late_req)
+
+        coord.process_pending_full_payload_inputs(waiting, running, stage_recv_req_ids=set())
+
+        self.assertEqual(late_req.status, RequestStatus.WAITING)
+        self.assertEqual(coord.pending_input_registrations, [])
+        self.assertIn("late", coord._full_payload_input_received)
+        self.assertIn("late", coord.finished_requests)
+
 
 class TestTimeoutDetection(unittest.TestCase):
     """Regression tests for orphaned pending-recv timeout detection.
@@ -508,14 +630,18 @@ class TestTimeoutDetection(unittest.TestCase):
         self.assertEqual(len(coord._waiting_for_input), 0)
 
     def test_free_finished_request_clears_waiting_since(self):
-        """free_finished_request clears _waiting_since."""
+        """free_finished_request clears coordinator lifecycle markers."""
         coord = OmniSchedulingCoordinator(
             scheduler_max_num_seqs=10,
             stage_id=1,
         )
         coord._waiting_since["r1"] = 0.0
+        coord._full_payload_input_received.add("r1")
+        coord.finished_requests.add("r1")
         coord.free_finished_request("r1")
         self.assertNotIn("r1", coord._waiting_since)
+        self.assertNotIn("r1", coord._full_payload_input_received)
+        self.assertNotIn("r1", coord.finished_requests)
 
     def test_timeout_from_running_queue_full_lifecycle(self):
         """End-to-end: request from running → WAITING_FOR_CHUNK → restore →

tests/dfx/conftest.py

@@ -279,6 +279,10 @@ def create_benchmark_indices(
     return indices
 
 
+# Omni perf/stability: ``vllm bench serve --omni`` result template (used if bench exits without writing JSON).
+OMNI_RESULT_TEMPLATE_PATH = Path(__file__).resolve().parent / "perf" / "scripts" / "result_omni_template.json"
+
+
 def _safe_filename_token(value: Any | None, *, default: str = "na") -> str:
     """Make a single path segment safe for result filenames on common filesystems."""
     if value is None:
@@ -289,7 +293,7 @@ def _safe_filename_token(value: Any | None, *, default: str = "na") -> str:
     return s if s else default
 
 
-def _resolve_baseline_value(
+def resolve_baseline_value(
     baseline_raw: Any,
     *,
     sweep_index: int | None,
@@ -330,7 +334,7 @@ def _baseline_thresholds_for_step(
 ) -> dict[str, Any]:
     """Resolve baseline config to one threshold per metric for this iteration."""
     return {
-        metric_name: _resolve_baseline_value(
+        metric_name: resolve_baseline_value(
             baseline_raw,
             sweep_index=sweep_index,
             max_concurrency=max_concurrency,
@@ -354,7 +358,12 @@ def run_benchmark(
     random_input_len: Any | None = None,
     random_output_len: Any | None = None,
 ) -> dict[str, Any]:
-    """Run one ``vllm bench serve --omni`` iteration and return parsed metrics."""
+    """Run one ``vllm bench serve --omni`` iteration and return parsed metrics.
+
+    After ``vllm bench`` writes the JSON, ``result["baseline"]`` holds the resolved per-metric thresholds
+    (when ``baseline_config`` is provided). If the benchmark exits without writing a result file,
+    ``result_omni_template.json`` is used as a fallback.
+    """
     current_dt = datetime.now().strftime("%Y%m%d-%H%M%S")
     ri = _safe_filename_token(random_input_len)
     ro = _safe_filename_token(random_output_len)
@@ -403,8 +412,17 @@ def run_benchmark(
         result_dir = "./"
 
     result_path = os.path.join(result_dir, result_filename)
-    with open(result_path, encoding="utf-8") as f:
-        result = json.load(f)
+    if not os.path.exists(result_path):
+        with open(OMNI_RESULT_TEMPLATE_PATH, encoding="utf-8") as f:
+            template_result: dict[str, Any] = json.load(f)
+        Path(result_path).parent.mkdir(parents=True, exist_ok=True)
+        with open(result_path, "w", encoding="utf-8") as f:
+            json.dump(template_result, f, ensure_ascii=False, indent=2)
+        print(f"Benchmark result file not generated, fallback to template: {result_path}")
+        result = template_result
+    else:
+        with open(result_path, encoding="utf-8") as f:
+            result = json.load(f)
 
     if baseline_config:
         result["baseline"] = _baseline_thresholds_for_step(
@@ -432,3 +450,12 @@ def pytest_addoption(parser: pytest.Parser) -> None:
         default=None,
         help=("Path to benchmark config JSON. Example: --test-config-file tests/dfx/perf/tests/test_tts.json"),
     )
+    parser.addoption(
+        "--assert-baseline",
+        action="store_true",
+        default=False,
+        help=(
+            "When set, omni/diffusion perf runners compare metrics against the baseline block in the JSON config "
+            "(default: off)."
+        ),
+    )

tests/dfx/perf/scripts/diffusion_result_template.json

@@ -1,7 +1,7 @@
 [
   {
     "test_name": null,
-    "backend": null,
+    "endpoint": null,
     "timestamp": null,
     "server_params": {
       "model": null,
@@ -43,7 +43,7 @@
       "stage_durations_mean": {},
       "stage_durations_p50": {},
       "stage_durations_p99": {},
-      "backend": null,
+      "endpoint": null,
       "model": null,
       "dataset": null,
       "task": null
@@ -51,6 +51,7 @@
     "log_file": null,
     "Model": null,
     "Framework": null,
+    "API Endpoint": null,
     "Hardware": null,
     "Deployment": null,
     "Task": null,

tests/dfx/perf/scripts/run_benchmark.py

@@ -1,8 +1,6 @@
 import json
 import os
-import subprocess
 import threading
-from datetime import datetime
 from pathlib import Path
 from typing import Any
 
@@ -14,11 +12,17 @@ from tests.dfx.conftest import (
     create_unique_server_params,
     get_benchmark_params_for_server,
     load_configs,
+    resolve_baseline_value,
+    run_benchmark,
 )
 from tests.helpers.runtime import OmniServer
 
 pytestmark = [pytest.mark.full_model]
 
+# Compare metrics to each test JSON ``baseline`` block only when pytest is run with ``--assert-baseline``
+# (registered in ``tests/dfx/conftest.py``; default: off). ``run_benchmark`` and ``_resolve_baseline_value`` are
+# defined in the same module.
+
 
 os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
 
@@ -47,7 +51,6 @@ if CONFIG_FILE_PATH is None:
     CONFIG_FILE_PATH = _DEFAULT_CONFIG_FILE
 
 BENCHMARK_CONFIGS = load_configs(CONFIG_FILE_PATH)
-OMNI_RESULT_TEMPLATE_PATH = Path(__file__).parent / "result_omni_template.json"
 
 
 DEPLOY_CONFIGS_DIR = Path(__file__).parent.parent / "deploy"
@@ -88,101 +91,6 @@ def omni_server(request):
         print("OmniServer stopped")
 
 
-def _safe_filename_token(value: Any | None, *, default: str = "na") -> str:
-    """Make a single path segment safe for result filenames on common filesystems."""
-    if value is None:
-        return default
-    s = str(value).strip()
-    for bad in ("/", "\\", ":", "*", "?", '"', "<", ">", "|"):
-        s = s.replace(bad, "_")
-    return s if s else default
-
-
-def run_benchmark(
-    args: list,
-    test_name: str,
-    flow,
-    dataset_name: str,
-    num_prompt,
-    *,
-    baseline_config: dict[str, Any] | None = None,
-    sweep_index: int | None = None,
-    request_rate: Any | None = None,
-    max_concurrency: Any | None = None,
-    random_input_len: Any | None = None,
-    random_output_len: Any | None = None,
-) -> Any:
-    """Run a single benchmark iteration and return the parsed result JSON.
-
-    After ``vllm bench`` writes the JSON, ``result["baseline"]`` holds the same
-    per-metric resolved thresholds as ``assert_result`` (via ``_baseline_thresholds_for_step``).
-    When ``random_input_len`` / ``random_output_len`` are set, they are also written into the result JSON;
-    omitted keys when not configured.
-    """
-    current_dt = datetime.now().strftime("%Y%m%d-%H%M%S")
-    ri = _safe_filename_token(random_input_len)
-    ro = _safe_filename_token(random_output_len)
-    result_filename = f"result_{test_name}_{dataset_name}_{flow}_{num_prompt}_in{ri}_out{ro}_{current_dt}.json"
-    if "--result-filename" in args:
-        print(f"The result file will be overwritten by {result_filename}")
-    command = (
-        ["vllm", "bench", "serve", "--omni"]
-        + args
-        + [
-            "--save-result",
-            "--result-dir",
-            os.environ.get("BENCHMARK_DIR", "tests"),
-            "--result-filename",
-            result_filename,
-        ]
-    )
-    process = subprocess.Popen(
-        command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, bufsize=1, universal_newlines=True
-    )
-
-    for line in iter(process.stdout.readline, ""):
-        print(line, end=" ")
-
-    for line in iter(process.stderr.readline, ""):
-        print(line, end=" ")
-
-    if "--result-dir" in command:
-        index = command.index("--result-dir")
-        result_dir = command[index + 1]
-    else:
-        result_dir = "./"
-
-    result_path = os.path.join(result_dir, result_filename)
-    if not os.path.exists(result_path):
-        with open(OMNI_RESULT_TEMPLATE_PATH, encoding="utf-8") as f:
-            template_result: dict[str, Any] = json.load(f)
-        Path(result_path).parent.mkdir(parents=True, exist_ok=True)
-        with open(result_path, "w", encoding="utf-8") as f:
-            json.dump(template_result, f, ensure_ascii=False, indent=2)
-        print(f"Benchmark result file not generated, fallback to template: {result_path}")
-        result = template_result
-    else:
-        with open(result_path, encoding="utf-8") as f:
-            result = json.load(f)
-
-    if baseline_config:
-        result["baseline"] = _baseline_thresholds_for_step(
-            baseline_config,
-            sweep_index=sweep_index,
-            request_rate=request_rate,
-            max_concurrency=max_concurrency,
-        )
-    else:
-        result["baseline"] = {}
-    if random_input_len is not None:
-        result["random_input_len"] = random_input_len
-    if random_output_len is not None:
-        result["random_output_len"] = random_output_len
-    with open(result_path, "w", encoding="utf-8") as f:
-        json.dump(result, f, ensure_ascii=False, indent=2)
-    return result
-
-
 benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping)
 
 
@@ -212,76 +120,23 @@ def benchmark_params(request, omni_server):
     }
 
 
-def _resolve_baseline_value(
-    baseline_raw: Any,
-    *,
-    sweep_index: int | None,
-    max_concurrency: Any = None,
-    request_rate: Any = None,
-) -> Any:
-    """Pick the baseline threshold for this sweep step.
-
-    Supported shapes per metric:
-    - **Scalar** — same threshold for every concurrency / QPS.
-    - **List** — aligned with ``max_concurrency`` / ``request_rate`` sweep order; use ``sweep_index``.
-    - **Dict** — keyed by concurrency or rate, e.g. ``{"1": 500, "4": 800}`` (keys are strings in JSON).
-
-    For dict lookup, ``max_concurrency`` is preferred when both are set (concurrency sweep).
-    """
-    if baseline_raw is None:
-        # If no baseline is set, the maximum value will be used.
-        return 100000
-    if isinstance(baseline_raw, dict):
-        if max_concurrency is not None:
-            for key in (max_concurrency, str(max_concurrency)):
-                if key in baseline_raw:
-                    return baseline_raw[key]
-        if request_rate is not None:
-            for key in (request_rate, str(request_rate)):
-                if key in baseline_raw:
-                    return baseline_raw[key]
-        raise KeyError(
-            f"baseline dict has no key for max_concurrency={max_concurrency!r} "
-            f"or request_rate={request_rate!r}; keys={list(baseline_raw.keys())!r}"
-        )
-    if isinstance(baseline_raw, (list, tuple)):
-        return baseline_raw[sweep_index]
-    return baseline_raw
-
-
-def _baseline_thresholds_for_step(
-    baseline_data: dict[str, Any],
-    *,
-    sweep_index: int | None = None,
-    max_concurrency: Any = None,
-    request_rate: Any = None,
-) -> dict[str, Any]:
-    """Resolve ``test.json`` ``baseline`` block to one threshold per metric (same as ``assert_result``)."""
-    return {
-        metric_name: _resolve_baseline_value(
-            baseline_raw,
-            sweep_index=sweep_index,
-            max_concurrency=max_concurrency,
-            request_rate=request_rate,
-        )
-        for metric_name, baseline_raw in baseline_data.items()
-    }
-
-
 def assert_result(
     result,
     params,
     num_prompt,
     *,
+    assert_baseline: bool,
     sweep_index: int | None = None,
-    max_concurrency: Any = None,
-    request_rate: Any = None,
+    max_concurrency: Any | None = None,
+    request_rate: Any | None = None,
 ) -> None:
     assert result["completed"] == num_prompt, "Request failures exist"
+    if not assert_baseline:
+        return
     baseline_data = params.get("baseline", {})
     for metric_name, baseline_raw in baseline_data.items():
         current_value = result[metric_name]
-        baseline_value = _resolve_baseline_value(
+        baseline_value = resolve_baseline_value(
             baseline_raw,
             sweep_index=sweep_index,
             max_concurrency=max_concurrency,
@@ -300,7 +155,7 @@ def assert_result(
 @pytest.mark.benchmark
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 @pytest.mark.parametrize("benchmark_params", benchmark_indices, indirect=True)
-def test_performance_benchmark(omni_server, benchmark_params):
+def test_performance_benchmark(omni_server, benchmark_params, request):
     test_name = benchmark_params["test_name"]
     params = benchmark_params["params"]
     dataset_name = params.get("dataset_name", "")
@@ -312,6 +167,8 @@ def test_performance_benchmark(omni_server, benchmark_params):
     print(f"Running benchmark for model: {model}")
     print(f"Benchmark parameters: {benchmark_params}")
 
+    assert_baseline = request.config.getoption("--assert-baseline", default=False)
+
     def to_list(value, default=None):
         if value is None:
             return [] if default is None else [default]
@@ -350,7 +207,7 @@ def test_performance_benchmark(omni_server, benchmark_params):
         elif not isinstance(value, bool):
             args.extend([arg_name, str(value)])
 
-    # QPS test (sweep_index aligns with qps_list / num_prompt_list for this loop)
+    # QPS / request-rate sweep
     for i, (qps, num_prompt) in enumerate(zip(qps_list, num_prompt_list)):
         args = args + ["--request-rate", str(qps), "--num-prompts", str(num_prompt)]
         result = run_benchmark(
@@ -369,12 +226,13 @@ def test_performance_benchmark(omni_server, benchmark_params):
         assert_result(
             result,
             params,
-            num_prompt=num_prompt,
+            num_prompt,
+            assert_baseline=assert_baseline,
             sweep_index=i,
             request_rate=qps,
         )
 
-    # concurrency test (sweep_index aligns with max_concurrency_list for separate thresholds per concurrency)
+    # concurrency test
     for i, (concurrency, num_prompt) in enumerate(zip(max_concurrency_list, num_prompt_list)):
         args = args + ["--max-concurrency", str(concurrency), "--num-prompts", str(num_prompt), "--request-rate", "inf"]
         result = run_benchmark(
@@ -393,7 +251,8 @@ def test_performance_benchmark(omni_server, benchmark_params):
         assert_result(
             result,
             params,
-            num_prompt=num_prompt,
+            num_prompt,
+            assert_baseline=assert_baseline,
             sweep_index=i,
             max_concurrency=concurrency,
         )

tests/dfx/perf/scripts/run_diffusion_benchmark.py

@@ -5,16 +5,17 @@ This runner separates two concepts:
 
 1. ``server_type``: how the serving process is started.
    Currently only ``vllm-omni`` is supported here.
-2. ``benchmark_backend``: which serving API the benchmark client calls.
-   Examples: ``vllm-omni`` for ``/v1/chat/completions`` and ``v1/videos``
-   for async video jobs.
+2. ``benchmark_endpoint``: which serving API the benchmark client calls.
+   Examples: ``/v1/chat/completions`` and ``/v1/videos``.
 
 A config JSON file is REQUIRED via --test-config-file:
   pytest run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
 
+Optional: ``--assert-baseline`` compares metrics to the ``baseline`` block in each benchmark entry (default: off).
+
 All benchmark results for a session are consolidated into a single JSON file under
 BENCHMARK_RESULT_DIR (override via the DIFFUSION_BENCHMARK_DIR environment variable).
-Each entry in the file contains the test metadata (test_name, backend, benchmark_params,
+Each entry in the file contains the test metadata (test_name, endpoint, benchmark_params,
 timestamp) together with the raw metrics returned by the benchmark script.
 """
 
@@ -33,6 +34,8 @@ from typing import Any, cast
 import psutil
 import pytest
 
+from benchmarks.diffusion.backends import endpoint_filename_token, normalize_endpoint
+
 pytestmark = [pytest.mark.diffusion, pytest.mark.full_model]
 
 os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
@@ -400,7 +403,7 @@ def _unique_server_params(configs: list[dict[str, Any]]) -> list[dict[str, Any]]
                 "model": cfg["server_params"]["model"],
                 "serve_args_dict": serve_args_dict,
                 "serve_args": _build_serve_args(serve_args_dict),
-                "benchmark_backend": cfg.get("benchmark_backend"),
+                "benchmark_endpoint": cfg.get("benchmark_endpoint", cfg.get("benchmark_backend")),
                 "server_params": cfg["server_params"],
             }
         )
@@ -482,7 +485,7 @@ def run_benchmark(
     model: str,
     params: dict[str, Any],
     test_name: str,
-    backend: str = "vllm-omni",
+    endpoint: str = "/v1/chat/completions",
     server_cfg: dict[str, Any] | None = None,
     source_file: str = "",
 ) -> dict[str, Any]:
@@ -490,18 +493,19 @@ def run_benchmark(
 
     The raw metrics are written to a temporary file by the subprocess.  After
     the run completes the metrics are merged with full metadata (test_name,
-    backend, benchmark_params, timestamp, flat reporting fields) and appended
+    endpoint, benchmark_params, timestamp, flat reporting fields) and appended
     to the session-wide aggregated JSON file (AGGREGATED_RESULT_FILE).  The
     temporary file is removed afterwards.  Subprocess stdout/stderr are tee'd
     to a .log file under BENCHMARK_RESULT_DIR/logs/; its path is stored in
     the record.
     """
     timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
+    endpoint = normalize_endpoint(endpoint)
 
     log_dir = BENCHMARK_RESULT_DIR / "logs"
     log_dir.mkdir(parents=True, exist_ok=True)
-    backend_label = backend.replace("/", "_")
-    log_file = log_dir / f"{test_name}_{backend_label}_{timestamp}.log"
+    endpoint_label = endpoint_filename_token(endpoint)
+    log_file = log_dir / f"{test_name}_{endpoint_label}_{timestamp}.log"
 
     with tempfile.NamedTemporaryFile(mode="w", suffix=".json", prefix="diffusion_bench_tmp_", delete=False) as tmp:
         tmp_result_file = Path(tmp.name)
@@ -517,8 +521,8 @@ def run_benchmark(
         str(port),
         "--model",
         model,
-        "--backend",
-        backend,
+        "--endpoint",
+        endpoint,
         "--dataset",
         params.get("dataset", "random"),
         "--task",
@@ -545,7 +549,7 @@ def run_benchmark(
     # cause truncated or out-of-order log output when stdout is piped).
     cmd = [cmd[0], "-u"] + cmd[1:]
 
-    print(f"\nRunning benchmark (backend={backend}): {' '.join(cmd)}")
+    print(f"\nRunning benchmark (endpoint={endpoint}): {' '.join(cmd)}")
     print(f"  Log file: {log_file}")
 
     # Redirect stdout + stderr directly to the log file at the OS level
@@ -597,7 +601,7 @@ def run_benchmark(
 
     record: dict[str, Any] = {
         "test_name": test_name,
-        "backend": backend,
+        "endpoint": endpoint,
         "timestamp": timestamp,
         "server_params": server_cfg.get("server_params"),
         "benchmark_params": params,
@@ -605,7 +609,7 @@ def run_benchmark(
         "log_file": str(log_file),
         "Model": model,
         "Framework": server_type,
-        "API Backend": backend,
+        "API Endpoint": endpoint,
         "Hardware": "",
         "Deployment": "",
         "Task": params.get("task", "t2i"),
@@ -650,12 +654,14 @@ def run_benchmark(
 # ---------------------------------------------------------------------------
 
 
-def assert_result(result: dict[str, Any], params: dict[str, Any]) -> None:
-    """Assert that benchmark metrics satisfy the configured baselines."""
+def assert_result(result: dict[str, Any], params: dict[str, Any], *, assert_baseline: bool) -> None:
+    """Assert request completion; optionally assert metrics against ``baseline`` in *params*."""
     num_prompts = params.get("num-prompts", 10)
     completed = result.get("completed_requests", result.get("completed", 0))
     assert completed == num_prompts, f"Expected {num_prompts} completed requests, got {completed}"
 
+    if not assert_baseline:
+        return
     if params.get("skip-performance-assertion", False):
         print("Skipping performance assertions.")
         return
@@ -669,21 +675,21 @@ def assert_result(result: dict[str, Any], params: dict[str, Any]) -> None:
             assert current <= threshold, f"{metric}: {current:.4f} > baseline {threshold}"
 
 
-def _default_benchmark_backend_for_task(task: str) -> str:
-    """Return the default client-side benchmark backend for a diffusion task."""
+def _default_benchmark_endpoint_for_task(task: str) -> str:
+    """Return the default client-side benchmark endpoint for a diffusion task."""
     if task in {"t2v", "i2v", "ti2v"}:
-        return "v1/videos"
+        return "/v1/videos"
     if task in {"t2i", "i2i", "ti2i"}:
-        return "vllm-omni"
-    raise ValueError(f"Unsupported task for benchmark backend resolution: {task}")
+        return "/v1/chat/completions"
+    raise ValueError(f"Unsupported task for benchmark endpoint resolution: {task}")
 
 
-def _resolve_benchmark_backend(server_cfg: dict[str, Any], params: dict[str, Any]) -> str:
+def _resolve_benchmark_endpoint(server_cfg: dict[str, Any], params: dict[str, Any]) -> str:
     """Resolve which serving API the benchmark client should call."""
-    configured = server_cfg.get("benchmark_backend")
+    configured = server_cfg.get("benchmark_endpoint")
     if configured:
-        return cast(str, configured)
-    return _default_benchmark_backend_for_task(cast(str, params.get("task", "t2i")))
+        return normalize_endpoint(cast(str, configured))
+    return _default_benchmark_endpoint_for_task(cast(str, params.get("task", "t2i")))
 
 
 # ---------------------------------------------------------------------------
@@ -697,21 +703,19 @@ def _resolve_benchmark_backend(server_cfg: dict[str, Any], params: dict[str, Any
     indirect=True,
 )
 @pytest.mark.parametrize("benchmark_params", benchmark_indices, indirect=True)
-def test_diffusion_performance_benchmark(diffusion_server, benchmark_params):
-    """Run the diffusion performance benchmark and assert against baselines.
+def test_diffusion_performance_benchmark(diffusion_server, benchmark_params, request):
+    """Run the diffusion performance benchmark and verify request completion.
 
     One server is started per unique parallel configuration (module scope).
     For each server, all benchmark parameter sets defined in the config JSON
-    are executed sequentially; results are asserted against the baselines.
+    are executed sequentially; metrics are recorded to the aggregated result file.
 
-    Tracked metrics:
-        - throughput_qps          (higher is better)
-        - latency_p50, latency_p99 (lower is better)
+    Pass ``--assert-baseline`` to compare ``throughput_qps``, ``latency_mean``, etc. to the JSON ``baseline`` block.
     """
     test_name = benchmark_params["test_name"]
     params = benchmark_params["params"]
     server_cfg = getattr(diffusion_server, "server_cfg", {})
-    backend = _resolve_benchmark_backend(server_cfg, params)
+    endpoint = _resolve_benchmark_endpoint(server_cfg, params)
 
     result = run_benchmark(
         host=diffusion_server.host,
@@ -719,13 +723,13 @@ def test_diffusion_performance_benchmark(diffusion_server, benchmark_params):
         model=diffusion_server.model,
         params=params,
         test_name=test_name,
-        backend=backend,
+        endpoint=endpoint,
         server_cfg=server_cfg,
         source_file=cast(str, CONFIG_FILE_PATH),
     )
 
     print(f"\n{'=' * 60}")
-    print(f"Results for {test_name} (server={diffusion_server.server_type}, backend={backend}):")
+    print(f"Results for {test_name} (server={diffusion_server.server_type}, endpoint={endpoint}):")
     for key in (
         "throughput_qps",
         "latency_mean",
@@ -742,4 +746,5 @@ def test_diffusion_performance_benchmark(diffusion_server, benchmark_params):
     print(f"\n  Aggregated results: {AGGREGATED_RESULT_FILE}")
     print("=" * 60)
 
-    assert_result(result, params)
+    assert_baseline = request.config.getoption("--assert-baseline", default=False)
+    assert_result(result, params, assert_baseline=assert_baseline)

tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json

@@ -3,31 +3,67 @@
         "test_name": "test_qwen3_omni_chunk",
         "server_params": {
             "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
-            "extra_cli_args": ["--async-chunk"]
+            "extra_cli_args": [
+                "--async-chunk"
+            ]
         },
         "benchmark_params": [
             {
                 "dataset_name": "random",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [4, 16, 32, 64],
-                "max_concurrency": [1, 4, 8, 16],
+                "num_prompts": [
+                    4,
+                    16,
+                    32,
+                    64,
+                    128
+                ],
+                "max_concurrency": [
+                    1,
+                    4,
+                    8,
+                    16,
+                    32
+                ],
                 "random_input_len": 2500,
                 "random_output_len": 900,
                 "ignore_eos": true,
                 "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
                 "baseline": {
-                    "mean_ttft_ms": [1000, 3000, 5000, 7000],
-                    "mean_audio_ttfp_ms": [1000, 3000, 5000, 7000],
-                    "mean_audio_rtf": [0.2, 0.35, 0.6, 0.85]
+                    "mean_ttft_ms": [
+                        400,
+                        300,
+                        400,
+                        600,
+                        900
+                    ],
+                    "mean_e2el_ms": [
+                        26000,
+                        43000,
+                        52000,
+                        72000,
+                        109000
+                    ],
+                    "mean_audio_rtf": [
+                        0.15,
+                        0.30,
+                        0.35,
+                        0.50,
+                        0.75
+                    ]
                 }
             },
             {
                 "dataset_name": "random-mm",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [10],
-                "request_rate": [0.1],
+                "num_prompts": [
+                    10
+                ],
+                "request_rate": [
+                    0.1
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
@@ -42,17 +78,27 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
                 "baseline": {
-                    "mean_ttft_ms": [2000],
-                    "mean_audio_ttfp_ms": [2000],
-                    "mean_audio_rtf": [0.25]
+                    "mean_ttft_ms": [
+                        600
+                    ],
+                    "mean_e2el_ms": [
+                        6000
+                    ],
+                    "mean_audio_rtf": [
+                        0.20
+                    ]
                 }
             },
             {
                 "dataset_name": "random-mm",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [40],
-                "request_rate": [0.5],
+                "num_prompts": [
+                    40
+                ],
+                "request_rate": [
+                    0.5
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
@@ -69,17 +115,27 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
                 "baseline": {
-                    "mean_ttft_ms": [6000],
-                    "mean_audio_ttfp_ms": [6000],
-                    "mean_audio_rtf": [0.7]
+                    "mean_ttft_ms": [
+                        300
+                    ],
+                    "mean_e2el_ms": [
+                        7000
+                    ],
+                    "mean_audio_rtf": [
+                        0.25
+                    ]
                 }
             },
             {
                 "dataset_name": "random-mm",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [100],
-                "request_rate": [1.0],
+                "num_prompts": [
+                    100
+                ],
+                "request_rate": [
+                    1.0
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
@@ -98,40 +154,79 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
                 "baseline": {
-                    "mean_ttft_ms": [12000],
-                    "mean_audio_ttfp_ms": [12000],
-                    "mean_audio_rtf": [1.0]
+                    "mean_ttft_ms": [
+                        700
+                    ],
+                    "mean_e2el_ms": [
+                        11000
+                    ],
+                    "mean_audio_rtf": [
+                        0.35
+                    ]
                 }
             },
             {
                 "dataset_name": "random",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [4, 16, 32, 64, 128],
-                "max_concurrency": [1, 4, 8, 16, 32],
+                "num_prompts": [
+                    4,
+                    16,
+                    32,
+                    64,
+                    128
+                ],
+                "max_concurrency": [
+                    1,
+                    4,
+                    8,
+                    16,
+                    32
+                ],
                 "random_input_len": 2500,
                 "random_output_len": 900,
                 "ignore_eos": true,
                 "extra_body": {
-                    "modalities": ["text"]
+                    "modalities": [
+                        "text"
+                    ]
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el",
                 "baseline": {
-                    "mean_ttft_ms": [1000, 3000, 5000, 7000, 9000]
+                    "mean_ttft_ms": [
+                        100,
+                        300,
+                        500,
+                        800,
+                        1500
+                    ],
+                    "mean_e2el_ms": [
+                        5000,
+                        8000,
+                        10000,
+                        14000,
+                        20000
+                    ]
                 }
             },
             {
                 "dataset_name": "random-mm",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [10],
-                "request_rate": [0.1],
+                "num_prompts": [
+                    10
+                ],
+                "request_rate": [
+                    0.1
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
                 "ignore_eos": true,
                 "extra_body": {
-                    "modalities": ["text"]
+                    "modalities": [
+                        "text"
+                    ]
                 },
                 "random_mm_base_items_per_request": 1,
                 "random_mm_num_mm_items_range_ratio": 0.5,
@@ -143,21 +238,32 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el",
                 "baseline": {
-                    "mean_ttft_ms": [2000]
+                    "mean_ttft_ms": [
+                        500
+                    ],
+                    "mean_e2el_ms": [
+                        1000
+                    ]
                 }
             },
             {
                 "dataset_name": "random-mm",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [40],
-                "request_rate": [0.5],
+                "num_prompts": [
+                    40
+                ],
+                "request_rate": [
+                    0.5
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
                 "ignore_eos": true,
                 "extra_body": {
-                    "modalities": ["text"]
+                    "modalities": [
+                        "text"
+                    ]
                 },
                 "random_mm_base_items_per_request": 2,
                 "random_mm_num_mm_items_range_ratio": 0.5,
@@ -171,21 +277,32 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el",
                 "baseline": {
-                    "mean_ttft_ms": [6000]
+                    "mean_ttft_ms": [
+                        200
+                    ],
+                    "mean_e2el_ms": [
+                        800
+                    ]
                 }
             },
             {
                 "dataset_name": "random-mm",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [100],
-                "request_rate": [1.0],
+                "num_prompts": [
+                    100
+                ],
+                "request_rate": [
+                    1.0
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
                 "ignore_eos": true,
                 "extra_body": {
-                    "modalities": ["text"]
+                    "modalities": [
+                        "text"
+                    ]
                 },
                 "random_mm_base_items_per_request": 3,
                 "random_mm_num_mm_items_range_ratio": 0.5,
@@ -201,7 +318,12 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el",
                 "baseline": {
-                    "mean_ttft_ms": [6000]
+                    "mean_ttft_ms": [
+                        600
+                    ],
+                    "mean_e2el_ms": [
+                        1300
+                    ]
                 }
             }
         ]

tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json

@@ -3,31 +3,67 @@
         "test_name": "test_qwen3_omni",
         "server_params": {
             "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
-            "extra_cli_args": ["--no-async-chunk"]
+            "extra_cli_args": [
+                "--no-async-chunk"
+            ]
         },
         "benchmark_params": [
             {
                 "dataset_name": "random",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [4, 16, 32, 64, 128],
-                "max_concurrency": [1, 4, 8, 16, 32],
+                "num_prompts": [
+                    4,
+                    16,
+                    32,
+                    64,
+                    128
+                ],
+                "max_concurrency": [
+                    1,
+                    4,
+                    8,
+                    16,
+                    32
+                ],
                 "random_input_len": 2500,
                 "random_output_len": 900,
                 "ignore_eos": true,
                 "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
                 "baseline": {
-                    "mean_ttft_ms": [1000, 3000, 5000, 7000, 9000],
-                    "mean_audio_ttfp_ms": [30000, 60000, 90000, 120000, 150000],
-                    "mean_audio_rtf": [0.35, 0.45, 0.55, 0.65, 0.75]
+                    "mean_ttft_ms": [
+                        400,
+                        300,
+                        400,
+                        600,
+                        900
+                    ],
+                    "mean_e2el_ms": [
+                        32000,
+                        34000,
+                        45000,
+                        61000,
+                        90000
+                    ],
+                    "mean_audio_rtf": [
+                        0.20,
+                        0.25,
+                        0.30,
+                        0.45,
+                        0.60
+                    ]
                 }
             },
             {
                 "dataset_name": "random-mm",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [10],
-                "request_rate": [0.1],
+                "num_prompts": [
+                    10
+                ],
+                "request_rate": [
+                    0.1
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
@@ -42,17 +78,27 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
                 "baseline": {
-                    "mean_ttft_ms": [2000],
-                    "mean_audio_ttfp_ms": [10000],
-                    "mean_audio_rtf": [0.25]
+                    "mean_ttft_ms": [
+                        500
+                    ],
+                    "mean_e2el_ms": [
+                        6000
+                    ],
+                    "mean_audio_rtf": [
+                        0.20
+                    ]
                 }
             },
             {
                 "dataset_name": "random-mm",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [40],
-                "request_rate": [0.5],
+                "num_prompts": [
+                    40
+                ],
+                "request_rate": [
+                    0.5
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
@@ -69,17 +115,27 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
                 "baseline": {
-                    "mean_ttft_ms": [6000],
-                    "mean_audio_ttfp_ms": [15000],
-                    "mean_audio_rtf": [0.45]
+                    "mean_ttft_ms": [
+                        300
+                    ],
+                    "mean_e2el_ms": [
+                        7000
+                    ],
+                    "mean_audio_rtf": [
+                        0.20
+                    ]
                 }
             },
             {
                 "dataset_name": "random-mm",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [100],
-                "request_rate": [1.0],
+                "num_prompts": [
+                    100
+                ],
+                "request_rate": [
+                    1.0
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
@@ -98,9 +154,15 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
                 "baseline": {
-                    "mean_ttft_ms": [12000],
-                    "mean_audio_ttfp_ms": [18000],
-                    "mean_audio_rtf": [0.9]
+                    "mean_ttft_ms": [
+                        700
+                    ],
+                    "mean_e2el_ms": [
+                        9000
+                    ],
+                    "mean_audio_rtf": [
+                        0.30
+                    ]
                 }
             }
         ]

tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json

@@ -4,7 +4,9 @@
         "server_params": {
             "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
             "use_omni": false,
-            "extra_cli_args": ["--no-enable-prefix-caching"]
+            "extra_cli_args": [
+                "--no-enable-prefix-caching"
+            ]
         },
         "benchmark_params": [
             {
@@ -12,8 +14,20 @@
                 "backend": "vllm",
                 "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
                 "endpoint": "/v1/completions",
-                "num_prompts": [4, 16, 32, 64, 128],
-                "max_concurrency": [1, 4, 8, 16, 32],
+                "num_prompts": [
+                    4,
+                    16,
+                    32,
+                    64,
+                    128
+                ],
+                "max_concurrency": [
+                    1,
+                    4,
+                    8,
+                    16,
+                    32
+                ],
                 "random_input_len": 2500,
                 "random_output_len": 900,
                 "temperature": 0.4,
@@ -23,11 +37,26 @@
                 "repetition_penalty": 1.05,
                 "ignore_eos": true,
                 "extra_body": {
-                    "modalities": ["text"]
+                    "modalities": [
+                        "text"
+                    ]
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el",
                 "baseline": {
-                    "mean_ttft_ms": [1000, 3000, 5000, 7000, 9000]
+                    "mean_ttft_ms": [
+                        300,
+                        300,
+                        1000,
+                        600,
+                        800
+                    ],
+                    "mean_e2el_ms": [
+                        6000,
+                        8000,
+                        11000,
+                        13000,
+                        17000
+                    ]
                 }
             },
             {
@@ -35,8 +64,12 @@
                 "backend": "vllm",
                 "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
                 "endpoint": "/v1/completions",
-                "num_prompts": [10],
-                "request_rate": [0.1],
+                "num_prompts": [
+                    10
+                ],
+                "request_rate": [
+                    0.1
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
@@ -47,7 +80,9 @@
                 "repetition_penalty": 1.05,
                 "ignore_eos": true,
                 "extra_body": {
-                    "modalities": ["text"]
+                    "modalities": [
+                        "text"
+                    ]
                 },
                 "random_mm_base_items_per_request": 1,
                 "random_mm_num_mm_items_range_ratio": 0.5,
@@ -59,7 +94,12 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el",
                 "baseline": {
-                    "mean_ttft_ms": [2000]
+                    "mean_ttft_ms": [
+                        30
+                    ],
+                    "mean_e2el_ms": [
+                        540
+                    ]
                 }
             },
             {
@@ -67,8 +107,12 @@
                 "backend": "vllm",
                 "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
                 "endpoint": "/v1/completions",
-                "num_prompts": [40],
-                "request_rate": [0.5],
+                "num_prompts": [
+                    40
+                ],
+                "request_rate": [
+                    0.5
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
@@ -79,7 +123,9 @@
                 "repetition_penalty": 1.05,
                 "ignore_eos": true,
                 "extra_body": {
-                    "modalities": ["text"]
+                    "modalities": [
+                        "text"
+                    ]
                 },
                 "random_mm_base_items_per_request": 2,
                 "random_mm_num_mm_items_range_ratio": 0.5,
@@ -93,7 +139,12 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el",
                 "baseline": {
-                    "mean_ttft_ms": [6000]
+                    "mean_ttft_ms": [
+                        30
+                    ],
+                    "mean_e2el_ms": [
+                        600
+                    ]
                 }
             },
             {
@@ -101,8 +152,12 @@
                 "backend": "vllm",
                 "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
                 "endpoint": "/v1/completions",
-                "num_prompts": [100],
-                "request_rate": [1.0],
+                "num_prompts": [
+                    100
+                ],
+                "request_rate": [
+                    1.0
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
@@ -113,7 +168,9 @@
                 "repetition_penalty": 1.05,
                 "ignore_eos": true,
                 "extra_body": {
-                    "modalities": ["text"]
+                    "modalities": [
+                        "text"
+                    ]
                 },
                 "random_mm_base_items_per_request": 3,
                 "random_mm_num_mm_items_range_ratio": 0.5,
@@ -129,7 +186,12 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el",
                 "baseline": {
-                    "mean_ttft_ms": [6000]
+                    "mean_ttft_ms": [
+                        40
+                    ],
+                    "mean_e2el_ms": [
+                        700
+                    ]
                 }
             }
         ]

tests/dfx/perf/tests/test_qwen_image_edit_2509_vllm_omni.json

@@ -22,10 +22,10 @@
                 "num-input-images": 2,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.05,
-                    "latency_mean": 18,
-                    "peak_memory_mb_max": 78500,
-                    "peak_memory_mb_mean": 78500
+                    "throughput_qps": 0.06,
+                    "latency_mean": 20,
+                    "peak_memory_mb_max": 57000,
+                    "peak_memory_mb_mean": 57000
                 }
             },
             {
@@ -41,9 +41,9 @@
                 "enable-negative-prompt": true,
                 "baseline": {
                     "throughput_qps": 0.01,
-                    "latency_mean": 70,
-                    "peak_memory_mb_max": 81000,
-                    "peak_memory_mb_mean": 81000
+                    "latency_mean": 60,
+                    "peak_memory_mb_max": 67000,
+                    "peak_memory_mb_mean": 67000
                 }
             }
         ]
@@ -75,10 +75,10 @@
                 "num-input-images": 2,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.1,
-                    "latency_mean": 12,
-                    "peak_memory_mb_max": 69000,
-                    "peak_memory_mb_mean": 69000
+                    "throughput_qps": 0.15,
+                    "latency_mean": 10,
+                    "peak_memory_mb_max": 55000,
+                    "peak_memory_mb_mean": 55000
                 }
             },
             {
@@ -93,10 +93,10 @@
                 "num-input-images": 2,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.03,
-                    "latency_mean": 28,
-                    "peak_memory_mb_max": 69000,
-                    "peak_memory_mb_mean": 69000
+                    "throughput_qps": 0.05,
+                    "latency_mean": 20,
+                    "peak_memory_mb_max": 55000,
+                    "peak_memory_mb_mean": 55000
                 }
             }
         ]
@@ -138,10 +138,10 @@
                 "num-input-images": 2,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.10,
-                    "latency_mean": 12,
-                    "peak_memory_mb_max": 73000,
-                    "peak_memory_mb_mean": 73000
+                    "throughput_qps": 0.21,
+                    "latency_mean": 10,
+                    "peak_memory_mb_max": 57000,
+                    "peak_memory_mb_mean": 57000
                 }
             },
             {
@@ -156,10 +156,10 @@
                 "num-input-images": 2,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.05,
+                    "throughput_qps": 0.08,
                     "latency_mean": 20,
-                    "peak_memory_mb_max": 81000,
-                    "peak_memory_mb_mean": 81000
+                    "peak_memory_mb_max": 66000,
+                    "peak_memory_mb_mean": 66000
                 }
             }
         ]

tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json

@@ -21,10 +21,10 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.05,
-                    "latency_mean": 15.0,
-                    "peak_memory_mb_max": 72500,
-                    "peak_memory_mb_mean": 72500
+                    "throughput_qps": 0.1,
+                    "latency_mean": 10,
+                    "peak_memory_mb_max": 57000,
+                    "peak_memory_mb_mean": 57000
                 }
             },
             {
@@ -38,10 +38,10 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.01,
-                    "latency_mean": 65.6,
-                    "peak_memory_mb_max": 80777,
-                    "peak_memory_mb_mean": 80777
+                    "throughput_qps": 0.02,
+                    "latency_mean": 50,
+                    "peak_memory_mb_max": 65000,
+                    "peak_memory_mb_mean": 65000
                 }
             }
         ]
@@ -72,10 +72,10 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.10,
-                    "latency_mean": 7.2,
-                    "peak_memory_mb_max": 68100,
-                    "peak_memory_mb_mean": 68100
+                    "throughput_qps": 0.21,
+                    "latency_mean": 10,
+                    "peak_memory_mb_max": 56000,
+                    "peak_memory_mb_mean": 56000
                 }
             },
             {
@@ -89,10 +89,10 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.03,
-                    "latency_mean": 24.0,
-                    "peak_memory_mb_max": 68100,
-                    "peak_memory_mb_mean": 68100
+                    "throughput_qps": 0.06,
+                    "latency_mean": 20,
+                    "peak_memory_mb_max": 56000,
+                    "peak_memory_mb_mean": 56000
                 }
             }
         ]
@@ -133,10 +133,10 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.1,
-                    "latency_mean": 6.5,
-                    "peak_memory_mb_max": 72600,
-                    "peak_memory_mb_mean": 72600
+                    "throughput_qps": 0.26,
+                    "latency_mean": 10,
+                    "peak_memory_mb_max": 58000,
+                    "peak_memory_mb_mean": 58000
                 }
             },
             {
@@ -150,10 +150,10 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.05,
-                    "latency_mean": 16.0,
-                    "peak_memory_mb_max": 81000,
-                    "peak_memory_mb_mean": 81000
+                    "throughput_qps": 0.1,
+                    "latency_mean": 10,
+                    "peak_memory_mb_max": 67000,
+                    "peak_memory_mb_mean": 67000
                 }
             }
         ]

tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json

@@ -21,10 +21,10 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.02,
-                    "latency_mean": 40.0,
-                    "peak_memory_mb_max": 70000,
-                    "peak_memory_mb_mean": 70000
+                    "throughput_qps": 0.05,
+                    "latency_mean": 20,
+                    "peak_memory_mb_max": 63000,
+                    "peak_memory_mb_mean": 63000
                 }
             },
             {
@@ -38,10 +38,10 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.005,
-                    "latency_mean": 80.0,
-                    "peak_memory_mb_max": 70000,
-                    "peak_memory_mb_mean": 70000
+                    "throughput_qps": 0.03,
+                    "latency_mean": 30,
+                    "peak_memory_mb_max": 63000,
+                    "peak_memory_mb_mean": 63000
                 }
             }
         ]

tests/dfx/perf/tests/test_qwen_image_vllm_omni.json

@@ -21,9 +21,9 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.30,
-                    "latency_mean": 3.50,
-                    "peak_memory_mb_mean": 67000
+                    "throughput_qps": 0.35,
+                    "latency_mean": 3,
+                    "peak_memory_mb_mean": 56000
                 }
             },
             {
@@ -37,9 +37,9 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.037,
-                    "latency_mean": 27.0,
-                    "peak_memory_mb_mean": 74000
+                    "throughput_qps": 0.03,
+                    "latency_mean": 26,
+                    "peak_memory_mb_mean": 65000
                 }
             }
         ]
@@ -67,9 +67,9 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.30,
-                    "latency_mean": 3.50,
-                    "peak_memory_mb_mean": 67000
+                    "throughput_qps": 0.35,
+                    "latency_mean": 3,
+                    "peak_memory_mb_mean": 56000
                 }
             },
             {
@@ -83,9 +83,9 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.037,
-                    "latency_mean": 27.0,
-                    "peak_memory_mb_mean": 74000
+                    "throughput_qps": 0.03,
+                    "latency_mean": 26,
+                    "peak_memory_mb_mean": 65000
                 }
             }
         ]
@@ -116,9 +116,9 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.1,
-                    "latency_mean": 9.1,
-                    "peak_memory_mb_mean": 61000
+                    "throughput_qps": 0.11,
+                    "latency_mean": 9,
+                    "peak_memory_mb_mean": 55000
                 }
             }
         ]
@@ -159,9 +159,9 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.1,
-                    "latency_mean": 3.50,
-                    "peak_memory_mb_mean": 67000
+                    "throughput_qps": 0.46,
+                    "latency_mean": 3,
+                    "peak_memory_mb_mean": 56000
                 }
             },
             {
@@ -175,9 +175,9 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.1,
-                    "latency_mean": 6.15,
-                    "peak_memory_mb_mean": 74000
+                    "throughput_qps": 0.18,
+                    "latency_mean": 6,
+                    "peak_memory_mb_mean": 67000
                 }
             }
         ]

tests/dfx/perf/tests/test_tts.json

@@ -2,40 +2,73 @@
   {
     "test_name": "test_qwen3_tts_base",
     "server_params": {
-      "model": "Qwen/Qwen3-TTS-12Hz-1.7B-Base"
+      "model": "Qwen/Qwen3-TTS-12Hz-1.7B-Base",
+      "stage_overrides": {
+        "1": {
+          "devices": "1"
+        }
+      }
     },
     "benchmark_params": [
       {
         "task": "voice_clone",
         "eval_phase": "latency",
-        "enabled": false,
         "dataset_name": "seed-tts",
         "backend": "openai-audio-speech",
         "endpoint": "/v1/audio/speech",
-        "num_prompts": [20],
-        "max_concurrency": [1],
+        "dataset_path": "linyueqian/seed-tts-eval-subset",
+        "num_prompts": [
+          20
+        ],
+        "max_concurrency": [
+          1
+        ],
         "seed_tts_locale": "en",
         "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
         "baseline": {
-          "median_audio_ttfp_ms": [350],
-          "median_audio_rtf": [0.25]
+          "median_audio_ttfp_ms": [
+            3000
+          ],
+          "median_audio_rtf": [
+            0.6
+          ]
         }
       },
       {
         "task": "voice_clone",
         "eval_phase": "throughput",
-        "enabled": false,
         "dataset_name": "seed-tts",
         "backend": "openai-audio-speech",
         "endpoint": "/v1/audio/speech",
-        "num_prompts": [80],
-        "max_concurrency": [8],
+        "dataset_path": "linyueqian/seed-tts-eval-subset",
+        "num_prompts": [
+          80,
+          128,
+          128
+        ],
+        "max_concurrency": [
+          8,
+          16,
+          64
+        ],
         "seed_tts_locale": "en",
         "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
         "baseline": {
-          "median_audio_ttfp_ms": [3500],
-          "median_audio_rtf": [0.75],
-          "audio_throughput": [10.0]
+          "median_audio_ttfp_ms": [
+            1000,
+            3000,
+            14000
+          ],
+          "median_audio_rtf": [
+            0.70,
+            1.15,
+            3.90
+          ],
+          "audio_throughput": [
+            12,
+            14,
+            14
+          ]
         }
       },
       {
@@ -45,13 +78,20 @@
         "dataset_name": "seed-tts",
         "backend": "openai-audio-speech",
         "endpoint": "/v1/audio/speech",
-        "num_prompts": [200],
-        "max_concurrency": [4],
+        "dataset_path": "linyueqian/seed-tts-eval-subset",
+        "num_prompts": [
+          200
+        ],
+        "max_concurrency": [
+          4
+        ],
         "seed_tts_locale": "en",
         "seed_tts_wer_eval": true,
         "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
         "baseline": {
-          "mean_audio_rtf": [0.45]
+          "mean_audio_rtf": [
+            0.45
+          ]
         }
       }
     ]
@@ -59,7 +99,12 @@
   {
     "test_name": "test_qwen3_tts_customvoice",
     "server_params": {
-      "model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"
+      "model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
+      "stage_overrides": {
+        "1": {
+          "devices": "1"
+        }
+      }
     },
     "benchmark_params": [
       {
@@ -69,14 +114,26 @@
         "backend": "openai-audio-speech",
         "endpoint": "/v1/audio/speech",
         "dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
-        "num_prompts": [20],
-        "max_concurrency": [1],
+        "num_prompts": [
+          20
+        ],
+        "max_concurrency": [
+          1
+        ],
         "seed_tts_locale": "en",
-        "extra_body": {"voice": "Vivian", "language": "English", "task_type": "CustomVoice"},
+        "extra_body": {
+          "voice": "Vivian",
+          "language": "English",
+          "task_type": "CustomVoice"
+        },
         "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
         "baseline": {
-          "median_audio_ttfp_ms": [150],
-          "median_audio_rtf": [0.15]
+          "median_audio_ttfp_ms": [
+            2000
+          ],
+          "median_audio_rtf": [
+            0.40
+          ]
         }
       },
       {
@@ -86,15 +143,39 @@
         "backend": "openai-audio-speech",
         "endpoint": "/v1/audio/speech",
         "dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
-        "num_prompts": [80],
-        "max_concurrency": [8],
+        "num_prompts": [
+          80,
+          128,
+          128
+        ],
+        "max_concurrency": [
+          8,
+          16,
+          64
+        ],
         "seed_tts_locale": "en",
-        "extra_body": {"voice": "Vivian", "language": "English", "task_type": "CustomVoice"},
+        "extra_body": {
+          "voice": "Vivian",
+          "language": "English",
+          "task_type": "CustomVoice"
+        },
         "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
         "baseline": {
-          "median_audio_ttfp_ms": [1500],
-          "median_audio_rtf": [0.30],
-          "audio_throughput": [30.0]
+          "median_audio_ttfp_ms": [
+            300,
+            1000,
+            5000
+          ],
+          "median_audio_rtf": [
+            0.25,
+            0.35,
+            1.10
+          ],
+          "audio_throughput": [
+            30,
+            40,
+            40
+          ]
         }
       },
       {
@@ -105,14 +186,24 @@
         "backend": "openai-audio-speech",
         "endpoint": "/v1/audio/speech",
         "dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
-        "num_prompts": [200],
-        "max_concurrency": [4],
+        "num_prompts": [
+          200
+        ],
+        "max_concurrency": [
+          4
+        ],
         "seed_tts_locale": "en",
-        "extra_body": {"voice": "Vivian", "language": "English", "task_type": "CustomVoice"},
+        "extra_body": {
+          "voice": "Vivian",
+          "language": "English",
+          "task_type": "CustomVoice"
+        },
         "seed_tts_wer_eval": true,
         "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
         "baseline": {
-          "mean_audio_rtf": [0.35]
+          "mean_audio_rtf": [
+            0.35
+          ]
         }
       },
       {
@@ -122,15 +213,29 @@
         "backend": "openai-audio-speech",
         "endpoint": "/v1/audio/speech",
         "dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
-        "num_prompts": [100],
-        "request_rate": [2.0],
+        "num_prompts": [
+          100
+        ],
+        "request_rate": [
+          2.0
+        ],
         "seed_tts_locale": "en",
-        "extra_body": {"voice": "Vivian", "language": "English", "task_type": "CustomVoice"},
+        "extra_body": {
+          "voice": "Vivian",
+          "language": "English",
+          "task_type": "CustomVoice"
+        },
         "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
         "baseline": {
-          "median_audio_ttfp_ms": [3000],
-          "median_audio_rtf": [0.25],
-          "audio_throughput": [4.0]
+          "median_audio_ttfp_ms": [
+            100
+          ],
+          "median_audio_rtf": [
+            0.20
+          ],
+          "audio_throughput": [
+            10
+          ]
         }
       },
       {
@@ -140,14 +245,25 @@
         "backend": "openai-audio-speech",
         "endpoint": "/v1/audio/speech",
         "dataset_path": "benchmarks/build_dataset/seed_tts_design",
-        "num_prompts": [20],
-        "max_concurrency": [1],
+        "num_prompts": [
+          20
+        ],
+        "max_concurrency": [
+          1
+        ],
         "seed_tts_locale": "en",
-        "extra_body": {"task_type": "VoiceDesign", "language": "English"},
+        "extra_body": {
+          "task_type": "VoiceDesign",
+          "language": "English"
+        },
         "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
         "baseline": {
-          "median_audio_ttfp_ms": [150],
-          "median_audio_rtf": [0.15]
+          "median_audio_ttfp_ms": [
+            100
+          ],
+          "median_audio_rtf": [
+            0.15
+          ]
         }
       },
       {
@@ -157,15 +273,38 @@
         "backend": "openai-audio-speech",
         "endpoint": "/v1/audio/speech",
         "dataset_path": "benchmarks/build_dataset/seed_tts_design",
-        "num_prompts": [80],
-        "max_concurrency": [8],
+        "num_prompts": [
+          80,
+          128,
+          128
+        ],
+        "max_concurrency": [
+          8,
+          16,
+          64
+        ],
         "seed_tts_locale": "en",
-        "extra_body": {"task_type": "VoiceDesign", "language": "English"},
+        "extra_body": {
+          "task_type": "VoiceDesign",
+          "language": "English"
+        },
         "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
         "baseline": {
-          "median_audio_ttfp_ms": [1500],
-          "median_audio_rtf": [0.35],
-          "audio_throughput": [25.0]
+          "median_audio_ttfp_ms": [
+            100,
+            1000,
+            5000
+          ],
+          "median_audio_rtf": [
+            0.25,
+            0.40,
+            1.20
+          ],
+          "audio_throughput": [
+            30,
+            40,
+            40
+          ]
         }
       },
       {
@@ -175,15 +314,28 @@
         "backend": "openai-audio-speech",
         "endpoint": "/v1/audio/speech",
         "dataset_path": "benchmarks/build_dataset/seed_tts_design",
-        "num_prompts": [100],
-        "request_rate": [2.0],
+        "num_prompts": [
+          100
+        ],
+        "request_rate": [
+          2.0
+        ],
         "seed_tts_locale": "en",
-        "extra_body": {"task_type": "VoiceDesign", "language": "English"},
+        "extra_body": {
+          "task_type": "VoiceDesign",
+          "language": "English"
+        },
         "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
         "baseline": {
-          "median_audio_ttfp_ms": [3500],
-          "median_audio_rtf": [0.30],
-          "audio_throughput": [4.0]
+          "median_audio_ttfp_ms": [
+            100
+          ],
+          "median_audio_rtf": [
+            0.20
+          ],
+          "audio_throughput": [
+            9
+          ]
         }
       }
     ]

tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json

@@ -30,9 +30,9 @@
                     }
                 ],
                 "baseline": {
-                    "throughput_qps": 0.034,
-                    "latency_mean": 26.0,
-                    "peak_memory_mb_mean": 80000
+                    "throughput_qps": 0.04,
+                    "latency_mean": 30,
+                    "peak_memory_mb_mean": 75000
                 }
             }
         ]
@@ -72,9 +72,9 @@
                     }
                 ],
                 "baseline": {
-                    "throughput_qps": 0.042,
-                    "latency_mean": 21.6,
-                    "peak_memory_mb_mean": 55300
+                    "throughput_qps": 0.06,
+                    "latency_mean": 20,
+                    "peak_memory_mb_mean": 48000
                 }
             },
             {
@@ -97,9 +97,9 @@
                     }
                 ],
                 "baseline": {
-                    "throughput_qps": 0.0085,
-                    "latency_mean": 101.6,
-                    "peak_memory_mb_mean": 65200
+                    "throughput_qps": 0.01,
+                    "latency_mean": 90,
+                    "peak_memory_mb_mean": 56000
                 }
             }
         ]

tests/dfx/reliability/invalid_param_test/conftest.py

@@ -0,0 +1,23 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Shared env for ``http_invalid`` (real :func:`omni_server`)."""
+
+from __future__ import annotations
+
+import io
+import os
+
+import pytest
+from PIL import Image
+
+# Match ``tests/e2e/online_serving/*`` module-level env for subprocess serve.
+os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
+os.environ.setdefault("VLLM_TEST_CLEAN_GPU_MEMORY", "0")
+
+
+@pytest.fixture
+def tiny_png_bytes() -> bytes:
+    img = Image.new("RGB", (32, 32), color="gray")
+    buf = io.BytesIO()
+    img.save(buf, format="PNG")
+    return buf.getvalue()

tests/dfx/reliability/invalid_param_test/test_invalid_audio_diffusion.py

@@ -0,0 +1,118 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""``POST /v1/audio/generate`` validation (live Stable Audio Open server)."""
+
+from __future__ import annotations
+
+import pytest
+
+from tests.helpers.mark import hardware_marks
+from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
+
+pytestmark = [pytest.mark.slow, pytest.mark.diffusion]
+
+# https://github.com/vllm-project/vllm-omni/issues/3649 — invalid inputs still HTTP 200 / timeout / missing validation.
+_SKIP_ISSUE_3649 = pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/3649")
+
+_PARAMS = [
+    pytest.param(
+        OmniServerParams(
+            model="stabilityai/stable-audio-open-1.0",
+            server_args=[
+                "--trust-remote-code",
+                "--enforce-eager",
+                "--model-class-name",
+                "StableAudioPipeline",
+            ],
+        ),
+        id="stable_audio_open",
+        marks=hardware_marks(res={"cuda": "L4"}),
+    ),
+]
+
+
+@pytest.mark.parametrize(
+    "overrides, err_message",
+    [
+        pytest.param({"input": ""}, ("cannot be empty", "input"), id="input_empty", marks=_SKIP_ISSUE_3649),
+        pytest.param(
+            {"input": "   "}, ("cannot be empty", "input"), id="input_whitespace_only", marks=_SKIP_ISSUE_3649
+        ),
+        pytest.param({"input": 123}, ("input", "a valid string"), id="input_wrong_type"),
+        pytest.param(
+            {"response_format": "mpeg"},
+            ("response_format", "literal_error", "Input should be"),
+            id="response_format_invalid",
+        ),
+        pytest.param({"speed": 0.1}, ("speed", "greater_than_equal", "0.25"), id="speed_too_low"),
+        pytest.param({"speed": 99.0}, ("speed", "less_than_equal", "4"), id="speed_too_high"),
+        pytest.param({"stream_format": 1}, ("stream_format", "literal_error", "audio"), id="stream_format_wrong_type"),
+        pytest.param(
+            {"stream_format": "sse"},
+            ("sse", "stream_format", "value_error", "not a supported stream_format"),
+            id="stream_format_sse_blocked",
+        ),
+        pytest.param(
+            {"audio_length": -1.0}, ("audio_end_in_s", "higher than", "audio_start_in_s"), id="audio_length_negative"
+        ),
+        pytest.param({"audio_length": 0}, "audio_length", id="audio_length_zero", marks=_SKIP_ISSUE_3649),
+        # ``OmniOpenAIServingAudioGenerate`` only forwards ``audio_start`` when ``audio_length`` is set.
+        pytest.param(
+            {"audio_start": -0.5, "audio_length": 5.0},
+            ("audio_start_in_s", "between", "0", "512"),
+            id="audio_start_negative",
+        ),
+        pytest.param(
+            {"negative_prompt": ["noise"]}, ("negative_prompt", "a valid string"), id="negative_prompt_wrong_type"
+        ),
+        pytest.param({"guidance_scale": -1.0}, "guidance_scale", id="guidance_scale_negative", marks=_SKIP_ISSUE_3649),
+        pytest.param({"guidance_scale": 0}, "guidance_scale", id="guidance_scale_zero", marks=_SKIP_ISSUE_3649),
+        pytest.param(
+            {"num_inference_steps": 0}, "num_inference_steps", id="num_inference_steps_zero", marks=_SKIP_ISSUE_3649
+        ),
+        pytest.param(
+            {"num_inference_steps": -1}, ("number of steps", "non-negative"), id="num_inference_steps_negative"
+        ),
+        pytest.param(
+            {"num_inference_steps": 6000},
+            "num_inference_steps",
+            id="num_inference_steps_above_max",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            {"guidance_scale": 1001.0}, "guidance_scale", id="guidance_scale_above_max", marks=_SKIP_ISSUE_3649
+        ),
+        pytest.param(
+            {"audio_length": 86401.0}, ("Requested audio length", "exceeds maximum"), id="audio_length_above_max"
+        ),
+        pytest.param(
+            {"audio_start": 86401.0, "audio_length": 1.0},
+            ("audio_start_in_s", "between", "0", "512"),
+            id="audio_start_above_max",
+        ),
+    ],
+)
+@pytest.mark.parametrize("omni_server", _PARAMS, indirect=True)
+def test_audio_generate_invalid_field_values(
+    omni_server: OmniServer,
+    openai_client: OpenAIClientHandler,
+    overrides: dict[str, object],
+    err_message: str | tuple[str, ...],
+) -> None:
+    body = {"model": omni_server.model, "input": "ambient electronic pad"}
+    body.update(overrides)
+    openai_client.send_audio_generate_http_request(
+        {"json": body, "timeout": 120, "err_code": 400, "err_message": err_message}
+    )
+
+
+@pytest.mark.parametrize("omni_server", _PARAMS, indirect=True)
+def test_audio_generate_missing_input(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
+    openai_client.send_audio_generate_http_request(
+        {
+            "json": {"model": omni_server.model},
+            "timeout": 120,
+            "err_code": 400,
+            "err_message": ("missing", "input", "field required"),
+        }
+    )

tests/dfx/reliability/invalid_param_test/test_invalid_audio_speech.py

@@ -0,0 +1,862 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""HTTP + WebSocket validation for Qwen3-TTS Base: ``/v1/audio/speech``, ``/v1/audio/speech/stream``, batch, voices."""
+
+from __future__ import annotations
+
+import io
+import json
+import uuid
+import wave
+from typing import Any
+
+import pytest
+
+from tests.helpers.mark import hardware_marks
+from tests.helpers.media import load_test_audio_data_url
+from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
+from tests.helpers.stage_config import get_deploy_config_path
+
+# Speech / voice-upload numeric caps for these tests only (not exported from vllm_omni).
+# Align where possible with serving_speech defaults: _TTS_MAX_INSTRUCTIONS_LENGTH (500),
+# _TTS_MAX_NEW_TOKENS_MAX (4096). Voice form max lengths are not public constants in prod;
+# values below keep tests self-contained and deterministic.
+_SPEECH_API_MAX_INSTRUCTIONS_CHARS = 500
+_SPEECH_API_MAX_NEW_TOKENS = 4096
+_VOICE_UPLOAD_MAX_CONSENT_LEN = 1024
+_VOICE_UPLOAD_MAX_REF_TEXT_CHARS = 8192
+_VOICE_UPLOAD_MAX_SPEAKER_DESCRIPTION_CHARS = 2048
+REF_AUDIO_URL = load_test_audio_data_url("qwen3_tts/clone_2.wav")
+REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."
+# Qwen3-TTS 0.6B expects 1024-dim speaker embeddings (see serving_speech / talker hidden_size).
+_QWEN3_TTS_06B_SPEAKER_EMBEDDING_DIM = 1024
+# Pop-only marker for ``test_speech_invalid_field_values``: omit ``ref_audio`` / ``ref_text`` so Base
+# ``speaker_embedding`` cases mirror standalone embedding-only JSON (no ref_audio clash).
+_SPEECH_INVALID_EMBEDDING_ONLY_BODY = "__speech_invalid_embedding_only_body__"
+# Batch-only: merged items inherit batch ``ref_audio`` / ``ref_text`` because ``_pick`` treats item ``None``
+# as "unset". Strip clone fields from the batch envelope before asserting CustomVoice preset errors.
+_SPEECH_BATCH_NO_CLONE_FIELDS = "__speech_batch_no_clone_fields__"
+
+_SKIP_ISSUE_3649 = pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/3649")
+
+
+pytestmark = [pytest.mark.slow, pytest.mark.tts]
+
+_L4_SPEECH_HW = hardware_marks(res={"cuda": "L4"})
+_SPEECH_SERVER_ARGS = ["--trust-remote-code", "--disable-log-stats"]
+
+_QWEN3_TTS_SPEECH = [
+    pytest.param(
+        OmniServerParams(
+            model="Qwen/Qwen3-TTS-12Hz-0.6B-Base",
+            stage_config_path=get_deploy_config_path("qwen3_tts.yaml"),
+            server_args=_SPEECH_SERVER_ARGS,
+        ),
+        id="qwen3_tts",
+        marks=_L4_SPEECH_HW,
+    ),
+]
+
+
+def _apply_batch_overrides(body: dict[str, object], *, loc: str, overrides: dict[str, object]) -> None:
+    if loc == "batch":
+        body.update(overrides)
+        return
+    items = body["items"]
+    assert isinstance(items, list) and len(items) >= 1
+    first = items[0]
+    assert isinstance(first, dict)
+    body["items"] = [{**first, **overrides}]
+
+
+def _pcm_wav_mono_bytes(*, duration_s: float = 1.1, sample_rate: int = 44100) -> bytes:
+    buf = io.BytesIO()
+    with wave.open(buf, "wb") as wf:
+        wf.setnchannels(1)
+        wf.setsampwidth(2)
+        wf.setframerate(sample_rate)
+        nframes = max(1, int(sample_rate * duration_s))
+        wf.writeframes(b"\x00\x00" * nframes)
+    return buf.getvalue()
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# POST /v1/audio/speech (JSON)
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
+def test_speech_malformed_json(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
+    openai_client.send_audio_speech_http_request(
+        {
+            "raw_body": "{",
+            "timeout": 120,
+            "err_code": 400,
+            "err_message": ("json", "decode", "invalid"),
+        }
+    )
+
+
+@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
+def test_speech_missing_required_fields(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
+    openai_client.send_audio_speech_http_request(
+        {
+            "json": {"model": omni_server.model},
+            "timeout": 120,
+            "err_code": 400,
+            "err_message": ("input", "field required", "missing"),
+        }
+    )
+
+
+@pytest.mark.parametrize(
+    "overrides, err_message",
+    [
+        pytest.param({"input": ""}, ("input", "empty"), id="input_empty"),
+        pytest.param({"input": "   "}, ("input", "empty"), id="input_whitespace_only"),
+        pytest.param({"voice": ""}, "Invalid voice", id="voice_empty", marks=_SKIP_ISSUE_3649),
+        pytest.param(
+            {"instructions": 123}, ("instructions", "string_type", "valid string"), id="instructions_wrong_type"
+        ),
+        pytest.param(
+            {"response_format": "mpeg"}, ("response_format", "literal_error", "wav"), id="response_format_invalid"
+        ),
+        pytest.param({"stream_format": 1}, ("stream_format", "literal_error", "audio"), id="stream_format_wrong_type"),
+        pytest.param(
+            {"stream_format": "sse"}, ("stream_format", "value_error", "audio"), id="stream_format_sse_blocked"
+        ),
+        pytest.param({"stream": "wrong_type"}, ("stream", "bool_parsing", "validation error"), id="stream_wrong_type"),
+        pytest.param(
+            {
+                "voice": "vivian",
+                "task_type": "CustomVoice",
+                "ref_audio": None,
+                "ref_text": None,
+            },
+            ("CustomVoice", "does not support"),
+            id="customvoice_task_not_supported",
+        ),
+        pytest.param(
+            {"task_type": "InvalidEnum"}, ("task_type", "literal_error", "CustomVoice"), id="task_type_invalid"
+        ),
+        pytest.param({"language": ""}, ("language", "invalid language", "chinese"), id="language_empty"),
+        pytest.param({"ref_audio": "ftp://example.com/a.wav"}, ("ref_audio", "URL"), id="ref_audio_bad_scheme"),
+        pytest.param(
+            {"ref_audio": "not_a_valid_uri"},
+            ("ref_audio", "url"),
+            id="ref_audio_invalid_uri",
+        ),
+        pytest.param({"ref_text": ["x"]}, ("ref_text", "string_type", "valid string"), id="ref_text_wrong_type"),
+        pytest.param(
+            {"x_vector_only_mode": "wrong"},
+            ("x_vector_only_mode", "valid boolean"),
+            id="x_vector_only_mode_string_not_boolean",
+        ),
+        pytest.param(
+            {"speaker_embedding": [0.01] * _QWEN3_TTS_06B_SPEAKER_EMBEDDING_DIM},
+            ("mutually exclusive", "speaker_embedding", "ref_audio"),
+            id="speaker_embedding_with_ref_audio_mutually_exclusive",
+        ),
+        pytest.param({"max_new_tokens": 0}, ("max_new_tokens", "least 1"), id="max_new_tokens_below_min"),
+        pytest.param(
+            {"max_new_tokens": _SPEECH_API_MAX_NEW_TOKENS + 1},
+            ("max_new_tokens", "exceed 4096"),
+            id="max_new_tokens_above_max",
+        ),
+        pytest.param({"seed": -1}, ("seed", "greater_than_equal", "0"), id="seed_negative"),
+        pytest.param({"seed": 2**63}, ("seed", "less_than_equal", "9223372036854775807"), id="seed_above_max"),
+        pytest.param(
+            {"initial_codec_chunk_frames": -1},
+            ("initial_codec_chunk_frames", "greater_than_equal", "0"),
+            id="initial_codec_chunk_frames_negative",
+        ),
+        pytest.param(
+            {"instructions": "x" * (_SPEECH_API_MAX_INSTRUCTIONS_CHARS + 1)},
+            ("instructions", "too long"),
+            id="instructions_exceed_api_limit",
+        ),
+        # Base clone: whitespace-only ref_text fails serving validation (not pydantic).
+        pytest.param(
+            {
+                "task_type": "Base",
+                "ref_audio": "data:audio/wav;base64,UklGRiQAAABXQVZFZm10IBAAAAABAAEARKwAAIhYAQACABAAZGF0YQAAAAA=",
+                "ref_text": "   ",
+            },
+            ("ref_text", "base task", "non-empty"),
+            id="ref_text_whitespace_only_base_clone",
+        ),
+        pytest.param(
+            {
+                _SPEECH_INVALID_EMBEDDING_ONLY_BODY: True,
+                "task_type": "Base",
+                "speaker_embedding": [],
+            },
+            ("speaker_embedding", "non-empty"),
+            id="base_speaker_embedding_empty",
+        ),
+        pytest.param(
+            {
+                _SPEECH_INVALID_EMBEDDING_ONLY_BODY: True,
+                "task_type": "Base",
+                "x_vector_only_mode": True,
+                "speaker_embedding": [0.01] * 512,
+            },
+            ("speaker_embedding", "dimensions", str(_QWEN3_TTS_06B_SPEAKER_EMBEDDING_DIM)),
+            id="base_speaker_embedding_wrong_dim_short",
+        ),
+        pytest.param(
+            {
+                _SPEECH_INVALID_EMBEDDING_ONLY_BODY: True,
+                "task_type": "Base",
+                "x_vector_only_mode": True,
+                "speaker_embedding": [0.01] * 2048,
+            },
+            ("speaker_embedding", "2048", "dimensions", str(_QWEN3_TTS_06B_SPEAKER_EMBEDDING_DIM)),
+            id="base_speaker_embedding_wrong_dim_long",
+        ),
+    ],
+)
+@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
+def test_speech_invalid_field_values(
+    omni_server: OmniServer,
+    openai_client: OpenAIClientHandler,
+    overrides: dict[str, object],
+    err_message: str | tuple[str, ...],
+) -> None:
+    ov = dict(overrides)
+    embedding_only = bool(ov.pop(_SPEECH_INVALID_EMBEDDING_ONLY_BODY, False))
+    body = {
+        "model": omni_server.model,
+        "input": "Hello.",
+        "voice": "clone",
+        "language": "English",
+        "response_format": "wav",
+        "ref_audio": REF_AUDIO_URL,
+        "ref_text": REF_TEXT,
+    }
+    if embedding_only:
+        body.pop("ref_audio", None)
+        body.pop("ref_text", None)
+    body.update(ov)
+    openai_client.send_audio_speech_http_request(
+        {"json": body, "timeout": 120, "err_code": 400, "err_message": err_message}
+    )
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# POST /v1/audio/speech/batch (JSON)
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
+def test_speech_batch_empty_items(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
+    openai_client.send_audio_speech_batch_http_request(
+        {"json": {"items": []}, "timeout": 120, "err_code": (400, 422), "err_message": ("items", "least 1 item")}
+    )
+
+
+@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
+def test_speech_batch_missing_items(omni_server: OmniServer, openai_client: OpenAIClientHandler) -> None:
+    openai_client.send_audio_speech_batch_http_request(
+        {
+            "json": {"model": omni_server.model},
+            "timeout": 120,
+            "err_code": (400, 422),
+            "err_message": ("items", "field required", "missing"),
+        }
+    )
+
+
+# Align field-level checks with ``POST /v1/audio/speech`` where ``BatchSpeechRequest`` /
+# ``SpeechBatchItem`` expose the same knobs. Not on batch schema (single-speech only):
+# ``stream``, ``stream_format``, ``seed``, ``speaker_embedding`` — see dedicated speech tests.
+@pytest.mark.parametrize(
+    "loc, overrides, err_message",
+    [
+        pytest.param("item", {"input": ""}, ("input", "empty"), id="item_input_empty", marks=_SKIP_ISSUE_3649),
+        pytest.param(
+            "item",
+            {"input": "   "},
+            ("input", "empty"),
+            id="item_input_whitespace_only",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param("batch", {"voice": ""}, "Invalid voice", id="batch_voice_empty", marks=_SKIP_ISSUE_3649),
+        pytest.param("item", {"voice": ""}, "Invalid voice", id="item_voice_empty", marks=_SKIP_ISSUE_3649),
+        pytest.param(
+            "batch",
+            {
+                "voice": "nonexistent_voice_xyz",
+                "task_type": "CustomVoice",
+                "ref_audio": None,
+                "ref_text": None,
+            },
+            "Invalid voice",
+            id="batch_voice_unknown_preset",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "item",
+            {
+                _SPEECH_BATCH_NO_CLONE_FIELDS: True,
+                "voice": "nonexistent_voice_xyz",
+                "task_type": "CustomVoice",
+            },
+            "Invalid voice",
+            id="item_voice_unknown_preset",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "batch",
+            {"instructions": 123},
+            ("instructions", "string_type", "valid string"),
+            id="batch_instructions_wrong_type",
+        ),
+        pytest.param(
+            "item",
+            {"instructions": 123},
+            ("instructions", "string_type", "valid string"),
+            id="item_instructions_wrong_type",
+        ),
+        pytest.param(
+            "batch",
+            {"instructions": "x" * (_SPEECH_API_MAX_INSTRUCTIONS_CHARS + 1)},
+            ("instructions", "too long"),
+            id="batch_instructions_exceed_api_limit",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "item",
+            {"instructions": "x" * (_SPEECH_API_MAX_INSTRUCTIONS_CHARS + 1)},
+            ("instructions", "too long"),
+            id="item_instructions_exceed_api_limit",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "batch",
+            {"response_format": "mpeg"},
+            ("response_format", "literal_error", "wav"),
+            id="batch_response_format_invalid",
+        ),
+        pytest.param(
+            "item",
+            {"response_format": "mpeg"},
+            ("response_format", "literal_error", "wav"),
+            id="item_response_format_invalid",
+        ),
+        pytest.param("batch", {"speed": 0.1}, ("speed", "greater_than_equal", "0.25"), id="batch_speed_too_low"),
+        pytest.param("item", {"speed": 10.0}, ("speed", "less_than_equal", "4"), id="item_speed_too_high"),
+        pytest.param(
+            "batch",
+            {"task_type": "InvalidEnum"},
+            ("task_type", "literal_error", "CustomVoice"),
+            id="batch_task_type_invalid",
+        ),
+        pytest.param(
+            "item",
+            {"task_type": "InvalidEnum"},
+            ("task_type", "literal_error", "CustomVoice"),
+            id="item_task_type_invalid",
+        ),
+        pytest.param(
+            "batch",
+            {"language": ""},
+            ("language", "invalid language", "chinese"),
+            id="batch_language_empty",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "item",
+            {"language": ""},
+            ("language", "invalid language", "chinese"),
+            id="item_language_empty",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "batch",
+            {"ref_audio": "ftp://example.com/a.wav"},
+            ("ref_audio", "URL"),
+            id="batch_ref_audio_bad_scheme",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "item",
+            {"ref_audio": "ftp://example.com/a.wav"},
+            ("ref_audio", "URL"),
+            id="item_ref_audio_bad_scheme",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "batch",
+            {"ref_audio": "not_a_valid_uri"},
+            ("ref_audio", "url"),
+            id="batch_ref_audio_invalid_uri",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "item",
+            {"ref_audio": "not_a_valid_uri"},
+            ("ref_audio", "url"),
+            id="item_ref_audio_invalid_uri",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "batch",
+            {"ref_text": ["x"]},
+            ("ref_text", "string_type", "valid string"),
+            id="batch_ref_text_wrong_type",
+        ),
+        pytest.param(
+            "item",
+            {"ref_text": ["x"]},
+            ("ref_text", "string_type", "valid string"),
+            id="item_ref_text_wrong_type",
+        ),
+        pytest.param(
+            "batch",
+            {"x_vector_only_mode": "wrong_type"},
+            ("x_vector_only_mode", "bool_parsing", "validation error"),
+            id="batch_x_vector_only_mode_wrong_type",
+        ),
+        pytest.param(
+            "item",
+            {"x_vector_only_mode": "wrong_type"},
+            ("x_vector_only_mode", "bool_parsing", "validation error"),
+            id="item_x_vector_only_mode_wrong_type",
+        ),
+        pytest.param(
+            "batch",
+            {"max_new_tokens": 0},
+            ("max_new_tokens", "least 1"),
+            id="batch_max_new_tokens_below_min",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "item",
+            {"max_new_tokens": 0},
+            ("max_new_tokens", "least 1"),
+            id="item_max_new_tokens_below_min",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "batch",
+            {"max_new_tokens": _SPEECH_API_MAX_NEW_TOKENS + 1},
+            ("max_new_tokens", "exceed 4096"),
+            id="batch_max_new_tokens_above_max",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "item",
+            {"max_new_tokens": _SPEECH_API_MAX_NEW_TOKENS + 1},
+            ("max_new_tokens", "exceed 4096"),
+            id="item_max_new_tokens_above_max",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "batch",
+            {"initial_codec_chunk_frames": -1},
+            ("initial_codec_chunk_frames", "greater_than_equal", "0"),
+            id="batch_initial_codec_chunk_frames_negative",
+        ),
+        pytest.param(
+            "item",
+            {"initial_codec_chunk_frames": -1},
+            ("initial_codec_chunk_frames", "greater_than_equal", "0"),
+            id="item_initial_codec_chunk_frames_negative",
+        ),
+        # Base clone: whitespace-only ref_text (same serving check as single speech).
+        pytest.param(
+            "batch",
+            {
+                "task_type": "Base",
+                "ref_audio": "data:audio/wav;base64,UklGRiQAAABXQVZFZm10IBAAAAABAAEARKwAAIhYAQACABAAZGF0YQAAAAA=",
+                "ref_text": "   ",
+            },
+            ("ref_text", "base task", "non-empty"),
+            id="batch_ref_text_whitespace_only_base_clone",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "item",
+            {
+                "task_type": "Base",
+                "ref_audio": "data:audio/wav;base64,UklGRiQAAABXQVZFZm10IBAAAAABAAEARKwAAIhYAQACABAAZGF0YQAAAAA=",
+                "ref_text": "   ",
+            },
+            ("ref_text", "base task", "non-empty"),
+            id="item_ref_text_whitespace_only_base_clone",
+            marks=_SKIP_ISSUE_3649,
+        ),
+    ],
+)
+@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
+def test_speech_batch_invalid_field_values(
+    omni_server: OmniServer,
+    openai_client: OpenAIClientHandler,
+    loc: str,
+    overrides: dict[str, object],
+    err_message: str | tuple[str, ...],
+) -> None:
+    ov = dict(overrides)
+    strip_clone = bool(ov.pop(_SPEECH_BATCH_NO_CLONE_FIELDS, False))
+    body = {
+        "model": omni_server.model,
+        "voice": "clone",
+        "ref_audio": REF_AUDIO_URL,
+        "ref_text": REF_TEXT,
+        "language": "English",
+        "response_format": "wav",
+        "items": [{"input": "Hello."}],
+    }
+    if strip_clone:
+        body.pop("ref_audio", None)
+        body.pop("ref_text", None)
+    _apply_batch_overrides(body, loc=loc, overrides=ov)
+    openai_client.send_audio_speech_batch_http_request(
+        {"json": body, "timeout": 120, "err_code": 400, "err_message": err_message}
+    )
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# WebSocket /v1/audio/speech/stream
+# ─────────────────────────────────────────────────────────────────────────────
+
+# ``test_speech_stream_invalid_requests``: build ``session.config`` JSON with ``omni_server.model``.
+_SPEECH_STREAM_WS_SESSION_STREAM_AUDIO_WAV_FRAMES = object()
+
+
+@pytest.mark.parametrize(
+    "send_frames_spec, err_message",
+    [
+        pytest.param("{not-json", "Invalid JSON", id="invalid_json"),
+        pytest.param(
+            json.dumps({"type": "input.text", "text": "hello"}),
+            "Expected session.config",
+            id="wrong_first_message_type",
+        ),
+        pytest.param(
+            _SPEECH_STREAM_WS_SESSION_STREAM_AUDIO_WAV_FRAMES,
+            ("invalid session config", "response_format", "pcm"),
+            id="session_config_stream_audio_requires_pcm",
+        ),
+    ],
+)
+@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
+def test_speech_stream_invalid_requests(
+    omni_server: OmniServer,
+    openai_client: OpenAIClientHandler,
+    send_frames_spec: Any,
+    err_message: str | tuple[str, ...],
+) -> None:
+    """Bad first WebSocket text frames: non-JSON, wrong ``type``, or invalid ``session.config``."""
+    if send_frames_spec is _SPEECH_STREAM_WS_SESSION_STREAM_AUDIO_WAV_FRAMES:
+        send_frames = json.dumps(
+            {
+                "type": "session.config",
+                "model": omni_server.model,
+                "response_format": "wav",
+                "stream_audio": True,
+            }
+        )
+    else:
+        send_frames = send_frames_spec
+    assert isinstance(send_frames, str)
+    openai_client.send_audio_speech_stream_ws_request(
+        {
+            "send_frames": send_frames,
+            "timeout": 120,
+            "ws_max_size": None,
+            "ws_json_type": "error",
+            "err_message": err_message,
+        }
+    )
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# POST /v1/audio/voices (multipart + embedding-only) and DELETE /v1/audio/voices/{name}
+# ─────────────────────────────────────────────────────────────────────────────
+
+# Templates for ``test_voices_create_invalid_requests``: resolved in ``_finalize_voices_form_data``.
+_VF_LONG_CONSENT = object()
+_VF_LONG_REF_TEXT = object()
+_VF_LONG_SPEAKER_DESC = object()
+_VF_EMBEDDING_NAN_JSON = object()
+
+
+def _finalize_voices_form_data(template: dict[str, Any], uuid_hex: str) -> dict[str, Any]:
+    out: dict[str, Any] = {}
+    for key, val in template.items():
+        if val is _VF_LONG_CONSENT:
+            out[key] = "x" * (_VOICE_UPLOAD_MAX_CONSENT_LEN + 1)
+        elif val is _VF_LONG_REF_TEXT:
+            out[key] = "y" * (_VOICE_UPLOAD_MAX_REF_TEXT_CHARS + 1)
+        elif val is _VF_LONG_SPEAKER_DESC:
+            out[key] = "z" * (_VOICE_UPLOAD_MAX_SPEAKER_DESCRIPTION_CHARS + 1)
+        elif val is _VF_EMBEDDING_NAN_JSON:
+            out[key] = json.dumps([0.1] * 1023 + [float("nan")])
+        elif isinstance(val, str):
+            out[key] = val.replace("{uuid}", uuid_hex)
+        else:
+            out[key] = val
+    return out
+
+
+def _voices_upload_multipart_files(kind: str | None) -> dict[str, Any] | None:
+    """``None`` => no multipart file (embedding-only or neither)."""
+    if kind is None:
+        return None
+    if kind == "wav_ok":
+        return {"audio_sample": ("clip.wav", _pcm_wav_mono_bytes(), "audio/wav")}
+    if kind == "wav_short":
+        return {"audio_sample": ("clip.wav", _pcm_wav_mono_bytes(duration_s=0.3), "audio/wav")}
+    if kind == "wav_bad_body":
+        return {"audio_sample": ("clip.wav", b"not valid wav content", "audio/wav")}
+    if kind == "wav_pdf_type":
+        return {"audio_sample": ("clip.wav", _pcm_wav_mono_bytes(), "application/pdf")}
+    raise AssertionError(f"unknown multipart kind {kind!r}")
+
+
+@pytest.mark.parametrize(
+    "multipart_kind, form_template, err_message",
+    [
+        pytest.param(
+            None,
+            {"consent": "c1", "name": "v1"},
+            ("audio_sample", "speaker_embedding", "provided"),
+            id="neither_audio_nor_embedding",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "wav_ok",
+            {"name": "v_miss_consent_{uuid}"},
+            ("missing", "consent", "field required"),
+            id="missing_consent",
+        ),
+        pytest.param(
+            "wav_ok",
+            {"consent": "consent_ok"},
+            ("missing", "name", "field required"),
+            id="missing_name",
+        ),
+        pytest.param(
+            "wav_ok",
+            {"consent": "", "name": "v_empty_consent_{uuid}"},
+            ("missing", "consent", "field required"),
+            id="empty_consent",
+        ),
+        pytest.param(
+            "wav_ok",
+            {"consent": "   ", "name": "v_ws_consent_{uuid}"},
+            "consent",
+            id="whitespace_consent",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "wav_ok",
+            {"consent": "bad/consent", "name": "v_bad_cons_{uuid}"},
+            "consent",
+            id="consent_path_sep",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "wav_ok",
+            {"consent": _VF_LONG_CONSENT, "name": "v_long_cons_{uuid}"},
+            ("consent", "too long", "Failed to save"),
+            id="consent_too_long",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "wav_ok",
+            {"consent": "consent_ok", "name": ""},
+            ("missing", "name", "field required"),
+            id="empty_name",
+        ),
+        pytest.param(
+            "wav_ok",
+            {"consent": "consent_ok", "name": "evil/name"},
+            ("voice name", "invalid voice name"),
+            id="invalid_name_path_sep",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "wav_ok",
+            {"consent": "consent_ok", "name": "v_long_ref_{uuid}", "ref_text": _VF_LONG_REF_TEXT},
+            "ref_text",
+            id="ref_text_too_long",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "wav_ok",
+            {
+                "consent": "consent_ok",
+                "name": "v_long_desc_{uuid}",
+                "speaker_description": _VF_LONG_SPEAKER_DESC,
+            },
+            "speaker_description",
+            id="speaker_description_too_long",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "wav_pdf_type",
+            {"consent": "consent_ok", "name": "v_bad_mime_{uuid}"},
+            ("mime", "unsupported", "audio/mp4"),
+            id="audio_unsupported_mime",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "wav_short",
+            {"consent": "consent_ok", "name": "v_short_{uuid}"},
+            ("too short", "reference audio"),
+            id="audio_too_short",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "wav_bad_body",
+            {"consent": "consent_ok", "name": "v_bad_wav_{uuid}"},
+            ("decode", "audio", "Format not recognised"),
+            id="audio_decode_error",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            None,
+            {
+                "speaker_embedding": "not valid json [[[",
+                "consent": "consent_emb",
+                "name": "v_bad_json_{uuid}",
+            },
+            ("speaker_embedding", "valid JSON"),
+            id="speaker_embedding_invalid_json",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            None,
+            {
+                "speaker_embedding": json.dumps([]),
+                "consent": "consent_emb",
+                "name": "v_empty_emb_{uuid}",
+            },
+            ("speaker_embedding", "non-empty"),
+            id="speaker_embedding_empty",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            None,
+            {
+                "speaker_embedding": _VF_EMBEDDING_NAN_JSON,
+                "consent": "consent_emb",
+                "name": "v_nan_emb_{uuid}",
+            },
+            ("speaker_embedding", "finite"),
+            id="speaker_embedding_nan",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            None,
+            {
+                "speaker_embedding": json.dumps([0.01] * 512),
+                "consent": "consent_emb",
+                "name": "v_wrong_dim_{uuid}",
+            },
+            ("expected", "dimensions", "1024"),
+            id="speaker_embedding_wrong_dim",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            "wav_ok",
+            {
+                "consent": "consent_emb",
+                "name": "v_both_{uuid}",
+                "speaker_embedding": json.dumps([0.01] * 1024),
+            },
+            ("mutually exclusive", "audio_sample", "speaker_embedding"),
+            id="audio_and_embedding_mutually_exclusive",
+            marks=_SKIP_ISSUE_3649,
+        ),
+    ],
+)
+@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
+def test_voices_create_invalid_requests(
+    omni_server: OmniServer,
+    openai_client: OpenAIClientHandler,
+    multipart_kind: str | None,
+    form_template: dict[str, Any],
+    err_message: str | tuple[str, ...],
+) -> None:
+    """Invalid ``POST /v1/audio/voices`` bodies (multipart or embedding-only)."""
+    uid = uuid.uuid4().hex[:8]
+    cfg: dict[str, Any] = {
+        "data": _finalize_voices_form_data(form_template, uid),
+        "timeout": 120,
+        "err_code": 400,
+        "err_message": err_message,
+    }
+    files = _voices_upload_multipart_files(multipart_kind)
+    if files is not None:
+        cfg["files"] = files
+    openai_client.send_audio_voices_create_http_request(cfg)
+
+
+@pytest.mark.parametrize(
+    "voice_name, err_code, err_message",
+    [
+        pytest.param(
+            "missing-voice-404",
+            404,
+            "not found",
+            id="not_found",
+        ),
+        pytest.param(
+            "   ",
+            404,
+            "not found",
+            id="whitespace_only",
+        ),
+        pytest.param(
+            "evil/name",
+            404,
+            "not found",
+            id="path_separator",
+        ),
+        pytest.param(
+            ".",
+            405,
+            ("Method Not Allowed", "detail"),
+            id="dot_single",
+        ),
+        pytest.param(
+            "..",
+            404,
+            "not found",
+            id="dot_dot",
+        ),
+        pytest.param(
+            "bad\x00voice",
+            404,
+            "not found",
+            id="nul_byte",
+        ),
+        pytest.param(
+            "",
+            405,
+            "Method Not Allowed",
+            id="empty_name_path",
+        ),
+    ],
+)
+@pytest.mark.parametrize("omni_server", _QWEN3_TTS_SPEECH, indirect=True)
+def test_voices_delete_invalid_requests(
+    omni_server: OmniServer,
+    openai_client: OpenAIClientHandler,
+    voice_name: str,
+    err_code: int | tuple[int, ...],
+    err_message: str | tuple[str, ...],
+) -> None:
+    """DELETE ``/v1/audio/voices/{name}``: missing voice (404), odd segments (often 404), ``.``paths (405)."""
+    openai_client.send_audio_voices_delete_http_request(
+        {
+            "name": voice_name,
+            "timeout": 120,
+            "err_code": err_code,
+            "err_message": err_message,
+        }
+    )

tests/dfx/reliability/invalid_param_test/test_invalid_image_editing.py

@@ -0,0 +1,90 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""``POST /v1/images/edits`` multipart validation (live ``Qwen/Qwen-Image-Edit``)."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pytest
+
+from tests.helpers.mark import hardware_marks
+from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
+
+pytestmark = [pytest.mark.slow, pytest.mark.diffusion]
+
+_PARAMS = [
+    pytest.param(
+        OmniServerParams(model="Qwen/Qwen-Image-Edit"),
+        id="qwen_image_edit",
+        marks=hardware_marks(res={"cuda": "H100"}),
+    ),
+]
+
+# Populated in ``test_images_edits_invalid_requests`` from ``omni_server.model``.
+_IMAGE_EDITS_SERVER_MODEL = object()
+
+
+def _finalize_edits_form_data(template: dict[str, Any], server_model: str) -> dict[str, Any]:
+    return {k: (server_model if v is _IMAGE_EDITS_SERVER_MODEL else v) for k, v in template.items()}
+
+
+@pytest.mark.parametrize(
+    "include_image, data_template, err_message, err_code",
+    [
+        pytest.param(
+            False,
+            {"prompt": "make it brighter", "model": _IMAGE_EDITS_SERVER_MODEL},
+            ("image", "required", "Unprocessable Entity"),
+            422,
+            id="missing_image",
+        ),
+        pytest.param(
+            True,
+            {"prompt": "edit", "model": "wrong-model-id"},
+            ("model", "mismatch"),
+            400,
+            id="model_mismatch",
+        ),
+        pytest.param(
+            True,
+            {"prompt": "edit", "response_format": "url"},
+            ("response_format", "b64_json", "supported"),
+            400,
+            id="unsupported_response_format",
+        ),
+        pytest.param(
+            True,
+            {"prompt": "edit", "model": _IMAGE_EDITS_SERVER_MODEL, "output_compression": "101"},
+            ("less_than_equal", "output_compression", "100"),
+            400,
+            id="output_compression_above_max",
+        ),
+        pytest.param(
+            True,
+            {"prompt": "edit", "model": _IMAGE_EDITS_SERVER_MODEL, "layers": "2"},
+            ("Invalid layers", "3", "1"),
+            400,
+            id="invalid_layers",
+        ),
+    ],
+)
+@pytest.mark.parametrize("omni_server", _PARAMS, indirect=True)
+def test_images_edits_invalid_requests(
+    omni_server: OmniServer,
+    openai_client: OpenAIClientHandler,
+    tiny_png_bytes: bytes,
+    include_image: bool,
+    data_template: dict[str, Any],
+    err_message: str | tuple[str, ...],
+    err_code: int | tuple[int, ...],
+) -> None:
+    cfg: dict[str, Any] = {
+        "data": _finalize_edits_form_data(data_template, omni_server.model),
+        "timeout": 300,
+        "err_code": err_code,
+        "err_message": err_message,
+    }
+    if include_image:
+        cfg["files"] = {"image": ("x.png", tiny_png_bytes, "image/png")}
+    openai_client.send_images_edits_http_request(cfg)

tests/dfx/reliability/invalid_param_test/test_invalid_image_generation.py

@@ -0,0 +1,142 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""``POST /v1/images/generations`` validation (live ``Qwen/Qwen-Image``)."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pytest
+
+from tests.helpers.mark import hardware_marks
+from tests.helpers.runtime import OmniServer, OmniServerParams, OpenAIClientHandler
+
+pytestmark = [pytest.mark.slow, pytest.mark.diffusion]
+
+_SKIP_ISSUE_3649 = pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/3649")
+
+_PARAMS = [
+    pytest.param(
+        OmniServerParams(model="Qwen/Qwen-Image"),
+        id="qwen_image",
+        marks=hardware_marks(res={"cuda": "H100"}),
+    ),
+]
+
+# Full replacement body for ``test_images_generations_invalid_requests`` (no minimal prompt/model merge).
+_IMAGES_GEN_MISSING_PROMPT_JSON = object()
+
+
+def _minimal_images_gen_json(omni_server: OmniServer) -> dict[str, object]:
+    return {"prompt": "a simple red apple icon", "model": omni_server.model}
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# POST /v1/images/generations (JSON)
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+@pytest.mark.parametrize(
+    "body_spec, err_message",
+    [
+        pytest.param(_IMAGES_GEN_MISSING_PROMPT_JSON, ("prompt", "Field required", "Missing"), id="missing_prompt"),
+        pytest.param(
+            {"prompt": "cat", "model": "not-the-loaded-checkpoint"},
+            ("model", "mismatch"),
+            id="model_mismatch",
+        ),
+        pytest.param({"prompt": ""}, ("prompt", "empty"), id="prompt_empty", marks=_SKIP_ISSUE_3649),
+        pytest.param({"prompt": "   "}, ("prompt", "empty"), id="prompt_whitespace_only", marks=_SKIP_ISSUE_3649),
+        pytest.param({"prompt": 123}, ("prompt", "string_type", "valid string"), id="prompt_wrong_type"),
+        pytest.param({"n": 0}, ("n", "greater_than_equal", "1"), id="n_below_min"),
+        pytest.param({"n": 11}, ("n", "less_than_equal", "10"), id="n_above_max"),
+        pytest.param({"size": "1024"}, ("size", "value_error", "WIDTHxHEIGHT"), id="size_missing_height_token"),
+        pytest.param({"size": "not-a-size"}, ("size", "value_error", "WIDTHxHEIGHT"), id="size_missing_separator"),
+        pytest.param({"size": "1024xabc"}, ("Invalid size format", "integers"), id="size_non_integer_height"),
+        pytest.param({"size": "1024x"}, ("Invalid size format", "integers"), id="size_incomplete_height"),
+        pytest.param({"size": "0x1024"}, ("Invalid size", "positive integers"), id="size_non_positive_width"),
+        pytest.param(
+            {"response_format": "url"},
+            ("response_format", "value_error", "b64_json"),
+            id="response_format_not_b64_json",
+        ),
+        pytest.param(
+            {"use_system_prompt": "bogus"},
+            ("use_system_prompt", "value_error", "dynamic"),
+            id="use_system_prompt_invalid",
+        ),
+        pytest.param(
+            {"num_inference_steps": 0},
+            ("num_inference_steps", "'greater_than_equal", "1"),
+            id="num_inference_steps_below_min",
+        ),
+        pytest.param(
+            {"num_inference_steps": 201},
+            ("num_inference_steps", "less_than_equal", "200"),
+            id="num_inference_steps_above_max",
+        ),
+        pytest.param(
+            {"guidance_scale": -1.0}, ("guidance_scale", "greater_than_equal", "0"), id="guidance_scale_negative"
+        ),
+        pytest.param(
+            {"guidance_scale": 21.0}, ("guidance_scale", "less_than_equal", "20"), id="guidance_scale_above_max"
+        ),
+        pytest.param(
+            {"true_cfg_scale": -1.0}, ("true_cfg_scale", "greater_than_equal", "0"), id="true_cfg_scale_negative"
+        ),
+        pytest.param(
+            {"true_cfg_scale": 21.0}, ("true_cfg_scale", "less_than_equal", "20"), id="true_cfg_scale_above_max"
+        ),
+        pytest.param({"layers": 2}, ("layers", "value_error", "3", "10"), id="layers_below_min"),
+        pytest.param({"layers": 11}, ("layers", "value_error", "3", "10"), id="layers_above_max"),
+        pytest.param({"seed": -1}, ("seed", "greater_than_equal", "0"), id="seed_negative", marks=_SKIP_ISSUE_3649),
+        pytest.param(
+            {"seed": 2**32},
+            ("seed", "less_than_equal", "4294967295"),
+            id="seed_above_uint32",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            {"output_format": "gif"},
+            ("output_format", "value_error", "b64_json"),
+            id="output_format_invalid",
+            marks=_SKIP_ISSUE_3649,
+        ),
+        pytest.param(
+            {"vae_use_slicing": "wrong_type"},
+            ("vae_use_slicing", "bool_parsing", "validation error"),
+            id="vae_use_slicing_wrong_type",
+        ),
+        pytest.param({"user": 123}, ("user", "string_type", "valid string"), id="user_wrong_type"),
+        pytest.param(
+            {"negative_prompt": ["noise"]},
+            ("negative_prompt", "string_type", "valid string"),
+            id="negative_prompt_wrong_type",
+        ),
+        pytest.param(
+            {"generator_device": 1},
+            ("generator_device", "string_type", "valid string"),
+            id="generator_device_wrong_type",
+        ),
+        pytest.param(
+            {"lora": {"foo": "bar"}}, ("lora", "both name and path", "required"), id="lora_missing_required_fields"
+        ),
+    ],
+)
+@pytest.mark.parametrize("omni_server", _PARAMS, indirect=True)
+def test_images_generations_invalid_requests(
+    omni_server: OmniServer,
+    openai_client: OpenAIClientHandler,
+    body_spec: Any,
+    err_message: str | tuple[str, ...],
+) -> None:
+    """Invalid JSON bodies for ``POST /v1/images/generations`` (missing prompt, model mismatch, bad fields)."""
+    if body_spec is _IMAGES_GEN_MISSING_PROMPT_JSON:
+        body: dict[str, object] = {"size": "1024x1024"}
+    else:
+        assert isinstance(body_spec, dict)
+        body = _minimal_images_gen_json(omni_server)
+        body.update(body_spec)
+    openai_client.send_images_generations_http_request(
+        {"json": body, "timeout": 300, "err_code": 400, "err_message": err_message}
+    )